Repository: canonical/microk8s Branch: master Commit: 5d5036d4fadf Files: 330 Total size: 1.5 MB Directory structure: gitextract_osiht2xu/ ├── .github/ │ ├── .jira_sync_config.yaml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ ├── feature_request.md │ │ └── question.yml │ ├── PULL_REQUEST_TEMPLATE.md │ ├── actions/ │ │ └── test-prep/ │ │ └── action.yaml │ ├── dependabot.yml │ └── workflows/ │ ├── backport.yml │ ├── build-installer.yml │ ├── build-snap.yml │ ├── check-formatting.yml │ ├── check-unit-tests.yml │ ├── cla-check.yml │ ├── stale-cron.yaml │ └── update-images.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── build-scripts/ │ ├── .gitignore │ ├── addons/ │ │ └── repositories.sh │ ├── build-component.sh │ ├── components/ │ │ ├── README.md │ │ ├── cluster-agent/ │ │ │ ├── build.sh │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── cni/ │ │ │ ├── build.sh │ │ │ ├── patches/ │ │ │ │ └── default/ │ │ │ │ └── 0001-single-entrypoint-for-cni-tools.patch │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── containerd/ │ │ │ ├── build.sh │ │ │ ├── patches/ │ │ │ │ ├── default/ │ │ │ │ │ └── 0001-microk8s-sideload-images-plugin.patch │ │ │ │ └── v2.1.3/ │ │ │ │ └── 0001-microk8s-sideload-images-plugin.patch │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── etcd/ │ │ │ ├── build.sh │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── flannel-cni-plugin/ │ │ │ ├── build.sh │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── flanneld/ │ │ │ ├── build.sh │ │ │ ├── patches/ │ │ │ │ └── default/ │ │ │ │ └── 0001-disable-udp-backend.patch │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── helm/ │ │ │ ├── build.sh │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── k8s-dqlite/ │ │ │ ├── build.sh │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── kubernetes/ │ │ │ ├── build.sh │ │ │ ├── patches/ │ │ │ │ ├── v1.27.0/ │ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ │ └── 0001-Unix-socket-skip-validation-in-component-status.patch │ │ │ │ ├── v1.27.4/ │ │ │ │ │ └── 0000-Kubelite-integration.patch │ │ │ │ ├── v1.28.0/ │ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ │ └── 0001-Set-log-reapply-handling-to-ignore-unchanged.patch │ │ │ │ ├── v1.31.0/ │ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ │ └── 0001-Set-log-reapply-handling-to-ignore-unchanged.patch │ │ │ │ ├── v1.32.0/ │ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ │ └── 0001-Set-log-reapply-handling-to-ignore-unchanged.patch │ │ │ │ ├── v1.33.0/ │ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ │ └── 0001-Set-log-reapply-handling-to-ignore-unchanged.patch │ │ │ │ ├── v1.34.0/ │ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ │ ├── 0001-Set-log-reapply-handling-to-ignore-unchanged.patch │ │ │ │ │ └── 0002-fix-allow-node-to-get-endpointslices.patch │ │ │ │ └── v1.35.0/ │ │ │ │ ├── 0000-Kubelite-integration.patch │ │ │ │ ├── 0001-Set-log-reapply-handling-to-ignore-unchanged.patch │ │ │ │ └── 0002-fix-allow-node-to-get-endpointslices.patch │ │ │ ├── pre-patch.sh │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── microk8s-completion/ │ │ │ ├── build.sh │ │ │ ├── patches/ │ │ │ │ └── default/ │ │ │ │ └── 0001-microk8s-autocompleter-script.patch │ │ │ ├── repository │ │ │ └── version.sh │ │ ├── python/ │ │ │ └── requirements.txt │ │ └── runc/ │ │ ├── build.sh │ │ ├── patches/ │ │ │ └── default/ │ │ │ └── 0001-Disable-static-PIE-on-arm64.patch │ │ ├── repository │ │ ├── strict-patches/ │ │ │ ├── v1.1.12/ │ │ │ │ ├── 0001-apparmor-change-profile-immediately-not-on-exec.patch │ │ │ │ ├── 0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch │ │ │ │ └── 0003-standard_init_linux-change-AppArmor-profile-as-late-.patch │ │ │ ├── v1.1.15/ │ │ │ │ ├── 0001-apparmor-change-profile-immediately-not-on-exec.patch │ │ │ │ ├── 0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch │ │ │ │ └── 0003-standard_init_linux-change-AppArmor-profile-as-late-.patch │ │ │ ├── v1.2.6/ │ │ │ │ ├── 0001-apparmor-change-profile-immediately-not-on-exec.patch │ │ │ │ ├── 0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch │ │ │ │ └── 0003-standard_init_linux-change-AppArmor-profile-as-late-.patch │ │ │ └── v1.3.0/ │ │ │ ├── 0001-apparmor-change-profile-immediately-not-on-exec.patch │ │ │ ├── 0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch │ │ │ └── 0003-standard_init_linux-change-AppArmor-profile-as-late-.patch │ │ └── version.sh │ ├── generate-bom.py │ ├── images.txt │ ├── print-patches-for.py │ └── update-images.sh ├── docs/ │ ├── build.md │ ├── community.md │ └── k8s-patches.md ├── installer/ │ ├── .gitignore │ ├── __init__.py │ ├── build-linux.sh │ ├── cli/ │ │ ├── __init__.py │ │ ├── echo.py │ │ └── microk8s.py │ ├── common/ │ │ ├── __init__.py │ │ ├── auxiliary.py │ │ ├── definitions.py │ │ ├── errors.py │ │ └── file_utils.py │ ├── microk8s.py │ ├── microk8s.spec │ ├── requirements.txt │ ├── setup.cfg │ ├── setup.py │ ├── tests/ │ │ ├── __init__.py │ │ ├── integration/ │ │ │ └── test_cli.py │ │ └── unit/ │ │ ├── test_auxiliary.py │ │ └── test_cli.py │ ├── vm_providers/ │ │ ├── __init__.py │ │ ├── _base_provider.py │ │ ├── _multipass/ │ │ │ ├── __init__.py │ │ │ ├── _instance_info.py │ │ │ ├── _multipass.py │ │ │ ├── _multipass_command.py │ │ │ └── _windows.py │ │ ├── errors.py │ │ ├── factory.py │ │ └── repo/ │ │ ├── __init__.py │ │ ├── errors.py │ │ └── snaps.py │ └── windows/ │ ├── README.md │ └── microk8s.nsi ├── microk8s-resources/ │ ├── actions/ │ │ └── common/ │ │ └── utils.sh │ ├── basic_auth.csv │ ├── certs/ │ │ ├── csr-dqlite.conf.template │ │ └── csr.conf.template │ ├── client-x509.config.template │ ├── client.config │ ├── client.config.template │ ├── containerd-profile │ ├── default-args/ │ │ ├── admission-control-config-file.yaml │ │ ├── apiserver-proxy │ │ ├── certs.d/ │ │ │ ├── docker.io/ │ │ │ │ └── hosts.toml │ │ │ └── localhost__32000/ │ │ │ └── hosts.toml │ │ ├── cluster-agent │ │ ├── cni-env │ │ ├── cni-network/ │ │ │ └── flannel.conflist │ │ ├── containerd │ │ ├── containerd-env │ │ ├── containerd-template.toml │ │ ├── ctr │ │ ├── etcd │ │ ├── eventconfig.yaml │ │ ├── flannel-network-mgr-config │ │ ├── flannel-template.conflist │ │ ├── flanneld │ │ ├── git/ │ │ │ └── .gitconfig │ │ ├── ha-conf │ │ ├── k8s-dqlite │ │ ├── k8s-dqlite-env │ │ ├── kube-apiserver │ │ ├── kube-controller-manager │ │ ├── kube-proxy │ │ ├── kube-scheduler │ │ ├── kubectl │ │ ├── kubectl-env │ │ ├── kubelet │ │ ├── kubelite │ │ └── traefik/ │ │ ├── provider-template.yaml │ │ └── traefik-template.yaml │ ├── default-hooks/ │ │ ├── post-refresh.d/ │ │ │ └── 30-helm │ │ ├── reconcile.d/ │ │ │ ├── 10-pods-restart │ │ │ └── 90-calico-apply │ │ └── remove.d/ │ │ ├── 10-cni-link │ │ ├── 10-cni-link-cilium │ │ ├── 20-cni-netns │ │ └── 90-containers │ ├── kubelet.config │ ├── kubelet.config.template │ ├── kubeproxy.config │ ├── microk8s.default.yaml │ └── wrappers/ │ ├── apiservice-kicker │ ├── git.wrapper │ ├── microk8s-add-node.wrapper │ ├── microk8s-addons.wrapper │ ├── microk8s-config.wrapper │ ├── microk8s-ctr.wrapper │ ├── microk8s-dashboard-proxy.wrapper │ ├── microk8s-dbctl.wrapper │ ├── microk8s-disable.wrapper │ ├── microk8s-enable.wrapper │ ├── microk8s-helm.wrapper │ ├── microk8s-helm3.wrapper │ ├── microk8s-images.wrapper │ ├── microk8s-istioctl.wrapper │ ├── microk8s-join.wrapper │ ├── microk8s-kubectl.wrapper │ ├── microk8s-leave.wrapper │ ├── microk8s-linkerd.wrapper │ ├── microk8s-refresh-certs.wrapper │ ├── microk8s-remove-node.wrapper │ ├── microk8s-reset.wrapper │ ├── microk8s-start.wrapper │ ├── microk8s-status.wrapper │ ├── microk8s-stop.wrapper │ ├── microk8s-version.wrapper │ ├── microk8s.wrapper │ ├── openssl.wrapper │ ├── run-apiserver-proxy-with-args │ ├── run-cluster-agent-with-args │ ├── run-containerd-with-args │ ├── run-etcd-with-args │ ├── run-flanneld-with-args │ ├── run-k8s-dqlite-with-args │ └── run-kubelite-with-args ├── pyproject.toml ├── scripts/ │ ├── calico/ │ │ └── upgrade.py │ ├── find-resolv-conf.py │ ├── generate-cni.sh │ ├── inspect.sh │ ├── kill-host-pods.py │ ├── run-lifecycle-hooks.py │ └── wrappers/ │ ├── add_token.py │ ├── addons.py │ ├── common/ │ │ ├── __init__.py │ │ ├── cluster/ │ │ │ ├── __init__.py │ │ │ └── utils.py │ │ └── utils.py │ ├── dashboard_proxy.py │ ├── dbctl.py │ ├── disable.py │ ├── distributed_op.py │ ├── enable.py │ ├── images.py │ ├── join.py │ ├── leave.py │ ├── refresh_certs.py │ ├── remove_node.py │ ├── reset.py │ ├── status.py │ ├── upgrade.py │ └── version.py ├── snap/ │ ├── hooks/ │ │ ├── configure │ │ ├── connect-plug-network-control │ │ ├── disconnect-plug-network-control │ │ ├── install │ │ ├── post-refresh │ │ └── remove │ └── snapcraft.yaml ├── tests/ │ ├── libs/ │ │ ├── addons-upgrade.sh │ │ ├── addons.sh │ │ ├── airgap.sh │ │ ├── clustering.sh │ │ ├── spread.sh │ │ ├── upgrade-path.sh │ │ └── utils.sh │ ├── lxc/ │ │ ├── install-deps/ │ │ │ ├── images_almalinux-8 │ │ │ ├── images_archlinux │ │ │ ├── images_centos-7 │ │ │ ├── images_centos-8-Stream │ │ │ ├── images_debian-10 │ │ │ ├── images_debian-11 │ │ │ ├── images_debian-12 │ │ │ ├── images_fedora-37 │ │ │ ├── images_fedora-38 │ │ │ ├── images_rockylinux-8 │ │ │ ├── ubuntu_16.04 │ │ │ ├── ubuntu_18.04 │ │ │ ├── ubuntu_20.04 │ │ │ └── ubuntu_22.04 │ │ ├── microk8s-zfs.profile │ │ └── microk8s.profile │ ├── requirements.txt │ ├── smoke-test.sh │ ├── templates/ │ │ ├── bbox-local.yaml │ │ ├── dual-stack.yaml │ │ ├── ingress.yaml │ │ ├── nginx-pod.yaml │ │ ├── pvc.yaml │ │ ├── registry-sc.yaml │ │ └── simple-deploy.yaml │ ├── test-cluster-agent.py │ ├── test-cluster.py │ ├── test-distro.sh │ ├── test-simple.py │ ├── test-upgrade-path.py │ ├── test-upgrade.py │ ├── unit/ │ │ ├── cluster/ │ │ │ ├── test_join.py │ │ │ └── test_leave.py │ │ ├── test_addons.py │ │ ├── test_addtoken.py │ │ ├── test_dashboard_proxy.py │ │ ├── test_disable.py │ │ ├── test_enable.py │ │ ├── test_refresh_certs.py │ │ ├── test_reset.py │ │ ├── test_upgrade_calico_cni.py │ │ ├── test_version.py │ │ └── yamls/ │ │ ├── calico-new.yaml │ │ ├── cni.yaml │ │ └── invalid.yaml │ ├── utils.py │ ├── validators.py │ └── verify-branches.py ├── tox.ini └── upgrade-scripts/ ├── 000-switch-to-calico/ │ ├── commit-master.sh │ ├── commit-node.sh │ ├── description.txt │ ├── prepare-master.sh │ ├── prepare-node.sh │ ├── resources/ │ │ └── calico.yaml │ ├── rollback-master.sh │ └── rollback-node.sh ├── 001-switch-to-dqlite/ │ ├── commit-master.sh │ ├── commit-node.sh │ ├── description.txt │ ├── prepare-master.sh │ ├── prepare-node.sh │ ├── rollback-master.sh │ └── rollback-node.sh └── 002-switch-to-flannel-etcd/ ├── commit-master.sh ├── commit-node.sh ├── description.txt ├── prepare-master.sh ├── prepare-node.sh ├── rollback-master.sh └── rollback-node.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/.jira_sync_config.yaml ================================================ settings: # Jira project key to create the issue in jira_project_key: "KU" # Dictionary mapping GitHub issue status to Jira issue status status_mapping: opened: Untriaged closed: done # (Optional) Jira project components that should be attached to the created issue # Component names are case-sensitive components: - Microk8s snap # (Optional) GitHub labels. Only issues with one of those labels will be synchronized. # If not specified, all issues will be synchronized # labels: [] # (Optional) (Default: false) Add a new comment in GitHub with a link to Jira created issue add_gh_comment: false # (Optional) (Default: true) Synchronize issue description from GitHub to Jira sync_description: true # (Optional) (Default: true) Synchronize comments from GitHub to Jira sync_comments: true # (Optional) (Default: None) Parent Epic key to link the issue to epic_key: "KU-925" # (Optional) Dictionary mapping GitHub issue labels to Jira issue types. # If label on the issue is not in specified list, this issue will be created as a Bug label_mapping: enhancement: Story ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug Report about: Something is not working --- #### Summary #### What Should Happen Instead? #### Reproduction Steps 1. ... 2. ... #### Introspection Report #### Can you suggest a fix? #### Are you interested in contributing with a fix? ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature Request about: Suggest a new feature --- #### Summary #### Why is this important? #### Are you interested in contributing to this feature? ================================================ FILE: .github/ISSUE_TEMPLATE/question.yml ================================================ contact_links: - name: Ask a question url: https://kubernetes.slack.com/archives/CAUNWQ85V about: "For discussions and/or other questions related to MicroK8s, please use the #microk8s channel on the Kubernetes Slack" ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ #### Summary #### Changes #### Testing #### Possible Regressions #### Checklist * [ ] Read the [contributions](https://github.com/canonical/microk8s/blob/master/CONTRIBUTING.md) page. * [ ] Submitted the [CLA form](https://ubuntu.com/legal/contributors/agreement), if you are a first time contributor. * [ ] The introduced changes are covered by unit and/or integration tests. #### Notes ================================================ FILE: .github/actions/test-prep/action.yaml ================================================ name: Prepare test prerequisites runs: using: "composite" steps: - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install test dependencies shell: bash run: | set -x sudo pip3 install -r ./tests/requirements.txt # Docker sets iptables rules that interfere with LXD and K8s. # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker - name: Apply Docker iptables workaround shell: bash run: sudo iptables -I DOCKER-USER -j ACCEPT - name: Fetch snap uses: actions/download-artifact@v4 with: name: microk8s.snap path: build ================================================ FILE: .github/dependabot.yml ================================================ # Set update schedule for GitHub Actions version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: # Check for updates to GitHub Actions every weekday interval: "daily" ================================================ FILE: .github/workflows/backport.yml ================================================ name: Backport merged pull request on: pull_request_target: types: [closed] issue_comment: types: [created] permissions: contents: write # so it can comment pull-requests: write # so it can create pull requests jobs: backport: name: Backport pull request runs-on: ubuntu-latest # Only run when pull request is merged # or when a comment containing `/backport` is created by someone other than the # https://github.com/backport-action bot user (user id: 97796249). Note that if you use your # own PAT as `github_token`, that you should replace this id with yours. # cdkbot's user ID is 99445902. if: > ( github.event_name == 'pull_request_target' && github.event.pull_request.merged ) || ( github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.user.id != 99445902 && contains(github.event.comment.body, '/backport') ) steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Create backport pull requests uses: korthout/backport-action@v3 with: # Set (default) action parameters explicitly. branch_name: backport-${pull_number}-to-${target_branch} cherry_picking: auto copy_assignees: false copy_milestone: false copy_requested_reviewers: false experimental: > { "conflict_resolution": "fail" } github_token: ${{ secrets.BOT_TOKEN }} github_workspace: ${{ github.workspace }} label_pattern: ^backport ([^ ]+)$ merge_commits: fail pull_description: |- # Description Backport of #${pull_number} to `${target_branch}`. pull_title: >- [Backport ${target_branch}] ${pull_title} ================================================ FILE: .github/workflows/build-installer.yml ================================================ name: Build MicroK8s Installers on: push: branches: - "**install**" pull_request: branches: - "**install**" jobs: windows: runs-on: windows-latest defaults: run: working-directory: ${{ github.workspace }}/installer/windows steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python 3.8 uses: actions/setup-python@v5.5.0 with: python-version: 3.8 - name: Install Python requirements run: python -m pip install -r ../requirements.txt - name: Build exe working-directory: ${{ github.workspace }}/installer run: pyinstaller.exe ./microk8s.spec - name: Move exe to installer build directory working-directory: ${{ github.workspace }}/installer run: move microk8s.exe ./windows/microk8s.exe - name: Download EnVar plugin for NSIS uses: carlosperate/download-file-action@v2.0.2 with: file-url: https://github.com/GsNSIS/EnVar/releases/download/v0.3.1/EnVar-Plugin.zip file-name: envar_plugin.zip location: ${{ github.workspace }} - name: Extract EnVar plugin run: 7z x -o"C:/Program Files (x86)/NSIS" "${{ github.workspace }}/envar_plugin.zip" - name: Download Multipass installer uses: carlosperate/download-file-action@v2.0.2 with: file-url: https://github.com/canonical/multipass/releases/download/v1.12.2/multipass-1.12.2+win-win64.exe file-name: multipass.exe location: ${{ github.workspace }}/installer/windows - name: Download kubectl uses: carlosperate/download-file-action@v2.0.2 with: file-url: https://storage.googleapis.com/kubernetes-release/release/v1.28.3/bin/windows/amd64/kubectl.exe file-name: kubectl.exe location: ${{ github.workspace }}/installer/windows - name: Create installer run: makensis.exe ${{ github.workspace }}/installer/windows/microk8s.nsi - name: Upload installer uses: actions/upload-artifact@v4 with: name: Windows installer path: ${{ github.workspace }}/installer/windows/microk8s-installer.exe ================================================ FILE: .github/workflows/build-snap.yml ================================================ name: Build and test MicroK8s snap on: pull_request: branches: - master jobs: build: name: Create snap package runs-on: ubuntu-latest steps: - name: Checking out repo uses: actions/checkout@v4 - name: Install lxd run: | sudo lxd init --auto sudo usermod --append --groups lxd $USER # `newgrp` does not work in GitHub Actions; use `sudo --user` instead # See https://github.com/actions/runner-images/issues/9932#issuecomment-2573170305 sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc version # Docker sets iptables rules that interfere with LXD or K8s. # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker - name: Apply Docker iptables workaround shell: bash run: sudo iptables -I DOCKER-USER -j ACCEPT - name: Install snapcraft run: | sudo snap install snapcraft --classic - name: Install snapd from candidate run: | # TODO(neoaggelos): revert this after latest/beta is working again sudo snap refresh snapd --channel=latest/stable - name: Build snap run: | sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- snapcraft --use-lxd sudo mv microk8s*.snap microk8s.snap - name: Uploading snap uses: actions/upload-artifact@v4 with: name: microk8s.snap path: microk8s.snap test-upgrade: name: Upgrade path test runs-on: ubuntu-latest needs: build timeout-minutes: 30 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Prepare test prerequisites uses: ./.github/actions/test-prep - name: Running upgrade path test run: | sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade-path.py test-addons-core: name: Test core addons runs-on: ubuntu-latest needs: build timeout-minutes: 60 env: # Avoid truncated "ps" output COLUMNS: 2048 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Prepare test prerequisites uses: ./.github/actions/test-prep - name: Running addons tests env: UNDER_TIME_PRESSURE: ${{ !contains(github.event.pull_request.labels.*.name, 'run-all-tests') }} run: | set -x sudo snap install build/microk8s.snap --classic --dangerous ./tests/smoke-test.sh # The GitHub runner is using the 10.1.0.0/16 CIDR, which would conflict with # kube-ovn's default POD_CIDR. They have to be different. export POD_CIDR="10.200.0.0/16" export POD_GATEWAY="10.200.0.1" export SKIP_PROMETHEUS="False" export UNDER_TIME_PRESSURE=${UNDER_TIME_PRESSURE@u} sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py" test-addons-community: name: Test community addons runs-on: ubuntu-latest needs: build timeout-minutes: 60 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Prepare test prerequisites uses: ./.github/actions/test-prep - name: Running addons tests env: UNDER_TIME_PRESSURE: ${{ !contains(github.event.pull_request.labels.*.name, 'run-all-tests') }} run: | set -x sudo snap install build/microk8s.snap --classic --dangerous sudo microk8s enable community export UNDER_TIME_PRESSURE=${UNDER_TIME_PRESSURE@u} sudo -E bash -c "cd /var/snap/microk8s/common/addons/community/; pytest -s -ra ./tests/" test-addons-core-upgrade: name: Test core addons upgrade runs-on: ubuntu-latest needs: build timeout-minutes: 30 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Prepare test prerequisites uses: ./.github/actions/test-prep - name: Running upgrade tests env: UNDER_TIME_PRESSURE: ${{ !contains(github.event.pull_request.labels.*.name, 'run-all-tests') }} run: | set -x export UNDER_TIME_PRESSURE=${UNDER_TIME_PRESSURE@u} sudo -E bash -c "UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade.py" test-cluster-agent: name: Cluster agent health check runs-on: ubuntu-latest needs: build timeout-minutes: 30 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Prepare test prerequisites uses: ./.github/actions/test-prep - name: Running cluster agent health check run: | set -x sudo snap install build/microk8s.snap --classic --dangerous sudo -E bash -c "pytest -s ./tests/test-cluster-agent.py" test-airgap: name: Test airgap installation runs-on: ubuntu-latest needs: build timeout-minutes: 30 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Prepare test prerequisites uses: ./.github/actions/test-prep - name: Initialize LXD run: | sudo lxd init --auto sudo lxc network set lxdbr0 ipv6.address=none sudo usermod --append --groups lxd $USER # `newgrp` does not work in GitHub Actions; use `sudo --user` instead # See https://github.com/actions/runner-images/issues/9932#issuecomment-2573170305 sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc version - name: Run airgap tests run: | sudo -E bash -x -c "./tests/libs/airgap.sh --distro ubuntu:22.04 --channel $PWD/build/microk8s.snap" security-scan: name: Security scan outputs: sarif_files: ${{ steps.get_sarif_files.outputs.sarif-files }} runs-on: ubuntu-latest needs: build timeout-minutes: 30 steps: - name: Checking out repo uses: actions/checkout@v4 - name: Fetch snap uses: actions/download-artifact@v4 with: name: microk8s.snap path: build - name: Create sarifs directory run: | mkdir -p sarifs - name: Install Trivy vulnerability scanner uses: aquasecurity/setup-trivy@v0.2.2 - name: Run Trivy vulnerability scanner on codebase run: | trivy fs . --format sarif --severity CRITICAL > sarifs/trivy-microk8s-repo-scan--results.sarif - name: Run Trivy vulnerability scanner on images run: | for i in $(cat ./build-scripts/images.txt) ; do name=$(echo $i | awk -F ':|/' '{print $(NF-1)}') trivy image $i --format sarif > sarifs/$name.sarif done - name: Run Trivy vulnerability scanner on the snap run: | cp build/microk8s.snap . unsquashfs microk8s.snap trivy rootfs ./squashfs-root/ --format sarif > sarifs/snap.sarif - name: Generate list of SARIF files id: get_sarif_files run: | sarif_files=$(find sarifs -name "*.sarif" -printf "%P\n" | jq -R -s -c 'split("\n") | map(select(length > 0))') echo "sarif-files=$sarif_files" >> "$GITHUB_OUTPUT" - name: Upload SARIF files artifact uses: actions/upload-artifact@v4 with: name: sarifs path: sarifs retention-days: 1 upload_sarifs_matrix: needs: security-scan runs-on: ubuntu-latest strategy: fail-fast: true matrix: sarif_file_path: ${{ fromJson(needs.security-scan.outputs.sarif_files) }} steps: - name: Checkout repository uses: actions/checkout@v4 - name: Download SARIF files artifact uses: actions/download-artifact@v4 with: name: sarifs path: sarifs - name: Prepare SARIF category id: prepare_category run: | sarif_file="${{ matrix.sarif_file_path }}" base_name=$(basename "$sarif_file" .sarif) echo "category=$base_name" >> "$GITHUB_OUTPUT" - name: Upload SARIF file uses: github/codeql-action/upload-sarif@v3 with: sarif_file: sarifs/${{ matrix.sarif_file_path }} category: ${{ steps.prepare_category.outputs.category }} ================================================ FILE: .github/workflows/check-formatting.yml ================================================ name: Lint Code on: - pull_request jobs: check-formatting: name: Check Formatting runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt-get update sudo apt-get install tox --fix-missing sudo snap install node --classic sudo npm install --save-dev --save-exact -g prettier - name: Check Python formatting run: | tox -e lint - name: Check YAML formatting run: | set -eux prettier --check $(find . -name "*.yaml" -o -name "*.yml" | \ grep -v "./microk8s-resources/actions/ingress.yaml" | \ grep -v "./microk8s-resources/actions/metallb.yaml" | \ grep -v invalid.yaml | \ grep -v calico) ================================================ FILE: .github/workflows/check-unit-tests.yml ================================================ name: Unit Tests on: - pull_request jobs: check-unit-tests: name: Check Unit Tests runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt-get update sudo apt-get install tox --fix-missing sudo pip3 install -U pytest==7.1.3 sh==1.14.3 - name: Check Units run: | tox -e scripts tox -e wrappers tox -e cluster - name: Verify branches run: | pytest -s ./tests/verify-branches.py ================================================ FILE: .github/workflows/cla-check.yml ================================================ name: cla-check on: pull_request: branches: [master, default] jobs: cla-check: runs-on: ubuntu-latest steps: - name: Check if CLA signed uses: canonical/has-signed-canonical-cla@v2 ================================================ FILE: .github/workflows/stale-cron.yaml ================================================ name: Close inactive issues or PRs on: schedule: - cron: "0 0 * * *" # Runs every midnight pull_request: paths: - .github/workflows/stale-cron.yaml jobs: close-issues: runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: actions/stale@v9 with: days-before-stale: 330 days-before-close: 30 stale-issue-label: inactive stale-issue-message: > This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions. close-issue-message: > This issue was closed because it has been inactive for 30 days since being marked as stale. exempt-pr-labels: pinned,security exempt-issue-labels: pinned,security repo-token: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/update-images.yml ================================================ name: Update list of images on: workflow_dispatch: schedule: - cron: "0 10 * * *" permissions: contents: write pull-requests: write jobs: update: runs-on: ubuntu-latest strategy: matrix: include: # Latest branches - { branch: master, channel: latest/edge } # Stable branches - { branch: "1.32", channel: "1.32" } - { branch: "1.31", channel: "1.31" } - { branch: "1.30", channel: "1.30" } - { branch: "1.29", channel: "1.29" } - { branch: "1.28", channel: "1.28" } - { branch: "1.27", channel: "1.27" } # Stable strict branches - { branch: 1.32-strict, channel: 1.32-strict } - { branch: 1.31-strict, channel: 1.31-strict } - { branch: 1.30-strict, channel: 1.30-strict } - { branch: 1.29-strict, channel: 1.29-strict } - { branch: 1.28-strict, channel: 1.28-strict } - { branch: 1.27-strict, channel: 1.27-strict } steps: - name: Checkout uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - name: Update image list run: | ./build-scripts/update-images.sh ${{ matrix.channel }} build-scripts/images.txt - name: Create pull request uses: peter-evans/create-pull-request@v7 with: commit-message: update list of images used by ${{ matrix.channel }} title: "[${{ matrix.channel }}] Update MicroK8s images" body: update list of images used by ${{ matrix.channel }} reviewers: berkayoz,ktsakalozos branch: auto-update-images/${{ matrix.branch }} delete-branch: true base: ${{ matrix.branch }} ================================================ FILE: .gitignore ================================================ /build/ /parts/ /prime/ /stage/ !/snap/hooks/ *.snap microk8s.egg-info/ microk8s_source.tar.bz2 tests/__pycache__ .idea/ **/tests/*.pyc .venv/ .vscode/ /installer/.venv/ /installer/**/__pycache__ /installer/dist/ /installer/build/ .tox/ .tox_env/ __pycache__/ microk8s_*.txt #Remote build log ================================================ FILE: CODE_OF_CONDUCT.md ================================================ MicroK8s has adopted the [Ubuntu Code of Conduct v2.0](https://ubuntu.com/community/ethos/code-of-conduct) ================================================ FILE: CONTRIBUTING.md ================================================ # Contributor Guide MicroK8s is open source ([Apache License 2.0](./LICENSE)) and actively seeks any community contributions for code, add-ons, suggestions and documentation. Many of the features currently part of MicroK8s originated in the community, and we are very keen for that to continue. This page details a few notes, workflows and suggestions for how to make contributions most effective and help us all build a better MicroK8s for everyone - please give them a read before working on any contributions. ## Licensing MicroK8s has been created under the [Apache License 2.0](./LICENSE), which will cover any contributions you may make to this project. Please familiarise yourself with the terms of the license. Additionally, MicroK8s uses the Harmony CLA agreement. It’s the easiest way for you to give us permission to use your contributions. In effect, you’re giving us a licence, but you still own the copyright — so you retain the right to modify your code and use it in other projects. Please [sign the CLA here](https://ubuntu.com/legal/contributors/agreement) before making any contributions. ## Code of conduct MicroK8s has adopted the Ubuntu code of Conduct. You can read this in full [here](https://ubuntu.com/community/code-of-conduct). ## Contributing code The workflow for contributing code is as follows: 1. **Create/choose an issue**: MicroK8s tracks issues at [https://github.com/canonical/microk8s/issues](https://github.com/canonical/microk8s/issues). If you want to work on a new feature, create an issue first! This gives everyone a place to discuss scope and implementation. 2. **Create a fork of the MicroK8s repo** 3. **Make a new branch for your contribution**. Write your code there. 4. For details on how to **build and test MicroK8s**, see the [build instructions](./docs/build.md). Add new tests as needed, and make sure the existing tests continue to pass when your changes are complete. 5. **Submit a pull request** to get changes from your branch into master. You can add "#Fixes xxx" where `xxx` is the issue number to automatically link to the issue you chose or created earlier. 6. Someone will review your PR and may make suggestions or have comments, so **keep an eye on the PR status** in case there are changes to make 7. **Please make sure you have submitted your [CLA form](https://ubuntu.com/legal/contributors/agreement) if you are a first time contributor**. 8. Thanks! ## Documentation Docs for MicroK8s are published online at [https://microk8s.io/docs](https://microk8s.io/docs). You can make suggestions and edit the pages themselves by joining the Kubernetes discourse at [discuss.kubernetes.io](https://discuss.kubernetes.io/t/introduction-to-microk8s/11243) or follow the link at the bottom of any of the pages published at [https://microk8s.io/docs](https://microk8s.io/docs) There is a documentation page which describes how to write and edit docs, [published as part of the documentation](https://microk8s.io/docs/docs). ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2018 Canonical, Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ [![](https://github.com/canonical/microk8s/actions/workflows/build-snap.yml/badge.svg)](https://github.com/canonical/microk8s/actions/workflows/build-snap.yml) [![](https://snapcraft.io/microk8s/badge.svg)](https://snapcraft.io/microk8s) ## The smallest, fastest Kubernetes Single-package fully conformant lightweight Kubernetes that works on [42 flavours of Linux](https://snapcraft.io/microk8s). Perfect for: - Developer workstations - IoT - Edge - CI/CD > Canonical might have assembled the easiest way to provision a single node Kubernetes cluster - [Kelsey Hightower](https://twitter.com/kelseyhightower/status/1120834594138406912) ## Why MicroK8s? - **Small**. Developers want the smallest K8s for laptop and workstation development. MicroK8s provides a standalone K8s compatible with Azure AKS, Amazon EKS, Google GKE when you run it on Ubuntu. - **Simple**. Minimize administration and operations with a single-package install that has no moving parts for simplicity and certainty. All dependencies and batteries included. - **Secure**. Updates are available for all security issues and can be applied immediately or scheduled to suit your maintenance cycle. - **Current**. MicroK8s tracks upstream and releases beta, RC and final bits the same day as upstream K8s. You can track latest K8s or stick to any release version from 1.10 onwards. - **Comprehensive**. MicroK8s includes a curated collection of manifests for common K8s capabilities and services: - Service Mesh: Istio, Linkerd - Serverless: Knative - Monitoring: Fluentd, Prometheus, Grafana, Metrics - Ingress, DNS, Dashboard, Clustering - Automatic updates to the latest Kubernetes version - GPGPU bindings for AI/ML Drop us a line at [MicroK8s in the Wild](docs/community.md) if you are doing something fun with MicroK8s! ## Quickstart Install MicroK8s with: ``` snap install microk8s --classic ``` MicroK8s includes a `microk8s kubectl` command: ``` sudo microk8s kubectl get nodes sudo microk8s kubectl get services ``` To use MicroK8s with your existing kubectl: ``` sudo microk8s kubectl config view --raw > $HOME/.kube/config ``` #### User access without sudo The *microk8s* user group is created during the snap installation. Users in that group are granted access to `microk8s` commands. To add a user to that group: ``` sudo usermod -a -G microk8s ``` #### Kubernetes add-ons MicroK8s installs a barebones upstream Kubernetes. Additional services like dns and the Kubernetes dashboard can be enabled using the `microk8s enable` command. ``` sudo microk8s enable dns sudo microk8s enable dashboard ``` Use `microk8s status` to see a list of enabled and available addons. You can find the addon manifests and/or scripts under `${SNAP}/actions/`, with `${SNAP}` pointing by default to `/snap/microk8s/current`. ## Documentation The [official docs](https://microk8s.io/docs/) are maintained in the Kubernetes upstream Discourse. Take a look at the [build instructions](docs/build.md) if you want to contribute to MicroK8s. Get it from the Snap Store ================================================ FILE: build-scripts/.gitignore ================================================ .build .install ================================================ FILE: build-scripts/addons/repositories.sh ================================================ #!/bin/bash -x # List of addon repositories to bundle in the snap # (name),(repository),(reference) ADDONS_REPOS=" core,https://github.com/canonical/microk8s-core-addons,main community,https://github.com/canonical/microk8s-community-addons,main " # List of addon repositories to automatically enable ADDONS_REPOS_ENABLED="core" INSTALL="${1}" if [ -d "${INSTALL}/addons" ]; then rm -rf "${INSTALL}/addons" fi if [ -d addons ]; then rm -rf addons fi IFS=';' echo "${ADDONS_REPOS}" | while read line; do if [ -z "${line}" ]; then continue fi name="$(echo ${line} | cut -f1 -d',')" repository="$(echo ${line} | cut -f2 -d',')" reference="$(echo ${line} | cut -f3 -d',')" git clone "${repository}" -b "${reference}" "addons/${name}" done echo "${ADDONS_REPOS_ENABLED}" > addons/.auto-add cp -r "addons" "${INSTALL}/addons" ================================================ FILE: build-scripts/build-component.sh ================================================ #!/bin/bash set -ex DIR=`realpath $(dirname "${0}")` BUILD_DIRECTORY="${SNAPCRAFT_PART_BUILD:-${DIR}/.build}" INSTALL_DIRECTORY="${SNAPCRAFT_PART_INSTALL:-${DIR}/.install}" mkdir -p "${BUILD_DIRECTORY}" "${INSTALL_DIRECTORY}" COMPONENT_NAME="${1}" COMPONENT_DIRECTORY="${DIR}/components/${COMPONENT_NAME}" GIT_REPOSITORY="$(cat "${COMPONENT_DIRECTORY}/repository")" GIT_TAG="$("${COMPONENT_DIRECTORY}/version.sh")" COMPONENT_BUILD_DIRECTORY="${BUILD_DIRECTORY}/${COMPONENT_NAME}" # cleanup git repository if we cannot git checkout to the build tag if [ -d "${COMPONENT_BUILD_DIRECTORY}" ]; then cd "${COMPONENT_BUILD_DIRECTORY}" if ! git checkout "${GIT_TAG}"; then cd "${BUILD_DIRECTORY}" rm -rf "${COMPONENT_BUILD_DIRECTORY}" fi fi if [ ! -d "${COMPONENT_BUILD_DIRECTORY}" ]; then git clone "${GIT_REPOSITORY}" --depth 1 -b "${GIT_TAG}" "${COMPONENT_BUILD_DIRECTORY}" fi cd "${COMPONENT_BUILD_DIRECTORY}" git config user.name "MicroK8s builder bot" git config user.email "microk8s-builder-bot@canonical.com" if [ -e "${COMPONENT_DIRECTORY}/pre-patch.sh" ]; then bash -xe "${COMPONENT_DIRECTORY}/pre-patch.sh" fi for patch in $(python3 "${DIR}/print-patches-for.py" "${COMPONENT_NAME}" "${GIT_TAG}"); do git am "${patch}" done bash -xe "${COMPONENT_DIRECTORY}/build.sh" "${INSTALL_DIRECTORY}" "${GIT_TAG}" ================================================ FILE: build-scripts/components/README.md ================================================ # Parts directory This directory contains the build scripts for Go components built into MicroK8s. The directory structure looks like this: ``` build-scripts/ build-component.sh <-- runs as `build-component.sh $component_name` - checks out the git repository - runs the `pre-patch.sh` script (if any) - applies the patches (if any) - runs the `build.sh` script to build the component component/ $component_name/ repository <-- git repository to clone version.sh <-- prints the repository tag or commit to checkout build.sh <-- runs as `build.sh $output $version` first argument is the output directory where binaries should be placed, second is the component version pre-patch.sh <-- runs as `pre-patch.sh`. takes any action needed before applying the component patches patches/ <-- list of patches to apply after checkout (see section below) ... strict-patches/ <-- list of extra patches to apply when building strictly confined snap ... ``` ## Applying patches Most MicroK8s components are retrieved from an upstream source (specified in the `repository`), with a specific tag (specified in `version.sh`), have some patches applied to them (from the `patches/` and `strict-patches/` directories) and are then built (using `build.sh`). This section explains the directory format for the `patches` and `strict-patches` directories. The same rules apply for both. Note that the `strict-patches` (if any) are applied **after** any `patches` have been applied. Our patches do not frequently change between versions, but they do have to be rebased from time to time, which breaks compatibility with older versions. For that reason, we maintain a set of patches for each version that introduces a breaking change. Consider the following directory structure for the Kubernetes component. ``` patches/default/0.patch patches/v1.27.0/a.patch patches/v1.27.0/b.patch patches/v1.27.4/c.patch patches/v1.28.0/d.patch patches/v1.28.0-beta.0/e.patch ``` The Kubernetes version to build may be decided dynamically while building the snap, or be pinned to a specified version. The following table shows which patches we would apply depending on the Kubernetes version that we build: | Kubernetes version | Applied patches | Explanation | | ------------------ | ----------------------- | ------------------------------------------------------------------------------------------ | | `v1.27.0` | `a.patch` and `b.patch` | | | `v1.27.1` | `a.patch` and `b.patch` | In case there is no exact match, find the most recent older version | | `v1.27.4` | `c.patch` | Older patches are not applied | | `v1.27.12` | `c.patch` | In semver, `v1.27.12 > v1.27.4` so we again must get the most recent patches | | `v1.28.0-rc.0` | `d.patch` | Extra items from semver are ignored, so we can define the `v1.28.0` patch and be done | | `v1.28.0-beta.0` | `e.patch` | Extra items from semver are ignored, but due to exact match this patch is used instead | | `v1.28.0` | `d.patch` | Extra items from semver are ignored, so we can define the `v1.28.0` patch and be done | | `v1.28.4` | `d.patch` | Picks the patches from the stable versions only, not from beta | | `v1.29.1` | `d.patch` | Uses patches from most recent version, even if on a different minor | | `hack/branch` | `0.patch` | If not semver and no match, any patches from the `default/` directory are applied (if any) | Same logic applies for all other components as well. ### Testing which patches would be applied You can verify which set of patches would be applied in any case using the `print-patches-for.py` script directly: ```bash $ ./build-scripts/print-patches-for.py kubernetes v1.27.4 /home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.27.4/0000-Kubelite-integration.patch $ ./build-scripts/print-patches-for.py kubernetes v1.27.3 /home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.27.0/0000-Kubelite-integration.patch /home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.27.0/0001-Unix-socket-skip-validation-in-component-status.patch $ ./build-scripts/print-patches-for.py kubernetes v1.28.1 /home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.28.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch /home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.28.0/0000-Kubelite-integration.patch ``` ### How to add support for newer versions When a new release comes out which is no longer compatible with the existing latest patches, simply create a new directory under `patches/` with the new version number. This ensures that previous versions will still work, and newer ones will pick up the fixed patches. ================================================ FILE: build-scripts/components/cluster-agent/build.sh ================================================ #!/bin/bash export INSTALL="${1}/bin" mkdir -p "${INSTALL}" make cluster-agent cp cluster-agent "${INSTALL}" ================================================ FILE: build-scripts/components/cluster-agent/repository ================================================ https://github.com/canonical/microk8s-cluster-agent ================================================ FILE: build-scripts/components/cluster-agent/version.sh ================================================ #!/bin/bash echo "main" ================================================ FILE: build-scripts/components/cni/build.sh ================================================ #!/bin/bash VERSION="${2}" INSTALL="${1}/opt/cni/bin" mkdir -p "${INSTALL}" # these would very tedious to apply with a patch go get github.com/docker/docker/pkg/reexec go mod vendor sed -i 's/^package main/package plugin_main/' plugins/*/*/*.go sed -i 's/^func main()/func Main()/' plugins/*/*/*.go export CGO_ENABLED=0 go build -o cni -ldflags "-s -w -extldflags -static -X github.com/containernetworking/plugins/pkg/utils/buildversion.BuildVersion=${VERSION}" ./cni.go cp cni "${INSTALL}/" for plugin in dhcp host-local static bridge host-device ipvlan loopback macvlan ptp vlan bandwidth firewall portmap sbr tuning vrf; do ln -f -s ./cni "${INSTALL}/${plugin}" done ================================================ FILE: build-scripts/components/cni/patches/default/0001-single-entrypoint-for-cni-tools.patch ================================================ From 3d0636d0ad86c9050da190b50bc01387d71dc80a Mon Sep 17 00:00:00 2001 From: MicroK8s builder bot Date: Sun, 12 Feb 2023 13:34:45 +0000 Subject: [PATCH] single entrypoint for cni tools --- cni.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 cni.go diff --git a/cni.go b/cni.go new file mode 100644 index 0000000..93828f9 --- /dev/null +++ b/cni.go @@ -0,0 +1,54 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/docker/docker/pkg/reexec" + + ipam_dhcp "github.com/containernetworking/plugins/plugins/ipam/dhcp" + ipam_host_local "github.com/containernetworking/plugins/plugins/ipam/host-local" + ipam_static "github.com/containernetworking/plugins/plugins/ipam/static" + + main_bridge "github.com/containernetworking/plugins/plugins/main/bridge" + main_host_device "github.com/containernetworking/plugins/plugins/main/host-device" + main_ipvlan "github.com/containernetworking/plugins/plugins/main/ipvlan" + main_loopback "github.com/containernetworking/plugins/plugins/main/loopback" + main_macvlan "github.com/containernetworking/plugins/plugins/main/macvlan" + main_ptp "github.com/containernetworking/plugins/plugins/main/ptp" + main_vlan "github.com/containernetworking/plugins/plugins/main/vlan" + + meta_bandwidth "github.com/containernetworking/plugins/plugins/meta/bandwidth" + meta_firewall "github.com/containernetworking/plugins/plugins/meta/firewall" + meta_portmap "github.com/containernetworking/plugins/plugins/meta/portmap" + meta_sbr "github.com/containernetworking/plugins/plugins/meta/sbr" + meta_tuning "github.com/containernetworking/plugins/plugins/meta/tuning" + meta_vrf "github.com/containernetworking/plugins/plugins/meta/vrf" +) + +func main() { + os.Args[0] = filepath.Base(os.Args[0]) + if reexec.Init() { + return + } + panic("invalid entrypoint name") +} + +func init() { + reexec.Register("dhcp", ipam_dhcp.Main) + reexec.Register("host-local", ipam_host_local.Main) + reexec.Register("static", ipam_static.Main) + reexec.Register("bridge", main_bridge.Main) + reexec.Register("host-device", main_host_device.Main) + reexec.Register("ipvlan", main_ipvlan.Main) + reexec.Register("loopback", main_loopback.Main) + reexec.Register("macvlan", main_macvlan.Main) + reexec.Register("ptp", main_ptp.Main) + reexec.Register("vlan", main_vlan.Main) + reexec.Register("bandwidth", meta_bandwidth.Main) + reexec.Register("firewall", meta_firewall.Main) + reexec.Register("portmap", meta_portmap.Main) + reexec.Register("sbr", meta_sbr.Main) + reexec.Register("tuning", meta_tuning.Main) + reexec.Register("vrf", meta_vrf.Main) +} -- 2.25.1 ================================================ FILE: build-scripts/components/cni/repository ================================================ https://github.com/containernetworking/plugins ================================================ FILE: build-scripts/components/cni/version.sh ================================================ #!/bin/bash # Match https://github.com/kubernetes/kubernetes/blob/master/build/dependencies.yaml#L20 echo "v1.8.0" ================================================ FILE: build-scripts/components/containerd/build.sh ================================================ #!/bin/bash INSTALL="${1}/bin" mkdir -p "${INSTALL}" VERSION="${2}" REVISION=$(git rev-parse HEAD) sed -i "s,^VERSION.*$,VERSION=${VERSION}," Makefile sed -i "s,^REVISION.*$,REVISION=${REVISION}," Makefile export STATIC=1 for bin in ctr containerd containerd-shim-runc-v2; do make "bin/${bin}" cp "bin/${bin}" "${INSTALL}/${bin}" done ================================================ FILE: build-scripts/components/containerd/patches/default/0001-microk8s-sideload-images-plugin.patch ================================================ From d703811ab64963a6d52e6ac98b6a33b26b13e020 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Mon, 10 Jul 2023 12:15:34 +0300 Subject: [PATCH] microk8s sideload images plugin --- cmd/containerd/builtins_microk8s.go | 6 ++ microk8s_plugins/sideload.go | 132 ++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 cmd/containerd/builtins_microk8s.go create mode 100644 microk8s_plugins/sideload.go diff --git a/cmd/containerd/builtins_microk8s.go b/cmd/containerd/builtins_microk8s.go new file mode 100644 index 0000000..d9afc6f --- /dev/null +++ b/cmd/containerd/builtins_microk8s.go @@ -0,0 +1,6 @@ +package main + +// register containerd microk8s plugins here +import ( + _ "github.com/containerd/containerd/microk8s_plugins" +) diff --git a/microk8s_plugins/sideload.go b/microk8s_plugins/sideload.go new file mode 100644 index 0000000..3ac632e --- /dev/null +++ b/microk8s_plugins/sideload.go @@ -0,0 +1,132 @@ +package microk8s + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/plugin" +) + +const pluginName = "sideload-images" + +var logger = log.L.WithField("plugin", pluginName) + +type Config struct { + // Interval configures how frequently the plugin will look for new images found + // in the sources. If set to zero, images are only loaded during initial start. + Interval *time.Duration `toml:"interval"` + + // Sources is a list of paths to look for .tar images. + // For example, `/var/snap/microk8s/common/etc/sideload` + Sources []string `toml:"sources"` + + // Namespace the images will be loaded into, e.g. "k8s.io" + Namespace string `toml:"namespace"` +} + +func (c *Config) SetDefaults() { + if c.Namespace == "" { + c.Namespace = "k8s.io" + } + if len(c.Sources) == 0 { + snapCommon := os.Getenv("SNAP_COMMON") + if snapCommon == "" { + snapCommon = "/var/snap/microk8s/common" + } + c.Sources = []string{filepath.Join(snapCommon, "etc", "sideload")} + } + if c.Interval == nil { + t := 5 * time.Second + c.Interval = &t + } +} + +func init() { + c := &Config{} + plugin.Register(&plugin.Registration{ + Type: plugin.ServicePlugin, + ID: pluginName, + Config: c, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + config := ic.Config.(*Config) + config.SetDefaults() + + logger.Debugf("Loaded config %#v", config) + + if len(config.Sources) == 0 { + return nil, fmt.Errorf("no sources configured: %w", plugin.ErrSkipPlugin) + } + + go func() { + // get a containerd client + var ( + cl *containerd.Client + err error + ) + for cl == nil { + select { + case <-ic.Context.Done(): + return + default: + } + + cl, err = containerd.New(ic.Address, containerd.WithDefaultNamespace(config.Namespace), containerd.WithTimeout(2*time.Second)) + if err != nil { + logger.Info("Failed to create containerd client") + } + } + + for { + nextDir: + for _, dir := range c.Sources { + logger := logger.WithField("dir", dir) + logger.Debug("Looking for images") + files, err := filepath.Glob(filepath.Join(dir, "*.tar")) + if err != nil { + logger.WithError(err).Warn("Failed to look for images") + continue nextDir + } + + nextFile: + for _, file := range files { + logger := logger.WithField("file", file) + r, err := os.Open(file) + if err != nil { + logger.WithError(err).Warn("Failed to open file") + continue nextFile + } + images, err := cl.Import(ic.Context, r, containerd.WithImportPlatform(platforms.Default())) + if err != nil { + logger.WithError(err).Error("Failed to import images") + } else { + logger.Infof("Imported %d images", len(images)) + os.Rename(file, file+".loaded") + } + if closeErr := r.Close(); closeErr != nil { + logger.WithError(closeErr).Error("Failed to close reader") + } + } + } + + // retry after interval, finish if interval is zero + if *c.Interval == 0 { + logger.Info("Plugin terminating") + return + } + select { + case <-ic.Context.Done(): + return + case <-time.After(*c.Interval): + } + } + }() + + return nil, nil + }, + }) +} -- 2.34.1 ================================================ FILE: build-scripts/components/containerd/patches/v2.1.3/0001-microk8s-sideload-images-plugin.patch ================================================ From 7f26b3e013169510867383f09358b2d91641ad9f Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Mon, 10 Jul 2023 12:15:34 +0300 Subject: [PATCH] microk8s sideload images plugin --- cmd/containerd/builtins_microk8s.go | 6 ++ microk8s_plugins/sideload.go | 142 ++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 cmd/containerd/builtins_microk8s.go create mode 100644 microk8s_plugins/sideload.go diff --git a/cmd/containerd/builtins_microk8s.go b/cmd/containerd/builtins_microk8s.go new file mode 100644 index 000000000..c215987fa --- /dev/null +++ b/cmd/containerd/builtins_microk8s.go @@ -0,0 +1,6 @@ +package main + +// register containerd microk8s plugins here +import ( + _ "github.com/containerd/containerd/v2/microk8s_plugins" +) diff --git a/microk8s_plugins/sideload.go b/microk8s_plugins/sideload.go new file mode 100644 index 000000000..a6d97c8a3 --- /dev/null +++ b/microk8s_plugins/sideload.go @@ -0,0 +1,142 @@ +package microk8s + +import ( + "fmt" + "os" + "path/filepath" + "time" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/plugins" + "github.com/containerd/log" + "github.com/containerd/platforms" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" +) + +const pluginName = "sideload-images" + +var logger = log.L.WithField("plugin", pluginName) + +type Config struct { + // Interval configures how frequently the plugin will look for new images found + // in the sources. If set to zero, images are only loaded during initial start. + Interval *time.Duration `toml:"interval"` + + // Sources is a list of paths to look for .tar images. + // For example, `/var/snap/microk8s/common/etc/sideload` + Sources []string `toml:"sources"` + + // Namespace the images will be loaded into, e.g. "k8s.io" + Namespace string `toml:"namespace"` +} + +func (c *Config) SetDefaults() { + if c.Namespace == "" { + c.Namespace = "k8s.io" + } + if len(c.Sources) == 0 { + snapCommon := os.Getenv("SNAP_COMMON") + if snapCommon == "" { + snapCommon = "/var/snap/microk8s/common" + } + c.Sources = []string{filepath.Join(snapCommon, "etc", "sideload")} + } + if c.Interval == nil { + t := 5 * time.Second + c.Interval = &t + } +} + +func init() { + c := &Config{} + registry.Register(&plugin.Registration{ + Type: plugins.ServicePlugin, + ID: pluginName, + Config: c, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + config := ic.Config.(*Config) + config.SetDefaults() + + logger.Debugf("Loaded config %#v", config) + + if len(config.Sources) == 0 { + return nil, fmt.Errorf("no sources configured: %w", plugin.ErrSkipPlugin) + } + + go func() { + // get a containerd client + var ( + cl *containerd.Client + err error + ) + for cl == nil { + select { + case <-ic.Context.Done(): + return + default: + } + + cl, err = containerd.New( + "", + containerd.WithDefaultNamespace(config.Namespace), + containerd.WithDefaultPlatform(platforms.Default()), + containerd.WithInMemoryServices(ic), + containerd.WithTimeout(2*time.Second), + ) + if err != nil { + logger.Info("Failed to create containerd client") + } + } + + for { + nextDir: + for _, dir := range c.Sources { + logger := logger.WithField("dir", dir) + logger.Debug("Looking for images") + files, err := filepath.Glob(filepath.Join(dir, "*.tar")) + if err != nil { + logger.WithError(err).Warn("Failed to look for images") + continue nextDir + } + + nextFile: + for _, file := range files { + logger := logger.WithField("file", file) + r, err := os.Open(file) + if err != nil { + logger.WithError(err).Warn("Failed to open file") + continue nextFile + } + ctx := namespaces.WithNamespace(ic.Context, config.Namespace) + images, err := cl.Import(ctx, r, containerd.WithImportPlatform(platforms.DefaultStrict())) + if err != nil { + logger.WithError(err).Error("Failed to import images") + } else { + logger.Infof("Imported %d images", len(images)) + os.Rename(file, file+".loaded") + } + if closeErr := r.Close(); closeErr != nil { + logger.WithError(closeErr).Error("Failed to close reader") + } + } + } + + // retry after interval, finish if interval is zero + if *c.Interval == 0 { + logger.Info("Plugin terminating") + return + } + select { + case <-ic.Context.Done(): + return + case <-time.After(*c.Interval): + } + } + }() + + return nil, nil + }, + }) +} -- 2.43.0 ================================================ FILE: build-scripts/components/containerd/repository ================================================ https://github.com/containerd/containerd ================================================ FILE: build-scripts/components/containerd/version.sh ================================================ #!/bin/bash echo "v2.1.3" ================================================ FILE: build-scripts/components/etcd/build.sh ================================================ #!/bin/bash export INSTALL="${1}" mkdir -p "${INSTALL}" GO_LDFLAGS="-s -w" GO_BUILD_FLAGS="-v" make build for bin in etcd etcdctl; do cp "bin/${bin}" "${INSTALL}/${bin}" done ================================================ FILE: build-scripts/components/etcd/repository ================================================ https://github.com/etcd-io/etcd ================================================ FILE: build-scripts/components/etcd/version.sh ================================================ #!/bin/bash echo "v3.6.6" ================================================ FILE: build-scripts/components/flannel-cni-plugin/build.sh ================================================ #!/bin/bash INSTALL="${1}/opt/cni/bin" mkdir -p "${INSTALL}" VERSION="${2}" export CGO_ENABLED=0 go build -o dist/flannel -ldflags "-s -w -X github.com/flannel-io/cni-plugin/version.Version=${VERSION} -extldflags -static" cp dist/flannel "${INSTALL}/flannel" ================================================ FILE: build-scripts/components/flannel-cni-plugin/repository ================================================ https://github.com/flannel-io/cni-plugin ================================================ FILE: build-scripts/components/flannel-cni-plugin/version.sh ================================================ #!/bin/bash echo "v1.8.0-flannel2" ================================================ FILE: build-scripts/components/flanneld/build.sh ================================================ #!/bin/bash INSTALL="${1}/opt/cni/bin" mkdir -p "${INSTALL}" VERSION="${2}" export CGO_ENABLED=0 go build -o dist/flanneld -ldflags "-s -w -X github.com/flannel-io/flannel/version.Version=${VERSION} -extldflags -static" cp dist/flanneld "${INSTALL}/flanneld" ================================================ FILE: build-scripts/components/flanneld/patches/default/0001-disable-udp-backend.patch ================================================ From 45ec777a0d113089453eca7fd2f7cb195555c6c9 Mon Sep 17 00:00:00 2001 From: MicroK8s builder bot Date: Wed, 15 Feb 2023 15:52:51 +0000 Subject: [PATCH] disable udp backend --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 064f58d..b591458 100644 --- a/main.go +++ b/main.go @@ -50,7 +50,7 @@ import ( _ "github.com/flannel-io/flannel/pkg/backend/ipip" _ "github.com/flannel-io/flannel/pkg/backend/ipsec" _ "github.com/flannel-io/flannel/pkg/backend/tencentvpc" - _ "github.com/flannel-io/flannel/pkg/backend/udp" + // _ "github.com/flannel-io/flannel/pkg/backend/udp" _ "github.com/flannel-io/flannel/pkg/backend/vxlan" _ "github.com/flannel-io/flannel/pkg/backend/wireguard" ) -- 2.25.1 ================================================ FILE: build-scripts/components/flanneld/repository ================================================ https://github.com/flannel-io/flannel ================================================ FILE: build-scripts/components/flanneld/version.sh ================================================ #!/bin/bash echo "v0.27.4" ================================================ FILE: build-scripts/components/helm/build.sh ================================================ #!/bin/bash VERSION="${2}" INSTALL="${1}" mkdir -p "${INSTALL}/bin" make VERSION="${VERSION}" cp bin/helm "${INSTALL}/bin/helm" ./bin/helm completion bash \ | sed "s/complete -o default -F __start_helm helm/complete -o default -F __start_helm microk8s.helm/g" \ | sed "s/complete -o default -o nospace -F __start_helm helm/complete -o default -o nospace -F __start_helm microk8s.helm/g" \ > bin/helm.bash ./bin/helm completion bash \ | sed "s/complete -o default -F __start_helm helm/complete -o default -F __start_helm microk8s.helm3/g" \ | sed "s/complete -o default -o nospace -F __start_helm helm/complete -o default -o nospace -F __start_helm microk8s.helm3/g" \ > bin/helm3.bash cp bin/helm.bash "${INSTALL}/helm.bash" cp bin/helm3.bash "${INSTALL}/helm3.bash" ================================================ FILE: build-scripts/components/helm/repository ================================================ https://github.com/helm/helm ================================================ FILE: build-scripts/components/helm/version.sh ================================================ #!/bin/bash echo "v3.19.2" ================================================ FILE: build-scripts/components/k8s-dqlite/build.sh ================================================ #!/bin/bash INSTALL="${1}/bin" mkdir -p "${INSTALL}" make static -j cp bin/static/dqlite "${INSTALL}/dqlite" cp bin/static/k8s-dqlite "${INSTALL}/k8s-dqlite" ================================================ FILE: build-scripts/components/k8s-dqlite/repository ================================================ https://github.com/canonical/k8s-dqlite ================================================ FILE: build-scripts/components/k8s-dqlite/version.sh ================================================ #!/bin/bash echo "v1.8.1" ================================================ FILE: build-scripts/components/kubernetes/build.sh ================================================ #!/bin/bash -x INSTALL="${1}" export KUBE_GIT_VERSION_FILE="${PWD}/.version.sh" for app in kubectl kubelite; do make WHAT="cmd/${app}" KUBE_STATIC_OVERRIDES=kubelite cp _output/bin/"${app}" "${INSTALL}/${app}" done _output/bin/kubectl completion bash \ | sed "s/complete -o default -F __start_kubectl kubectl/complete -o default -F __start_kubectl microk8s.kubectl/g" \ | sed "s/complete -o default -o nospace -F __start_kubectl kubectl/complete -o default -o nospace -F __start_kubectl microk8s.kubectl/g" \ > _output/kubectl.bash cp _output/kubectl.bash "${INSTALL}/kubectl.bash" ================================================ FILE: build-scripts/components/kubernetes/patches/v1.27.0/0000-Kubelite-integration.patch ================================================ From d0ae18d074db5ff361f363073f32b2f30c7a3686 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 +++- cmd/kube-scheduler/app/server.go | 6 ++- cmd/kubelet/app/server.go | 13 +++-- cmd/kubelite/app/daemons/daemon.go | 84 +++++++++++++++++++++++++++++ cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/app/server.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/kubelite.go | 28 ++++++++++ pkg/volume/csi/csi_plugin.go | 10 ++-- 8 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index fc36d044dbe..cffb7c35a3c 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -89,7 +89,7 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(stopCh... <- chan struct{}) *cobra.Command { s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "kube-apiserver", @@ -129,7 +129,12 @@ cluster's shared state through which all other components interact.`, } // add feature enablement metrics utilfeature.DefaultMutableFeatureGate.AddMetrics() - return Run(completedOptions, genericapiserver.SetupSignalHandler()) + + if len(stopCh) != 0 { + return Run(completedOptions, stopCh[0]) + } else { + return Run(completedOptions, genericapiserver.SetupSignalHandler()) + } }, Args: func(cmd *cobra.Command, args []string) error { for _, arg := range args { diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 8d01f3b7670..44ac7f69328 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -132,7 +132,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 9444f136866..8ca88f64d04 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -120,7 +120,7 @@ const ( ) // NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { +func NewKubeletCommand(ctx ...context.Context) *cobra.Command { cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() @@ -250,6 +250,12 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API if err := checkPermissions(); err != nil { klog.ErrorS(err, "kubelet running with insufficient permissions") } + runctx := context.Background() + if len(ctx) == 0 { + runctx = genericapiserver.SetupSignalContext() + } else { + runctx = ctx[0] + } // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() @@ -259,12 +265,9 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API // log the kubelet's config for inspection klog.V(5).InfoS("KubeletConfiguration", "configuration", config) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet - return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) + return Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) }, } diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..dbef03cf07e --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,84 @@ +package daemon + +import ( + "context" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + genericcontrollermanager "k8s.io/controller-manager/app" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx <-chan struct{}) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} \ No newline at end of file diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..e7452a09e3e --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,79 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" + "os" + "time" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx.Done()) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360 * time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile , "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile , "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..667b24f68e6 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + "math/rand" + "time" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + rand.Seed(time.Now().UnixNano()) + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index ce7a543c94f..a8094f878d6 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -240,18 +240,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host); err != nil { + if err := initializeCSINode(host, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost) error { +func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.27.0/0001-Unix-socket-skip-validation-in-component-status.patch ================================================ From dd1db952eab13912a55207c81a2ac267909677ac Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Tue, 24 Aug 2021 11:17:19 +0300 Subject: [PATCH] Unix socket skip validation in component status --- pkg/registry/core/rest/storage_core.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index 1f915c32d4b..0bb7f1a9bf9 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -350,6 +350,12 @@ func (s componentStatusStorage) serversToValidate() map[string]*componentstatus. klog.Errorf("Failed to parse etcd url for validation: %v", err) continue } + + if etcdUrl.Scheme == "unix" { + klog.Infof("Socket etcd endpoint detected. Will not validate") + continue + } + var port int var addr string if strings.Contains(etcdUrl.Host, ":") { -- 2.25.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.27.4/0000-Kubelite-integration.patch ================================================ From 3162aa9df25819b60a3c0a3b044394639d55280c Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 +++- cmd/kube-scheduler/app/server.go | 6 ++- cmd/kubelet/app/server.go | 13 +++-- cmd/kubelite/app/daemons/daemon.go | 84 +++++++++++++++++++++++++++++ cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/app/server.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/kubelite.go | 28 ++++++++++ pkg/volume/csi/csi_plugin.go | 10 ++-- 8 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index b021bac2574..8d5a1bad8ed 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -76,7 +76,7 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(stopCh... <- chan struct{}) *cobra.Command { s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "kube-apiserver", @@ -116,7 +116,12 @@ cluster's shared state through which all other components interact.`, } // add feature enablement metrics utilfeature.DefaultMutableFeatureGate.AddMetrics() - return Run(completedOptions, genericapiserver.SetupSignalHandler()) + + if len(stopCh) != 0 { + return Run(completedOptions, stopCh[0]) + } else { + return Run(completedOptions, genericapiserver.SetupSignalHandler()) + } }, Args: func(cmd *cobra.Command, args []string) error { for _, arg := range args { diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c48b09a420d..b7f273b02ac 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -132,7 +132,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 742524e325f..561cbf68868 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -122,7 +122,7 @@ const ( ) // NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { +func NewKubeletCommand(ctx ...context.Context) *cobra.Command { cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() @@ -249,6 +249,12 @@ is checked every 20 seconds (also configurable with a flag).`, if err := checkPermissions(); err != nil { klog.ErrorS(err, "kubelet running with insufficient permissions") } + runctx := context.Background() + if len(ctx) == 0 { + runctx = genericapiserver.SetupSignalContext() + } else { + runctx = ctx[0] + } // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() @@ -258,12 +264,9 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection klog.V(5).InfoS("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet - return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) + return Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) }, } diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..dbef03cf07e --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,84 @@ +package daemon + +import ( + "context" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + genericcontrollermanager "k8s.io/controller-manager/app" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx <-chan struct{}) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} \ No newline at end of file diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..e7452a09e3e --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,79 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" + "os" + "time" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx.Done()) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360 * time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile , "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile , "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..667b24f68e6 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + "math/rand" + "time" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + rand.Seed(time.Now().UnixNano()) + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 2556517276e..0b5ef45d083 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -243,18 +243,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host); err != nil { + if err := initializeCSINode(host, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost) error { +func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.28.0/0000-Kubelite-integration.patch ================================================ From 3162aa9df25819b60a3c0a3b044394639d55280c Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 +++- cmd/kube-scheduler/app/server.go | 6 ++- cmd/kubelet/app/server.go | 13 +++-- cmd/kubelite/app/daemons/daemon.go | 84 +++++++++++++++++++++++++++++ cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/app/server.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/kubelite.go | 28 ++++++++++ pkg/volume/csi/csi_plugin.go | 10 ++-- 8 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index b021bac2574..8d5a1bad8ed 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -76,7 +76,7 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(stopCh... <- chan struct{}) *cobra.Command { s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "kube-apiserver", @@ -116,7 +116,12 @@ cluster's shared state through which all other components interact.`, } // add feature enablement metrics utilfeature.DefaultMutableFeatureGate.AddMetrics() - return Run(completedOptions, genericapiserver.SetupSignalHandler()) + + if len(stopCh) != 0 { + return Run(completedOptions, stopCh[0]) + } else { + return Run(completedOptions, genericapiserver.SetupSignalHandler()) + } }, Args: func(cmd *cobra.Command, args []string) error { for _, arg := range args { diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c48b09a420d..b7f273b02ac 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -132,7 +132,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 742524e325f..561cbf68868 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -122,7 +122,7 @@ const ( ) // NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { +func NewKubeletCommand(ctx ...context.Context) *cobra.Command { cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() @@ -249,6 +249,12 @@ is checked every 20 seconds (also configurable with a flag).`, if err := checkPermissions(); err != nil { klog.ErrorS(err, "kubelet running with insufficient permissions") } + runctx := context.Background() + if len(ctx) == 0 { + runctx = genericapiserver.SetupSignalContext() + } else { + runctx = ctx[0] + } // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() @@ -258,12 +264,9 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection klog.V(5).InfoS("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet - return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) + return Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) }, } diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..dbef03cf07e --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,84 @@ +package daemon + +import ( + "context" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + genericcontrollermanager "k8s.io/controller-manager/app" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx <-chan struct{}) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} \ No newline at end of file diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..e7452a09e3e --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,79 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" + "os" + "time" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx.Done()) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360 * time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile , "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile , "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..667b24f68e6 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + "math/rand" + "time" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + rand.Seed(time.Now().UnixNano()) + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 2556517276e..0b5ef45d083 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -243,18 +243,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host); err != nil { + if err := initializeCSINode(host, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost) error { +func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.28.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch ================================================ From 55f4864d816c8e7ca0ebb39571dc88dbdf05eff2 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 27 Jul 2023 18:08:00 +0300 Subject: [PATCH] Set log reapply handling to ignore unchanged --- staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go index 2db9b1f5382..e0824dcdc4e 100644 --- a/staging/src/k8s.io/component-base/logs/api/v1/options.go +++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go @@ -64,7 +64,7 @@ func NewLoggingConfiguration() *LoggingConfiguration { // are no goroutines which might call logging functions. The default for ValidateAndApply // and ValidateAndApplyWithOptions is to return an error when called more than once. // Binaries and unit tests can override that behavior. -var ReapplyHandling = ReapplyHandlingError +var ReapplyHandling = ReapplyHandlingIgnoreUnchanged type ReapplyHandlingType int -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.31.0/0000-Kubelite-integration.patch ================================================ From d261b947963b9e808725a21e3d55e52c20826e22 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 ++- cmd/kube-scheduler/app/server.go | 6 +- cmd/kubelet/app/server.go | 13 +++-- cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++++++++++++++ cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/app/server.go | 80 +++++++++++++++++++++++++++ cmd/kubelite/kubelite.go | 25 +++++++++ pkg/volume/csi/csi_plugin.go | 10 +++- 8 files changed, 296 insertions(+), 11 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index fad0f1e9579..5e291005671 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -63,7 +63,7 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(ctx ...context.Context) *cobra.Command { _, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) s := options.NewServerRunOptions() @@ -119,7 +119,12 @@ cluster's shared state through which all other components interact.`, return nil }, } - cmd.SetContext(genericapiserver.SetupSignalContext()) + + if len(ctx) == 0 { + cmd.SetContext(genericapiserver.SetupSignalContext()) + } else { + cmd.SetContext(ctx[0]) + } fs := cmd.Flags() namedFlagSets := s.Flags() diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 7b08b119b4b..5d2a5a5f51a 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -145,7 +145,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 2d90030d210..666996e2ecb 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -129,7 +129,7 @@ const ( ) // NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { +func NewKubeletCommand(ctx ...context.Context) *cobra.Command { cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() @@ -266,6 +266,12 @@ is checked every 20 seconds (also configurable with a flag).`, if err := checkPermissions(); err != nil { klog.ErrorS(err, "kubelet running with insufficient permissions") } + runctx := context.Background() + if len(ctx) == 0 { + runctx = genericapiserver.SetupSignalContext() + } else { + runctx = ctx[0] + } // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() @@ -275,12 +281,9 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection klog.V(5).InfoS("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet - return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) + return Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) }, } diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..46c1af7fdb9 --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,85 @@ +package daemon + +import ( + "context" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/klog/v2" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx context.Context) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..4ff36cd6432 --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,80 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile, "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile, "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..30ab604f480 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index fec8a34b4d3..523eb78f05c 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -253,18 +253,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host); err != nil { + if err := initializeCSINode(host, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost) error { +func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.31.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch ================================================ From 55f4864d816c8e7ca0ebb39571dc88dbdf05eff2 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 27 Jul 2023 18:08:00 +0300 Subject: [PATCH] Set log reapply handling to ignore unchanged --- staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go index 2db9b1f5382..e0824dcdc4e 100644 --- a/staging/src/k8s.io/component-base/logs/api/v1/options.go +++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go @@ -64,7 +64,7 @@ func NewLoggingConfiguration() *LoggingConfiguration { // are no goroutines which might call logging functions. The default for ValidateAndApply // and ValidateAndApplyWithOptions is to return an error when called more than once. // Binaries and unit tests can override that behavior. -var ReapplyHandling = ReapplyHandlingError +var ReapplyHandling = ReapplyHandlingIgnoreUnchanged type ReapplyHandlingType int -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.32.0/0000-Kubelite-integration.patch ================================================ From 819b718ecfee8c4f6fb503d0dea80a43e86cdb6f Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 ++- cmd/kube-scheduler/app/server.go | 6 +- cmd/kubelet/app/server.go | 13 +++-- cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++++++++++++++ cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/app/server.go | 80 +++++++++++++++++++++++++++ cmd/kubelite/kubelite.go | 25 +++++++++ pkg/volume/csi/csi_plugin.go | 10 +++- 8 files changed, 296 insertions(+), 11 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 8aa05a4d8f8..75b982aaff9 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -63,11 +63,16 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command { _, featureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) s := options.NewServerRunOptions() - ctx := genericapiserver.SetupSignalContext() + ctx := context.Background() + if len(ctxs) == 0 { + ctx = genericapiserver.SetupSignalContext() + } else { + ctx = ctxs[0] + } cmd := &cobra.Command{ Use: "kube-apiserver", diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 1785bbdcc91..64f01ba5c93 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -144,7 +144,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index cfcf6e7d5cd..9c3853173ff 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -138,7 +138,7 @@ const ( ) // NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { +func NewKubeletCommand(ctx ...context.Context) *cobra.Command { cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() @@ -275,6 +275,12 @@ is checked every 20 seconds (also configurable with a flag).`, if err := checkPermissions(); err != nil { klog.ErrorS(err, "kubelet running with insufficient permissions") } + runctx := context.Background() + if len(ctx) == 0 { + runctx = genericapiserver.SetupSignalContext() + } else { + runctx = ctx[0] + } // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() @@ -284,12 +290,9 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection klog.V(5).InfoS("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet - return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) + return Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) }, } diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..46c1af7fdb9 --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,85 @@ +package daemon + +import ( + "context" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/klog/v2" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx context.Context) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..4ff36cd6432 --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,80 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile, "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile, "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..30ab604f480 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 59fa6245b7f..6443dffd695 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -256,18 +256,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host); err != nil { + if err := initializeCSINode(host, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost) error { +func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.43.0 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.32.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch ================================================ From 55f4864d816c8e7ca0ebb39571dc88dbdf05eff2 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 27 Jul 2023 18:08:00 +0300 Subject: [PATCH] Set log reapply handling to ignore unchanged --- staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go index 2db9b1f5382..e0824dcdc4e 100644 --- a/staging/src/k8s.io/component-base/logs/api/v1/options.go +++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go @@ -64,7 +64,7 @@ func NewLoggingConfiguration() *LoggingConfiguration { // are no goroutines which might call logging functions. The default for ValidateAndApply // and ValidateAndApplyWithOptions is to return an error when called more than once. // Binaries and unit tests can override that behavior. -var ReapplyHandling = ReapplyHandlingError +var ReapplyHandling = ReapplyHandlingIgnoreUnchanged type ReapplyHandlingType int -- 2.34.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.33.0/0000-Kubelite-integration.patch ================================================ From aa3c3a427082e2a1ddd9f72790a8977efefee1cf Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH 1/2] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 ++- cmd/kube-scheduler/app/server.go | 6 +- cmd/kubelet/app/server.go | 13 +++-- cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++++++++++++++ cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++ cmd/kubelite/app/server.go | 80 +++++++++++++++++++++++++++ cmd/kubelite/kubelite.go | 25 +++++++++ pkg/volume/csi/csi_plugin.go | 11 +++- 8 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 042ddc9714a..6a6d4b07f40 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -65,9 +65,14 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command { s := options.NewServerRunOptions() - ctx := genericapiserver.SetupSignalContext() + ctx := context.Background() + if len(ctxs) == 0 { + ctx = genericapiserver.SetupSignalContext() + } else { + ctx = ctxs[0] + } featureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) cmd := &cobra.Command{ diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index c9ab27027cf..aade3f81dc4 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -149,7 +149,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 3bd3ac16eb7..70fa7fd86b7 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -135,7 +135,7 @@ func init() { } // NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { +func NewKubeletCommand(ctx ...context.Context) *cobra.Command { cleanFlagSet := pflag.NewFlagSet(server.ComponentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() @@ -279,6 +279,12 @@ is checked every 20 seconds (also configurable with a flag).`, if err := checkPermissions(); err != nil { klog.ErrorS(err, "kubelet running with insufficient permissions") } + runctx := context.Background() + if len(ctx) == 0 { + runctx = genericapiserver.SetupSignalContext() + } else { + runctx = ctx[0] + } // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() @@ -288,12 +294,9 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection klog.V(5).InfoS("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet - return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) + return Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) }, } diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..46c1af7fdb9 --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,85 @@ +package daemon + +import ( + "context" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/klog/v2" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx context.Context) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..4ff36cd6432 --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,80 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile, "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile, "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..30ab604f480 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 902f31e9911..95d20ba48d8 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -345,18 +345,23 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) PluginHandler.csiPlugin = p // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host, p.csiDriverInformer); err != nil { + if err := initializeCSINode(host, p.csiDriverInformer, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer) error { +func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.49.0 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.33.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch ================================================ From beaeb73228b7e4a836b3bd5d73e49402279a47a6 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 27 Jul 2023 18:08:00 +0300 Subject: [PATCH 2/2] Set log reapply handling to ignore unchanged --- staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go index 95c0a2cba98..4482564ad95 100644 --- a/staging/src/k8s.io/component-base/logs/api/v1/options.go +++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go @@ -65,7 +65,7 @@ func NewLoggingConfiguration() *LoggingConfiguration { // are no goroutines which might call logging functions. The default for ValidateAndApply // and ValidateAndApplyWithOptions is to return an error when called more than once. // Binaries and unit tests can override that behavior. -var ReapplyHandling = ReapplyHandlingError +var ReapplyHandling = ReapplyHandlingIgnoreUnchanged type ReapplyHandlingType int -- 2.49.0 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.34.0/0000-Kubelite-integration.patch ================================================ From 241fc2a00be6fde3f7059ae61842cba44dff82a5 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration --- cmd/kube-apiserver/app/server.go | 9 +- .../app/options/options.go | 10 -- cmd/kube-scheduler/app/server.go | 6 +- cmd/kubelet/app/server.go | 4 - cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++ cmd/kubelite/app/options/options.go | 79 ++++++++++++++++ cmd/kubelite/app/server.go | 91 +++++++++++++++++++ cmd/kubelite/kubelite.go | 25 +++++ pkg/volume/csi/csi_plugin.go | 11 ++- 9 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 71ebb317461..86dc50ce671 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -67,9 +67,14 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command { s := options.NewServerRunOptions() - ctx := genericapiserver.SetupSignalContext() + ctx := context.Background() + if len(ctxs) == 0 { + ctx = genericapiserver.SetupSignalContext() + } else { + ctx = ctxs[0] + } featureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) cmd := &cobra.Command{ diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 176b2a14bc1..5048b04014e 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -25,11 +25,9 @@ import ( v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/version" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" - clientgofeaturegate "k8s.io/client-go/features" clientset "k8s.io/client-go/kubernetes" clientgokubescheme "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" @@ -40,7 +38,6 @@ import ( cpoptions "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" basecompatibility "k8s.io/component-base/compatibility" - "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" @@ -298,13 +295,6 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).") - if !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) { - ver := version.MustParse("1.34") - if err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil { - panic(fmt.Sprintf("unable to set %s feature gate, err: %v", clientgofeaturegate.WatchListClient, err)) - } - } - s.ComponentGlobalsRegistry.AddFlags(fss.FlagSet("generic")) return fss diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 803e0c37861..5b58d437d30 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -150,7 +150,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index f069f6b08b9..62f3ca58dc7 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -63,7 +63,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/wait" - genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" @@ -291,9 +290,6 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection logger.V(5).Info("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..46c1af7fdb9 --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,85 @@ +package daemon + +import ( + "context" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/klog/v2" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx context.Context) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..858a7c1094c --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,91 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/version" + genericapiserver "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" + clientgofeaturegate "k8s.io/client-go/features" + "k8s.io/component-base/featuregate" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + if !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) { + ver := version.MustParse("1.34") + if err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil { + panic(fmt.Sprintf("unable to set %s feature gate, err: %v", clientgofeaturegate.WatchListClient, err)) + } + } + + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile, "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile, "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..30ab604f480 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index d2d830376da..fa6dbc91fef 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -347,18 +347,23 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) PluginHandler.csiPlugin = p // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host, p.csiDriverInformer); err != nil { + if err := initializeCSINode(host, p.csiDriverInformer, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer) error { +func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.43.0 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.34.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch ================================================ From e2e2155d05b3ddd16640272bd1360425a3883c78 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 27 Jul 2023 18:08:00 +0300 Subject: [PATCH 2/2] Set log reapply handling to ignore unchanged --- staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go index 4c8a0d2c53f..6cb6ae1f42d 100644 --- a/staging/src/k8s.io/component-base/logs/api/v1/options.go +++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go @@ -65,7 +65,7 @@ func NewLoggingConfiguration() *LoggingConfiguration { // are no goroutines which might call logging functions. The default for ValidateAndApply // and ValidateAndApplyWithOptions is to return an error when called more than once. // Binaries and unit tests can override that behavior. -var ReapplyHandling = ReapplyHandlingError +var ReapplyHandling = ReapplyHandlingIgnoreUnchanged type ReapplyHandlingType int -- 2.43.0 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.34.0/0002-fix-allow-node-to-get-endpointslices.patch ================================================ From dea2abd80878be1eff519216c0bad5a0e35462ec Mon Sep 17 00:00:00 2001 From: Mateo Florido Date: Thu, 11 Sep 2025 17:36:10 -0500 Subject: [PATCH] fix: allow node to get endpointslices --- plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 447b0bc2e99..daa3bde6b1c 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -228,6 +228,7 @@ func NodeRules() []rbacv1.PolicyRule { // TODO: add to the Node authorizer and restrict to endpoints referenced by pods or PVs bound to the node // Needed for glusterfs volumes rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), + rbacv1helpers.NewRule("get", "list", "watch").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie(), // Used to create a certificatesigningrequest for a node-specific client certificate, and watch // for it to be signed. This allows the kubelet to rotate it's own certificate. rbacv1helpers.NewRule("create", "get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(), -- 2.48.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.35.0/0000-Kubelite-integration.patch ================================================ From afd6e0c5da37c3ae02c22c91ef898c1169e8d657 Mon Sep 17 00:00:00 2001 From: Homayoon Alimohammadi Date: Thu, 27 Nov 2025 18:47:43 +0400 Subject: [PATCH] Kubelite integration Signed-off-by: Homayoon Alimohammadi --- cmd/kube-apiserver/app/server.go | 9 +- .../app/options/options.go | 10 -- cmd/kube-scheduler/app/server.go | 6 +- cmd/kubelet/app/server.go | 4 - cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++ cmd/kubelite/app/options/options.go | 79 ++++++++++++++++ cmd/kubelite/app/server.go | 91 +++++++++++++++++++ cmd/kubelite/kubelite.go | 25 +++++ pkg/volume/csi/csi_plugin.go | 11 ++- 9 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 cmd/kubelite/app/daemons/daemon.go create mode 100644 cmd/kubelite/app/options/options.go create mode 100644 cmd/kubelite/app/server.go create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index a0ba37e0103..32ae1a53496 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -67,9 +67,14 @@ func init() { } // NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { +func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command { s := options.NewServerRunOptions() - ctx := genericapiserver.SetupSignalContext() + ctx := context.Background() + if len(ctxs) == 0 { + ctx = genericapiserver.SetupSignalContext() + } else { + ctx = ctxs[0] + } featureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) cmd := &cobra.Command{ diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 867cd8d3250..8c4f50a923e 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -26,12 +26,10 @@ import ( v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/server/flagz" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" - clientgofeaturegate "k8s.io/client-go/features" clientset "k8s.io/client-go/kubernetes" clientgokubescheme "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" @@ -42,7 +40,6 @@ import ( cpoptions "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" basecompatibility "k8s.io/component-base/compatibility" - "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" @@ -311,13 +308,6 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fss.FlagSet("generic").DurationVar(&s.ControllerShutdownTimeout, "controller-shutdown-timeout", s.ControllerShutdownTimeout, "Time to wait for the controllers to shut down before terminating the executable") - if !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) { - ver := version.MustParse("1.34") - if err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil { - panic(fmt.Sprintf("unable to set %s feature gate, err: %v", clientgofeaturegate.WatchListClient, err)) - } - } - s.ComponentGlobalsRegistry.AddFlags(fss.FlagSet("generic")) return fss diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 834213e337f..1eb9c630220 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -150,7 +150,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { - stopCh := server.SetupSignalHandler() + c := cmd.Context() + if c == nil { + c = server.SetupSignalContext() + } + stopCh := c.Done() <-stopCh cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 54ff8d6a26b..203fe982a9a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -64,7 +64,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/wait" - genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/flagz" "k8s.io/apiserver/pkg/server/healthz" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -287,9 +286,6 @@ is checked every 20 seconds (also configurable with a flag).`, // log the kubelet's config for inspection logger.V(5).Info("KubeletConfiguration", "configuration", klog.Format(config)) - // set up signal context for kubelet shutdown - ctx := genericapiserver.SetupSignalContext() - utilfeature.DefaultMutableFeatureGate.AddMetrics() // run the kubelet return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate) diff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go new file mode 100644 index 00000000000..46c1af7fdb9 --- /dev/null +++ b/cmd/kubelite/app/daemons/daemon.go @@ -0,0 +1,85 @@ +package daemon + +import ( + "context" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/klog/v2" + apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" + controller "k8s.io/kubernetes/cmd/kube-controller-manager/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + scheduler "k8s.io/kubernetes/cmd/kube-scheduler/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" + + "time" +) + +func StartControllerManager(args []string, ctx context.Context) { + command := controller.NewControllerManagerCommand() + command.SetArgs(args) + + klog.Info("Starting Controller Manager") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Controller Manager exited %v", err) + } + klog.Info("Stopping Controller Manager") +} + +func StartScheduler(args []string, ctx context.Context) { + command := scheduler.NewSchedulerCommand() + command.SetArgs(args) + + klog.Info("Starting Scheduler") + if err := command.ExecuteContext(ctx); err != nil { + klog.Fatalf("Scheduler exited %v", err) + } + klog.Info("Stopping Scheduler") +} + +func StartProxy(args []string) { + command := proxy.NewProxyCommand() + command.SetArgs(args) + + klog.Info("Starting Proxy") + if err := command.Execute(); err != nil { + klog.Fatalf("Proxy exited %v", err) + } + klog.Info("Stopping Proxy") +} + +func StartKubelet(args []string, ctx context.Context) { + command := kubelet.NewKubeletCommand(ctx) + command.SetArgs(args) + + klog.Info("Starting Kubelet") + if err := command.Execute(); err != nil { + klog.Fatalf("Kubelet exited %v", err) + } + klog.Info("Stopping Kubelet") +} + +func StartAPIServer(args []string, ctx context.Context) { + command := apiserver.NewAPIServerCommand(ctx) + command.SetArgs(args) + klog.Info("Starting API Server") + if err := command.Execute(); err != nil { + klog.Fatalf("API Server exited %v", err) + } + klog.Info("Stopping API Server") +} + +func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) { + klog.Info("Waiting for the API server") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + klog.Fatalf("could not find the cluster's kubeconfig file %v", err) + } + // create the client + client, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("could not create client to the cluster %v", err) + } + genericcontrollermanager.WaitForAPIServer(client, timeout) +} diff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go new file mode 100644 index 00000000000..80f1d8b09fc --- /dev/null +++ b/cmd/kubelite/app/options/options.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "bufio" + "k8s.io/klog/v2" + "os" + "strings" +) + +// Options has all the params needed to run a Kubelite +type Options struct { + SchedulerArgsFile string + ControllerManagerArgsFile string + ProxyArgsFile string + KubeletArgsFile string + APIServerArgsFile string + KubeconfigFile string + StartControlPlane bool +} + +func NewOptions() (*Options){ + o := Options{ + "/var/snap/microk8s/current/args/kube-scheduler", + "/var/snap/microk8s/current/args/kube-controller-manager", + "/var/snap/microk8s/current/args/kube-proxy", + "/var/snap/microk8s/current/args/kubelet", + "/var/snap/microk8s/current/args/kube-apiserver", + "/var/snap/microk8s/current/credentials/client.config", + true, + } + return &o +} + +func ReadArgsFromFile(filename string) []string { + var args []string + file, err := os.Open(filename) + if err != nil { + klog.Fatalf("Failed to open arguments file %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + // ignore lines with # and empty lines + if len(line) <= 0 || strings.HasPrefix(line, "#") { + continue + } + // remove " and ' + for _, r := range "\"'" { + line = strings.ReplaceAll(line, string(r), "") + } + for _, part := range strings.Split(line, " ") { + + args = append(args, os.ExpandEnv(part)) + } + } + if err := scanner.Err(); err != nil { + klog.Fatalf("Failed to read arguments file %v", err) + } + return args +} diff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go new file mode 100644 index 00000000000..858a7c1094c --- /dev/null +++ b/cmd/kubelite/app/server.go @@ -0,0 +1,91 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/version" + genericapiserver "k8s.io/apiserver/pkg/server" + utilfeature "k8s.io/apiserver/pkg/util/feature" + clientgofeaturegate "k8s.io/client-go/features" + "k8s.io/component-base/featuregate" + daemon "k8s.io/kubernetes/cmd/kubelite/app/daemons" + "k8s.io/kubernetes/cmd/kubelite/app/options" +) + +var opts = options.NewOptions() + +// liteCmd represents the base command when called without any subcommands +var liteCmd = &cobra.Command{ + Use: "kubelite", + Short: "Single server kubernetes", + Long: `A single server that spawns all other kubernetes servers as threads`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + ctx := genericapiserver.SetupSignalContext() + + if opts.StartControlPlane { + if !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) { + ver := version.MustParse("1.34") + if err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil { + panic(fmt.Sprintf("unable to set %s feature gate, err: %v", clientgofeaturegate.WatchListClient, err)) + } + } + + apiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile) + go daemon.StartAPIServer(apiserverArgs, ctx) + daemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second) + + controllerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile) + go daemon.StartControllerManager(controllerArgs, ctx) + + schedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile) + go daemon.StartScheduler(schedulerArgs, ctx) + } + + proxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile) + go daemon.StartProxy(proxyArgs) + + kubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile) + daemon.StartKubelet(kubeletArgs, ctx) + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the liteCmd. +func Execute() { + if err := liteCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize() + + liteCmd.Flags().StringVar(&opts.SchedulerArgsFile, "scheduler-args-file", opts.SchedulerArgsFile, "file with the arguments for the scheduler") + liteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, "controller-manager-args-file", opts.ControllerManagerArgsFile, "file with the arguments for the controller manager") + liteCmd.Flags().StringVar(&opts.ProxyArgsFile, "proxy-args-file", opts.ProxyArgsFile, "file with the arguments for kube-proxy") + liteCmd.Flags().StringVar(&opts.KubeletArgsFile, "kubelet-args-file", opts.KubeletArgsFile, "file with the arguments for kubelet") + liteCmd.Flags().StringVar(&opts.APIServerArgsFile, "apiserver-args-file", opts.APIServerArgsFile, "file with the arguments for the API server") + liteCmd.Flags().StringVar(&opts.KubeconfigFile, "kubeconfig-file", opts.KubeconfigFile, "the kubeconfig file to use to healthcheck the API server") + liteCmd.Flags().BoolVar(&opts.StartControlPlane, "start-control-plane", opts.StartControlPlane, "start the control plane (API server, scheduler and controller manager)") +} diff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go new file mode 100644 index 00000000000..30ab604f480 --- /dev/null +++ b/cmd/kubelite/kubelite.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "k8s.io/component-base/logs" + _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/kubernetes/cmd/kubelite/app" +) + +func main() { + println("Starting kubelite") + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + // utilflag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + app.Execute() + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index d2d830376da..fa6dbc91fef 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -347,18 +347,23 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + localNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) PluginHandler.csiPlugin = p // This function prevents Kubelet from posting Ready status until CSINode // is both installed and initialized - if err := initializeCSINode(host, p.csiDriverInformer); err != nil { + if err := initializeCSINode(host, p.csiDriverInformer, localNim); err != nil { return errors.New(log("failed to initialize CSINode: %v", err)) } + + if _, ok := host.(volume.KubeletVolumeHost); ok { + nim = localNim + } + return nil } -func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer) error { +func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer, nim nodeinfomanager.Interface) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") -- 2.48.1 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.35.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch ================================================ From e2e2155d05b3ddd16640272bd1360425a3883c78 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 27 Jul 2023 18:08:00 +0300 Subject: [PATCH 2/2] Set log reapply handling to ignore unchanged --- staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go index 4c8a0d2c53f..6cb6ae1f42d 100644 --- a/staging/src/k8s.io/component-base/logs/api/v1/options.go +++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go @@ -65,7 +65,7 @@ func NewLoggingConfiguration() *LoggingConfiguration { // are no goroutines which might call logging functions. The default for ValidateAndApply // and ValidateAndApplyWithOptions is to return an error when called more than once. // Binaries and unit tests can override that behavior. -var ReapplyHandling = ReapplyHandlingError +var ReapplyHandling = ReapplyHandlingIgnoreUnchanged type ReapplyHandlingType int -- 2.43.0 ================================================ FILE: build-scripts/components/kubernetes/patches/v1.35.0/0002-fix-allow-node-to-get-endpointslices.patch ================================================ From dea2abd80878be1eff519216c0bad5a0e35462ec Mon Sep 17 00:00:00 2001 From: Mateo Florido Date: Thu, 11 Sep 2025 17:36:10 -0500 Subject: [PATCH] fix: allow node to get endpointslices --- plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 447b0bc2e99..daa3bde6b1c 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -228,6 +228,7 @@ func NodeRules() []rbacv1.PolicyRule { // TODO: add to the Node authorizer and restrict to endpoints referenced by pods or PVs bound to the node // Needed for glusterfs volumes rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), + rbacv1helpers.NewRule("get", "list", "watch").Groups(discoveryGroup).Resources("endpointslices").RuleOrDie(), // Used to create a certificatesigningrequest for a node-specific client certificate, and watch // for it to be signed. This allows the kubelet to rotate it's own certificate. rbacv1helpers.NewRule("create", "get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(), -- 2.48.1 ================================================ FILE: build-scripts/components/kubernetes/pre-patch.sh ================================================ #!/bin/bash -x # Ensure clean Kubernetes version KUBE_ROOT="${PWD}" source "${KUBE_ROOT}/hack/lib/version.sh" kube::version::get_version_vars kube::version::save_version_vars "${PWD}/.version.sh" ================================================ FILE: build-scripts/components/kubernetes/repository ================================================ https://github.com/kubernetes/kubernetes ================================================ FILE: build-scripts/components/kubernetes/version.sh ================================================ #!/bin/bash KUBE_TRACK="${KUBE_TRACK:-}" # example: "1.24" KUBE_VERSION="${KUBE_VERSION:-}" # example: "v1.24.2" if [ -z "${KUBE_VERSION}" ]; then if [ -z "${KUBE_TRACK}" ]; then KUBE_VERSION="$(curl -L --silent "https://dl.k8s.io/release/stable.txt")" else KUBE_TRACK="${KUBE_TRACK#v}" KUBE_VERSION="$(curl -L --silent "https://dl.k8s.io/release/stable-${KUBE_TRACK}.txt")" fi fi echo "${KUBE_VERSION}" ================================================ FILE: build-scripts/components/microk8s-completion/build.sh ================================================ #!/bin/bash INSTALL="${1}" go mod tidy -compat=1.17 go run -tags microk8s_hack ./cmd/helm 2> /dev/null cp microk8s.bash "${INSTALL}/microk8s.bash" ================================================ FILE: build-scripts/components/microk8s-completion/patches/default/0001-microk8s-autocompleter-script.patch ================================================ From 18e46f3fe5fdb6bb90c5c39b923c57d255733e1e Mon Sep 17 00:00:00 2001 From: Homayoon Alimohammadi Date: Thu, 27 Nov 2025 11:39:44 +0400 Subject: [PATCH] microk8s autocompleter script Signed-off-by: Homayoon Alimohammadi --- cmd/helm/hack_microk8s_autocompleter.go | 51 +++++++++++++++++++++++++ cmd/helm/helm.go | 2 +- go.mod | 6 +++ go.sum | 12 ++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 cmd/helm/hack_microk8s_autocompleter.go diff --git a/cmd/helm/hack_microk8s_autocompleter.go b/cmd/helm/hack_microk8s_autocompleter.go new file mode 100644 index 000000000..e819f44be --- /dev/null +++ b/cmd/helm/hack_microk8s_autocompleter.go @@ -0,0 +1,51 @@ +// go:build microk8s_hack + +package main + +import ( + "bytes" + "log" + "os" + "strings" + + "github.com/spf13/cobra" + "helm.sh/helm/v3/pkg/action" + "k8s.io/kubectl/pkg/cmd" +) + +func main() { + command := &cobra.Command{Use: "microk8s"} + command.AddCommand(cmd.NewDefaultKubectlCommand()) + helmCmd, _ := newRootCmd(&action.Configuration{}, nil, nil) + helmCmd.Use = "helm" + helm3Cmd, _ := newRootCmd(&action.Configuration{}, nil, nil) + helm3Cmd.Use = "helm3" + command.AddCommand(&cobra.Command{Use: "addons", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "add-node", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "config", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "enable", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "disable", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(helmCmd) + command.AddCommand(helm3Cmd) + command.AddCommand(&cobra.Command{Use: "helm3", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "images", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "inspect", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "join", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "refresh-certs", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "remove-node", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "reset", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "start", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "status", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "stop", Run: func(cmd *cobra.Command, args []string) {}}) + command.AddCommand(&cobra.Command{Use: "version", Run: func(cmd *cobra.Command, args []string) {}}) + + var b bytes.Buffer + if err := command.GenBashCompletion(&b); err != nil { + log.Fatalf("failed to generate completion script: %v", err) + } + completionScript := b.String() + completionScript = strings.ReplaceAll(completionScript, `args=("${words[@]:1}")`, `args=("${words[@]:2}")`) + completionScript = strings.ReplaceAll(completionScript, `requestComp="${words[0]} __completeNoDesc ${args[*]}"`, `requestComp="${words[0]} ${words[1]} __completeNoDesc ${args[*]}"`) + + os.WriteFile("microk8s.bash", []byte(completionScript), 0644) +} diff --git a/cmd/helm/helm.go b/cmd/helm/helm.go index 7bca93358..2008c1b08 100644 --- a/cmd/helm/helm.go +++ b/cmd/helm/helm.go @@ -62,7 +62,7 @@ func hookOutputWriter(_, _, _ string) io.Writer { return log.Writer() } -func main() { +func Main() { // Setting the name of the app for managedFields in the Kubernetes client. // It is set here to the full name of "helm" so that renaming of helm to // another name (e.g., helm2 or helm3) does not change the name of the diff --git a/go.mod b/go.mod index f2cecf187..534dfe524 100644 --- a/go.mod +++ b/go.mod @@ -73,6 +73,7 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect @@ -98,12 +99,14 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jonboulle/clockwork v0.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/lithammer/dedent v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect @@ -173,10 +176,13 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.34.0 // indirect + k8s.io/component-helpers v0.34.0 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/metrics v0.34.0 // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/kustomize/api v0.20.1 // indirect + sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 // indirect sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect diff --git a/go.sum b/go.sum index 217119fcb..bd7f38567 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -177,6 +179,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -208,6 +212,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -533,12 +539,16 @@ k8s.io/client-go v0.34.0 h1:YoWv5r7bsBfb0Hs2jh8SOvFbKzzxyNo0nSb0zC19KZo= k8s.io/client-go v0.34.0/go.mod h1:ozgMnEKXkRjeMvBZdV1AijMHLTh3pbACPvK7zFR+QQY= k8s.io/component-base v0.34.0 h1:bS8Ua3zlJzapklsB1dZgjEJuJEeHjj8yTu1gxE2zQX8= k8s.io/component-base v0.34.0/go.mod h1:RSCqUdvIjjrEm81epPcjQ/DS+49fADvGSCkIP3IC6vg= +k8s.io/component-helpers v0.34.0 h1:5T7P9XGMoUy1JDNKzHf0p/upYbeUf8ZaSf9jbx0QlIo= +k8s.io/component-helpers v0.34.0/go.mod h1:kaOyl5tdtnymriYcVZg4uwDBe2d1wlIpXyDkt6sVnt4= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= k8s.io/kubectl v0.34.0 h1:NcXz4TPTaUwhiX4LU+6r6udrlm0NsVnSkP3R9t0dmxs= k8s.io/kubectl v0.34.0/go.mod h1:bmd0W5i+HuG7/p5sqicr0Li0rR2iIhXL0oUyLF3OjR4= +k8s.io/metrics v0.34.0 h1:nYSfG2+tnL6/MRC2I+sGHjtNEGoEoM/KktgGOoQFwws= +k8s.io/metrics v0.34.0/go.mod h1:KCuXmotE0v4AvoARKUP8NC4lUnbK/Du1mluGdor5h4M= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= @@ -547,6 +557,8 @@ sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7np sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= +sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 h1:sYJsarwy/SDJfjjLMUqwFDGPwzUtMOQ1i1Ed49+XSbw= +sigs.k8s.io/kustomize/kustomize/v5 v5.7.1/go.mod h1:+5/SrBcJ4agx1SJknGuR/c9thwRSKLxnKoI5BzXFaLU= sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78= sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= -- 2.48.1 ================================================ FILE: build-scripts/components/microk8s-completion/repository ================================================ https://github.com/helm/helm ================================================ FILE: build-scripts/components/microk8s-completion/version.sh ================================================ #!/bin/bash echo "v3.19.2" ================================================ FILE: build-scripts/components/python/requirements.txt ================================================ PyYAML==6.0.1 netifaces==0.10.9 jsonschema==4.0.0 ================================================ FILE: build-scripts/components/runc/build.sh ================================================ #!/bin/bash VERSION="${2}" export INSTALL="${1}/bin" mkdir -p "${INSTALL}" # Ensure `runc --version` prints the correct release commit export COMMIT="$(git describe --always --long "${VERSION}")" make BUILDTAGS="seccomp apparmor" EXTRA_LDFLAGS="-s -w" static cp runc "${INSTALL}/runc" ================================================ FILE: build-scripts/components/runc/patches/default/0001-Disable-static-PIE-on-arm64.patch ================================================ From 7b7171f0f5048225e0d914cbffd47295af1fbfc5 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Fri, 28 Mar 2025 13:14:34 +0000 Subject: [PATCH] Disable static PIE on arm64 Ubuntu does not currently have the rcrt1.o file on arm64 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0a15fd90..d7e6209a 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ LDFLAGS_STATIC := -extldflags -static # Enable static PIE executables on supported platforms. # This (among the other things) requires libc support (rcrt1.o), which seems # to be available only for arm64 and amd64 (Debian Bullseye). -ifneq (,$(filter $(GOARCH),arm64 amd64)) +ifneq (,$(filter $(GOARCH),amd64)) ifeq (,$(findstring -race,$(EXTRA_FLAGS))) GO_BUILDMODE_STATIC := -buildmode=pie LDFLAGS_STATIC := -linkmode external -extldflags -static-pie -- 2.43.0 ================================================ FILE: build-scripts/components/runc/repository ================================================ https://github.com/opencontainers/runc ================================================ FILE: build-scripts/components/runc/strict-patches/v1.1.12/0001-apparmor-change-profile-immediately-not-on-exec.patch ================================================ From a367e391600dfab0d9eb3deaec4db300a2fb1fa1 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Wed, 16 Jun 2021 15:04:16 +0300 Subject: [PATCH 1/3] apparmor: change profile immediately, not on exec --- libcontainer/apparmor/apparmor_linux.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go index 8b1483c..292cfa6 100644 --- a/libcontainer/apparmor/apparmor_linux.go +++ b/libcontainer/apparmor/apparmor_linux.go @@ -48,9 +48,9 @@ func setProcAttr(attr, value string) error { return err } -// changeOnExec reimplements aa_change_onexec from libapparmor in Go -func changeOnExec(name string) error { - if err := setProcAttr("exec", "exec "+name); err != nil { +// changeProfile reimplements aa_change_profile from libapparmor in Go +func changeProfile(name string) error { + if err := setProcAttr("current", "changeprofile "+name); err != nil { return fmt.Errorf("apparmor failed to apply profile: %w", err) } return nil @@ -64,5 +64,5 @@ func applyProfile(name string) error { return nil } - return changeOnExec(name) + return changeProfile(name) } -- 2.34.1 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.1.12/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch ================================================ From 5351ef6f5b592472e077512714b2516cdbae1b51 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 1 Feb 2024 11:23:08 +0200 Subject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the apparmor profile With the current version of the AppArmor kernel module, it's not possible to switch the AppArmor profile if the NoNewPrivileges flag is set. So, we invert the order of the two operations. Adjusts the previous patch for runc version v1.1.12 Co-Authored-By: Alberto Mardegan --- libcontainer/setns_init_linux.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go index d1bb122..00407ce 100644 --- a/libcontainer/setns_init_linux.go +++ b/libcontainer/setns_init_linux.go @@ -56,11 +56,6 @@ func (l *linuxSetnsInit) Init() error { return err } } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return err - } - } if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { return err } @@ -84,6 +79,11 @@ func (l *linuxSetnsInit) Init() error { if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { return err } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + } // Check for the arg before waiting to make sure it exists and it is // returned as a create time error. -- 2.34.1 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.1.12/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch ================================================ From 103a94a51ea334d25bf573f2f20cd4d9a099d827 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Thu, 17 Jun 2021 14:31:35 +0300 Subject: [PATCH 3/3] standard_init_linux: change AppArmor profile as late as possible --- libcontainer/standard_init_linux.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go index d1d9435..7097571 100644 --- a/libcontainer/standard_init_linux.go +++ b/libcontainer/standard_init_linux.go @@ -127,10 +127,6 @@ func (l *linuxStandardInit) Init() error { return &os.SyscallError{Syscall: "sethostname", Err: err} } } - if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { - return fmt.Errorf("unable to apply apparmor profile: %w", err) - } - for key, value := range l.config.Config.Sysctl { if err := writeSystemProperty(key, value); err != nil { return err @@ -150,17 +146,21 @@ func (l *linuxStandardInit) Init() error { if err != nil { return fmt.Errorf("can't get pdeath signal: %w", err) } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return &os.SyscallError{Syscall: "prctl(SET_NO_NEW_PRIVS)", Err: err} - } - } // Tell our parent that we're ready to Execv. This must be done before the // Seccomp rules have been applied, because we need to be able to read and // write to a socket. if err := syncParentReady(l.pipe); err != nil { return fmt.Errorf("sync ready: %w", err) } + if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { + return fmt.Errorf("apply apparmor profile: %w", err) + } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return fmt.Errorf("set nonewprivileges: %w", err) + } + } + if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { return fmt.Errorf("can't set process label: %w", err) } -- 2.34.1 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.1.15/0001-apparmor-change-profile-immediately-not-on-exec.patch ================================================ From a367e391600dfab0d9eb3deaec4db300a2fb1fa1 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Wed, 16 Jun 2021 15:04:16 +0300 Subject: [PATCH 1/3] apparmor: change profile immediately, not on exec --- libcontainer/apparmor/apparmor_linux.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go index 8b1483c..292cfa6 100644 --- a/libcontainer/apparmor/apparmor_linux.go +++ b/libcontainer/apparmor/apparmor_linux.go @@ -48,9 +48,9 @@ func setProcAttr(attr, value string) error { return err } -// changeOnExec reimplements aa_change_onexec from libapparmor in Go -func changeOnExec(name string) error { - if err := setProcAttr("exec", "exec "+name); err != nil { +// changeProfile reimplements aa_change_profile from libapparmor in Go +func changeProfile(name string) error { + if err := setProcAttr("current", "changeprofile "+name); err != nil { return fmt.Errorf("apparmor failed to apply profile: %w", err) } return nil @@ -64,5 +64,5 @@ func applyProfile(name string) error { return nil } - return changeOnExec(name) + return changeProfile(name) } -- 2.34.1 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.1.15/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch ================================================ From b145a4ac9e9cd09e82d35e0998c6ddee80854275 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 1 Feb 2024 11:23:08 +0200 Subject: [PATCH] setns_init_linux: set the NNP flag after changing the apparmor profile With the current version of the AppArmor kernel module, it's not possible to switch the AppArmor profile if the NoNewPrivileges flag is set. So, we invert the order of the two operations. Adjusts the previous patch for runc version v1.1.12 Co-Authored-By: Alberto Mardegan --- libcontainer/setns_init_linux.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go index bb358901..6c1b16bd 100644 --- a/libcontainer/setns_init_linux.go +++ b/libcontainer/setns_init_linux.go @@ -57,11 +57,6 @@ func (l *linuxSetnsInit) Init() error { return err } } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return err - } - } // Tell our parent that we're ready to exec. This must be done before the // Seccomp rules have been applied, because we need to be able to read and @@ -93,6 +88,11 @@ func (l *linuxSetnsInit) Init() error { if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { return err } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + } // Check for the arg before waiting to make sure it exists and it is // returned as a create time error. -- 2.43.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.1.15/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch ================================================ From f9e0ca2f29c6c77ea9bc9c52929dac3915545dd9 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Thu, 17 Jun 2021 14:31:35 +0300 Subject: [PATCH] standard_init_linux: change AppArmor profile as late as possible --- libcontainer/standard_init_linux.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go index d9a6a224..e4d603e4 100644 --- a/libcontainer/standard_init_linux.go +++ b/libcontainer/standard_init_linux.go @@ -127,10 +127,6 @@ func (l *linuxStandardInit) Init() error { return &os.SyscallError{Syscall: "sethostname", Err: err} } } - if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { - return fmt.Errorf("unable to apply apparmor profile: %w", err) - } - for key, value := range l.config.Config.Sysctl { if err := writeSystemProperty(key, value); err != nil { return err @@ -150,11 +146,6 @@ func (l *linuxStandardInit) Init() error { if err != nil { return fmt.Errorf("can't get pdeath signal: %w", err) } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return &os.SyscallError{Syscall: "prctl(SET_NO_NEW_PRIVS)", Err: err} - } - } // Tell our parent that we're ready to exec. This must be done before the // Seccomp rules have been applied, because we need to be able to read and @@ -162,6 +153,15 @@ func (l *linuxStandardInit) Init() error { if err := syncParentReady(l.pipe); err != nil { return fmt.Errorf("sync ready: %w", err) } + if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { + return fmt.Errorf("apply apparmor profile: %w", err) + } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return fmt.Errorf("set nonewprivileges: %w", err) + } + } + if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { return fmt.Errorf("can't set process label: %w", err) } -- 2.43.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.2.6/0001-apparmor-change-profile-immediately-not-on-exec.patch ================================================ From 5cdb43bdc26e81be36d93fd8b81b7de6ad152c22 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Wed, 16 Jun 2021 15:04:16 +0300 Subject: [PATCH 1/3] apparmor: change profile immediately, not on exec --- libcontainer/apparmor/apparmor_linux.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go index 17d36ed1..fb159f3c 100644 --- a/libcontainer/apparmor/apparmor_linux.go +++ b/libcontainer/apparmor/apparmor_linux.go @@ -53,9 +53,9 @@ func setProcAttr(attr, value string) error { return err } -// changeOnExec reimplements aa_change_onexec from libapparmor in Go -func changeOnExec(name string) error { - if err := setProcAttr("exec", "exec "+name); err != nil { +// changeProfile reimplements aa_change_profile from libapparmor in Go +func changeProfile(name string) error { + if err := setProcAttr("current", "changeprofile "+name); err != nil { return fmt.Errorf("apparmor failed to apply profile: %w", err) } return nil @@ -69,5 +69,5 @@ func applyProfile(name string) error { return nil } - return changeOnExec(name) + return changeProfile(name) } -- 2.49.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.2.6/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch ================================================ From 259ebdf71e84433f55c1b28efb206ccc3a1b2736 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 1 Feb 2024 11:23:08 +0200 Subject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the apparmor profile With the current version of the AppArmor kernel module, it's not possible to switch the AppArmor profile if the NoNewPrivileges flag is set. So, we invert the order of the two operations. Adjusts the previous patch for runc version v1.1.12 Co-Authored-By: Alberto Mardegan --- libcontainer/setns_init_linux.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go index 92c6ef77..e9a55e31 100644 --- a/libcontainer/setns_init_linux.go +++ b/libcontainer/setns_init_linux.go @@ -62,11 +62,6 @@ func (l *linuxSetnsInit) Init() error { return fmt.Errorf("failed to setup pidfd: %w", err) } } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return err - } - } if l.config.Config.Umask != nil { unix.Umask(int(*l.config.Config.Umask)) } @@ -106,6 +101,11 @@ func (l *linuxSetnsInit) Init() error { if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { return err } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + } if l.config.Config.Personality != nil { if err := setupPersonality(l.config.Config); err != nil { return err -- 2.49.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.2.6/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch ================================================ From 3f82798f6081f060b244d7346524c2aced231287 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Thu, 17 Jun 2021 14:31:35 +0300 Subject: [PATCH 3/3] standard_init_linux: change AppArmor profile as late as possible --- libcontainer/standard_init_linux.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go index 9f7fa45d..c674d927 100644 --- a/libcontainer/standard_init_linux.go +++ b/libcontainer/standard_init_linux.go @@ -126,9 +126,6 @@ func (l *linuxStandardInit) Init() error { return &os.SyscallError{Syscall: "setdomainname", Err: err} } } - if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { - return fmt.Errorf("unable to apply apparmor profile: %w", err) - } for key, value := range l.config.Config.Sysctl { if err := writeSystemProperty(key, value); err != nil { @@ -149,11 +146,6 @@ func (l *linuxStandardInit) Init() error { if err != nil { return fmt.Errorf("can't get pdeath signal: %w", err) } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return &os.SyscallError{Syscall: "prctl(SET_NO_NEW_PRIVS)", Err: err} - } - } if l.config.Config.Scheduler != nil { if err := setupScheduler(l.config.Config); err != nil { @@ -172,6 +164,15 @@ func (l *linuxStandardInit) Init() error { if err := syncParentReady(l.pipe); err != nil { return fmt.Errorf("sync ready: %w", err) } + if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { + return fmt.Errorf("apply apparmor profile: %w", err) + } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return fmt.Errorf("set nonewprivileges: %w", err) + } + } + if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { return fmt.Errorf("can't set process label: %w", err) } -- 2.49.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.3.0/0001-apparmor-change-profile-immediately-not-on-exec.patch ================================================ From 5cdb43bdc26e81be36d93fd8b81b7de6ad152c22 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Wed, 16 Jun 2021 15:04:16 +0300 Subject: [PATCH 1/3] apparmor: change profile immediately, not on exec --- libcontainer/apparmor/apparmor_linux.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go index 17d36ed1..fb159f3c 100644 --- a/libcontainer/apparmor/apparmor_linux.go +++ b/libcontainer/apparmor/apparmor_linux.go @@ -53,9 +53,9 @@ func setProcAttr(attr, value string) error { return err } -// changeOnExec reimplements aa_change_onexec from libapparmor in Go -func changeOnExec(name string) error { - if err := setProcAttr("exec", "exec "+name); err != nil { +// changeProfile reimplements aa_change_profile from libapparmor in Go +func changeProfile(name string) error { + if err := setProcAttr("current", "changeprofile "+name); err != nil { return fmt.Errorf("apparmor failed to apply profile: %w", err) } return nil @@ -69,5 +69,5 @@ func applyProfile(name string) error { return nil } - return changeOnExec(name) + return changeProfile(name) } -- 2.49.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.3.0/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch ================================================ From 259ebdf71e84433f55c1b28efb206ccc3a1b2736 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 1 Feb 2024 11:23:08 +0200 Subject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the apparmor profile With the current version of the AppArmor kernel module, it's not possible to switch the AppArmor profile if the NoNewPrivileges flag is set. So, we invert the order of the two operations. Adjusts the previous patch for runc version v1.1.12 Co-Authored-By: Alberto Mardegan --- libcontainer/setns_init_linux.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go index 92c6ef77..e9a55e31 100644 --- a/libcontainer/setns_init_linux.go +++ b/libcontainer/setns_init_linux.go @@ -62,11 +62,6 @@ func (l *linuxSetnsInit) Init() error { return fmt.Errorf("failed to setup pidfd: %w", err) } } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return err - } - } if l.config.Config.Umask != nil { unix.Umask(int(*l.config.Config.Umask)) } @@ -106,6 +101,11 @@ func (l *linuxSetnsInit) Init() error { if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { return err } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + } if l.config.Config.Personality != nil { if err := setupPersonality(l.config.Config); err != nil { return err -- 2.49.0 ================================================ FILE: build-scripts/components/runc/strict-patches/v1.3.0/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch ================================================ From 7f91e445a8731856e2d22b2295d8438e07cf2bf7 Mon Sep 17 00:00:00 2001 From: Alberto Mardegan Date: Thu, 17 Jun 2021 14:31:35 +0300 Subject: [PATCH] standard_init_linux: change AppArmor profile as late as possible --- libcontainer/standard_init_linux.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go index 384750bf..ccd9297a 100644 --- a/libcontainer/standard_init_linux.go +++ b/libcontainer/standard_init_linux.go @@ -126,9 +126,6 @@ func (l *linuxStandardInit) Init() error { return &os.SyscallError{Syscall: "setdomainname", Err: err} } } - if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { - return fmt.Errorf("unable to apply apparmor profile: %w", err) - } for key, value := range l.config.Config.Sysctl { if err := writeSystemProperty(key, value); err != nil { @@ -149,11 +146,6 @@ func (l *linuxStandardInit) Init() error { if err != nil { return fmt.Errorf("can't get pdeath signal: %w", err) } - if l.config.NoNewPrivileges { - if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { - return &os.SyscallError{Syscall: "prctl(SET_NO_NEW_PRIVS)", Err: err} - } - } if err := setupScheduler(l.config); err != nil { return err @@ -169,6 +161,15 @@ func (l *linuxStandardInit) Init() error { if err := syncParentReady(l.pipe); err != nil { return fmt.Errorf("sync ready: %w", err) } + if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { + return fmt.Errorf("apply apparmor profile: %w", err) + } + if l.config.NoNewPrivileges { + if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return fmt.Errorf("set nonewprivileges: %w", err) + } + } + if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { return fmt.Errorf("can't set process label: %w", err) } -- 2.43.0 ================================================ FILE: build-scripts/components/runc/version.sh ================================================ #!/bin/bash echo "v1.3.3" ================================================ FILE: build-scripts/generate-bom.py ================================================ #!/usr/bin/env python3 import json import os from pathlib import Path import subprocess import sys import yaml DIR = Path(__file__).absolute().parent SNAPCRAFT_PART_BUILD = Path(os.getenv("SNAPCRAFT_PART_BUILD", "")) SNAPCRAFT_PART_INSTALL = Path(os.getenv("SNAPCRAFT_PART_INSTALL", "")) BUILD_DIRECTORY = SNAPCRAFT_PART_BUILD.exists() and SNAPCRAFT_PART_BUILD or DIR / ".build" INSTALL_DIRECTORY = SNAPCRAFT_PART_INSTALL.exists() and SNAPCRAFT_PART_INSTALL or DIR / ".install" # Location of Python binary PYTHON = INSTALL_DIRECTORY / ".." / ".." / "python-runtime" / "install" / "usr" / "bin" / "python3" # Location of MicroK8s addons MICROK8S_ADDONS = INSTALL_DIRECTORY / ".." / ".." / "microk8s-addons" / "install" / "addons" # List of tools used to build or bundled in the snap TOOLS = { "go": ["go", "version"], "gcc": ["gcc", "--version"], "python": [PYTHON, "-B", "-VV"], "python-requirements": [PYTHON, "-B", "-m", "pip", "freeze"], } # Retrieve list of components we care about from the snapcraft.yaml file with open(DIR / ".." / "snap" / "snapcraft.yaml") as fin: COMPONENTS = yaml.safe_load(fin)["parts"]["bom"]["after"] def _listdir(dir: Path): try: return sorted(os.listdir(dir)) except OSError: return [] def _parse_output(*args, **kwargs): return subprocess.check_output(*args, **kwargs).decode().strip() def _read_file(path: Path) -> str: return path.read_text().strip() if __name__ == "__main__": BOM = { "microk8s": { "version": _parse_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]), "revision": _parse_output(["git", "rev-parse", "HEAD"]), }, "tools": {}, "components": {}, "addons": {}, } for tool_name, version_cmd in TOOLS.items(): BOM["tools"][tool_name] = _parse_output(version_cmd).split("\n") for component in COMPONENTS: component_dir = DIR / "components" / component try: version = _parse_output([component_dir / "version.sh"]) patches = _parse_output([PYTHON, DIR / "print-patches-for.py", component, version]) clean_patches = [] if patches: clean_patches = [p[p.find("build-scripts/") :] for p in patches.split("\n")] BOM["components"][component] = { "repository": _read_file(component_dir / "repository"), "version": version, "revision": _parse_output( ["git", "rev-parse", "HEAD"], cwd=BUILD_DIRECTORY / ".." / ".." / component / "build" / component, ), "patches": clean_patches, } except OSError as e: print(f"Could not get info for {component}: {e}", file=sys.stderr) for repo in _listdir(MICROK8S_ADDONS): repo_dir = MICROK8S_ADDONS / repo if not repo_dir.is_dir(): continue BOM["addons"][repo] = { "repository": _parse_output(["git", "remote", "get-url", "origin"], cwd=repo_dir), "version": _parse_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=repo_dir), "revision": _parse_output(["git", "rev-parse", "HEAD"], cwd=repo_dir), } print(json.dumps(BOM, indent=2)) ================================================ FILE: build-scripts/images.txt ================================================ docker.io/calico/cni:v3.29.3 docker.io/calico/kube-controllers:v3.29.3 docker.io/calico/node:v3.29.3 docker.io/cdkbot/hostpath-provisioner:1.5.0 docker.io/coredns/coredns:1.12.3 docker.io/library/busybox:1.28.4 docker.io/traefik:v3.6.2 registry.k8s.io/metrics-server/metrics-server:v0.8.0 registry.k8s.io/pause:3.10 ================================================ FILE: build-scripts/print-patches-for.py ================================================ #!/usr/bin/env python3 import argparse import os from pathlib import Path DIR = Path(__file__).absolute().parent # SNAPCRAFT_PROJECT_DIR is set when building the snap. If unset, resolve based on current file path _SNAPCRAFT_PROJECT_DIR = os.getenv("SNAPCRAFT_PROJECT_DIR") or "" SNAPCRAFT_PROJECT_DIR = _SNAPCRAFT_PROJECT_DIR and Path(_SNAPCRAFT_PROJECT_DIR) or Path(DIR / "..") STRICT = "confinement: strict" in (SNAPCRAFT_PROJECT_DIR / "snap" / "snapcraft.yaml").read_text() class Version: def __init__(self, version_string: str): self.str = version_string if version_string[0] == "v": version_string = version_string[1:] if "-" in version_string: version_string = version_string[: version_string.rfind("-")] try: self.version = [int(x) for x in version_string.split(".")] self.type = "semver" except (TypeError, ValueError): self.version = self.str self.type = "string" def equal_or_older_than(self, v: "Version") -> bool: """Consider the following cases: - `v1.1.0` is equal or older than `v1.2.0`. - `v1.2.0` is equal or older than `v1.2.0`. - `v1.28.0-rc.0` is equal or older than v1.28.0`. - `fix/mybug` is not equal or older than `v1.28.0`. """ if self.type == v.type == "semver" and self.version <= v.version: return True if {self.type, v.type} & {"string"} and v.str.startswith(self.str): return True return False def find_suitable_patch_version(candidates: list, target_version: Version) -> str: """pick the version string from a list of candidate versions""" result = None has_default = False for candidate in candidates: if candidate.str == target_version.str: return target_version.str has_default = has_default or candidate.str == "default" if not candidate.equal_or_older_than(target_version): continue if result is None or result.equal_or_older_than(candidate): result = candidate # found a suitable patch directory if result: return result.str # no suitable patch directory found, but there is a default if has_default: return "default" # component does not have any patches return None def get_patches_for(component: str, version_string: str, strict: bool) -> list: """Return a list of patches that must be applied when building 'component' with target 'version'. """ component_version = Version(version_string) component_dir = SNAPCRAFT_PROJECT_DIR / "build-scripts" / "components" / component patches = [] patch_directories = ["patches"] if strict: patch_directories += ["strict-patches"] for patch_dir_name in patch_directories: patches_dir = component_dir / patch_dir_name if not patches_dir.is_dir(): continue candidates = [ Version(path.name) for path in sorted( patches_dir.iterdir(), reverse=True ) # sort by reverse to handle exact matches if path.is_dir() ] patch_version = find_suitable_patch_version(candidates, component_version) if patch_version is not None: patches.extend( [p.resolve().as_posix() for p in (patches_dir / patch_version).iterdir()] ) return patches def main(): parser = argparse.ArgumentParser() parser.add_argument("component", type=str) parser.add_argument("version", type=str) args = parser.parse_args() print("\n".join(get_patches_for(args.component, args.version, STRICT))) if __name__ == "__main__": main() ================================================ FILE: build-scripts/update-images.sh ================================================ #!/bin/bash -x ## Description: # # Install MicroK8s from a specific channel and update the list of images required by MicroK8s # and the core addons. # ## Example: # # $ ./build-scripts/update-images.sh latest/edge build-scripts/images.txt sudo snap install microk8s --classic --channel $1 sudo microk8s status --wait-ready sudo microk8s enable storage ingress metrics-server dns sudo microk8s kubectl apply -f - < $2 ================================================ FILE: docs/build.md ================================================ # Building and testing MicroK8s ## Building the snap from source To build the MicroK8s snap, you need to install Snapcraft. ```shell sudo snap install snapcraft --classic ``` [Building a snap](https://snapcraft.io/docs/snapcraft-overview) is done by running `snapcraft` in the root of the project. Snapcraft spawn a VM managed by [Multipass](https://multipass.run/) and builds MicroK8s inside of it. If you don’t have Multipass installed, snapcraft will first prompt for its automatic installation. After `snapcraft` finishes, you can install the newly compiled snap: ```shell sudo snap install microk8s_*_amd64.snap --classic --dangerous ``` ## Building the snap using LXD Alternatively, you can build the snap in an LXC container. This is useful if you are working in a virtual machine without nested virtualization. Snapcraft and LXD are needed in this case: ```shell sudo snap install lxd sudo apt-get remove lxd* -y sudo apt-get remove lxc* -y sudo lxd init sudo usermod -a -G lxd ${USER} ``` Build the snap with: ```shell git clone http://github.com/canonical/microk8s cd ./microk8s/ snapcraft --use-lxd ``` Finally, install it with: ```shell sudo snap install microk8s_*_amd64.snap --classic --dangerous ``` ## Building a custom MicroK8s package To produce a custom build with specific component versions you cannot use the snapcraft build process on the host OS. You need to [prepare an LXC](https://forum.snapcraft.io/t/how-to-create-a-lxd-container-for-snap-development/4658) container with Ubuntu 16:04 and snapcraft: ```shell lxc launch ubuntu:16.04 --ephemeral test-build lxc exec test-build -- snap install snapcraft --classic lxc exec test-build -- apt update lxc exec test-build -- git clone https://github.com/canonical/microk8s ``` You can then set the following environment variables prior to building: - KUBE_VERSION: Kubernetes release to package. Defaults to latest stable. - ETCD_VERSION: version of etcd. - CNI_VERSION: version of CNI tools. - KUBE_TRACK: Kubernetes release series (e.g., 1.10) to package. Defaults to latest stable. - ISTIO_VERSION: istio release. - KNATIVE_SERVING_VERSION: Knative Serving release. - KNATIVE_EVENTING_VERSION: Knative Eventing release. - RUNC_COMMIT: the commit hash from which to build runc. - CONTAINERD_COMMIT: the commit hash from which to build containerd - KUBERNETES_REPOSITORY: build the Kubernetes binaries from this repository instead of getting them from upstream - KUBERNETES_COMMIT: commit to be used from KUBERNETES_REPOSITORY for building the Kubernetes binaries For building you prepend the variables you need as well as `SNAPCRAFT_BUILD_ENVIRONMENT=host` so the current LXC container is used. For example to build the MicroK8s snap for Kubernetes v1.9.6, run: ```shell lxc exec test-build -- sh -c "cd microk8s && SNAPCRAFT_BUILD_ENVIRONMENT=host KUBE_VERSION=v1.9.6 snapcraft" ``` The produced snap is inside the ephemeral LXC container, you need to copy it to the host: ```shell lxc file pull test-build/root/microk8s/microk8s_v1.9.6_amd64.snap . ``` After copying it, you can install it with: ```shell snap install microk8s_*_amd64.snap --classic --dangerous ``` ## Assembling the Calico CNI manifest The calico CNI manifest can be found under `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`. Building the manifest is subject to the upstream calico project k8s installation process. The `calico.yaml` manifest is a slightly modified version of: `https://projectcalico.docs.tigera.io/archive/v3.25/manifests/calico.yaml`: - Set the calico backend from "bird" to "vxlan": `calico_backend: "vxlan"` - `CALICO_IPV4POOL_CIDR` was set to "10.1.0.0/16" - `CNI_NET_DIR` had to be added and set to "/var/snap/microk8s/current/args/cni-network" - We set the following mount paths: 1. `var-run-calico` to `/var/snap/microk8s/current/var/run/calico` 1. `var-lib-calico` to `/var/snap/microk8s/current/var/lib/calico` 1. `cni-bin-dir` to `/var/snap/microk8s/current/opt/cni/bin` 1. `cni-net-dir` to `/var/snap/microk8s/current/args/cni-network` 1. `cni-log-dir` to `/var/snap/microk8s/common/var/log/calico/cni` 1. `host-local-net-dir` to `/var/snap/microk8s/current/var/lib/cni/networks` 1. `policysync` to `/var/snap/microk8s/current/var/run/nodeagent` - We enabled vxlan following the instructions in [the official docs.](https://docs.projectcalico.org/getting-started/kubernetes/installation/config-options#switching-from-ip-in-ip-to-vxlan) - The `liveness` and `readiness` probes of `bird-live` was commented out - We set the IP autodetection method to ```dtd - name: IP_AUTODETECTION_METHOD value: "first-found" ``` - In the `cni_network_config` in the calico manifest we also set: ```dtd "log_file_path": "/var/snap/microk8s/common/var/log/calico/cni/cni.log", "nodename_file": "/var/snap/microk8s/current/var/lib/calico/nodename", ``` - The `mount-bpffs` pod is commented out. This disables eBPF support but allows the CNI to deploy inside an LXC container. - The bpffs mount is commented out: ```dtd - name: bpffs mountPath: /sys/fs/bpf ``` end ```dtd - name: bpffs hostPath: path: /sys/fs/bpf type: Directory ``` ## Running the tests locally To successfully run the tests you need to install: 1. From your distribution's repository: - python3 - pytest - pip3 - docker.io - tox 1. From pip3: - requests - pyyaml - sh On Ubuntu 20.04, for example, run the following commands to get these dependencies. ```shell sudo apt install python3-pip docker.io tox -y pip3 install -U pytest requests pyyaml sh ``` First, run the static analysis using Tox. Run the following command in the root of the project. ```shell tox -e lint ``` Then, use the "Building the snap from source" instructions to build the snap and install it locally. The tests check this locally installed MicroK8s instance. Finally, run the tests themselves. The `test-simple.py` and `test-upgrade.py` files under the `tests` directory are the two main files of our test suite. Running the tests is done with pytest: ```shell pytest -s tests/test-simple.py pytest -s tests/test-upgrade.py ``` Note: the `ingress` and `dashboard-ingress` tests make use of nip.io for wildcard ingress domains on localhost. [DNS rebinding protection](https://en.wikipedia.org/wiki/DNS_rebinding) may prevent the resolution of the domains used in the tests. A workaround is adding these entries to `/etc/hosts`: ``` 127.0.0.1 kubernetes-dashboard.127.0.0.1.nip.io 127.0.0.1 microbot.127.0.0.1.nip.io ``` ================================================ FILE: docs/community.md ================================================ # MicroK8s in the Wild People are doing amazing things with MicroK8s. Feel free to submit a pull request with updates to this page. | Event/Mention | Author(s) | Source | Date | |---------------------------------------------------------------------|--------------|----------|---------------| | [Kubernetes Automation Toolkit](https://github.com/BigBitBusInc/kubernetes-automation-toolkit) | Sachin Agarwal (BigBitBus) | github.com | 6-Aug-2021 | | [IoT/edge: Akri (MSFT) on MicroK8s](https://github.com/didier-durand/microk8s-akri) | Didier Durand | github.com | 27-Nov-2020 | | [Kata Containers on MicroK8s](https://github.com/didier-durand/microk8s-kata-containers) | Didier Durand | github.com | 13-Nov-2020 | | [MicroK8s analyzed with kube-bench](https://github.com/didier-durand/microk8s-kube-bench) | Didier Durand | github.com | 23-Oct-2020 | | [MicroK8s on Raspberry Pi](https://github.com/trulede/raspberry.microk8s) | Tim Rule | github.com | 27-April-2020 | | [Getting started with Kubernetes MicroK8s & Linode](https://www.youtube.com/watch?v=RiV2TNcjAFw) | Egee | Youtube.com | 6-Oct-2019 | | [How do I set up Tilt to use MicroK8s?](https://docs.tilt.dev/faq.html#q-how-do-i-set-up-tilt-to-use-microk8s) | The tilt.dev team | docs.tilt.dev | 8-Aug-2019 | | [Microk8s + Flask (Part 2): Exposing Flask app in a Microk8s service](https://medium.com/@giorgos.apost.1994/microk8s-flask-part-2-280ac2d9eff) | George Apostolopoulos | medium.com | 19-Aug-2019 | | [Microk8s + Flask (Part 1): Dockerize a Flask Application](https://medium.com/@giorgos.apost.1994/microk8s-flask-part-1-efa3cc4e0e00) | George Apostolopoulos | medium.com | 19-Jul-2019 | | [Getting Started with Cilium on Microk8s](https://docs.cilium.io/en/stable/gettingstarted/microk8s/)| Joe Stringer | docs.cilium.io | 23-Apr-2019 | | [Single-User Production on Microk8s](https://transcrob.es/post/single-user-prod-microk8s/)| Anton Melser | transcrob.es | 05-Feb-2019 | | [MicroK8s, Part 3: How To Deploy a Pod in Kubernetes](https://virtualizationreview.com/articles/2019/02/01/microk8s-part-3-how-to-deploy-a-pod-in-kubernetes.aspx)| Tom Fenton | virtualizationreview.com | 02-Feb-2019 | | [MicroK8s, Part 2: How To Monitor and Manage Kubernetes](https://virtualizationreview.com/articles/2019/01/30/microk8s-part-2-how-to-monitor-and-manage-kubernetes.aspx)| Tom Fenton | virtualizationreview.com | 30-Jan-2019 | | [MicroK8s: How To Install and Use Kubernetes](https://virtualizationreview.com/articles/2019/01/28/microk8s-how-to-install-and-use-kubernetes.aspx)| Tom Fenton | virtualizationreview.com | 28-Jan-2019 | | [SUSE Cloud Application Platform (CAP) on Microk8s](https://dimitris.karakasilis.me/2019/01/27/scf-on-microk8s.html)| Dimitris Karakasilis | dimitris.karakasilis.me | 27-Jan-2019 | | [Files upload from Kubeless on MicroK8s to Minio](https://itnext.io/files-upload-from-kubeless-on-microk8s-to-minio-607e06598a4b) | Rafał bluszcz Zawadzki | itnext.io | 07-Jan-2019 | | [Get into Kubernetes with MicroK8s](https://medium.com/devopslinks/get-into-kubernetes-with-microk8s-85f0e0231c4a) | Milindu S. Kumarage | medium.com | 06-Jan-2019 | | [Serverless MicroK8s Kubernetes](https://medium.com/@bluszcz/serverless-microk8s-kubernetes-fcd6b875cd33) | Rafał bluszcz Zawadzki | medium.com | 06-Jan-2019 | | [Deploy OpenFaaS with MicroK8s](https://johnmccabe.net/technology/projects/openfaas-on-microk8s/) | John McCabe | johnmccabe.net | 01-Jan-2019 | | [Docker desktop, microk8s and the battle for the k8s laptop](https://blog.linnovate.net/docker-desktop-microk8s-and-the-battle-for-the-k8s-laptop-8d3fb50dd543) | liorkesos | blog.linnovate.net | 18-Dec-2018 | | [Canonical Launches MicroK8s](http://www.linux-magazine.com/Online/News/Canonical-Launches-MicroK8s) | Swapnil Bhartiya | linux-magazine.com | 12-Dec-2018 | | [Kubernetes single node cluster using microk8s](https://www.youtube.com/watch?v=FEELm-o__gU) | Just me and Opensource | youtube.com | 02-Dec-2018 | | [Getting started with Kubeflow (machine learning toolkit for Kubernetes) via Microk8s and Multipass](https://github.com/leigh-johnson/kubeflow-microk8s-demo) | Leigh Johnson | github.com | 20-Nov-2018 | | [Kubernetes CICD Hacks with MicroK8s and Kubeflow](https://www.youtube.com/watch?v=1SSvS2w5OMQ), KubeCon China 2018 | Land Lu, Zhang Lei Mao from Canonical | KubeCon 2018 | 14-Nov-2018 | | [Kubernetes On A Laptop](https://kubedex.com/local-kubernetes/) | Steven Acreman | kubedex.com | 03-Nov-2018 | | [Local Kubernetes for Linux – MiniKube vs MicroK8s](https://codefresh.io/kubernetes-tutorial/local-kubernetes-linux-minikube-vs-microk8s/) | Thomas Pliakas | codefresh.io | 01-Nov-2018 | | [A local Kubernetes with microk8s](https://itnext.io/a-local-kubernetes-with-microk8s-33ee31d1eed9) | Marco Ceppi | itnext.io | 22-May-2018 | ================================================ FILE: docs/k8s-patches.md ================================================ # Patches for Kubernetes The patch to add kubelite is applied on top of the Kubernetes source code during the MicroK8s build process. The Kubernetes patches are under `build-scripts/patches/`. The patches are maintained in the [kubernetes-dqlite repository](https://github.com/canonical/kubernetes-dqlite). To produce them checkout the kubernetes-dqlite repository and head to the Kubernetes branch you would like to build. For example: ``` git clone https://github.com/canonical/kubernetes-dqlite.git cd kubernetes-dqlite/ git checkout release-1.18 ``` From `git log` note down the commits related to k8s patches. These commits are at the top. Form the patch with for example: ``` git format-patch c3c94660a58998dde628de7c716a63b695327016^..f0cc11e7a398d9454e8c4d3a526d6372cf1a1889 --stdout > k8s.patch ``` Remember to copy the produced patch file back to the `microk8s` source tree. ================================================ FILE: installer/.gitignore ================================================ .venv/ bin/ lib/ lib64 pyvenv.cfg share/ ================================================ FILE: installer/__init__.py ================================================ ================================================ FILE: installer/build-linux.sh ================================================ #!/bin/bash virtualenv -p python3 .venv source ./.venv/bin/activate pip install -r requirements.txt pyinstaller ./microk8s.spec deactivate rm -rf .venv ================================================ FILE: installer/cli/__init__.py ================================================ ================================================ FILE: installer/cli/echo.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2017 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Facilities to output to the terminal. These methods, which are named after common logging levels, wrap around click.echo adding the corresponding color codes for each level. """ import click import sys from typing import Any from common.definitions import MAX_CHARACTERS_WRAP class Echo: @staticmethod def is_tty_connected() -> bool: """Check to see if running under TTY.""" return sys.stdin.isatty() @staticmethod def wrapped(msg: str) -> None: """Output msg wrapped to the terminal width to stdout. The maximum wrapping is determined by MAX_CHARACTERS_WRAP """ click.echo( click.formatting.wrap_text(msg, width=MAX_CHARACTERS_WRAP, preserve_paragraphs=True) ) @staticmethod def info(msg: str) -> None: """Output msg as informative to stdout. If the terminal supports colors the output will be green. """ click.echo("\033[0;32m{}\033[0m".format(msg)) @staticmethod def warning(msg: str) -> None: """Output msg as a warning to stdout. If the terminal supports color the output will be yellow. """ click.echo("\033[1;33m{}\033[0m".format(msg)) @staticmethod def error(msg: str) -> None: """Output msg as an error to stdout. If the terminal supports color the output will be red. """ click.echo("\033[0;31m{}\033[0m".format(msg)) def confirm( self, msg: str, default: bool = False, abort: bool = False, prompt_suffix: str = ": ", show_default: bool = True, err: bool = False, ) -> bool: """Output message as a confirmation prompt. If not running on a tty, assume the default value. """ return ( click.confirm( msg, default=default, abort=abort, prompt_suffix=prompt_suffix, show_default=show_default, err=err, ) if self.is_tty_connected() else default ) def prompt( self, msg: str, default: Any = None, hide_input: bool = False, confirmation_prompt: bool = False, type=None, value_proc=None, prompt_suffix: str = ": ", show_default: bool = True, err: bool = False, ) -> Any: """Output message as a generic prompt. If not running on a tty, assume the default value. """ return ( click.prompt( msg, default=default, hide_input=hide_input, confirmation_prompt=confirmation_prompt, type=type, value_proc=value_proc, prompt_suffix=prompt_suffix, show_default=show_default, err=err, ) if self.is_tty_connected() else default ) ================================================ FILE: installer/cli/microk8s.py ================================================ import argparse import logging import traceback from typing import List from sys import exit, platform from os import getcwd import click from cli.echo import Echo from common import definitions from common.auxiliary import Windows, MacOS, Linux from common.errors import BaseError from common.file_utils import get_kubeconfig_path, clear_kubeconfig from vm_providers.factory import get_provider_for from vm_providers.errors import ProviderNotFound, ProviderInstanceNotFoundError logger = logging.getLogger(__name__) @click.command( name="microk8s", context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, ), ) @click.option("-h", "--help", is_flag=True) @click.pass_context def cli(ctx, help): try: if help and len(ctx.args) == 0: show_help() exit(0) elif help: ctx.args.append("--help") if len(ctx.args) == 0: show_error() exit(1) if ctx.args[0] == "install": install(ctx.args[1:]) exit(0) elif ctx.args[0] == "uninstall": uninstall() exit(0) elif ctx.args[0] == "start": start() run(ctx.args) exit(0) elif ctx.args[0] == "stop": run(ctx.args) stop() exit(0) elif ctx.args[0] == "kubectl": exit(kubectl(ctx.args[1:])) elif ctx.args[0] == "dashboard-proxy": dashboard_proxy() exit(0) elif ctx.args[0] == "inspect": inspect() exit(0) else: run(ctx.args) exit(0) except BaseError as e: Echo.error(str(e)) exit(e.get_exit_code()) except Exception as e: Echo.error("An unexpected error occurred.") Echo.info(str(e)) Echo.info(traceback.print_exc()) exit(254) def show_error(): msg = """Usage: microk8s [OPTIONS] COMMAND [ARGS]... Options: --help Shows the available COMMANDS.""" click.echo(msg) def show_help(): msg = """Usage: microk8s [OPTIONS] COMMAND [ARGS]... Options: --help Show this message and exit. Commands: install Installs MicroK8s. Use --cpu, --mem, --disk, --channel, and --image to configure your setup. uninstall Removes MicroK8s""" click.echo(msg) commands = _get_microk8s_commands() for command in commands: if command in definitions.command_descriptions: click.echo(" {:<15} {}".format(command, definitions.command_descriptions[command])) else: click.echo(" {:<15}".format(command)) if len(commands) == 2: click.echo("") click.echo("Install and start MicroK8s to see the full list of commands.") def _show_install_help(): msg = f"""Usage: microk8s install OPTIONS Options: --help Show this message and exit. --cpu Cores used by MicroK8s (default={definitions.DEFAULT_CORES}, min={definitions.MIN_CORES}) --mem RAM in GB used by MicroK8s (default={definitions.DEFAULT_MEMORY_GB}, min={definitions.MIN_MEMORY_GB}) --disk Max volume in GB of the dynamically expandable hard disk to be used (default={definitions.DEFAULT_DISK_GB}, min={definitions.MIN_DISK_GB}) --channel Kubernetes version to install (default={definitions.DEFAULT_CHANNEL}) --image Ubuntu version to install (default={definitions.DEFAULT_IMAGE}) -y, --assume-yes Automatic yes to prompts""" # noqa Echo.info(msg) def memory(mem_gb: str) -> int: """ Validates the value in --mem parameter of the install command. """ mem_gb = int(mem_gb) if mem_gb < definitions.MIN_MEMORY_GB: raise ValueError("Out of valid memory range") return mem_gb def cpu(cpus: str) -> int: """ Validates the value in --cpu parameter of the install command. """ cpus = int(cpus) if cpus < definitions.MIN_CORES: raise ValueError("Invalid number of cpus") return cpus def disk(disk_gb: str) -> int: """ Validates the value in --disk parameter of the install command. """ disk_gb = int(disk_gb) if disk_gb < definitions.MIN_DISK_GB: raise ValueError("Out of valid disk range") return disk_gb def install(args) -> None: if "--help" in args or "-h" in args: _show_install_help() return parser = argparse.ArgumentParser("microk8s install") parser.add_argument("--cpu", default=definitions.DEFAULT_CORES, type=cpu) parser.add_argument("--mem", default=definitions.DEFAULT_MEMORY_GB, type=memory) parser.add_argument("--disk", default=definitions.DEFAULT_DISK_GB, type=disk) parser.add_argument("--channel", default=definitions.DEFAULT_CHANNEL, type=str) parser.add_argument("--image", default=definitions.DEFAULT_IMAGE, type=str) parser.add_argument( "-y", "--assume-yes", action="store_true", default=definitions.DEFAULT_ASSUME ) args = parser.parse_args(args) echo = Echo() if platform == "win32": host = Windows(args) elif platform == "darwin": host = MacOS(args) else: host = Linux(args) if not host.has_enough_cpus(): echo.error("VM cpus requested exceed number of available cores on host.") exit(1) if not host.has_enough_memory(): echo.warning("VM memory requested exceeds the total memory on host.") exit(1) if not host.has_enough_disk_space(): echo.warning("VM disk size requested exceeds free space on host.") vm_provider_name: str = "multipass" vm_provider_class = get_provider_for(vm_provider_name) try: vm_provider_class.ensure_provider() except ProviderNotFound as provider_error: if provider_error.prompt_installable: if args.assume_yes or ( echo.is_tty_connected() and echo.confirm( "Support for {!r} needs to be set up. " "Would you like to do that now?".format(provider_error.provider) ) ): vm_provider_class.setup_provider(echoer=echo) else: raise provider_error else: raise provider_error instance = vm_provider_class(echoer=echo) spec = vars(args) spec.update({"kubeconfig": get_kubeconfig_path()}) instance.launch_instance(spec) echo.info("MicroK8s is up and running. See the available commands with `microk8s --help`.") def uninstall() -> None: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) echo = Echo() try: vm_provider_class.ensure_provider() except ProviderNotFound as provider_error: if provider_error.prompt_installable: if echo.is_tty_connected(): echo.warning( ( "MicroK8s is not running. VM provider {!r} has been removed.".format( provider_error.provider ) ) ) return 1 else: raise provider_error instance = vm_provider_class(echoer=echo) instance.destroy() clear_kubeconfig() echo.info("Thank you for using MicroK8s!") def kubectl(args) -> int: if platform == "win32": return Windows(args).kubectl() if platform == "darwin": return MacOS(args).kubectl() else: return Linux(args).kubectl() def inspect() -> None: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) echo = Echo() try: vm_provider_class.ensure_provider() instance = vm_provider_class(echoer=echo) instance.get_instance_info() command = ["microk8s.inspect"] output = instance.run(command, hide_output=True) tarball_location = None host_destination = getcwd() if b"Report tarball is at" not in output: echo.error("Report tarball not generated") else: for line_out in output.split(b"\n"): line_out = line_out.decode() line = line_out.strip() if line.startswith("Report tarball is at "): tarball_location = line.split("Report tarball is at ")[1] break echo.wrapped(line_out) if not tarball_location: echo.error("Cannot find tarball file location") else: instance.pull_file(name=tarball_location, destination=host_destination) echo.wrapped( "The report tarball {} is stored on the current directory".format( tarball_location.split("/")[-1] ) ) except ProviderInstanceNotFoundError: _not_installed(echo) return 1 def dashboard_proxy() -> None: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) echo = Echo() try: vm_provider_class.ensure_provider() instance = vm_provider_class(echoer=echo) instance.get_instance_info() echo.info("Checking if Dashboard is running.") command = ["microk8s.enable", "dashboard"] output = instance.run(command, hide_output=True) if b"Addon dashboard is already enabled." not in output: echo.info("Waiting for Dashboard to come up.") command = [ "microk8s.kubectl", "-n", "kube-system", "wait", "--timeout=240s", "deployment", "kubernetes-dashboard", "--for", "condition=available", ] instance.run(command, hide_output=True) command = ["microk8s.kubectl", "-n", "kube-system", "create", "token", "default"] token = instance.run(command, hide_output=True).decode().strip() if not token: echo.error("Could not generate secret token to access dashboard.") exit(1) ip = instance.get_instance_info().ipv4[0] echo.info("Dashboard will be available at https://{}:10443".format(ip)) echo.info("Use the following token to login:") echo.info(token) command = [ "microk8s.kubectl", "port-forward", "-n", "kube-system", "service/kubernetes-dashboard", "10443:443", "--address", "0.0.0.0", ] try: instance.run(command) except KeyboardInterrupt: return except ProviderInstanceNotFoundError: _not_installed(echo) return 1 def start() -> None: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) vm_provider_class.ensure_provider() instance = vm_provider_class(echoer=Echo()) instance_info = instance.get_instance_info() if not instance_info.is_running(): instance.start() instance.run(["microk8s.start"]) def stop() -> None: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) vm_provider_class.ensure_provider() instance = vm_provider_class(echoer=Echo()) instance_info = instance.get_instance_info() if instance_info.is_running(): instance.stop() def run(cmd) -> None: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) echo = Echo() try: vm_provider_class.ensure_provider() instance = vm_provider_class(echoer=echo) instance_info = instance.get_instance_info() if not instance_info.is_running(): echo.warning("MicroK8s is not running. Please run `microk8s start`.") return 1 command = cmd[0] cmd[0] = "microk8s.{}".format(command) instance.run(cmd) except ProviderInstanceNotFoundError: _not_installed(echo) return 1 def _not_installed(echo) -> None: if echo.is_tty_connected(): echo.warning("MicroK8s is not installed. Please run `microk8s install`.") def _get_microk8s_commands() -> List: vm_provider_name = "multipass" vm_provider_class = get_provider_for(vm_provider_name) echo = Echo() try: vm_provider_class.ensure_provider() instance = vm_provider_class(echoer=echo) instance_info = instance.get_instance_info() if instance_info.is_running(): commands = instance.run("ls -1 /snap/bin/".split(), hide_output=True) mk8s = [ c.decode().replace("microk8s.", "") for c in commands.split() if c.decode().startswith("microk8s.") ] complete = mk8s if "dashboard-proxy" not in mk8s: complete += ["dashboard-proxy"] complete.sort() return complete else: return ["start", "stop"] except ProviderNotFound: return ["start", "stop"] if __name__ == "__main__": cli() ================================================ FILE: installer/common/__init__.py ================================================ ================================================ FILE: installer/common/auxiliary.py ================================================ import ctypes import logging import os import psutil import subprocess from abc import ABC from os.path import realpath from shutil import disk_usage from .file_utils import get_kubeconfig_path, get_kubectl_directory from . import definitions logger = logging.getLogger(__name__) class Auxiliary(ABC): """ Base OS auxiliary class. """ def __init__(self, args) -> None: """ :param args: ArgumentParser :return: None """ self._args = args try: self.requested_disk = self._args.disk * 1024 * 1024 * 1024 self.requested_memory = self._args.mem * 1024 * 1024 * 1024 self.requested_cores = self._args.cpu except AttributeError: self.requested_disk = definitions.DEFAULT_DISK_GB self.requested_memory = definitions.DEFAULT_MEMORY_GB self.requested_cores = definitions.DEFAULT_CORES @staticmethod def _free_disk_space() -> int: """ Get space free of disk that this script is installed to. :return: Integer free space """ return disk_usage(realpath("/")).free @staticmethod def _total_memory() -> int: """ Get available memory in machine this script is installed to. :return: Available memory in bytes """ return psutil.virtual_memory().total @staticmethod def _cpu_count() -> int: """ Get the number of cpus on the machine this script is installed to. :return: Number of cpus """ return psutil.cpu_count(logical=False) def has_enough_disk_space(self) -> bool: """ Compare free space with minimum. :return: Boolean """ return self._free_disk_space() > self.requested_disk def has_enough_memory(self) -> bool: """ Compare requested memory against available :return: Boolean """ return self._total_memory() > self.requested_memory def has_enough_cpus(self) -> bool: """ Compare requested cpus against available cores. :return: Boolean """ return self._cpu_count() >= self.requested_cores def get_kubectl_directory(self) -> str: """ Get the correct directory to install kubectl into, we can then call this when running `microk8s kubectl` without interfering with any systemwide install. :return: String """ return get_kubectl_directory() def get_kubeconfig_path(self) -> str: """ Get the correct path to write the kubeconfig file to. This is then read by the installed kubectl and won't interfere with one in the user's home. :return: String """ return get_kubeconfig_path() def kubectl(self) -> int: """ Run kubectl on the host, with the generated kubeconf. :return: None """ kctl_dir = self.get_kubectl_directory() try: exit_code = subprocess.check_call( [ os.path.join(kctl_dir, "kubectl"), "--kubeconfig={}".format(self.get_kubeconfig_path()), ] + self._args, ) except subprocess.CalledProcessError as e: return e.returncode else: return exit_code class Windows(Auxiliary): """ Windows auxiliary methods. """ def __init__(self, args) -> None: """ :param args: ArgumentParser :return: None """ super(Windows, self).__init__(args) @staticmethod def check_admin() -> bool: """ Check if running as admin. :return: Boolean """ return ctypes.windll.shell32.IsUserAnAdmin() == 1 @staticmethod def check_hyperv() -> bool: """ Check if Hyper V is already enabled. :return: Boolean """ try: out = subprocess.check_output( ["DISM", "/Online", "/Get-FeatureInfo", "/FeatureName:Microsoft-Hyper-V"] ) except subprocess.CalledProcessError: return False if "State : Disabled" in out.decode(): return False return True @staticmethod def enable_hyperv() -> None: """ Enable Hyper V feature. :return: None """ try: subprocess.check_call( [ "DISM", "/Online", "/Enable-Feature", "/All", "/NoRestart", "/FeatureName:Microsoft-Hyper-V", ] ) except subprocess.CalledProcessError as e: if e.returncode == 3010: pass # This is fine, because Windows. else: raise class Linux(Auxiliary): """ MacOS auxiliary methods. """ def __init__(self, args) -> None: """ :param args: ArgumentParser :return: None """ super(Linux, self).__init__(args) class MacOS(Linux): """ MacOS auxiliary methods. """ def __init__(self, args) -> None: """ :param args: ArgumentParser :return: None """ super(MacOS, self).__init__(args) ================================================ FILE: installer/common/definitions.py ================================================ MAX_CHARACTERS_WRAP: int = 120 command_descriptions = { "add-node": "Adds a node to a cluster", "ambassador": "Ambassador API Gateway and Ingress", "cilium": "The cilium client", "config": "Print the kubeconfig", "ctr": "The containerd client", "dashboard-proxy": "Enable the Kubernetes dashboard and proxy to host", "dbctl": "Backup and restore the Kubernetes datastore", "disable": "Disables running add-ons", "enable": "Enables useful add-ons", "helm": "The helm client", "helm3": "The helm3 client", "inspect": "Checks the cluster and gathers logs", "istioctl": "The istio client", "join": "Joins this instance as a node to a cluster", "kubectl": "The kubernetes client", "leave": "Disconnects this node from any cluster it has joined", "linkerd": "The linkerd client", "refresh-certs": "Refresh the CA certificates in this deployment", "remove-node": "Removes a node from the cluster", "reset": "Cleans the cluster from all workloads", "start": "Starts the kubernetes cluster", "status": "Displays the status of the cluster", "stop": "Stops the kubernetes cluster", } DEFAULT_CORES: int = 2 DEFAULT_MEMORY_GB: int = 4 DEFAULT_DISK_GB: int = 50 DEFAULT_ASSUME: bool = False DEFAULT_CHANNEL: str = "1.28/stable" DEFAULT_IMAGE: str = "22.04" MIN_CORES: int = 2 MIN_MEMORY_GB: int = 2 MIN_DISK_GB: int = 10 ================================================ FILE: installer/common/errors.py ================================================ class BaseError(Exception): """Base class for all exceptions. :cvar fmt: A format string that daughter classes override """ fmt = "Daughter classes should redefine this" def __init__(self, **kwargs) -> None: for key, value in kwargs.items(): setattr(self, key, value) def __str__(self): return self.fmt.format([], **self.__dict__) def get_exit_code(self): """Exit code to use if this exception causes the program to exit.""" return 2 ================================================ FILE: installer/common/file_utils.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2016-2019 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import hashlib import logging import os import shutil import sys if sys.version_info < (3, 6): pass logger = logging.getLogger(__name__) def _file_reader_iter(path: str, block_size=2 ** 20): with open(path, "rb") as f: block = f.read(block_size) while len(block) > 0: yield block block = f.read(block_size) def calculate_sha3_384(path: str) -> str: """Calculate sha3 384 hash, reading the file in 1MB chunks.""" return calculate_hash(path, algorithm="sha3_384") def calculate_hash(path: str, *, algorithm: str) -> str: """Calculate the hash for path with algorithm.""" # This will raise an AttributeError if algorithm is unsupported hasher = getattr(hashlib, algorithm)() for block in _file_reader_iter(path): hasher.update(block) return hasher.hexdigest() def is_dumb_terminal(): """Return True if on a dumb terminal.""" is_stdout_tty = os.isatty(1) is_term_dumb = os.environ.get("TERM", "") == "dumb" return not is_stdout_tty or is_term_dumb def get_kubectl_directory() -> str: """ Get the correct directory to install kubectl into, we can then call this when running `microk8s kubectl` without interfering with any systemwide install. :return: String """ if sys.platform == "win32": if getattr(sys, "frozen", None): d = os.path.dirname(sys.executable) else: d = os.path.dirname(os.path.abspath(__file__)) return os.path.join(d, "kubectl") else: full_path = shutil.which("kubectl") return os.path.dirname(full_path) def get_kubeconfig_path(): """Return a MicroK8s specific kubeconfig path.""" if sys.platform == "win32": return os.path.join(os.environ.get("LocalAppData"), "MicroK8s", "config") else: return os.path.join(os.path.expanduser("~"), ".microk8s", "config") def clear_kubeconfig(): """Clean kubeconfig file.""" if os.path.isdir(get_kubeconfig_path()): shutil.rmtree(os.path.dirname(get_kubeconfig_path())) ================================================ FILE: installer/microk8s.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2020 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from cli import microk8s if __name__ == "__main__": microk8s.cli() ================================================ FILE: installer/microk8s.spec ================================================ # -*- mode: python ; coding: utf-8 -*- import PyInstaller.config PyInstaller.config.CONF['distpath'] = "./" block_cipher = None a = Analysis( ['microk8s.py'], pathex=[''], binaries=[], datas=[], hiddenimports=[ "click", "responses", "pkg_resources.py2_warn" ], hookspath=[], runtime_hooks=[], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=False, cipher=block_cipher, noarchive=False ) pyz = PYZ( a.pure, a.zipped_data, cipher=block_cipher ) exe = EXE( pyz, a.scripts, a.binaries, a.zipfiles, a.datas, [], name='microk8s', debug=False, bootloader_ignore_signals=False, strip=False, upx=True, upx_exclude=[], runtime_tmpdir=None, icon="windows/microk8s.ico", console=True ) ================================================ FILE: installer/requirements.txt ================================================ urllib3==1.26.19 click==7.1.2 progressbar33==2.4 psutil==5.9.0 requests==2.32.2 requests_unixsocket==0.1.5 pysha3==1.0.2; python_version < '3.6' simplejson==3.8.2 toml==0.10.0 certifi==2024.7.4 chardet==3.0.4 idna==3.7 pyinstaller https://pyyaml.org/download/pyyaml/PyYAML-5.3.1-cp38-cp38-win_amd64.whl; sys_platform == 'win32' ================================================ FILE: installer/setup.cfg ================================================ [flake8] max-line-length = 120 ================================================ FILE: installer/setup.py ================================================ from setuptools import setup setup( name="microk8s", version="1.0.2", url="https://github.com/canonical/microk8s", license="Apache-2.0", author="Joe Borg", author_email="joseph.borg@canonical.com", description="MicroK8s is a small, fast, single-package Kubernetes for developers, IoT and edge", packages=["cli", "common", "vm_providers", "vm_providers._multipass", "vm_providers.repo"], install_requires=[ "click~=7.0", "progressbar33==2.4", "requests==2.32.2", "requests_unixsocket==0.1.5", "simplejson==3.8.2", "toml==0.10.0", "urllib3==1.26.19", ], platforms="any", entry_points={ "console_scripts": [ "microk8s=cli.microk8s:cli", ] }, ) ================================================ FILE: installer/tests/__init__.py ================================================ ================================================ FILE: installer/tests/integration/test_cli.py ================================================ import os import subprocess import platform from unittest import mock import pytest from click.testing import CliRunner from cli.microk8s import cli class TestClass: @pytest.mark.skipif( platform.system() != "Linux", reason="Add/remove multipass is available on Linux" ) @pytest.mark.skipif(os.getuid() != 0, reason="Add/remove multipass is possible to root") @mock.patch("sys.stdin.isatty", return_value=True) def test_install_remove_multipass(self, tty_mock): """ We are root on a Linux box. Test the install and remove of multipass """ runner = CliRunner() # making sure we start on a clean machine with multipass result = runner.invoke(cli, "uninstall") subprocess.check_call("sudo snap install multipass --classic".split()) assert os.path.isfile("/snap/bin/multipass") assert result.exit_code == 0 # making sure we start on a clean machine result = runner.invoke(cli, "install") assert result.exit_code == 0 assert os.path.isfile("/snap/bin/multipass") result = runner.invoke(cli, "status --wait-ready --timeout=60") assert result.exit_code == 0 result = runner.invoke(cli, "install") assert os.path.isfile("/snap/bin/multipass") assert result.exit_code == 0 def test_all_cli(self): runner = CliRunner() # Test no args. We should get an error. result = runner.invoke(cli) assert result.exit_code == 1 # Test no args. We should get an error. result = runner.invoke(cli, "--help") assert result.exit_code == 0 def test_install_argument_are_validated(self): runner = CliRunner() result = runner.invoke(cli, ["install", "--mem", "1"]) assert result.exit_code == 2 assert "invalid memory value" in result.output result = runner.invoke(cli, ["install", "--cpu", "1"]) assert result.exit_code == 2 assert "invalid cpu value" in result.output result = runner.invoke(cli, ["install", "--disk", "1"]) assert result.exit_code == 2 assert "invalid disk value" in result.output ================================================ FILE: installer/tests/unit/test_auxiliary.py ================================================ from common.auxiliary import Auxiliary from unittest.mock import Mock, patch def get_mocked_args(disk=1, mem=1, cpu=1): args = Mock(mem=mem, disk=disk, cpu=cpu) return args @patch("common.auxiliary.psutil.virtual_memory") def test_has_enough_memory(virtual_memory_mock): args = get_mocked_args(mem=50) host = Auxiliary(args) # 3 Bytes of memory virtual_memory_mock.return_value.total = 3 assert host.has_enough_memory() is False # 300 GB of memory virtual_memory_mock.return_value.total = 300 * 1024 * 1024 * 1024 assert host.has_enough_memory() is True @patch("common.auxiliary.psutil.cpu_count") def test_has_enough_cpus(cpu_count_mock): args = get_mocked_args(cpu=4) host = Auxiliary(args) cpu_count_mock.return_value = 3 assert host.has_enough_cpus() is False cpu_count_mock.return_value = 5 assert host.has_enough_cpus() is True @patch("common.auxiliary.disk_usage") def test_has_enough_disk_space(disk_usage_mock): args = get_mocked_args(disk=10) host = Auxiliary(args) disk_usage_mock.return_value.free = 3 assert host.has_enough_disk_space() is False disk_usage_mock.return_value.free = 50 * 1024 * 1024 * 1024 assert host.has_enough_disk_space() is True ================================================ FILE: installer/tests/unit/test_cli.py ================================================ from cli.microk8s import install from unittest.mock import patch, MagicMock import pytest @patch("common.auxiliary.Auxiliary.has_enough_cpus", return_value=False) def test_install_exits_on_cpus_requested_exceed_available_on_host(has_enough_cpus_mock): with pytest.raises(SystemExit) as exc: install(MagicMock()) assert exc.value.code == 1 has_enough_cpus_mock.assert_called_once() @patch("common.auxiliary.Auxiliary.has_enough_memory", return_value=False) @patch("common.auxiliary.Auxiliary.has_enough_cpus", return_value=True) def test_install_exits_on_memory_requested_exceed_available_on_host( has_enough_cpus_mock, has_enough_memory_mock, ): with pytest.raises(SystemExit) as exc: install(MagicMock()) assert exc.value.code == 1 has_enough_memory_mock.assert_called_once() ================================================ FILE: installer/vm_providers/__init__.py ================================================ ================================================ FILE: installer/vm_providers/_base_provider.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018-2019 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import abc import logging import os import pathlib import requests import shlex import sys from typing import Dict from typing import Optional, Sequence from . import errors from ._multipass._instance_info import InstanceInfo logger = logging.getLogger(__name__) class Provider(abc.ABC): def __init__( self, *, echoer, is_ephemeral: bool = False, build_provider_flags: Dict[str, str] = None, ) -> None: self.echoer = echoer self._is_ephemeral = is_ephemeral self.instance_name = "microk8s-vm" if build_provider_flags is None: build_provider_flags = dict() self.build_provider_flags = build_provider_flags.copy() self._cached_home_directory: Optional[pathlib.Path] = None @classmethod def ensure_provider(cls) -> None: """Necessary steps to ensure the provider is correctly setup.""" @classmethod def setup_provider(cls, *, echoer) -> None: """Necessary steps to install the provider on the host.""" @classmethod def _get_provider_name(cls) -> str: """Return the provider name.""" @classmethod def _get_is_snap_injection_capable(cls) -> bool: """Return whether the provider can install snaps from the host.""" @abc.abstractmethod def create(self) -> None: """Provider steps needed to create a fully functioning environment.""" @abc.abstractmethod def destroy(self) -> None: """Provider steps needed to ensure the instance is destroyed. This method should be safe to call multiple times and do nothing if the instance to destroy is already destroyed. """ @abc.abstractmethod def get_instance_info(self) -> InstanceInfo: """Return the instance info.""" @abc.abstractmethod def run(self, command: Sequence[str], hide_output: bool = False) -> Optional[bytes]: """Run a command on the instance.""" @abc.abstractmethod def _launch(self, specs: Dict): """Launch the instance.""" @abc.abstractmethod def _start(self): """Start an existing the instance.""" @abc.abstractmethod def _push_file(self, *, source: str, destination: str) -> None: """Push a file into the instance.""" @abc.abstractmethod def pull_file(self, name: str, destination: str, delete: bool = False) -> None: """ Provider steps needed to retrieve a file from the instance, optionally deleting the source file after a successful retrieval. :param name: the remote filename. :type name: str :param destination: the local filename. :type destination: str :param delete: whether the file should be deleted. :type delete: bool """ @abc.abstractmethod def shell(self) -> None: """Provider steps to provide a shell into the instance.""" def launch_instance(self, specs: Dict) -> None: try: # An ProviderStartError exception here means we need to create. self._start() except errors.ProviderInstanceNotFoundError: self._launch(specs) self._check_connectivity() # We need to setup MicroK8s and scan for cli commands. self._setup_microk8s(specs) self._copy_kubeconfig_to_kubectl(specs) def _check_connectivity(self) -> None: """Check that the VM can access the internet.""" try: requests.get("https://snapcraft.io") except requests.exceptions.RequestException: self.destroy() url = None if sys.platform == "win32": url = "https://multipass.run/docs/troubleshooting-networking-on-windows" elif sys.platform == "darwin": url = "https://multipass.run/docs/troubleshooting-networking-on-macos" if url: raise errors.ConnectivityError( "The VM cannot connect to snapcraft.io, please see {}".format(url) ) else: raise def _copy_kubeconfig_to_kubectl(self, specs: Dict): kubeconfig_path = specs.get("kubeconfig") kubeconfig = self.run(command=["microk8s", "config"], hide_output=True) if not os.path.isdir(os.path.dirname(kubeconfig_path)): os.mkdir(os.path.dirname(kubeconfig_path)) with open(kubeconfig_path, "wb") as f: f.write(kubeconfig) def _setup_microk8s(self, specs: Dict) -> None: self.run("snap install microk8s --classic --channel {}".format(specs["channel"]).split()) if sys.platform == "win32": self.run("snap install microk8s-integrator-windows".split()) elif sys.platform == "darwin": self.run("snap install microk8s-integrator-macos".split()) def _get_env_command(self) -> Sequence[str]: """Get command sequence for `env` with configured flags.""" env_list = ["env"] # Pass through configurable environment variables. for key in ["http_proxy", "https_proxy"]: value = self.build_provider_flags.get(key) if not value: continue # Ensure item is treated as string and append it. value = str(value) env_list.append(f"{key}={value}") return env_list def _get_home_directory(self) -> pathlib.Path: """Get user's home directory path.""" if self._cached_home_directory is not None: return self._cached_home_directory command = ["printenv", "HOME"] run_output = self.run(command=command, hide_output=True) # Shouldn't happen, but due to _run()'s return type as being Optional, # we need to check for it anyways for mypy. if not run_output: provider_name = self._get_provider_name() raise errors.ProviderExecError( provider_name=provider_name, command=command, exit_code=2 ) cached_home_directory = pathlib.Path(run_output.decode().strip()) self._cached_home_directory = cached_home_directory return cached_home_directory def _base_has_changed(self, base: str, provider_base: str) -> bool: # Make it backwards compatible with instances without project info if base == "core18" and provider_base is None: return False elif base != provider_base: return True return False def _log_run(self, command: Sequence[str]) -> None: cmd_string = " ".join([shlex.quote(c) for c in command]) logger.debug(f"Running: {cmd_string}") @abc.abstractmethod def stop(self) -> None: pass ================================================ FILE: installer/vm_providers/_multipass/__init__.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from ._multipass import Multipass # noqa: F401 from ._multipass_command import MultipassCommand # noqa: F401 ================================================ FILE: installer/vm_providers/_multipass/_instance_info.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import json from typing import Any, Dict, Type from vm_providers import errors class InstanceInfo: @classmethod def from_json( cls: Type["InstanceInfo"], *, instance_name: str, json_info: str ) -> "InstanceInfo": """Create an InstanceInfo from json_info retrieved from multipass. :param str instance_name: the name of the instance. :param str json_info: a json formatted string with the structure that would follow the output of a json formatted multipass info command. :returns: an InstanceInfo. :rtype: InstanceInfo :raises vm_providers.ProviderInfoDataKeyError: if the instance name cannot be found in the given json or if a required key is missing from that data structure for the instance. """ try: json_data = json.loads(json_info) except json.decoder.JSONDecodeError as decode_error: raise errors.ProviderBadDataError( provider_name="multipass", data=json_info ) from decode_error try: instance_info = json_data["info"][instance_name] except KeyError as missing_key: raise errors.ProviderInfoDataKeyError( provider_name="multipass", missing_key=str(missing_key), data=json_data ) from missing_key try: return cls( name=instance_name, state=instance_info["state"], image_release=instance_info["image_release"], mounts=instance_info["mounts"], ipv4=instance_info["ipv4"], ) except KeyError as missing_key: raise errors.ProviderInfoDataKeyError( provider_name="multipass", missing_key=str(missing_key), data=instance_info, ) from missing_key def __init__( self, *, name: str, state: str, image_release: str, mounts: Dict[str, Any], ipv4: str ) -> None: """Initialize an InstanceInfo. :param str name: the instance name. :param str state: the state of the instance which can be any one of RUNNING, STOPPED, DELETED. :param str image_release: the Operating System release string for the image. """ # We do not check for validity of state given that new states could # be introduced. self.name = name self.state = state self.image_release = image_release self.mounts = mounts self.ipv4 = ipv4 def is_mounted(self, mountpoint: str) -> bool: return mountpoint in self.mounts def is_stopped(self) -> bool: return self.state.upper() == "STOPPED" def is_running(self) -> bool: return self.state.upper() == "RUNNING" ================================================ FILE: installer/vm_providers/_multipass/_multipass.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018-2019 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import sys from typing import Dict, Optional, Sequence from time import sleep from vm_providers import errors from .._base_provider import Provider from ._instance_info import InstanceInfo from ._multipass_command import MultipassCommand logger = logging.getLogger(__name__) class Multipass(Provider): """A multipass provider for MicroK8s to execute its lifecycle.""" @classmethod def ensure_provider(cls): MultipassCommand.ensure_multipass(platform=sys.platform) @classmethod def setup_provider(cls, *, echoer) -> None: MultipassCommand.setup_multipass(echoer=echoer, platform=sys.platform) @classmethod def _get_is_snap_injection_capable(cls) -> bool: return True @classmethod def _get_provider_name(cls): return "multipass" def run(self, command: Sequence[str], hide_output: bool = False) -> Optional[bytes]: cmd = ["sudo"] cmd.extend(command) self._log_run(cmd) return self._multipass_cmd.execute( instance_name=self.instance_name, command=cmd, hide_output=hide_output ) def _launch(self, specs: Dict) -> None: # prepare core launch setting image = specs["image"] cpus = "{}".format(specs["cpu"]) mem = "{}G".format(specs["mem"]) disk = "{}G".format(specs["disk"]) try_for = 10 while True: try: self._multipass_cmd.launch( instance_name=self.instance_name, cpus=cpus, mem=mem, disk=disk, image=image ) except Exception: if try_for > 0: try_for -= 1 sleep(1) continue else: raise else: break def get_instance_info(self) -> InstanceInfo: try: instance_info = self._get_instance_info() return instance_info except errors.ProviderInfoError as instance_error: # Until we have proper multipass error codes to know if this # was a communication error we should keep this error tracking # and generation here. raise errors.ProviderInstanceNotFoundError( instance_name=self.instance_name ) from instance_error def _start(self): try: instance_info = self._get_instance_info() if not instance_info.is_running(): self._multipass_cmd.start(instance_name=self.instance_name) except errors.ProviderInfoError as instance_error: # Until we have proper multipass error codes to know if this # was a communication error we should keep this error tracking # and generation here. raise errors.ProviderInstanceNotFoundError( instance_name=self.instance_name ) from instance_error def _umount(self, *, mountpoint: str) -> None: mount = "{}:{}".format(self.instance_name, mountpoint) self._multipass_cmd.umount(mount=mount) def _push_file(self, *, source: str, destination: str) -> None: destination = "{}:{}".format(self.instance_name, destination) self._multipass_cmd.copy_files(source=source, destination=destination) def __init__( self, *, echoer, is_ephemeral: bool = False, build_provider_flags: Dict[str, str] = None, ) -> None: super().__init__( echoer=echoer, is_ephemeral=is_ephemeral, build_provider_flags=build_provider_flags, ) self._multipass_cmd = MultipassCommand(platform=sys.platform) self._instance_info: Optional[InstanceInfo] = None def create(self, specs: Dict) -> None: """Create the multipass instance and setup the build environment.""" self.echoer.info("Launching a VM.") self.launch_instance(specs) self._instance_info = self._get_instance_info() def destroy(self) -> None: """Destroy the instance, trying to stop it first.""" try: instance_info = self._instance_info = self._get_instance_info() except errors.ProviderInfoError: return if instance_info.is_stopped(): return self._multipass_cmd.stop(instance_name=self.instance_name) self._multipass_cmd.delete(instance_name=self.instance_name, purge=True) def pull_file(self, name: str, destination: str, delete: bool = False) -> None: # TODO add instance check. # check if file exists in instance self.run(command=["test", "-f", name]) # copy file from instance source = "{}:{}".format(self.instance_name, name) self._multipass_cmd.copy_files(source=source, destination=destination) if delete: self.run(command=["rm", name]) def shell(self) -> None: self.run(command=["/bin/bash"]) def _get_instance_info(self) -> InstanceInfo: instance_info_raw = self._multipass_cmd.info( instance_name=self.instance_name, output_format="json" ) return InstanceInfo.from_json( instance_name=self.instance_name, json_info=instance_info_raw.decode() ) def start(self) -> None: instance_info = self._instance_info = self._get_instance_info() if not instance_info.is_stopped(): return self._multipass_cmd.start(instance_name=self.instance_name) def stop(self) -> None: instance_info = self._instance_info = self._get_instance_info() if instance_info.is_stopped(): return self._multipass_cmd.stop(instance_name=self.instance_name) ================================================ FILE: installer/vm_providers/_multipass/_multipass_command.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import shutil import subprocess from time import sleep from typing import Dict, Optional, Sequence # noqa: F401 from ._windows import windows_reload_multipass_path_env, windows_install_multipass from vm_providers import errors from vm_providers.repo.snaps import install_snaps logger = logging.getLogger(__name__) def _run(command: Sequence[str], stdin=subprocess.DEVNULL) -> None: logger.debug("Running {}".format(" ".join(command))) subprocess.check_call(command, stdin=stdin) def _run_output(command: Sequence[str], **kwargs) -> bytes: logger.debug("Running {}".format(" ".join(command))) return subprocess.check_output(command, **kwargs) class MultipassCommand: """An object representation of common multipass cli commands.""" provider_name = "multipass" provider_cmd = "multipass" @classmethod def ensure_multipass(cls, platform: str) -> None: if platform == "win32": # Reload path env just in case multipass was installed without # launching a new command prompt / shell. windows_reload_multipass_path_env() if shutil.which(cls.provider_cmd): return if platform == "darwin": prompt_installable = True elif platform == "linux" and shutil.which("snap"): prompt_installable = True elif platform == "win32": prompt_installable = True else: prompt_installable = False raise errors.ProviderNotFound( provider=cls.provider_name, prompt_installable=prompt_installable, error_message="https://multipass.run", ) @classmethod def _wait_for_multipass_ready(cls, *, echoer): echoer.wrapped("Waiting for multipass...") retry_count = 60 while retry_count: try: output = subprocess.check_output([cls.provider_cmd, "version"]).decode() except subprocess.CalledProcessError: output = "" except FileNotFoundError: raise errors.ProviderStartError( provider_name=cls.provider_name, error_message="multipass not found - please check that it" " can be found in the configured PATH", ) # if multipassd is in the version information, it means the service is up # and we can carry on if "multipassd" in output: break retry_count -= 1 sleep(1) # No need to worry about getting to this point by exhausting our retry count, # the rest of the stack will handle the error appropriately. @classmethod def setup_multipass(cls, *, echoer, platform: str) -> None: if platform == "linux": install_snaps(["multipass/latest/stable"]) elif platform == "darwin": try: subprocess.check_call(["brew", "install", "multipass", "--cask"]) except subprocess.CalledProcessError: raise errors.ProviderStartError( provider_name=cls.provider_name, error_message="Failed to install multipass using homebrew.\n" "Verify your homebrew installation and try again.\n" "Alternatively, manually install multipass by running" " 'brew install multipass --cask'.", ) elif platform == "win32": windows_install_multipass(echoer) else: raise EnvironmentError( "Setting up multipass for {!r} is not supported.".format(platform) ) # wait for multipassd to be available cls._wait_for_multipass_ready(echoer=echoer) def __init__(self, *, platform: str) -> None: """Initialize a MultipassCommand instance. :raises errors.ProviderCommandNotFound: if the multipass command is not found. """ self.ensure_multipass(platform=platform) def launch( self, *, instance_name: str, image: str, cpus: str = None, mem: str = None, disk: str = None, remote: str = None, cloud_init: str = None ) -> None: """Passthrough for running multipass launch. :param str instance_name: the name the launched instance will have. :param str image: the image to create the instance with. :param str cpus: amount of virtual CPUs to assign to the launched instance. :param str mem: amount of RAM to assign to the launched instance. :param str disk: amount of disk space the instance will see. :param str remote: the remote server to retrieve the image from. :param str cloud_init: path to a user-data cloud-init configuration. """ if remote is not None: image = "{}:{}".format(remote, image) cmd = [self.provider_cmd, "launch", image, "--name", instance_name] if cloud_init is not None: cmd.extend(["--cloud-init", cloud_init]) if cpus is not None: cmd.extend(["--cpus", cpus]) if mem is not None: cmd.extend(["--memory", mem]) if disk is not None: cmd.extend(["--disk", disk]) try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderLaunchError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def start(self, *, instance_name: str) -> None: """Passthrough for running multipass start. :param str instance_name: the name of the instance to start. """ cmd = [self.provider_cmd, "start", instance_name] try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderStartError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def stop(self, *, instance_name: str, time: int = None) -> None: """Passthrough for running multipass stop. :param str instance_name: the name of the instance to stop. :param str time: time from now, in minutes, to delay shutdown of the instance. """ cmd = [self.provider_cmd, "stop"] if time: cmd.extend(["--time", str(time)]) cmd.append(instance_name) try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderStopError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def delete(self, *, instance_name: str, purge=True) -> None: """Passthrough for running multipass delete. :param str instance_name: the name of the instance to delete. :param bool purge: if true, purge the instance's image after deleting. """ cmd = [self.provider_cmd, "delete", instance_name] if purge: cmd.append("--purge") try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderDeleteError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def execute( self, *, command: Sequence[str], instance_name: str, hide_output: bool = False ) -> Optional[bytes]: """Passthrough for running multipass exec. :param list command: the command to execute on the instance. :param str instance_name: the name of the instance to execute command. :param bool hide_output: hide the output from stdout. """ cmd = [self.provider_cmd, "exec", instance_name, "--"] + list(command) output = None try: if hide_output: output = _run_output(cmd) else: _run(cmd, stdin=None) except subprocess.CalledProcessError as process_error: raise errors.ProviderExecError( provider_name=self.provider_name, command=command, exit_code=process_error.returncode, ) from process_error return output def shell(self, *, instance_name: str) -> None: """Passthrough for running multipass shell. :param str instance_name: the name of the instance to execute command. """ try: _run([self.provider_cmd, "shell", instance_name]) except subprocess.CalledProcessError as process_error: raise errors.ProviderShellError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def mount( self, *, source: str, target: str, uid_map: Dict[str, str] = None, gid_map: Dict[str, str] = None ) -> None: """Passthrough for running multipass mount. :param str source: path to the local directory to mount. :param str target: mountpoint inside the instance in the form of :path. :param dict uid_map: A mapping of user IDs for use in the mount of the form -> . File and folder ownership will be mapped from to inside the instance. :param dict gid_map: A mapping of group IDs for use in the mount of the form -> . File and folder ownership will be mapped from to inside the instance. :raises errors.ProviderMountError: when the mount operation fails. """ cmd = [self.provider_cmd, "mount", source, target] if uid_map is None: uid_map = dict() for host_map, instance_map in uid_map.items(): cmd.extend(["--uid-map", "{}:{}".format(host_map, instance_map)]) if gid_map is None: gid_map = dict() for host_map, instance_map in gid_map.items(): cmd.extend(["--gid-map", "{}:{}".format(host_map, instance_map)]) try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderMountError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def umount(self, *, mount: str) -> None: """Passthrough for running multipass mount. :param str mount: mountpoint inside the instance in the form of :path to unmount. :raises errors.ProviderUMountError: when the unmount operation fails. """ cmd = [self.provider_cmd, "umount", mount] try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderUnMountError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def copy_files(self, *, source: str, destination: str) -> None: """Passthrough for running multipass copy-files. :param str source: the source file to copy, using syntax expected by multipass. :param str destination: the destination of the copied file, using syntax expected by multipass. """ cmd = [self.provider_cmd, "copy-files", source, destination] try: _run(cmd) except subprocess.CalledProcessError as process_error: raise errors.ProviderFileCopyError( provider_name=self.provider_name, exit_code=process_error.returncode ) from process_error def info(self, *, instance_name: str, output_format: str = None) -> bytes: """Passthrough for running multipass info.""" cmd = [self.provider_cmd, "info", instance_name] if output_format is not None: cmd.extend(["--format", output_format]) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode != 0: raise errors.ProviderInfoError( provider_name=self.provider_name, exit_code=process.returncode, stderr=stderr, ) return stdout ================================================ FILE: installer/vm_providers/_multipass/_windows.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import logging import os.path import requests import shutil import simplejson import subprocess import sys import tempfile from progressbar import AnimatedMarker, Bar, Percentage, ProgressBar, UnknownLength from common.file_utils import calculate_sha3_384, is_dumb_terminal from vm_providers.errors import ( ProviderMultipassDownloadFailed, ProviderMultipassInstallationFailed, ) if sys.platform == "win32": import winreg logger = logging.getLogger(__name__) _MULTIPASS_RELEASES_API_URL = "https://api.github.com/repos/canonical/multipass/releases" _MULTIPASS_DL_VERSION = "1.12.2" _MULTIPASS_DL_NAME = "multipass-{version}+win-win64.exe".format(version=_MULTIPASS_DL_VERSION) # Download multipass installer and calculate hash: # python3 -c "from installer.common.file_utils import calculate_sha3_384; print(calculate_sha3_384('$HOME/Downloads/multipass-1.11.1+win-win64.exe'))" # noqa: E501 _MULTIPASS_DL_SHA3_384 = "9031c8fc98b941df1094a832c356e12f281c70d0eb10bee15b5576c61af4c8a17ef32b833f0043c8df0e04897e69c8bc" # noqa: E501 def windows_reload_multipass_path_env(): """Update PATH to include installed Multipass, if not already set.""" assert sys.platform == "win32" key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Environment") paths = os.environ["PATH"].split(";") # Drop empty placeholder for trailing comma, if present. if paths[-1] == "": del paths[-1] reg_user_path, _ = winreg.QueryValueEx(key, "Path") for path in reg_user_path.split(";"): if path not in paths and "Multipass" in path: paths.append(path) # Restore path with trailing comma. os.environ["PATH"] = ";".join(paths) + ";" def _run_installer(installer_path: str, echoer): """Execute multipass installer.""" echoer.info("Installing Multipass...") # Multipass requires administrative privileges to install, which requires # the use of `runas` functionality. Some of the options included: # (1) https://stackoverflow.com/a/34216774 # (2) ShellExecuteW and wait on installer by attempting to delete it. # Windows would prevent us from deleting installer with a PermissionError: # PermissionError: [WinError 32] The process cannot access the file because # it is being used by another process: # (3) Use PowerShell's "Start-Process" with RunAs verb as shown below. # None of the options are quite ideal, but #3 will do. cmd = """ & {{ try {{ $Output = Start-Process -FilePath {path!r} -Args /S -Verb RunAs -Wait -PassThru }} catch {{ [Environment]::Exit(1) }} }} """.format( path=installer_path ) try: subprocess.check_call(["powershell.exe", "-Command", cmd]) except subprocess.CalledProcessError: raise ProviderMultipassInstallationFailed("error launching installer") # Reload path environment to see if we can find multipass now. windows_reload_multipass_path_env() if not shutil.which("multipass.exe"): # Installation failed. raise ProviderMultipassInstallationFailed("installation did not complete successfully") echoer.info("Multipass installation completed successfully.") def _requests_exception_hint(e: requests.RequestException) -> str: # Use the __doc__ description to give the user a hint. It seems to be a # a decent option over trying to enumerate all of possible types. if e.__doc__: split_lines = e.__doc__.splitlines() if split_lines: return e.__doc__.splitlines()[0].decode().strip() # Should never get here. return "unknown download error" def _fetch_installer_url() -> str: """Verify version set is a valid ref in GitHub and return the full URL. """ try: resp = requests.get(_MULTIPASS_RELEASES_API_URL) except requests.RequestException as e: raise ProviderMultipassDownloadFailed(_requests_exception_hint(e)) try: data = resp.json() except simplejson.JSONDecodeError: raise ProviderMultipassDownloadFailed( "failed to fetch valid release data from {}".format(_MULTIPASS_RELEASES_API_URL) ) for assets in data: for asset in assets.get("assets", list()): # Find matching name. if asset.get("name") != _MULTIPASS_DL_NAME: continue return asset.get("browser_download_url") # Something changed we don't know about - we will simply categorize # all possible events as an updated version we do not yet know about. raise ProviderMultipassDownloadFailed("ref specified is not a valid ref in GitHub") def _download_multipass(dl_dir: str, echoer) -> str: """Creates temporary Downloads installer to temp directory.""" dl_url = _fetch_installer_url() dl_basename = os.path.basename(dl_url) dl_path = os.path.join(dl_dir, dl_basename) echoer.info("Downloading Multipass installer...\n{} -> {}".format(dl_url, dl_path)) try: request = requests.get(dl_url, stream=True, allow_redirects=True) request.raise_for_status() download_requests_stream(request, dl_path) except requests.RequestException as e: raise ProviderMultipassDownloadFailed(_requests_exception_hint(e)) digest = calculate_sha3_384(dl_path) if digest != _MULTIPASS_DL_SHA3_384: raise ProviderMultipassDownloadFailed( "download failed verification (expected={} but found={})".format( _MULTIPASS_DL_SHA3_384, digest ) ) echoer.info("Verified installer successfully...") return dl_path def windows_install_multipass(echoer) -> None: """Download and install multipass.""" assert sys.platform == "win32" dl_dir = tempfile.mkdtemp() dl_path = _download_multipass(dl_dir, echoer) _run_installer(dl_path, echoer) # Cleanup. shutil.rmtree(dl_dir) def _init_progress_bar(total_length, destination, message=None): if not message: message = "Downloading {!r}".format(os.path.basename(destination)) valid_length = total_length and total_length > 0 if valid_length and is_dumb_terminal(): widgets = [message, " ", Percentage()] maxval = total_length elif valid_length and not is_dumb_terminal(): widgets = [message, Bar(marker="=", left="[", right="]"), " ", Percentage()] maxval = total_length elif not valid_length and is_dumb_terminal(): widgets = [message] maxval = UnknownLength else: widgets = [message, AnimatedMarker()] maxval = UnknownLength return ProgressBar(widgets=widgets, maxval=maxval) def download_requests_stream(request_stream, destination, message=None, total_read=0): """This is a facility to download a request with nice progress bars.""" # Doing len(request_stream.content) may defeat the purpose of a # progress bar total_length = 0 if not request_stream.headers.get("Content-Encoding", ""): total_length = int(request_stream.headers.get("Content-Length", "0")) # Content-Length in the case of resuming will be # Content-Length - total_read so we add back up to have the feel of # resuming if os.path.exists(destination): total_length += total_read progress_bar = _init_progress_bar(total_length, destination, message) progress_bar.start() if os.path.exists(destination): mode = "ab" else: mode = "wb" with open(destination, mode) as destination_file: for buf in request_stream.iter_content(1024): destination_file.write(buf) if not is_dumb_terminal(): total_read += len(buf) progress_bar.update(total_read) progress_bar.finish() ================================================ FILE: installer/vm_providers/errors.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018-2019 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import shlex from typing import Any, Dict, Optional from typing import Sequence # noqa: F401 from common.errors import BaseError class ConnectivityError(BaseError): pass class ProviderBaseError(BaseError): pass class ProviderNotSupportedError(ProviderBaseError): fmt = ( "The {provider!r} provider is not supported, please choose a " "different one and try again." ) def __init__(self, *, provider: str) -> None: super().__init__(provider=provider) class ProviderNotFound(ProviderBaseError): fmt = "You need {provider!r} set-up to build snaps: {error_message}." def __init__(self, *, provider: str, prompt_installable: bool, error_message: str) -> None: super().__init__( provider=provider, prompt_installable=prompt_installable, error_message=error_message, ) self.prompt_installable = prompt_installable self.provider = provider class _GenericProviderError(ProviderBaseError): _FMT_ERROR_MESSAGE_AND_EXIT_CODE = ( "An error occurred with the instance when trying to {action} with " "{provider_name!r}: returned exit code {exit_code!r}: {error_message}.\n" "Ensure that {provider_name!r} is setup correctly and try again." ) _FMT_ERROR_MESSAGE = ( "An error occurred with the instance when trying to {action} with " "{provider_name!r}: {error_message}.\n" "Ensure that {provider_name!r} is setup correctly and try again." ) _FMT_EXIT_CODE = ( "An error occurred with the instance when trying to {action} with " "{provider_name!r}: returned exit code {exit_code!r}.\n" "Ensure that {provider_name!r} is setup correctly and try again." ) def __init__( self, *, provider_name: str, action: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: if exit_code is not None and error_message is not None: fmt = self._FMT_ERROR_MESSAGE_AND_EXIT_CODE elif error_message: fmt = self._FMT_ERROR_MESSAGE elif exit_code: fmt = self._FMT_EXIT_CODE else: raise RuntimeError("error_message nor exit_code are set") self.fmt = fmt super().__init__( provider_name=provider_name, action=action, error_message=error_message, exit_code=exit_code, ) class ProviderCommunicationError(ProviderBaseError): fmt = ( "An error occurred when trying to communicate with the " "{provider_name!r} provider: {message}." ) def __init__(self, *, provider_name: str, message: str) -> None: super().__init__(provider_name=provider_name, message=message) class ProviderLaunchError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="launch", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderStartError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="start", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderStopError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="stop", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderDeleteError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="delete", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderExecError(ProviderBaseError): fmt = ( "An error occurred when trying to execute {command_string!r} with " "{provider_name!r}: returned exit code {exit_code!r}." ) def __init__(self, *, provider_name: str, command: Sequence[str], exit_code: int) -> None: command_string = " ".join(shlex.quote(i) for i in command) super().__init__( provider_name=provider_name, command=command, command_string=command_string, exit_code=exit_code, ) class ProviderShellError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="shell", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderMountError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="mount", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderUnMountError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="unmount", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderFileCopyError(_GenericProviderError): def __init__( self, *, provider_name: str, error_message: Optional[str] = None, exit_code: Optional[int] = None ) -> None: super().__init__( action="copy files", provider_name=provider_name, error_message=error_message, exit_code=exit_code, ) class ProviderInfoError(ProviderBaseError): fmt = ( "An error occurred when using {provider_name!r} to " "query the status of the instance: returned exit code {exit_code!r}: {stderr!s}." ) def __init__(self, *, provider_name: str, exit_code: int, stderr: bytes) -> None: super().__init__(provider_name=provider_name, exit_code=exit_code, stderr=stderr.decode()) class ProviderInstanceNotFoundError(ProviderBaseError): fmt = "Cannot find an instance named {instance_name!r}." def __init__(self, *, instance_name: str) -> None: super().__init__(instance_name=instance_name) class ProviderInfoDataKeyError(ProviderBaseError): fmt = ( "The data returned by {provider_name!r} was not expected. " "It is missing a required key {missing_key!r} in {data!r}." ) def __init__(self, *, provider_name: str, missing_key: str, data: Dict[str, Any]) -> None: super().__init__(provider_name=provider_name, missing_key=missing_key, data=data) class ProviderBadDataError(ProviderBaseError): fmt = ( "The data returned by {provider_name!r} was not expected " "or in the wrong format: {data!r}." ) def __init__(self, *, provider_name: str, data: str) -> None: super().__init__(provider_name=provider_name, data=data) class ProviderMultipassDownloadFailed(ProviderBaseError): fmt = ( "Failed to download Multipass: {message!r}\n" "Please install manually. You can find the latest release at:\n" "https://multipass.run" ) def __init__(self, message): super().__init__(message=message) class ProviderMultipassInstallationFailed(ProviderBaseError): fmt = ( "Failed to install Multipass: {message!r}\n" "Please install manually. You can find the latest release at:\n" "https://multipass.run" ) def __init__(self, message): super().__init__(message=message) ================================================ FILE: installer/vm_providers/factory.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2018 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import TYPE_CHECKING from . import errors from ._multipass import Multipass if TYPE_CHECKING: from typing import Type # noqa: F401 from ._base_provider import Provider # noqa: F401 def get_provider_for(provider_name: str) -> "Type[Provider]": """Returns a Type that can build with provider_name.""" if provider_name == "multipass": return Multipass else: raise errors.ProviderNotSupportedError(provider=provider_name) ================================================ FILE: installer/vm_providers/repo/__init__.py ================================================ ================================================ FILE: installer/vm_providers/repo/errors.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2015-2019 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . from typing import Sequence from typing import List from common.errors import BaseError class RepoError(BaseError): pass class NoNativeBackendError(RepoError): fmt = "Native builds aren't supported on {distro}." class CacheUpdateFailedError(RepoError): fmt = ( "Failed to update the package cache: " "Some files could not be downloaded:{errors}" "Check that the sources on your host are configured correctly." ) def __init__(self, errors: str) -> None: if errors: errors = "\n\n{}\n\n".format(errors.replace(", ", "\n")) else: errors = " " super().__init__(errors=errors) class FileProviderNotFound(RepoError): fmt = "{file_path} is not provided by any package." def __init__(self, *, file_path: str) -> None: super().__init__(file_path=file_path) class BuildPackageNotFoundError(RepoError): fmt = "Could not find a required package in 'build-packages': {package}" def __init__(self, package): super().__init__(package=package) class BuildPackagesNotInstalledError(RepoError): fmt = "Could not install all requested build packages: {packages}" def __init__(self, *, packages: List[str]) -> None: super().__init__(packages=" ".join(packages)) class PackageFetchError(RepoError): fmt = "Package fetch error: {message}" def __init__(self, message: str) -> None: super().__init__(message=message) class PackageBrokenError(RepoError): fmt = "The package {package} has unmet dependencies: {deps}" def __init__(self, package: str, deps: List[str]) -> None: super().__init__(package=package, deps=" ".join(deps)) class PackageNotFoundError(RepoError): @property def message(self): message = "The package {!r} was not found.".format(self.package_name) # If the package was multiarch, try to help. return message def __init__(self, package_name): self.package_name = package_name def __str__(self): return self.message class UnpackError(RepoError): fmt = "Error while provisioning {package!r}" def __init__(self, package): super().__init__(package=package) class SnapUnavailableError(RepoError): fmt = ( "Failed to install or refresh a snap: {snap_name!r} does not exist " "or is not available on the desired channel {snap_channel!r}. " "Use `snap info {snap_name}` to get a list of channels the " "snap is available on." ) def __init__(self, *, snap_name: str, snap_channel: str) -> None: super().__init__(snap_name=snap_name, snap_channel=snap_channel) class SnapFindError(RepoError): fmt = ( "Could not find the snap {snap_name!r} installed on this host.\n" "Install the snap and try again." ) def __init__(self, *, snap_name): super().__init__(snap_name=snap_name) class SnapInstallError(RepoError): fmt = "Error while installing snap {snap_name!r} from channel {snap_channel!r}" def __init__(self, *, snap_name, snap_channel): super().__init__(snap_name=snap_name, snap_channel=snap_channel) class SnapDownloadError(RepoError): fmt = "Error while downloading snap {snap_name!r} from channel {snap_channel!r}" def __init__(self, *, snap_name, snap_channel): super().__init__(snap_name=snap_name, snap_channel=snap_channel) class SnapGetAssertionError(RepoError): fmt = ( "Error while retrieving assertion with parameters " "{assertion_params!r}\n" "Verify the assertion exists and try again." ) def __init__(self, *, assertion_params: Sequence[str]) -> None: super().__init__(assertion_params=assertion_params) class SnapRefreshError(RepoError): fmt = "Error while refreshing snap {snap_name!r} to channel {snap_channel!r}" def __init__(self, *, snap_name, snap_channel): super().__init__(snap_name=snap_name, snap_channel=snap_channel) class SnapdConnectionError(RepoError): fmt = "Failed to get information for snap {snap_name!r}: " "could not connect to {url!r}." def __init__(self, snap_name: str, url: str) -> None: super().__init__(snap_name=snap_name, url=url) ================================================ FILE: installer/vm_providers/repo/snaps.py ================================================ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # Copyright (C) 2017-2019 Canonical Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import contextlib import logging import os import sys from subprocess import check_call, check_output, CalledProcessError from typing import List, Sequence, Set, Union from urllib import parse import requests_unixsocket from requests import exceptions from . import errors _STORE_ASSERTION = [ "account-key", "public-key-sha3-384=BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0hqUel3m8ul", ] _CHANNEL_RISKS = ["stable", "candidate", "beta", "edge"] logger = logging.getLogger(__name__) class SnapPackage: """SnapPackage acts as a mediator to install or refresh a snap. It uses information provided by snapd implicitly referring to the local and remote stores to obtain information about the snap, such as its confinement value and channel availability. This information can also be used to determine if a snap should be installed or refreshed. There are risks of the data falling out of date between the query and the requested action given that it is not possible to hold a global lock on snapd and the store data can change in between validation and execution. """ @classmethod def is_valid_snap(cls, snap): return cls(snap).is_valid() @classmethod def is_snap_installed(cls, snap): return cls(snap).installed def __init__(self, snap): """Lifecycle handler for a snap of the format /.""" self.name, self.channel = _get_parsed_snap(snap) self._original_channel = self.channel if not self.channel or self.channel == "stable": self.channel = "latest/stable" # This store information from a local request self._local_snap_info = None # And this stores information from a remote request self._store_snap_info = None self._is_installed = None self._is_in_store = None @property def installed(self): if self._is_installed is None: self._is_installed = self.get_local_snap_info() is not None return self._is_installed @property def in_store(self): if self._is_in_store is None: try: self._is_in_store = self.get_store_snap_info() is not None except errors.SnapUnavailableError: self._is_in_store = False return self._is_in_store def get_local_snap_info(self): """Returns a local payload for the snap. Validity of the results are determined by checking self.installed.""" if self._is_installed is None: with contextlib.suppress(exceptions.HTTPError): self._local_snap_info = _get_local_snap_info(self.name) return self._local_snap_info def get_store_snap_info(self): """Returns a store payload for the snap.""" if self._is_in_store is None: # Some environments timeout often, like the armv7 testing # infrastructure. Given that constraint, we add some retry # logic. retry_count = 5 while retry_count > 0: try: self._store_snap_info = _get_store_snap_info(self.name) break except exceptions.HTTPError as http_error: logger.debug( "The http error when checking the store for " "{!r} is {!r} (retries left {})".format( self.name, http_error.response.status_code, retry_count ) ) if http_error.response.status_code == 404: raise errors.SnapUnavailableError( snap_name=self.name, snap_channel=self.channel ) retry_count -= 1 return self._store_snap_info def _get_store_channels(self): snap_store_info = self.get_store_snap_info() if not self.in_store: return dict() return snap_store_info["channels"] def get_current_channel(self): current_channel = "" if self.installed: local_snap_info = self.get_local_snap_info() current_channel = local_snap_info["channel"] if any([current_channel.startswith(risk) for risk in _CHANNEL_RISKS]): current_channel = "latest/{}".format(current_channel) return current_channel def has_assertions(self) -> bool: # A revision starting with x has been installed with # --dangerous. return not self.get_local_snap_info()["revision"].startswith("x") def is_classic(self) -> bool: store_channels = self._get_store_channels() try: return store_channels[self.channel]["confinement"] == "classic" except KeyError: # We have seen some KeyError issues when running tests that are # hard to debug as they only occur there, logging in debug mode # will help uncover the root cause if it happens again. logger.debug( "Current store channels are {!r} and the store" "payload is {!r}".format(store_channels, self._store_snap_info) ) raise def is_valid(self) -> bool: """Check if the snap is valid.""" if not self.in_store: return False store_channels = self._get_store_channels() return self.channel in store_channels.keys() def local_download(self, *, snap_path: str, assertion_path: str) -> None: assertions = list() # type: List[List[str]] # We write an empty assertions file for dangerous installs to # have a consistent interface. if self.has_assertions(): assertions.append(["snap-declaration", "snap-name={}".format(self.name)]) assertions.append( [ "snap-revision", "snap-revision={}".format(self._local_snap_info["revision"]), "snap-id={}".format(self._local_snap_info["id"]), ] ) if assertions: assertions.insert(0, _STORE_ASSERTION) with open(assertion_path, "wb") as assertion_file: for assertion in assertions: assertion_file.write(get_assertion(assertion)) assertion_file.write(b"\n") snap_file_iter = _get_local_snap_file_iter(self.name, chunk_size=1024) with open(snap_path, "wb") as snap_file: for buf in snap_file_iter: snap_file.write(buf) def download(self, *, directory: str = None): """Downloads a given snap.""" # We use the `snap download` command here on recommendation # of the snapd team. snap_download_cmd = ["snap", "download", self.name] if self._original_channel: snap_download_cmd.extend(["--channel", self._original_channel]) try: check_output(snap_download_cmd, cwd=directory) except CalledProcessError: raise errors.SnapDownloadError(snap_name=self.name, snap_channel=self.channel) def install(self): """Installs the snap onto the system.""" if not self.is_valid(): raise errors.SnapUnavailableError(snap_name=self.name, snap_channel=self.channel) snap_install_cmd = [] if _snap_command_requires_sudo(): snap_install_cmd = ["sudo"] snap_install_cmd.extend(["snap", "install", self.name]) if self._original_channel: snap_install_cmd.extend(["--channel", self._original_channel]) if self.is_classic(): # TODO make this a user explicit choice snap_install_cmd.append("--classic") try: check_call(snap_install_cmd) except CalledProcessError: raise errors.SnapInstallError(snap_name=self.name, snap_channel=self.channel) # Now that the snap is installed, invalidate the data we had on it. self._is_installed = None def refresh(self): """Refreshes a snap onto a channel on the system.""" if not self.is_valid(): raise errors.SnapUnavailableError(snap_name=self.name, snap_channel=self.channel) snap_refresh_cmd = [] if _snap_command_requires_sudo(): snap_refresh_cmd = ["sudo"] snap_refresh_cmd.extend(["snap", "refresh", self.name, "--channel", self.channel]) if self.is_classic(): # TODO make this a user explicit choice snap_refresh_cmd.append("--classic") try: check_call(snap_refresh_cmd) except CalledProcessError: raise errors.SnapRefreshError(snap_name=self.name, snap_channel=self.channel) # Now that the snap is refreshed, invalidate the data we had on it. self._is_installed = None def download_snaps(*, snaps_list: Sequence[str], directory: str) -> None: """ Download snaps of the format / into directory. The target directory is created if it does not exist. """ # TODO manifest.yaml with snap revision from future machine output # for `snap download`. os.makedirs(directory, exist_ok=True) for snap in snaps_list: snap_pkg = SnapPackage(snap) if not snap_pkg.is_valid(): raise errors.SnapUnavailableError( snap_name=snap_pkg.name, snap_channel=snap_pkg.channel ) # TODO: use dependency injected echoer logger.info("Downloading snap {!r}".format(snap_pkg.name)) snap_pkg.download(directory=directory) def install_snaps(snaps_list: Union[Sequence[str], Set[str]]) -> List[str]: """Install snaps of the format /. :return: a list of "name=revision" for the snaps installed. """ snaps_installed = [] for snap in snaps_list: snap_pkg = SnapPackage(snap) # Allow bases to be installed from non stable channels. snap_pkg_channel = snap_pkg.get_store_snap_info()["channel"] snap_pkg_type = snap_pkg.get_store_snap_info()["type"] if snap_pkg_channel != "stable" and snap_pkg_type == "base": snap_pkg = SnapPackage( "{snap_name}/latest/{channel}".format( snap_name=snap_pkg.name, channel=snap_pkg_channel ) ) if not snap_pkg.installed: snap_pkg.install() elif snap_pkg.get_current_channel() != snap_pkg.channel: snap_pkg.refresh() snaps_installed.append( "{}={}".format(snap_pkg.name, snap_pkg.get_local_snap_info()["revision"]) ) return snaps_installed def _snap_command_requires_sudo(): # snap whoami returns - if the user is not logged in. output = check_output(["snap", "whoami"]) whoami = output.decode(sys.getfilesystemencoding()) try: requires_root = whoami.split(":")[1].strip() == "-" # A safeguard if the output changes except IndexError: requires_root = True if requires_root: logger.warning("snapd is not logged in, snap install commands will use sudo") return requires_root def get_assertion(assertion_params: Sequence[str]) -> bytes: """Get assertion information. :param assertion_params: a sequence of strings to pass to 'snap known'. :returns: a stream of bytes from the assertion. :rtype: bytes """ try: return check_output(["snap", "known", *assertion_params]) except CalledProcessError as call_error: raise errors.SnapGetAssertionError(assertion_params=assertion_params) from call_error def _get_parsed_snap(snap): if "/" in snap: sep_index = snap.find("/") snap_name = snap[:sep_index] snap_channel = snap[sep_index + 1 :] else: snap_name = snap snap_channel = "" return snap_name, snap_channel def get_snapd_socket_path_template(): return "http+unix://%2Frun%2Fsnapd.socket/v2/{}" def _get_local_snap_file_iter(snap_name, *, chunk_size: int): slug = "snaps/{}/file".format(parse.quote(snap_name, safe="")) url = get_snapd_socket_path_template().format(slug) try: snap_file = requests_unixsocket.get(url) except exceptions.ConnectionError as e: raise errors.SnapdConnectionError(snap_name, url) from e snap_file.raise_for_status() return snap_file.iter_content(chunk_size) def _get_local_snap_info(snap_name): slug = "snaps/{}".format(parse.quote(snap_name, safe="")) url = get_snapd_socket_path_template().format(slug) try: snap_info = requests_unixsocket.get(url) except exceptions.ConnectionError as e: raise errors.SnapdConnectionError(snap_name, url) from e snap_info.raise_for_status() return snap_info.json()["result"] def _get_store_snap_info(snap_name): # This logic uses /v2/find returns an array of results, given that # we do a strict search either 1 result or a 404 will be returned. slug = "find?{}".format(parse.urlencode(dict(name=snap_name))) url = get_snapd_socket_path_template().format(slug) snap_info = requests_unixsocket.get(url) snap_info.raise_for_status() return snap_info.json()["result"][0] def get_installed_snaps(): """Return all the snaps installed in the system. :return: a list of "name=revision" for the snaps installed. """ slug = "snaps" url = get_snapd_socket_path_template().format(slug) try: snap_info = requests_unixsocket.get(url) snap_info.raise_for_status() local_snaps = snap_info.json()["result"] except exceptions.ConnectionError: local_snaps = [] return ["{}={}".format(snap["name"], snap["revision"]) for snap in local_snaps] ================================================ FILE: installer/windows/README.md ================================================ # MicroK8s Installer ## Windows Uses the NSIS package to create the installer. ### Prerequisites * Requires [NSIS 3](https://nsis.sourceforge.io/Download). * Requires the [EnVar plug in](https://nsis.sourceforge.io/EnVar_plug-in) to compile. ### Building 2 files must be placed in the working directory before compiling the installer. These are: * `microk8s.exe`: This is the file generated by PyInstaller. * `multipass.exe`: This is the Multipass installer exe, which is run at install time. Use the following command to compile the installer. ``` makensis microk8s.nsi ``` ================================================ FILE: installer/windows/microk8s.nsi ================================================ !include "MUI2.nsh" !include "nsDialogs.nsh" !include "LogicLib.nsh" !include "Sections.nsh" !define PRODUCT_NAME "MicroK8s" !define PRODUCT_VERSION "2.3.3" !define PRODUCT_PUBLISHER "Canonical" !define MUI_ICON ".\microk8s.ico" !define MUI_HEADERIMAGE !define MUI_HEADERIMAGE_BITMAP ".\microk8s.bmp" !define MUI_HEADERIMAGE_RIGHT Unicode True Name "${PRODUCT_NAME} for Windows ${PRODUCT_VERSION}" BrandingText "Canonical Ltd." Icon ".\microk8s.ico" OutFile "microk8s-installer.exe" InstallDir "$PROGRAMFILES64\${PRODUCT_NAME}" ShowInstDetails hide RequestExecutionLevel admin SetCompress auto !insertmacro MUI_PAGE_WELCOME !insertmacro MUI_PAGE_LICENSE "..\..\LICENSE" !define MUI_COMPONENTSPAGE_TEXT_COMPLIST " " !define MUI_COMPONENTSPAGE_TEXT_INSTTYPE " " !insertmacro MUI_PAGE_COMPONENTS !insertmacro MUI_PAGE_INSTFILES Page custom ConfigureVm LaunchVm !insertmacro MUI_PAGE_FINISH !insertmacro MUI_UNPAGE_CONFIRM !insertmacro MUI_UNPAGE_INSTFILES !insertmacro MUI_LANGUAGE "English" Var MultipassExitCode Var VmConfigureDialog Var VmConfigureDialogCpu Var VmConfigureDialogCpuLabel Var VmConfigureDialogMem Var VmConfigureDialogMemLabel Var VmConfigureDialogDisk Var VmConfigureDialogDiskLabel Var VmConfigureDialogTrack Var VmConfigureDialogTrackLabel Var VmConfigureDialogFootnote Section "Multipass (Required)" multipass_id ;Exit 0 : Multipass is installed and backing hypervisor is ready to roll (e.g. HyperV is already enabled). ;Exit 3010: Multipass is installed but backing hypervisor requires a reboot (e.g. HyperV just enabled). ;Exit 3011: Multipass is installed but no backing hypervisor has been (e.g. virtualbox needs installing). SectionIn RO IfFileExists $PROGRAMFILES64\Multipass\bin\multipass.exe endMultipass beginMultipass Goto endMultipass beginMultipass: SetOutPath $INSTDIR File "multipass.exe" ${If} ${Silent} ExecWait "multipass.exe /NoRestart /S" $0 ${Else} ExecWait "multipass.exe /NoRestart" $0 ${EndIf} StrCpy $MultipassExitCode $0 IfErrors failedMultipass Delete "$INSTDIR\multipass.exe" Goto endMultipass failedMultipass: Abort endMultipass: SectionEnd Section "Kubectl (Required)" kubectl_id SectionIn RO beginKubectl: SetOutPath $INSTDIR File "kubectl.exe" CopyFiles "$INSTDIR\kubectl.exe" "$INSTDIR\kubectl\kubectl.exe" Delete "$INSTDIR\kubectl.exe" Goto endKubectl endKubectl: SectionEnd Section -Install SectionIn RO SetOutPath $INSTDIR File "microk8s.exe" SectionEnd Section -WriteUninstaller SectionIn RO WriteUninstaller $INSTDIR\uninstall.exe WriteRegStr HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" \ "DisplayName" "${PRODUCT_NAME} ${PRODUCT_VERSION}" WriteRegStr HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" \ "UninstallString" "$INSTDIR\uninstall.exe" SectionEnd Section "Add 'microk8s' to PATH" add_to_path_id EnVar::AddValue "path" "$INSTDIR" SectionEnd Section /o "Add 'kubectl' to PATH" add_kubectl_to_path_id EnVar::AddValue "path" "$INSTDIR\kubectl" SectionEnd !insertmacro MUI_FUNCTION_DESCRIPTION_BEGIN !insertmacro MUI_DESCRIPTION_TEXT ${multipass_id} "REQUIRED: If already installed, will be unticked and skipped.$\n$\nSee https://multipass.run for more." !insertmacro MUI_DESCRIPTION_TEXT ${add_to_path_id} "Add the 'microk8s' executable to PATH.$\n$\nThis will allow you to run the command 'microk8s' in cmd and PowerShell in any directory." !insertmacro MUI_DESCRIPTION_TEXT ${add_kubectl_to_path_id} "Add the 'kubectl' executable to PATH.$\n$\nThis will set the bundled 'kubectl' as system default." !insertmacro MUI_FUNCTION_DESCRIPTION_END Function .onInit DeleteRegKey HKLM "Software\canonical\multipass" IfFileExists $PROGRAMFILES64\Multipass\bin\multipass.exe untickMultipass tickMultipass tickMultipass: IntOp $0 ${SF_SELECTED} | ${SF_RO} SectionSetFlags ${multipass_id} $0 Return untickMultipass: StrCpy $MultipassExitCode "0" ; Multipass is installed. SectionSetFlags ${multipass_id} ${SF_RO} Return FunctionEnd Function "ConfigureVm" ${IfNot} $MultipassExitCode == "0" ${If} $MultipassExitCode == "3010" MessageBox MB_ICONEXCLAMATION "Cannot configure the ${PRODUCT_NAME} VM as a reboot is required. Please re-run this wizard after reboot to configure the VM." SetRebootFlag true Abort ${ElseIf} $MultipassExitCode == "3011" MessageBox MB_ICONEXCLAMATION "Cannot configure the ${PRODUCT_NAME} VM as VirtualBox needs to be installed. Please re-run this wizard after installing VirtualBox to configure the VM." Abort ${EndIf} ${EndIf} MessageBox MB_YESNO "Do you want to configure and launch the ${PRODUCT_NAME} VM now?" /SD IDYES IDNO endLaunch nsDialogs::Create 1018 Pop $VmConfigureDialog ${NSD_CreateLabel} 19% 0 35u 10u "CPUs" Pop $VmConfigureDialogCpuLabel ${NSD_CreateLabel} 44% 0 35u 10u "Mem (GB)" Pop $VmConfigureDialogMemLabel ${NSD_CreateLabel} 69% 0 35u 10u "Disk (GB)" Pop $VmConfigureDialogDiskLabel ${NSD_CreateNumber} 19% 17.5 35u 10u "2" Pop $VmConfigureDialogCpu ${NSD_CreateNumber} 44% 17.5 35u 10u "4" Pop $VmConfigureDialogMem ${NSD_CreateNumber} 69% 17.5 35u 10u "50" Pop $VmConfigureDialogDisk ${NSD_CreateLabel} 42% 50 50u 10u "Snap Track" Pop $VmConfigureDialogTrackLabel ${NSD_CreateText} 42% 67.5 50u 10u "1.28/stable" Pop $VmConfigureDialogTrack ${NSD_CreateLabel} 8% 102.5 100% 10u "These are the minimum recommended parameters for the VM running ${PRODUCT_NAME}" Pop $VmConfigureDialogFootnote nsDialogs::Show endLaunch: Abort FunctionEnd Function "LaunchVM" ${NSD_GetText} $VmConfigureDialogCpu $0 ${NSD_GetText} $VmConfigureDialogMem $1 ${NSD_GetText} $VmConfigureDialogDisk $2 ${NSD_GetText} $VmConfigureDialogTrack $3 ExecWait "$INSTDIR\microk8s.exe install --cpu $0 --mem $1 --disk $2 --channel $3 --assume-yes" FunctionEnd Function un.onInit DeleteRegKey HKLM "Software\canonical\multipass" FunctionEnd Section "Uninstall" ExecWait "$INSTDIR\microk8s.exe uninstall" Delete $INSTDIR\uninstall.exe Delete $INSTDIR\microk8s.exe Delete $INSTDIR\kubectl\kubectl.exe RMDir $INSTDIR\kubectl RMDir $INSTDIR DeleteRegKey HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" EnVar::DeleteValue "path" "$INSTDIR" EnVar::DeleteValue "path" "$INSTDIR\kubectl" SectionEnd ================================================ FILE: microk8s-resources/actions/common/utils.sh ================================================ #!/usr/bin/env bash get_microk8s_group() { if is_strict then echo "snap_microk8s" else echo "microk8s" fi } get_microk8s_or_cis_group() { if [ -e $SNAP_DATA/var/lock/cis-hardening ] then echo "root" else get_microk8s_group fi } exit_if_no_permissions() { # test if we can access the default kubeconfig if [ ! -r $SNAP_DATA/credentials/client.config ]; then local group=$(get_microk8s_group) echo "Insufficient permissions to access MicroK8s." >&2 echo "You can either try again with sudo or add the user $USER to the '${group}' group:" >&2 echo "" >&2 echo " sudo usermod -a -G ${group} $USER" >&2 echo " sudo chown -R $USER ~/.kube" >&2 echo "" >&2 echo "After this, reload the user groups either via a reboot or by running 'newgrp ${group}'." >&2 exit 1 fi } exit_if_stopped() { # test if the snap is marked as stopped if [ -e ${SNAP_DATA}/var/lock/stopped.lock ] then echo "microk8s is not running, try microk8s start" >&2 exit 1 fi } exit_if_service_not_expected_to_start() { # exit if a lock is available for the service local service="$1" if [ -f ${SNAP_DATA}/var/lock/no-${service} ] then exit 0 fi } is_service_expected_to_start() { # return 1 if service is expected to start local service="$1" if [ -f ${SNAP_DATA}/var/lock/no-${service} ] then echo "0" else echo "1" fi } set_service_not_expected_to_start() { # mark service as not starting local service="$1" run_with_sudo $SNAP/bin/touch ${SNAP_DATA}/var/lock/no-${service} } set_service_expected_to_start() { # mark service as not starting local service="$1" rm -rf ${SNAP_DATA}/var/lock/no-${service} } remove_vxlan_interfaces() { links="$(${SNAP}/sbin/ip link show type vxlan | $SNAP/bin/grep -E 'flannel|cilium_vxlan' | $SNAP/usr/bin/gawk '{print $2}' | $SNAP/usr/bin/tr -d :)" for link in $links do if ! [ -z "$link" ] && $SNAP/sbin/ip link show ${link} &> /dev/null then echo "Deleting old ${link} link" >&2 run_with_sudo $SNAP/sbin/ip link delete ${link} fi done } list_env_vars() { env | awk -F= '{print $1}' | paste -sd, } run_with_sudo() { if [ "$1" == "preserve_env" ]; then shift fi if (is_strict); then "$@" else local SAVE_LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" LD_LIBRARY_PATH="" sudo --preserve-env="$(list_env_vars)" PATH="${PATH}" LD_LIBRARY_PATH="${SAVE_LD_LIBRARY_PATH}" PYTHONPATH="${PYTHONPATH:-}" "$@" fi } get_opt_in_config() { # return the value of an option in a configuration file or "" local opt="$1" local config_file="$SNAP_DATA/args/$2" val="" if $($SNAP/bin/grep -qE "^$opt=" $config_file); then val="$($SNAP/bin/grep -E "^$opt" "$config_file" | $SNAP/usr/bin/cut -d'=' -f2)" elif $($SNAP/bin/grep -qE "^$opt " $config_file); then val="$($SNAP/bin/grep -E "^$opt" "$config_file" | $SNAP/usr/bin/cut -d' ' -f2)" fi echo "$val" } refresh_opt_in_local_config() { # add or replace an option inside the local config file. # Create the file if doesn't exist local opt="--$1" local value="$2" local config_file="$SNAP_DATA/args/$3" local replace_line="$opt=$value" if $($SNAP/bin/grep -qE "^$opt=" $config_file); then run_with_sudo "$SNAP/bin/sed" -i "s@^$opt=.*@$replace_line@" $config_file else run_with_sudo "$SNAP/bin/sed" -i "1i$replace_line" "$config_file" fi } refresh_opt_in_config() { # add or replace an option inside the config file and propagate change. # Create the file if doesn't exist refresh_opt_in_local_config "$1" "$2" "$3" local opt="--$1" local value="$2" local config_file="$SNAP_DATA/args/$3" local replace_line="$opt=$value" if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" update_argument "$3" "$opt" "$value" fi if [ -e "${SNAP_DATA}/credentials/callback-tokens.txt" ] then tokens=$(run_with_sudo "$SNAP/bin/cat" "${SNAP_DATA}/credentials/callback-tokens.txt" | "$SNAP/usr/bin/wc" -l) if [[ "$tokens" -ge "0" ]] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" update_argument "$3" "$opt" "$value" fi fi } nodes_addon() { # Enable or disable a, addon across all nodes # state should be either 'enable' or 'disable' local addon="$1" local state="$2" if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" set_addon "$addon" "$state" fi if [ -e "${SNAP_DATA}/credentials/callback-tokens.txt" ] then tokens=$(run_with_sudo "$SNAP/bin/cat" "${SNAP_DATA}/credentials/callback-tokens.txt" | "$SNAP/usr/bin/wc" -l) if [[ "$tokens" -ge "0" ]] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" set_addon "$addon" "$state" fi fi } skip_opt_in_local_config() { # remove an option inside the config file. # argument $1 is the option to be removed # argument $2 is the configuration file under $SNAP_DATA/args local opt="--$1" local config_file="$SNAP_DATA/args/$2" # regex is "$opt[= ]", otherwise we remove all arguments with the same prefix run_with_sudo "${SNAP}/bin/sed" -i '/'"$opt[= ]"'/d' "${config_file}" } skip_opt_in_config() { # remove an option inside the config file. # argument $1 is the option to be removed # argument $2 is the configuration file under $SNAP_DATA/args skip_opt_in_local_config "$1" "$2" local opt="--$1" if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" remove_argument "$2" "$opt" fi if [ -e "${SNAP_DATA}/credentials/callback-tokens.txt" ] then tokens=$(run_with_sudo "$SNAP/bin/cat" "${SNAP_DATA}/credentials/callback-tokens.txt" | "$SNAP/usr/bin/wc" -l) if [[ "$tokens" -ge "0" ]] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" remove_argument "$2" "$opt" fi fi } remove_args() { # Removes arguments from respective service # argument $1: the service # rest of arguments: the arguments to be removed local service_name="$1" shift local args=("$@") for arg in "${args[@]}"; do if $SNAP/bin/grep -q "$arg" "$SNAP_DATA/args/$service_name"; then echo "Removing argument: $arg from $service_name" skip_opt_in_local_config "$arg" "$service_name" fi done } sanitise_args_kubeapi_server() { # Function to sanitise arguments for API server local args=( # Removed klog flags from 1.26+ # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md "log-dir" "log-file" "log-flush-frequency" "logtostderr" "alsologtostderr" "one-output" "stderrthreshold" "log-file-max-size" "skip-log-headers" "add-dir-header" "skip-headers" "log-backtrace-at" # Remove insecure-port from 1.24+ "insecure-port" "insecure-bind-address" "port" "address" # Remove service-account-api-audiences from 1.25+ # https://github.com/kubernetes/kubernetes/commit/92707cafbb67a5664324eb891ef70ab3d1dd4a97 "service-account-api-audiences" # extra "feature-gates=RemoveSelfLink" "experimental-encryption-provider-config" "target-ram-mb" ) remove_args "kube-apiserver" "${args[@]}" } sanitise_args_kubelet() { # Function to sanitise arguments for kubelet local args=( # Removed klog flags from 1.26+ # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md "log-dir" "log-file" "log-flush-frequency" "logtostderr" "alsologtostderr" "one-output" "stderrthreshold" "log-file-max-size" "skip-log-headers" "add-dir-header" "skip-headers" "log-backtrace-at" # Removed dockershim flags from 1.24+ # https://github.com/kubernetes/enhancements/issues/2221 "docker-endpoint" "image-pull-progress-deadline" "network-plugin" "cni-conf-dir" "cni-bin-dir" "cni-cache-dir" "network-plugin-mtu" # extra "experimental-kernel-memcg-notification" "pod-infra-container-image" "experimental-dockershim-root-directory" "non-masquerade-cidr" # Remove container-runtime flag from 1.27+ "container-runtime" ) remove_args "kubelet" "${args[@]}" # Remove 'DevicePlugins=true' from feature-gates from 1.28+ $SNAP/bin/sed -i 's,DevicePlugins=true,,' "$SNAP_DATA/args/kubelet" } sanitise_args_kube_proxy() { # Function to sanitise arguments for kube-proxy # userspace proxy-mode is not allowed on the 1.26+ k8s # https://kubernetes.io/blog/2022/11/18/upcoming-changes-in-kubernetes-1-26/#removal-of-kube-proxy-userspace-modes if $SNAP/bin/grep -- "--proxy-mode=userspace" $SNAP_DATA/args/kube-proxy then echo "Removing --proxy-mode=userspace flag from kube-proxy, since it breaks Calico." skip_opt_in_local_config "proxy-mode" "kube-proxy" fi local args=( # Removed klog flags from 1.26+ # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md "log-dir" "log-file" "log-flush-frequency" "logtostderr" "alsologtostderr" "one-output" "stderrthreshold" "log-file-max-size" "skip-log-headers" "add-dir-header" "skip-headers" "log-backtrace-at" ) remove_args "kube-proxy" "${args[@]}" } sanitise_args_kube_controller_manager() { # Function to sanitise arguments for kube-controller-manager local args=( # Removed klog flags from 1.26+ # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md "log-dir" "log-file" "log-flush-frequency" "logtostderr" "alsologtostderr" "one-output" "stderrthreshold" "log-file-max-size" "skip-log-headers" "add-dir-header" "skip-headers" "log-backtrace-at" # Remove insecure ports from 1.24+ # https://github.com/kubernetes/kubernetes/pull/96216/files "address" "port" # extra "experimental-cluster-signing-duration" ) remove_args "kube-controller-manager" "${args[@]}" } sanitise_args_kube_scheduler() { # Function to sanitise arguments for kube-scheduler local args=( # Removed klog flags from 1.26+ # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md "log-dir" "log-file" "log-flush-frequency" "logtostderr" "alsologtostderr" "one-output" "stderrthreshold" "log-file-max-size" "skip-log-headers" "add-dir-header" "skip-headers" "log-backtrace-at" # Remove insecure ports from 1.24+ # https://github.com/kubernetes/kubernetes/pull/96345/files "address" "port" ) remove_args "kube-scheduler" "${args[@]}" } restart_service() { # restart a systemd service # argument $1 is the service name if [ "$1" == "apiserver" ] || [ "$1" == "proxy" ] || [ "$1" == "kubelet" ] || [ "$1" == "scheduler" ] || [ "$1" == "controller-manager" ] then run_with_sudo preserve_env snapctl restart "microk8s.daemon-kubelite" else run_with_sudo preserve_env snapctl restart "microk8s.daemon-$1" fi if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" restart "$1" fi if [ -e "${SNAP_DATA}/credentials/callback-tokens.txt" ] then tokens=$(run_with_sudo "$SNAP/bin/cat" "${SNAP_DATA}/credentials/callback-tokens.txt" | "$SNAP/usr/bin/wc" -l) if [[ "$tokens" -ge "0" ]] then run_with_sudo preserve_env "$SNAP/usr/bin/python3" "$SNAP/scripts/wrappers/distributed_op.py" restart "$1" fi fi } arch() { case "${SNAP_ARCH}" in ppc64el) echo ppc64le ;; *) echo "${SNAP_ARCH}" ;; esac } snapshotter() { # Determine the underlying filesystem that containerd will be running on FSTYPE=$($SNAP/usr/bin/stat -f -c %T "${SNAP_COMMON}") # ZFS is supported through the native snapshotter if [ "$FSTYPE" = "zfs" ]; then echo "native" else echo "overlayfs" fi } use_manifest() { # Perform an action (apply or delete) on a manifest. # Optionally replace strings in the manifest # # Parameters: # $1 the name of the manifest. Should be ${SNAP}/actions/ and should not # include the trailing .yaml eg ingress, dns # $2 the action to be performed on the manifest, eg apply, delete # $3 (optional) an associative array with keys the string to be replaced and value what to # replace with. The string $ARCH is always injected to this array. # local manifest="$1.yaml"; shift local action="$1"; shift if ! [ "$#" = "0" ] then eval "declare -A items="${1#*=} else declare -A items fi local tmp_manifest="${SNAP_USER_DATA}/tmp/temp.yaml" items[\$ARCH]=$(arch) $SNAP/bin/mkdir -p ${SNAP_USER_DATA}/tmp $SNAP/bin/cp "${SNAP}/addons/core/addons/${manifest}" "${tmp_manifest}" for i in "${!items[@]}" do "$SNAP/bin/sed" -i 's@'$i'@'"${items[$i]}"'@g' "${tmp_manifest}" done "$SNAP/kubectl" "--kubeconfig=$SNAP_DATA/credentials/client.config" "$action" -f "${tmp_manifest}" use_manifest_result="$?" $SNAP/bin/rm "${tmp_manifest}" } addon_name() { # Extracts the addon from the argument. # addons can have arguments in the form of :=;= # Example: enable linkerd:proxy-auto-inject=on;other-args=xyz # Parameter: # $1 the full addon command # Returns: # local IFS=':' read -ra ADD_ON <<< "$1" echo "${ADD_ON[0]}" } addon_arguments() { # Extracts the addon arguments. # Example: enable linkerd:proxy-auto-inject=on;other-args=xyz # Parameter: # $1 the addon arguments in array # Returns: # add-on arguments array local IFS=':' read -ra ADD_ON <<< "$1" local IFS=';' read -ra ARGUMENTS <<< "${ADD_ON[1]}" echo "${ARGUMENTS[@]}" } wait_for_service() { # Wait for a service to start # Return fail if the service did not start in 30 seconds local service_name="$1" if [ "$1" == "apiserver" ] || [ "$1" == "proxy" ] || [ "$1" == "kubelet" ] || [ "$1" == "scheduler" ] || [ "$1" == "controller-manager" ] then if [ -e "${SNAP_DATA}/var/lock/lite.lock" ] then service_name="kubelite" fi fi local TRY_ATTEMPT=0 while ! (run_with_sudo preserve_env snapctl services ${SNAP_NAME}.daemon-${service_name} | grep active) && ! [ ${TRY_ATTEMPT} -eq 30 ] do TRY_ATTEMPT=$((TRY_ATTEMPT+1)) sleep 1 done if [ ${TRY_ATTEMPT} -eq 30 ] then echo "fail" fi } wait_for_service_shutdown() { # Wait for a service to stop # Return fail if the service did not stop in 30 seconds local namespace="$1" local labels="$2" local shutdown_timeout=30 local start_timer="$($SNAP/bin/date +%s)" KUBECTL="$SNAP/kubectl --kubeconfig=$SNAP/client.config" while ($KUBECTL get po -n "$namespace" -l "$labels" | $SNAP/bin/grep -z " Terminating") &> /dev/null do now="$($SNAP/bin/date +%s)" if [[ "$now" > "$(($start_timer + $shutdown_timeout))" ]] ; then echo "fail" break fi sleep 5 done } get_default_ip() { # Get the IP of the default interface local DEFAULT_INTERFACE="$($SNAP/sbin/ip route show default | $SNAP/usr/bin/gawk '{for(i=1; i/dev/null | $SNAP/bin/grep -v 'inet 169.254' | $SNAP/usr/bin/gawk '{print $4}' | $SNAP/usr/bin/cut -d/ -f1 | head -1)" CNI_IPS="/$CNI_IP/$CNI_IPS" done if [[ -z "$IP_ADDR" ]]; then echo "none" else local ips=""; for ip in $IP_ADDR; do # Append IP address only iff not in cni IP addresses (echo "$CNI_IPS" | $SNAP/bin/grep -q "/$ip/") || ips+="${ips:+ }$ip"; done echo "$ips" fi } gen_server_cert() ( "${SNAP}/openssl.wrapper" req -new -sha256 -key ${SNAP_DATA}/certs/server.key -out ${SNAP_DATA}/certs/server.csr -config ${SNAP_DATA}/certs/csr.conf "${SNAP}/openssl.wrapper" x509 -req -sha256 -in ${SNAP_DATA}/certs/server.csr -CA ${SNAP_DATA}/certs/ca.crt -CAkey ${SNAP_DATA}/certs/ca.key -CAcreateserial -out ${SNAP_DATA}/certs/server.crt -days 365 -extensions v3_ext -extfile ${SNAP_DATA}/certs/csr.conf ) gen_proxy_client_cert() ( "${SNAP}/openssl.wrapper" req -new -sha256 -key ${SNAP_DATA}/certs/front-proxy-client.key -out ${SNAP_DATA}/certs/front-proxy-client.csr -config <(sed '/^prompt = no/d' ${SNAP_DATA}/certs/csr.conf) -subj "/CN=front-proxy-client" "${SNAP}/openssl.wrapper" x509 -req -sha256 -in ${SNAP_DATA}/certs/front-proxy-client.csr -CA ${SNAP_DATA}/certs/front-proxy-ca.crt -CAkey ${SNAP_DATA}/certs/front-proxy-ca.key -CAcreateserial -out ${SNAP_DATA}/certs/front-proxy-client.crt -days 365 -extensions v3_ext -extfile ${SNAP_DATA}/certs/csr.conf ) create_user_certs_and_configs() { create_user_certificates create_user_kubeconfigs } create_user_certificates() { hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') generate_csr_with_sans "/CN=system:node:$hostname/O=system:nodes" "${SNAP_DATA}/certs/kubelet.key" | sign_certificate > "${SNAP_DATA}/certs/kubelet.crt" generate_csr /CN=admin/O=system:masters "${SNAP_DATA}/certs/client.key" | sign_certificate > "${SNAP_DATA}/certs/client.crt" generate_csr /CN=system:kube-proxy "${SNAP_DATA}/certs/proxy.key" | sign_certificate > "${SNAP_DATA}/certs/proxy.crt" generate_csr /CN=system:kube-scheduler "${SNAP_DATA}/certs/scheduler.key" | sign_certificate > "${SNAP_DATA}/certs/scheduler.crt" generate_csr /CN=system:kube-controller-manager "${SNAP_DATA}/certs/controller.key" | sign_certificate > "${SNAP_DATA}/certs/controller.crt" generate_csr /CN=kube-apiserver-kubelet-client/O=system:masters "${SNAP_DATA}/certs/apiserver-kubelet-client.key" | sign_certificate > "${SNAP_DATA}/certs/apiserver-kubelet-client.crt" } create_user_kubeconfigs() { hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') create_kubeconfig_x509 "client.config" "admin" ${SNAP_DATA}/certs/client.crt ${SNAP_DATA}/certs/client.key ${SNAP_DATA}/certs/ca.crt create_kubeconfig_x509 "controller.config" "system:kube-controller-manager" ${SNAP_DATA}/certs/controller.crt ${SNAP_DATA}/certs/controller.key ${SNAP_DATA}/certs/ca.crt create_kubeconfig_x509 "scheduler.config" "system:kube-scheduler" ${SNAP_DATA}/certs/scheduler.crt ${SNAP_DATA}/certs/scheduler.key ${SNAP_DATA}/certs/ca.crt create_kubeconfig_x509 "proxy.config" "system:kube-proxy" ${SNAP_DATA}/certs/proxy.crt ${SNAP_DATA}/certs/proxy.key ${SNAP_DATA}/certs/ca.crt create_kubeconfig_x509 "kubelet.config" "system:node:${hostname}" ${SNAP_DATA}/certs/kubelet.crt ${SNAP_DATA}/certs/kubelet.key ${SNAP_DATA}/certs/ca.crt } create_worker_kubeconfigs() { # $1: (Optional) API server IP, defaults to IP address from traefik-template.yaml (or empty) # $2: (Optional) API server port, defaults to port from traefik-template.yaml # NOTE(neoaggelos): Examples for how "${var%:*}" and ${var##*:} below behave # apiserver_proxy_listen=":16443" => apiserver_default="" port_default="16443" # apiserver_proxy_listen="1.1.1.1:6443" => apiserver_default="1.1.1.1" port_default="6443" # apiserver_proxy_listen="[::1]:16443" => apiserver_default="[::1]" port_default="16443" # apiserver_proxy_listen="" => apiserver_default="" port_default="" local apiserver_proxy_listen="$($SNAP/bin/cat $SNAP_DATA/args/traefik/traefik-template.yaml | $SNAP/bin/grep "address:" | $SNAP/usr/bin/cut -d'"' -f2)" local apiserver_default="${apiserver_proxy_listen%:*}" # drop last ':' and everything after local port_default="${apiserver_proxy_listen##*:}" # drop last ':' and everything before local apiserver="${1:-$apiserver_default}" local port="${2:-$port_default}" hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') create_kubeconfig_x509 "proxy.config" "system:kube-proxy" ${SNAP_DATA}/certs/proxy.crt ${SNAP_DATA}/certs/proxy.key ${SNAP_DATA}/certs/ca.remote.crt "${apiserver}" "${port}" create_kubeconfig_x509 "kubelet.config" "system:node:${hostname}" ${SNAP_DATA}/certs/kubelet.crt ${SNAP_DATA}/certs/kubelet.key ${SNAP_DATA}/certs/ca.remote.crt "${apiserver}" "${port}" } create_kubeconfig_x509() { # Create a kubeconfig file with x509 auth # $1: the name of the config file # $2: the user to use al login # $3: path to certificate file # $4: path to certificate key file # $5: path to ca file # $6: (optional) API server IP, optional defaults to 127.0.0.1 # $7: (optional) API server port, optional defaults to the port of the current node kubeconfig=$1 user=$2 cert=$3 key=$4 ca=$5 # optional arguments apiserver="${6:-"127.0.0.1"}" apiserver_port="$($SNAP/bin/cat $SNAP_DATA/args/kube-apiserver | $SNAP/bin/grep -- "--secure-port" | $SNAP/usr/bin/tr "=" " " | $SNAP/usr/bin/gawk '{print $2}')" port="${7:-${apiserver_port}}" ca_data=$($SNAP/bin/cat ${ca} | ${SNAP}/usr/bin/base64 -w 0) cert_data=$($SNAP/bin/cat ${cert} | ${SNAP}/usr/bin/base64 -w 0) key_data=$($SNAP/bin/cat ${key} | ${SNAP}/usr/bin/base64 -w 0) config_file="${SNAP_DATA}/credentials/${kubeconfig}" $SNAP/bin/cp "${SNAP}/client-x509.config.template" "${config_file}" $SNAP/bin/sed -i 's/CADATA/'"${ca_data}"'/g' "${config_file}" $SNAP/bin/sed -i 's/NAME/'"${user}"'/g' "${config_file}" $SNAP/bin/sed -i 's/PATHTOCERT/'"${cert_data}"'/g' "${config_file}" $SNAP/bin/sed -i 's/PATHTOKEYCERT/'"${key_data}"'/g' "${config_file}" $SNAP/bin/sed -i 's/client-certificate/client-certificate-data/g' "${config_file}" $SNAP/bin/sed -i 's/client-key/client-key-data/g' "${config_file}" $SNAP/bin/sed -i 's/127.0.0.1/'"${apiserver}"'/g' "${config_file}" $SNAP/bin/sed -i 's/16443/'"${port}"'/g' "${config_file}" } produce_certs() { # Generate RSA keys if not yet for key in serviceaccount.key ca.key server.key front-proxy-ca.key front-proxy-client.key; do if ! [ -f ${SNAP_DATA}/certs/$key ]; then "${SNAP}/openssl.wrapper" genrsa -out ${SNAP_DATA}/certs/$key 2048 fi done # Generate apiserver CA if ! [ -f ${SNAP_DATA}/certs/ca.crt ]; then "${SNAP}/openssl.wrapper" req -x509 -new -sha256 -nodes -days 3650 -key ${SNAP_DATA}/certs/ca.key -subj "/CN=10.152.183.1" -out ${SNAP_DATA}/certs/ca.crt fi # Generate front proxy CA if ! [ -f ${SNAP_DATA}/certs/front-proxy-ca.crt ]; then "${SNAP}/openssl.wrapper" req -x509 -new -sha256 -nodes -days 3650 -key ${SNAP_DATA}/certs/front-proxy-ca.key -subj "/CN=front-proxy-ca" -out ${SNAP_DATA}/certs/front-proxy-ca.crt fi # Produce certificates based on the rendered csr.conf.rendered. # The file csr.conf.rendered is compared with csr.conf to determine if a regeneration of the certs must be done. # # Returns # 0 if no change # 1 otherwise. render_csr_conf if ! [ -f "${SNAP_DATA}/certs/csr.conf" ]; then echo "changeme" > "${SNAP_DATA}/certs/csr.conf" fi local force if ! "${SNAP}/usr/bin/cmp" -s "${SNAP_DATA}/certs/csr.conf.rendered" "${SNAP_DATA}/certs/csr.conf"; then force=true $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.rendered ${SNAP_DATA}/certs/csr.conf else force=false fi if $force; then gen_server_cert gen_proxy_client_cert echo "1" elif [ ! -f "${SNAP_DATA}/certs/front-proxy-client.crt" ] || [ "$("${SNAP}/openssl.wrapper" < ${SNAP_DATA}/certs/front-proxy-client.crt x509 -noout -issuer)" == "issuer=CN = 127.0.0.1" ]; then gen_proxy_client_cert echo "1" else echo "0" fi } ensure_server_ca() { # ensure the server.crt is issued by ca.crt # in a ca chain it is only verified that the server.crt is issued by the intermediate ca # if current csr.conf is invalid, regenerate front-proxy-client certificates as well if ! "${SNAP}/openssl.wrapper" verify -no-CAfile -no-CApath -partial_chain -trusted ${SNAP_DATA}/certs/ca.crt ${SNAP_DATA}/certs/server.crt &>/dev/null then csr_modified="$(ensure_csr_conf_conservative)" gen_server_cert if [[ "$csr_modified" -eq "1" ]] then gen_proxy_client_cert fi echo "1" else echo "0" fi } check_csr_conf() { # if no argument is given, default csr.conf will be checked csr_conf="${1:-${SNAP_DATA}/certs/csr.conf}" "${SNAP}/openssl.wrapper" req -new -config $csr_conf -noout -nodes -keyout /dev/null &>/dev/null } refresh_csr_conf() { render_csr_conf $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.rendered ${SNAP_DATA}/certs/csr.conf } ensure_csr_conf_conservative() { # ensure csr.conf is a valid csr config file; if not: # copy csr.conf.rendered if valid, or render new if not if ! check_csr_conf then if ! check_csr_conf ${SNAP_DATA}/certs/csr.conf.rendered then render_csr_conf fi $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.rendered ${SNAP_DATA}/certs/csr.conf echo "1" else echo "0" fi } render_csr_conf() { # Render csr.conf.template to csr.conf.rendered local IP_ADDRESSES="$(get_ips)" $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.rendered if ! [ "$IP_ADDRESSES" == "127.0.0.1" ] && ! [ "$IP_ADDRESSES" == "none" ] then local ips='' sep='' local -i i=3 for IP_ADDR in $(echo "$IP_ADDRESSES"); do local ip_id="IP.$((i++))" while $SNAP/bin/grep '^'"${ip_id}" ${SNAP_DATA}/certs/csr.conf.template > /dev/null ; do ip_id="IP.$((i++))" done ips+="${sep}${ip_id} = ${IP_ADDR}" sep='\n' done "$SNAP/bin/sed" -i "s/#MOREIPS/${ips}/g" ${SNAP_DATA}/certs/csr.conf.rendered else "$SNAP/bin/sed" -i 's/#MOREIPS//g' ${SNAP_DATA}/certs/csr.conf.rendered fi } get_node() { # Returns the node name or no_node_found in case no node is present KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" timeout=60 start_timer="$($SNAP/bin/date +%s)" node_found="yes" while ! ($KUBECTL get no | $SNAP/bin/grep -z " Ready") &> /dev/null do now="$($SNAP/bin/date +%s)" if ! [ -z $timeout ] && [[ "$now" > "$(($start_timer + $timeout))" ]] ; then node_found="no" echo "no_node_found" break fi sleep 2 done if [ "${node_found}" == "yes" ] then node="$($KUBECTL get no | $SNAP/bin/grep ' Ready' | $SNAP/usr/bin/gawk '{print $1}')" echo $node fi } wait_for_node() { get_node &> /dev/null } drain_node() { # Drain node node="$(get_node)" KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" if ! [ "${node}" == "no_node_found" ] then $KUBECTL drain $node --timeout=120s --grace-period=60 --delete-local-data=true || true fi } uncordon_node() { # Un-drain node node="$(get_node)" KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" if ! [ "${node}" == "no_node_found" ] then $KUBECTL uncordon $node || true fi } function valid_ip() { # Test an IP address for validity: # Usage: # valid_ip IP_ADDRESS # if [[ $? -eq 0 ]]; then echo good; else echo bad; fi # OR # if valid_ip IP_ADDRESS; then echo good; else echo bad; fi # local ip=$1 local stat=1 if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then OIFS=$IFS IFS='.' ip=($ip) IFS=$OIFS [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \ && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]] stat=$? fi return $stat } init_cluster() { $SNAP/bin/mkdir -p ${SNAP_DATA}/var/kubernetes/backend IP="127.0.0.1" # To configure dqlite do: # echo "Address: 1.2.3.4:6364" > $STORAGE_DIR/update.yaml # after the initialisation but before connecting other nodes echo "Address: $IP:19001" > ${SNAP_DATA}/var/kubernetes/backend/init.yaml DNS=$($SNAP/bin/hostname) $SNAP/bin/mkdir -p $SNAP_DATA/var/tmp/ $SNAP/bin/cp $SNAP/certs/csr-dqlite.conf.template $SNAP_DATA/var/tmp/csr-dqlite.conf $SNAP/bin/sed -i 's/HOSTNAME/'"${DNS}"'/g' $SNAP_DATA/var/tmp/csr-dqlite.conf $SNAP/bin/sed -i 's/HOSTIP/'"${IP}"'/g' $SNAP_DATA/var/tmp/csr-dqlite.conf "${SNAP}/openssl.wrapper" req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout ${SNAP_DATA}/var/kubernetes/backend/cluster.key -out ${SNAP_DATA}/var/kubernetes/backend/cluster.crt -subj "/CN=k8s" -config $SNAP_DATA/var/tmp/csr-dqlite.conf -extensions v3_ext $SNAP/bin/chmod -R o-rwX ${SNAP_DATA}/var/kubernetes/backend/ local group=$(get_microk8s_group) if getent group ${group} >/dev/null 2>&1 then $SNAP/bin/chgrp ${group} -R --preserve=mode ${SNAP_DATA}/var/kubernetes/backend/ || true fi } function update_configs { # Create the basic tokens ca_data=$($SNAP/bin/cat ${SNAP_DATA}/certs/ca.crt | ${SNAP}/usr/bin/base64 -w 0) # Create the client kubeconfig run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/client.config $SNAP/bin/sed -i 's/CADATA/'"${ca_data}"'/g' ${SNAP_DATA}/credentials/client.config $SNAP/bin/sed -i 's/NAME/admin/g' ${SNAP_DATA}/credentials/client.config if $SNAP/bin/grep admin ${SNAP_DATA}/credentials/known_tokens.csv 2>&1 > /dev/null then admin_token=`$SNAP/bin/grep admin ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1` $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/client.config $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/client.config else admin_token=`$SNAP/bin/grep admin ${SNAP_DATA}/credentials/basic_auth.csv | $SNAP/usr/bin/cut -d, -f1` $SNAP/bin/sed -i 's/AUTHTYPE/password/g' ${SNAP_DATA}/credentials/client.config fi $SNAP/bin/sed -i 's/PASSWORD/'"${admin_token}"'/g' ${SNAP_DATA}/credentials/client.config # Create the known tokens proxy_token=`$SNAP/bin/grep kube-proxy ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1` hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') kubelet_token=`$SNAP/bin/grep kubelet-0, ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1` controller_token=`$SNAP/bin/grep kube-controller-manager ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1` scheduler_token=`$SNAP/bin/grep kube-scheduler ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1` # Create the client kubeconfig for the controller run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/controller.config $SNAP/bin/sed -i 's/CADATA/'"${ca_data}"'/g' ${SNAP_DATA}/credentials/controller.config $SNAP/bin/sed -i 's/NAME/controller/g' ${SNAP_DATA}/credentials/controller.config $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/controller.config $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/controller.config $SNAP/bin/sed -i 's/PASSWORD/'"${controller_token}"'/g' ${SNAP_DATA}/credentials/controller.config # Create the client kubeconfig for the scheduler run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/scheduler.config $SNAP/bin/sed -i 's/CADATA/'"${ca_data}"'/g' ${SNAP_DATA}/credentials/scheduler.config $SNAP/bin/sed -i 's/NAME/scheduler/g' ${SNAP_DATA}/credentials/scheduler.config $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/scheduler.config $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/scheduler.config $SNAP/bin/sed -i 's/PASSWORD/'"${scheduler_token}"'/g' ${SNAP_DATA}/credentials/scheduler.config # Create the proxy and kubelet kubeconfig run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/kubelet.config $SNAP/bin/sed -i 's/NAME/kubelet/g' ${SNAP_DATA}/credentials/kubelet.config $SNAP/bin/sed -i 's/CADATA/'"${ca_data}"'/g' ${SNAP_DATA}/credentials/kubelet.config $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/kubelet.config $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/kubelet.config $SNAP/bin/sed -i 's/PASSWORD/'"${kubelet_token}"'/g' ${SNAP_DATA}/credentials/kubelet.config run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/proxy.config $SNAP/bin/sed -i 's/NAME/kubeproxy/g' ${SNAP_DATA}/credentials/proxy.config $SNAP/bin/sed -i 's/CADATA/'"${ca_data}"'/g' ${SNAP_DATA}/credentials/proxy.config $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/proxy.config $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/proxy.config $SNAP/bin/sed -i 's/PASSWORD/'"${proxy_token}"'/g' ${SNAP_DATA}/credentials/proxy.config $SNAP/microk8s-stop.wrapper || true $SNAP/microk8s-start.wrapper } is_apiserver_ready() { if (${SNAP}/usr/bin/curl -L --cert ${SNAP_DATA}/certs/server.crt --key ${SNAP_DATA}/certs/server.key --cacert ${SNAP_DATA}/certs/ca.crt https://127.0.0.1:16443/readyz | $SNAP/bin/grep -z "ok") &> /dev/null then return 0 else return 1 fi } start_all_containers() { for task in $("${SNAP}/microk8s-ctr.wrapper" task ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}') do "${SNAP}/microk8s-ctr.wrapper" task resume $task &>/dev/null || true done } stop_all_containers() { for task in $("${SNAP}/microk8s-ctr.wrapper" task ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}') do "${SNAP}/microk8s-ctr.wrapper" task pause $task &>/dev/null || true "${SNAP}/microk8s-ctr.wrapper" task kill -s SIGKILL $task &>/dev/null || true done } remove_all_containers() { stop_all_containers for task in $("${SNAP}/microk8s-ctr.wrapper" task ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}') do "${SNAP}/microk8s-ctr.wrapper" task delete --force $task &>/dev/null || true done for container in $("${SNAP}/microk8s-ctr.wrapper" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}') do "${SNAP}/microk8s-ctr.wrapper" container delete $container &>/dev/null || true done run_with_sudo iptables-legacy -t nat -F CNI-HOSTPORT-DNAT &>/dev/null || true } get_container_shim_pids() { $SNAP/bin/ps -e -o pid= -o args= | $SNAP/bin/grep -v 'grep' | $SNAP/bin/sed -e 's/^ *//; s/\s\s*/\t/;' | $SNAP/bin/grep -w '/snap/microk8s/.*/bin/containerd-shim' | $SNAP/usr/bin/cut -f1 } kill_all_container_shims() { run_with_sudo systemctl kill snap.microk8s.daemon-kubelite.service --signal=SIGKILL &>/dev/null || true run_with_sudo systemctl kill snap.microk8s.daemon-containerd.service --signal=SIGKILL &>/dev/null || true } is_first_boot() { # Return 0 if this is the first start after the host booted. # The argument $1 is a directory that may contain a last-start-date file # The last-start-date file contains a date in seconds # if that date is prior to the creation date of /proc/1 we assume this is the first # time after the host booted # Note, lxc shares the same /proc/stat as the host if ! [ -e "$1/last-start-date" ] || ! [ -e /proc/1 ] then return 1 else last_start=$("$SNAP/bin/cat" "$1/last-start-date") if [ -e /proc/stat ] && $SNAP/bin/grep btime /proc/stat && ! $SNAP/bin/grep lxc /proc/1/environ then boot_time=$($SNAP/bin/grep btime /proc/stat | $SNAP/usr/bin/cut -d' ' -f2) else boot_time=$($SNAP/bin/date -r /proc/1 +%s) fi echo "Last time service started was $last_start and the host booted at $boot_time" if [ "$last_start" -le "$boot_time" ] then return 0 else return 1 fi fi } mark_boot_time() { # place the current time in the "$1"/last-start-date file now=$($SNAP/bin/date +%s) echo "$now" > "$1"/last-start-date } try_copy_users_to_snap_microk8s() { # try copy users from microk8s to snap_microk8s group if getent group microk8s >/dev/null 2>&1 && getent group snap_microk8s >/dev/null 2>&1 then for m in $($SNAP/usr/bin/members microk8s) do echo "Processing user $m" if ! usermod -a -G snap_microk8s $m then echo "Failed to migrate user $m to snap_microk8s group" fi done else echo "One of the microk8s or snap_microk8s groups is missing" fi } cluster_agent_port() { port="25000" if $SNAP/bin/grep -e port "${SNAP_DATA}"/args/cluster-agent &> /dev/null then port=$($SNAP/bin/cat "${SNAP_DATA}"/args/cluster-agent | "$SNAP"/usr/bin/gawk '{print $2}') fi echo "$port" } server_cert_check() { "${SNAP}/openssl.wrapper" x509 -in "$SNAP_DATA"/certs/server.crt -outform der | ${SNAP}/usr/bin/sha256sum | $SNAP/usr/bin/cut -d' ' -f1 | $SNAP/usr/bin/cut -c1-12 } generate_csr_with_sans() { # Description: # Generate CSR for component certificates, including hostname and node IP addresses # as SubjectAlternateNames. The CSR PEM is printed to stdout. Arguments are: # 1. The certificate subject, e.g. "/CN=system:node:$hostname/O=system:nodes" # 2. The path to write the private key, e.g. "$SNAP_DATA/certs/kubelet.key" # # Notes: # - Subject is /CN=system:node:$hostname/O=system:nodes # - Node hostname and IP addresses are added as Subject Alternate Names # # Example usage: # generate_csr_with_sans /CN=system:node:$hostname/O=system:nodes $SNAP_DATA/certs/kubelet.key > $SNAP_DATA/certs/kubelet.csr # Add DNS name and IP addresses as subjectAltName hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') subjectAltName="DNS:$hostname" for ip in $(get_ips); do subjectAltName="$subjectAltName, IP:$ip" done # generate key if it does not exist if [ ! -f "$2" ]; then "${SNAP}/openssl.wrapper" genrsa -out "$2" 2048 $SNAP/bin/chown 0:0 "$2" || true $SNAP/bin/chmod 0600 "$2" || true fi # generate csr "${SNAP}/openssl.wrapper" req -new -sha256 -subj "$1" -key "$2" -addext "subjectAltName = $subjectAltName" } generate_csr() { # Description: # Generate CSR for component certificates. The CSR PEM is written to stdout. Arguments are: # 1. The certificate subject, e.g. "/CN=system:kube-scheduler" # 2. The path to write the private key, e.g. "$SNAP_DATA/certs/scheduler.key" # # Example usage: # generate_csr /CN=system:kube-scheduler $SNAP_DATA/certs/scheduler.key > $SNAP_DATA/certs/scheduler.csr # generate key if it does not exist if [ ! -f "$2" ]; then "${SNAP}/openssl.wrapper" genrsa -out "$2" 2048 $SNAP/bin/chown 0:0 "$2" || true $SNAP/bin/chmod 0600 "$2" || true fi # generate csr "${SNAP}/openssl.wrapper" req -new -sha256 -subj "$1" -key "$2" } sign_certificate() { # Description: # Sign a certificate signing request (CSR) using the MicroK8s cluster CA. # The CSR is read through stdin, and the signed certificate is printed to stdout. # # Notes: # - Read from stdin and write to stdout, so no temporary files are required. # - Any SubjectAlternateNames that are included in the CSR are added to the certificate. # # Example usage: # cat component.csr | sign_certificate > component.crt # We need to use the request more than once, use this trick to grab stdin and save it in '$csr' csr="$($SNAP/bin/cat)" # Parse SANs from the CSR and add them to the certificate extensions (if any) extensions="" alt_names="$(echo "$csr" | "${SNAP}/openssl.wrapper" req -text | $SNAP/bin/grep "X509v3 Subject Alternative Name:" -A1 | $SNAP/usr/bin/tail -n 1 | $SNAP/bin/sed 's,IP Address:,IP:,g')" if test "x$alt_names" != "x"; then extensions="subjectAltName = $alt_names" fi # Sign certificate and print to stdout echo "$csr" | "${SNAP}/openssl.wrapper" x509 -req -sha256 -CA "${SNAP_DATA}/certs/ca.crt" -CAkey "${SNAP_DATA}/certs/ca.key" -CAcreateserial -days 3650 -extfile <(echo "${extensions}") } exit_if_low_memory_guard() { if [ -e ${SNAP_DATA}/var/lock/low-memory-guard.lock ] then echo '' echo 'This node does not have enough RAM to host the Kubernetes control plane services' echo 'and join the database quorum. You may consider joining this node as a worker' echo 'node to a cluster.' echo '' echo 'If you would still like to start the control plane services, start MicroK8s with:' echo '' echo ' microk8s start --disable-low-memory-guard' echo '' exit 1 fi } refresh_calico_if_needed() { # Call the python script that does the calico update if needed "$SNAP/usr/bin/python3" "$SNAP/scripts/calico/upgrade.py" } remove_docker_specific_args() { # Remove docker specific arguments and return 0 if kubelet needs to be restarted if $SNAP/bin/grep -e "\-\-network-plugin" ${SNAP_DATA}/args/kubelet || $SNAP/bin/grep -e "\-\-cni-conf-dir" ${SNAP_DATA}/args/kubelet || $SNAP/bin/grep -e "\-\-cni-bin-dir" ${SNAP_DATA}/args/kubelet then skip_opt_in_local_config network-plugin kubelet skip_opt_in_local_config cni-conf-dir kubelet skip_opt_in_local_config cni-bin-dir kubelet return 0 fi return 1 } fetch_as() { # download from location $1 to location $2 if is_strict then ARCH="$($SNAP/bin/uname -m)" LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" "${SNAP}/usr/bin/curl" -L $1 -o $2 else CA_CERT=/snap/core22/current/etc/ssl/certs/ca-certificates.crt run_with_sudo "${SNAP}/usr/bin/curl" --cacert $CA_CERT -L $1 -o $2 fi } ############################# Strict functions ###################################### log_init () { echo `$SNAP/bin/date +"[%m-%d %H:%M:%S]" start logging` > $SNAP_COMMON/var/log/microk8s.log } log () { echo -n `$SNAP/bin/date +"[%m-%d %H:%M:%S]"` >> $SNAP_COMMON/var/log/microk8s.log echo ": $@" >> $SNAP_COMMON/var/log/microk8s.log } is_strict() { # Return 0 if we are in strict mode if $SNAP/bin/cat $SNAP/meta/snap.yaml | $SNAP/bin/grep confinement | $SNAP/bin/grep -q strict then return 0 else return 1 fi } check_snap_interfaces() { # Check whether all of the required interfaces are connected before proceeding. # This is to address https://forum.snapcraft.io/t/mimic-sequence-of-hook-calls-with-auto-connected-interfaces/19618 declare -ra interfaces=( "account-control" "docker-privileged" "dot-kube" "dot-config-helm" "firewall-control" "hardware-observe" "home" "home-read-all" "k8s-journald" "k8s-kubelet" "k8s-kubeproxy" "kernel-module-observe" "kubernetes-support" "log-observe" "login-session-observe" "mount-observe" "network" "network-bind" "network-control" "network-observe" "opengl" "process-control" "system-observe" ) declare -a missing=() for interface in ${interfaces[@]} do if ! snapctl is-connected ${interface} then missing+=("${interface}") fi done if [ ${#missing[@]} -gt 0 ] then snapctl set-health blocked "You must connect ${missing[*]} before proceeding" exit 0 fi } enable_snap() { snapctl start --enable ${SNAP_NAME} snapctl set-health okay } exit_if_not_root() { # test if we run with sudo if (is_strict) && [ "$EUID" -ne 0 ] then echo "Elevated permissions are needed for this command. Please use sudo." exit 1 fi } is_first_boot_on_strict() { # Return 0 if this is the first start after the host booted. SENTINEL="/tmp/.containerd-first-book-check" # We rely on the fact that /tmp is cleared at every boot to determine if # this is the first call after boot: if the sentinel file exists, then it # means that no reboot occurred since last check; otherwise, return success # and create the sentinel file for the future check. if [ -f "$SENTINEL" ] then return 1 else $SNAP/bin/touch "$SENTINEL" return 0 fi } default_route_exists() { # test if we have a default route ( $SNAP/sbin/ip route; $SNAP/sbin/ip -6 route ) | $SNAP/bin/grep "^default" &>/dev/null } wait_for_default_route() { # wait 10 seconds for default route to appear n=0 until [ $n -ge 5 ] do default_route_exists && break echo "Waiting for default route to appear. (attempt $n)" n=$[$n+1] sleep 2 done } is_ec2_instance() { if [ -f "/sys/hypervisor/uuid" ] then EC2UID=$($SNAP/usr/bin/head -c 3 /sys/hypervisor/uuid | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') if [[ $EC2UID == *"ec2"* ]] then return 0 fi else if [ -f "/sys/devices/virtual/dmi/id/product_uuid" ] then EC2UID=$($SNAP/usr/bin/head -c 3 /sys/devices/virtual/dmi/id/product_uuid | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]') if [[ $EC2UID == *"ec2"* ]] then return 0 fi fi fi return 1 } increase_sysctl_parameter() { declare -a conf_locs=("/etc/sysctl.d/" "/run/sysctl.d/" "/usr/local/lib/sysctl.d/" "/usr/lib/sysctl.d/" "/lib/sysctl.d/" "/etc/sysctl.conf") local param_key=$1 local new_val=$2 val_max="0" if ! is_strict; then if [ -f "/etc/sysctl.d/10-microk8s.conf" ]; then run_with_sudo $SNAP/usr/bin/sort -u /etc/sysctl.d/10-microk8s.conf -o /etc/sysctl.d/10-microk8s.conf fi for loc in "${conf_locs[@]}" do if run_with_sudo $SNAP/bin/grep -qr "^$param_key=" $loc; then for val in $(run_with_sudo $SNAP/bin/grep -r "^$param_key=" $loc | $SNAP/bin/sed 's/^.*=//'); do if [ "$val" -ge "$val_max" ]; then val_max=$val; fi done fi done if [ "$val_max" -lt "$new_val" ]; then echo "$param_key=$new_val" | run_with_sudo $SNAP/usr/bin/tee -a /etc/sysctl.d/10-microk8s.conf if ! run_with_sudo sysctl --system; then echo "Could not refresh system parameters via sysctl" fi fi fi } use_snap_env() { # Configure PATH, LD_LIBRARY_PATH and PYTHONPATH export PATH="$SNAP/usr/bin:$SNAP/bin:$SNAP/usr/sbin:$SNAP/sbin:$REAL_PATH" export LD_LIBRARY_PATH="$SNAP_LIBRARY_PATH:$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$SNAPCRAFT_ARCH_TRIPLET:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/ceph:${REAL_LD_LIBRARY_PATH:-}" export PYTHONPATH="$SNAP/usr/lib/python3.10:$SNAP/lib/python3.10/site-packages:$SNAP/usr/lib/python3/dist-packages" # Python configuration export PYTHONNOUSERSITE=false # NOTE(neoaggelos/2023-08-14): # we cannot list system locales from snap. instead, we attempt # well-known locales for Ubuntu/Debian/CentOS and check whether # they are available on the system. # if they are, set them for the current shell. for locale in C.UTF-8 en_US.UTF-8 en_US.utf8; do if [ -z "$(export LC_ALL=$locale 2>&1)" ]; then export LC_ALL="${LC_ALL:-$locale}" export LANG="${LC_ALL:-$locale}" break fi done # Configure XDG_RUNTIME_DIR export XDG_RUNTIME_DIR="${SNAP_COMMON}/run" mkdir -p "${XDG_RUNTIME_DIR}" } # check if this file is run with arguments if [[ "$0" == "${BASH_SOURCE}" ]] && [[ ! -z "$1" ]] then # call help if echo "$*" | grep -q -- 'help'; then echo "usage: $0 [function]" echo "" echo "Run a utility function and return the output." echo "" echo "available functions:" declare -F | gawk '{print "- "$3}' exit 0 fi if declare -F "$1" > /dev/null then $1 ${@:2} exit $? else echo "Function does not exist: $1" >&2 exit 1 fi fi ================================================ FILE: microk8s-resources/basic_auth.csv ================================================ bOU1pQYT0Sjfyg6omGb4zzwmTHXH5Yrp,admin,admin,"system:masters" ================================================ FILE: microk8s-resources/certs/csr-dqlite.conf.template ================================================ [ req ] default_bits = 2048 prompt = no default_md = sha256 req_extensions = req_ext distinguished_name = dn [ dn ] C = GB ST = Canonical L = Canonical O = Canonical OU = Canonical CN = k8s [ req_ext ] subjectAltName = @alt_names [ alt_names ] DNS = HOSTNAME IP = HOSTIP [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=keyEncipherment,dataEncipherment,digitalSignature extendedKeyUsage=serverAuth,clientAuth subjectAltName=@alt_names ================================================ FILE: microk8s-resources/certs/csr.conf.template ================================================ [ req ] default_bits = 2048 prompt = no default_md = sha256 req_extensions = req_ext distinguished_name = dn [ dn ] C = GB ST = Canonical L = Canonical O = Canonical OU = Canonical CN = 127.0.0.1 [ req_ext ] subjectAltName = @alt_names [ alt_names ] DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.cluster DNS.5 = kubernetes.default.svc.cluster.local IP.1 = 127.0.0.1 IP.2 = 10.152.183.1 #MOREIPS [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=keyEncipherment,dataEncipherment,digitalSignature extendedKeyUsage=serverAuth,clientAuth subjectAltName=@alt_names ================================================ FILE: microk8s-resources/client-x509.config.template ================================================ apiVersion: v1 clusters: - cluster: certificate-authority-data: CADATA server: https://127.0.0.1:16443 name: microk8s-cluster contexts: - context: cluster: microk8s-cluster user: NAME name: microk8s current-context: microk8s kind: Config preferences: {} users: - name: NAME user: client-certificate: PATHTOCERT client-key: PATHTOKEYCERT ================================================ FILE: microk8s-resources/client.config ================================================ apiVersion: v1 clusters: - cluster: server: http://127.0.0.1:8080 name: microk8s-cluster contexts: - context: cluster: microk8s-cluster user: admin name: microk8s current-context: microk8s kind: Config preferences: {} users: - name: admin user: username: admin ================================================ FILE: microk8s-resources/client.config.template ================================================ apiVersion: v1 clusters: - cluster: certificate-authority-data: CADATA server: https://127.0.0.1:16443 name: microk8s-cluster contexts: - context: cluster: microk8s-cluster user: NAME name: microk8s current-context: microk8s kind: Config preferences: {} users: - name: NAME user: username: NAME AUTHTYPE: PASSWORD ================================================ FILE: microk8s-resources/containerd-profile ================================================ #include profile cri-containerd.apparmor.d flags=(attach_disconnected,mediate_deleted) { #include network, network inet, network inet6, network unix, network netlink, network raw, capability, file, umount, deny @{PROC}/* w, # deny write for all files directly in /proc (not in a subdir) # deny write to files not in /proc//** or /proc/sys/** deny @{PROC}/{[^1-9],[^1-9][^0-9],[^1-9s][^0-9y][^0-9s],[^1-9][^0-9][^0-9][^0-9]*}/** w, deny @{PROC}/sys/[^k]** w, # deny /proc/sys except /proc/sys/k* (effectively /proc/sys/kernel) deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} w, # deny everything except shm* in /proc/sys/kernel/ deny @{PROC}/sysrq-trigger rwklx, deny @{PROC}/mem rwklx, deny @{PROC}/kmem rwklx, deny @{PROC}/kcore rwklx, deny mount, deny /sys/[^f]*/** wklx, deny /sys/f[^s]*/** wklx, deny /sys/fs/[^c]*/** wklx, deny /sys/fs/c[^g]*/** wklx, deny /sys/fs/cg[^r]*/** wklx, deny /sys/firmware/efi/efivars/** rwklx, deny /sys/kernel/security/** rwklx, # suppress ptrace denials when using 'docker ps' or using 'ps' inside a container ptrace (trace,read) peer=cri-containerd.apparmor.d, signal (receive) peer=snap.microk8s.daemon-kubelite, signal (receive) peer=snap.microk8s.daemon-containerd, signal (send,receive) peer=cri-containerd.apparmor.d//&unconfined, } ================================================ FILE: microk8s-resources/default-args/admission-control-config-file.yaml ================================================ apiVersion: apiserver.config.k8s.io/v1 kind: AdmissionConfiguration plugins: - name: EventRateLimit path: eventconfig.yaml ================================================ FILE: microk8s-resources/default-args/apiserver-proxy ================================================ --traefik-config ${SNAP_DATA}/args/traefik/traefik.yaml --kubeconfig ${SNAP_DATA}/credentials/kubelet.config --refresh-interval 30s ================================================ FILE: microk8s-resources/default-args/certs.d/docker.io/hosts.toml ================================================ server = "https://docker.io" [host."https://registry-1.docker.io"] capabilities = ["pull", "resolve"] ================================================ FILE: microk8s-resources/default-args/certs.d/localhost__32000/hosts.toml ================================================ server = "http://localhost:32000" [host."http://localhost:32000"] capabilities = ["pull", "resolve"] ================================================ FILE: microk8s-resources/default-args/cluster-agent ================================================ --bind 0.0.0.0:25000 --keyfile "${SNAP_DATA}/certs/server.key" --certfile "${SNAP_DATA}/certs/server.crt" --timeout 240 ================================================ FILE: microk8s-resources/default-args/cni-env ================================================ # Choose CNI to deploy. Only 'calico' is supported at the moment CNI=calico # IPv4 configuration IPv4_SUPPORT=true IPv4_CLUSTER_CIDR=10.1.0.0/16 IPv4_SERVICE_CIDR=10.152.183.0/24 # IPv6 configuration IPv6_SUPPORT=false # IPv6_CLUSTER_CIDR=fd01::/64 # IPv6_SERVICE_CIDR=fd98::/108 # Calico-specific configuration settings # CALICO_VETH_MTU=1450 ================================================ FILE: microk8s-resources/default-args/cni-network/flannel.conflist ================================================ { "name": "microk8s-flannel-network", "plugins": [ { "type": "flannel", "delegate": { "hairpinMode": true, "isDefaultGateway": true } }, { "type": "portmap", "capabilities": {"portMappings": true}, "snat": true } ] } ================================================ FILE: microk8s-resources/default-args/containerd ================================================ --config ${SNAP_DATA}/args/containerd.toml --root ${SNAP_COMMON}/var/lib/containerd --state ${SNAP_COMMON}/run/containerd --address ${SNAP_COMMON}/run/containerd.sock ================================================ FILE: microk8s-resources/default-args/containerd-env ================================================ # Remember to restart MicroK8s after editing this file: # # sudo microk8s stop; sudo microk8s start # # To start containerd behind a proxy you need to add an HTTPS_PROXY # environment variable in this file. HTTPS_PROXY is of the following form: # HTTPS_PROXY=http://username:password@proxy:port/ # where username: and password@ are optional. eg: # # HTTPS_PROXY=https://squid.internal:3128 # # You may also want to set NO_PROXY to include the cluster-cidr and the services-cidr # as specified in /var/snap/microk8s/current/args/kube-proxy and # /var/snap/microk8s/current/args/kube-apiserver # # NO_PROXY=10.1.0.0/16,10.152.183.0/24 # # You can set the of the kata containers runtime here. # # KATA_PATH= # PATH=$PATH:$KATA_PATH # Attempt to change the maximum number of open file descriptors # this get inherited to the running containers # ulimit -n 65536 || true # Attempt to change the maximum locked memory limit # this get inherited to the running containers # ulimit -l 16384 || true ================================================ FILE: microk8s-resources/default-args/containerd-template.toml ================================================ # Use config version 2 to enable new configuration fields. version = 2 oom_score = 0 [grpc] uid = 0 gid = 0 max_recv_message_size = 16777216 max_send_message_size = 16777216 [debug] address = "" uid = 0 gid = 0 [metrics] address = "127.0.0.1:1338" grpc_histogram = false [cgroup] path = "" # The 'plugins."io.containerd.grpc.v1.cri"' table contains all of the server options. [plugins."io.containerd.grpc.v1.cri"] stream_server_address = "127.0.0.1" stream_server_port = "0" enable_selinux = false sandbox_image = "registry.k8s.io/pause:3.10" stats_collect_period = 10 enable_tls_streaming = false max_container_log_line_size = 16384 # 'plugins."io.containerd.grpc.v1.cri".containerd' contains config related to containerd [plugins."io.containerd.grpc.v1.cri".containerd] # snapshotter is the snapshotter used by containerd. snapshotter = "${SNAPSHOTTER}" # no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc. # This only works for runtime type "io.containerd.runtime.v1.linux". no_pivot = false # default_runtime_name is the default runtime name to use. default_runtime_name = "${RUNTIME}" # 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes' is a map from CRI RuntimeHandler strings, which specify types # of runtime configurations, to the matching configurations. # In this example, 'runc' is the RuntimeHandler string to match. [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux runtime_type = "${RUNTIME_TYPE}" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime] # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux runtime_type = "${RUNTIME_TYPE}" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime.options] BinaryName = "nvidia-container-runtime" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] BinaryName = "kata-runtime" # 'plugins."io.containerd.grpc.v1.cri".cni' contains config related to cni [plugins."io.containerd.grpc.v1.cri".cni] # bin_dir is the directory in which the binaries for the plugin is kept. bin_dir = "${SNAP_DATA}/opt/cni/bin" # conf_dir is the directory in which the admin places a CNI conf. conf_dir = "${SNAP_DATA}/args/cni-network" # 'plugins."io.containerd.grpc.v1.cri".registry' contains config related to the registry [plugins."io.containerd.grpc.v1.cri".registry] config_path = "${SNAP_DATA}/args/certs.d" ================================================ FILE: microk8s-resources/default-args/ctr ================================================ --address=${SNAP_COMMON}/run/containerd.sock --namespace k8s.io ================================================ FILE: microk8s-resources/default-args/etcd ================================================ --data-dir=${SNAP_COMMON}/var/run/etcd --advertise-client-urls=https://${DEFAULT_INTERFACE_IP_ADDR}:12379 --listen-client-urls=https://0.0.0.0:12379 --client-cert-auth --trusted-ca-file=${SNAP_DATA}/certs/ca.crt --cert-file=${SNAP_DATA}/certs/server.crt --key-file=${SNAP_DATA}/certs/server.key ================================================ FILE: microk8s-resources/default-args/eventconfig.yaml ================================================ apiVersion: eventratelimit.admission.k8s.io/v1alpha1 kind: Configuration limits: - type: Server qps: 5000 burst: 20000 ================================================ FILE: microk8s-resources/default-args/flannel-network-mgr-config ================================================ {"Network": "10.1.0.0/16", "Backend": {"Type": "vxlan"}} ================================================ FILE: microk8s-resources/default-args/flannel-template.conflist ================================================ { "name": "microk8s-flannel-network", "cniVersion": "0.3.1", "plugins": [ { "type": "flannel", "name": "flannel-plugin", "subnetFile": "${SNAP_COMMON}/run/flannel/subnet.env", "dataDir": "${SNAP_COMMON}/var/lib/cni/flannel", "delegate": { "hairpinMode": true, "isDefaultGateway": true } }, { "type": "portmap", "capabilities": {"portMappings": true}, "snat": true } ] } ================================================ FILE: microk8s-resources/default-args/flanneld ================================================ --iface="" --etcd-endpoints=https://127.0.0.1:12379 --etcd-cafile=${SNAP_DATA}/certs/ca.crt --etcd-certfile=${SNAP_DATA}/certs/server.crt --etcd-keyfile=${SNAP_DATA}/certs/server.key --subnet-file=${SNAP_COMMON}/run/flannel/subnet.env --ip-masq=true ================================================ FILE: microk8s-resources/default-args/git/.gitconfig ================================================ [safe] directory = /snap/microk8s/current/addons/community/.git directory = /snap/microk8s/current/addons/core/.git ================================================ FILE: microk8s-resources/default-args/ha-conf ================================================ failure-domain=1 ================================================ FILE: microk8s-resources/default-args/k8s-dqlite ================================================ --storage-dir=${SNAP_DATA}/var/kubernetes/backend/ --listen=unix://${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 ================================================ FILE: microk8s-resources/default-args/k8s-dqlite-env ================================================ # Environment variables k8s-dqlite will run with # # Enable dqlite debugging flags # LIBRAFT_TRACE="1" # LIBDQLITE_TRACE="1" ================================================ FILE: microk8s-resources/default-args/kube-apiserver ================================================ --cert-dir=${SNAP_DATA}/certs --service-cluster-ip-range=10.152.183.0/24 --authorization-mode=AlwaysAllow --service-account-key-file=${SNAP_DATA}/certs/serviceaccount.key --client-ca-file=${SNAP_DATA}/certs/ca.crt --tls-cert-file=${SNAP_DATA}/certs/server.crt --tls-private-key-file=${SNAP_DATA}/certs/server.key --tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 --kubelet-client-certificate=${SNAP_DATA}/certs/apiserver-kubelet-client.crt --kubelet-client-key=${SNAP_DATA}/certs/apiserver-kubelet-client.key --secure-port=16443 --etcd-servers="unix://${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379" --allow-privileged=true --service-account-issuer='https://kubernetes.default.svc' --service-account-signing-key-file=${SNAP_DATA}/certs/serviceaccount.key --event-ttl=5m --profiling=false --feature-gates=ListFromCacheSnapshot=false,SizeBasedListCostEstimate=false,DetectCacheInconsistency=false # Enable the aggregation layer --requestheader-client-ca-file=${SNAP_DATA}/certs/front-proxy-ca.crt --requestheader-allowed-names=front-proxy-client --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --proxy-client-cert-file=${SNAP_DATA}/certs/front-proxy-client.crt --proxy-client-key-file=${SNAP_DATA}/certs/front-proxy-client.key #~Enable the aggregation layer --enable-admission-plugins=EventRateLimit --admission-control-config-file=${SNAP_DATA}/args/admission-control-config-file.yaml --kubelet-certificate-authority=${SNAP_DATA}/certs/ca.crt --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP ================================================ FILE: microk8s-resources/default-args/kube-controller-manager ================================================ --kubeconfig=${SNAP_DATA}/credentials/controller.config --service-account-private-key-file=${SNAP_DATA}/certs/serviceaccount.key --root-ca-file=${SNAP_DATA}/certs/ca.crt --cluster-signing-cert-file=${SNAP_DATA}/certs/ca.crt --cluster-signing-key-file=${SNAP_DATA}/certs/ca.key --use-service-account-credentials --leader-elect-lease-duration=60s --leader-elect-renew-deadline=30s --profiling=false ================================================ FILE: microk8s-resources/default-args/kube-proxy ================================================ --kubeconfig=${SNAP_DATA}/credentials/proxy.config --cluster-cidr=10.1.0.0/16 --healthz-bind-address=127.0.0.1 --profiling=false ================================================ FILE: microk8s-resources/default-args/kube-scheduler ================================================ --kubeconfig=${SNAP_DATA}/credentials/scheduler.config --leader-elect-lease-duration=60s --leader-elect-renew-deadline=30s --profiling=false ================================================ FILE: microk8s-resources/default-args/kubectl ================================================ ================================================ FILE: microk8s-resources/default-args/kubectl-env ================================================ export KUBECONFIG=$SNAP_DATA/credentials/client.config ================================================ FILE: microk8s-resources/default-args/kubelet ================================================ --kubeconfig=${SNAP_DATA}/credentials/kubelet.config --cert-dir=${SNAP_DATA}/certs --client-ca-file=${SNAP_DATA}/certs/ca.crt --anonymous-auth=false --root-dir=${SNAP_COMMON}/var/lib/kubelet --fail-swap-on=false --eviction-hard="memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi" --container-runtime-endpoint=${SNAP_COMMON}/run/containerd.sock --containerd=${SNAP_COMMON}/run/containerd.sock --node-labels="microk8s.io/cluster=true,node.kubernetes.io/microk8s-controlplane=microk8s-controlplane" --authentication-token-webhook=true --read-only-port=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --serialize-image-pulls=false ================================================ FILE: microk8s-resources/default-args/kubelite ================================================ --scheduler-args-file=$SNAP_DATA/args/kube-scheduler --controller-manager-args-file=$SNAP_DATA/args/kube-controller-manager --proxy-args-file=$SNAP_DATA/args/kube-proxy --kubelet-args-file=$SNAP_DATA/args/kubelet --apiserver-args-file=$SNAP_DATA/args/kube-apiserver --kubeconfig-file=$SNAP_DATA/credentials/client.config --start-control-plane=true ================================================ FILE: microk8s-resources/default-args/traefik/provider-template.yaml ================================================ tcp: routers: Router-1: rule: "HostSNI(`*`)" service: "kube-apiserver" tls: passthrough: true services: kube-apiserver: loadBalancer: servers: # APISERVERS # - address: "10.130.0.2:16443" # - address: "10.130.0.3:16443" # - address: "10.130.0.4:16443" ================================================ FILE: microk8s-resources/default-args/traefik/traefik-template.yaml ================================================ entryPoints: apiserver: address: ":16443" providers: file: filename: ${SNAP_DATA}/args/traefik/provider.yaml watch: true ================================================ FILE: microk8s-resources/default-hooks/post-refresh.d/30-helm ================================================ #!/bin/bash # Install helm binaries in SNAP_DATA to maintain backwards-compatibility if [ -d "${SNAP_DATA}/bin" ]; then cp "${SNAP}/bin/helm" "${SNAP_DATA}/bin/helm3" fi ================================================ FILE: microk8s-resources/default-hooks/reconcile.d/10-pods-restart ================================================ #!/bin/bash . "${SNAP}/actions/common/utils.sh" if ! [ -e "${SNAP_DATA}/var/lock/no-cni-reload" ] && [ -e "${SNAP_DATA}/var/lock/snapdata-mounts-need-reload" ]; then if (is_apiserver_ready) && "${SNAP}/scripts/kill-host-pods.py" \ --with-snap-data-mounts --with-owner \ -- -A --field-selector spec.nodeName=$(hostname) ; then rm "${SNAP_DATA}/var/lock/snapdata-mounts-need-reload" fi fi ================================================ FILE: microk8s-resources/default-hooks/reconcile.d/90-calico-apply ================================================ #!/usr/bin/env bash . "${SNAP}/actions/common/utils.sh" use_snap_env KUBECTL="${SNAP}/microk8s-kubectl.wrapper" if [ -e "${SNAP_DATA}/args/cni-network/cni.yaml" ] && [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] && ! [ -e "${SNAP_DATA}/var/lock/cni-loaded" ] then echo "Setting up the CNI" if (is_apiserver_ready) && "${KUBECTL}" apply -f "${SNAP_DATA}/args/cni-network/cni.yaml" then touch "${SNAP_DATA}/var/lock/cni-loaded" # We just installed Calico, no need to refresh rm "${SNAP_DATA}/var/lock/cni-needs-reload" || true fi fi ================================================ FILE: microk8s-resources/default-hooks/remove.d/10-cni-link ================================================ #!/bin/bash . "${SNAP}/actions/common/utils.sh" if ! is_strict || (is_strict && snapctl is-connected network-control) then for link in cni0 do if "${SNAP}/sbin/ip" link show "${link}" then "${SNAP}/sbin/ip" link delete "${link}" || true fi done for calink in $("${SNAP}/sbin/ip" -j link show |\ "${SNAP}/usr/bin/jq" -r '.[].ifname | select(test("^vxlan[-v6]*.calico|cali[a-f0-9]*$"))') do "${SNAP}/sbin/ip" link delete "${calink}" || true done fi ================================================ FILE: microk8s-resources/default-hooks/remove.d/10-cni-link-cilium ================================================ #!/bin/bash . "${SNAP}/actions/common/utils.sh" if ! is_strict || (is_strict && snapctl is-connected network-control) then for link in cilium_host cilium_vxlan do if "${SNAP}/sbin/ip" link show "${link}" then "${SNAP}/sbin/ip" link delete "${link}" || true fi done fi ================================================ FILE: microk8s-resources/default-hooks/remove.d/20-cni-netns ================================================ #!/bin/bash . "${SNAP}/actions/common/utils.sh" if ! is_strict || (is_strict && snapctl is-connected network-control) then for ns in `"${SNAP}/sbin/ip" netns list | grep "^cni-" | awk '{print $1}'` do "${SNAP}/sbin/ip" netns delete "${ns}" || true done fi ================================================ FILE: microk8s-resources/default-hooks/remove.d/90-containers ================================================ #!/bin/bash . "${SNAP}/actions/common/utils.sh" remove_all_containers kill_all_container_shims ================================================ FILE: microk8s-resources/kubelet.config ================================================ apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURQakNDQWlhZ0F3SUJBZ0lKQU8xT0dNWkVmQlJETUEwR0NTcUdTSWIzRFFFQkN3VUFNQmt4RnpBVkJnTlYKQkFNTURqSTFNaTQxTUM0eE9EZ3VNakU0TUI0WERURTRNRFV3TVRFME5UWTFOMW9YRFRJNE1EUXlPREUwTlRZMQpOMW93R1RFWE1CVUdBMVVFQXd3T01qVXlMalV3TGpFNE9DNHlNVGd3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURYUHFsZ3VtMmhWTlI1R3BOdEkzbTF2YnFxNVpwNUNVZHhFZmZPcW1qK201NmUKSzFFQjVNdi9rRVUvWk9FYi9RdHFyeWtCZkFhSFZOY1c1eVJVcCtXTHlEOU5xY0tUQVZ5TFJ3YVRaOWdwTzFhdQo3Qk9VeklTOXZYZ255emUrZ2FPVTZ4Z3NNL2pncklINklmd0tQaDc2NU1rc2UxV2lSL3VuWnhZY0FjOFNwbWdyCm5JVlRidVJoeDZYWDhZdlFqOWoxVXc1RkQrejE0aE1vWHcyTFpYQUJwWW5NaFd1bUdYVldiNzluTkd0VTZPUzcKK2Eybi83Z01nbWdpZlFnWVhRZHBXbUgzWE8xdDVtZHBDSkYrdUxFTkRxUzIxYU1CS0prN080VmNRS1NRd2R2cQp2cVQ5ZmplUmNLMWEreGZiZU95NjNSblh2ZTEreGwwbyt2MndiMm9KQWdNQkFBR2pnWWd3Z1lVd0hRWURWUjBPCkJCWUVGSWoyL0szejlxS1RRSU83V3YvRVY4QzJpSzJBTUVrR0ExVWRJd1JDTUVDQUZJajIvSzN6OXFLVFFJTzcKV3YvRVY4QzJpSzJBb1Iya0d6QVpNUmN3RlFZRFZRUUREQTR5TlRJdU5UQXVNVGc0TGpJeE9JSUpBTzFPR01aRQpmQlJETUF3R0ExVWRFd1FGTUFNQkFmOHdDd1lEVlIwUEJBUURBZ0VHTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCCkFRQ3hUb0dFUXVFSUhwanJEQTIzakxOaVlVcmgxeEFwZlI4bWZxcmR4Ukp2NnN6ZTdIQks2QkRoelY2UURVQlcKRG1tWVNFMDlURkx0WGkyT1VWaGNsVkg3Qm90bVRseUEvTWlZRm5CS2x3ekE1S0VSdGRTU2l2V2tYL2xBQXkvYQpSTTNmZTVnR042RVROS3NvSEI0S1lLaU5DS2poVmR6RjdhajNkUktqanZHTmducEQ1R1hBd1AyY2NONXUxbXBjCkpPV2RzWlh4QmMyRkxyWmFaK1hBdWJWQ0V2amNJMSs0MzlMdEplNU1IdU94Q2UyS1k4N2lOU2gyaTl6cDAxc08KSVUxR0s0VHlCYU1Lc0Nhblh6ZmRyNGFoeXdvdUlVd3p3RDEzeTF6M1Q5RnRJRDltTUlxdlNPKzhOZFNDU3dINQpMUmJIa0xDdFZvMVF2alpDREV2eWFVSWsKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== server: http://127.0.0.1:8080 name: microk8s-cluster contexts: - context: cluster: microk8s-cluster user: kubelet name: microk8s current-context: microk8s kind: Config preferences: {} users: - name: kubelet user: token: tzviBa7HFzlDaGzVYezXdD528jbQPzgw ================================================ FILE: microk8s-resources/kubelet.config.template ================================================ apiVersion: v1 clusters: - cluster: certificate-authority-data: CADATA server: https://127.0.0.1:16443 name: microk8s-cluster contexts: - context: cluster: microk8s-cluster user: NAME name: microk8s current-context: microk8s kind: Config preferences: {} users: - name: NAME user: token: TOKEN ================================================ FILE: microk8s-resources/kubeproxy.config ================================================ apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURQakNDQWlhZ0F3SUJBZ0lKQU8xT0dNWkVmQlJETUEwR0NTcUdTSWIzRFFFQkN3VUFNQmt4RnpBVkJnTlYKQkFNTURqSTFNaTQxTUM0eE9EZ3VNakU0TUI0WERURTRNRFV3TVRFME5UWTFOMW9YRFRJNE1EUXlPREUwTlRZMQpOMW93R1RFWE1CVUdBMVVFQXd3T01qVXlMalV3TGpFNE9DNHlNVGd3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURYUHFsZ3VtMmhWTlI1R3BOdEkzbTF2YnFxNVpwNUNVZHhFZmZPcW1qK201NmUKSzFFQjVNdi9rRVUvWk9FYi9RdHFyeWtCZkFhSFZOY1c1eVJVcCtXTHlEOU5xY0tUQVZ5TFJ3YVRaOWdwTzFhdQo3Qk9VeklTOXZYZ255emUrZ2FPVTZ4Z3NNL2pncklINklmd0tQaDc2NU1rc2UxV2lSL3VuWnhZY0FjOFNwbWdyCm5JVlRidVJoeDZYWDhZdlFqOWoxVXc1RkQrejE0aE1vWHcyTFpYQUJwWW5NaFd1bUdYVldiNzluTkd0VTZPUzcKK2Eybi83Z01nbWdpZlFnWVhRZHBXbUgzWE8xdDVtZHBDSkYrdUxFTkRxUzIxYU1CS0prN080VmNRS1NRd2R2cQp2cVQ5ZmplUmNLMWEreGZiZU95NjNSblh2ZTEreGwwbyt2MndiMm9KQWdNQkFBR2pnWWd3Z1lVd0hRWURWUjBPCkJCWUVGSWoyL0szejlxS1RRSU83V3YvRVY4QzJpSzJBTUVrR0ExVWRJd1JDTUVDQUZJajIvSzN6OXFLVFFJTzcKV3YvRVY4QzJpSzJBb1Iya0d6QVpNUmN3RlFZRFZRUUREQTR5TlRJdU5UQXVNVGc0TGpJeE9JSUpBTzFPR01aRQpmQlJETUF3R0ExVWRFd1FGTUFNQkFmOHdDd1lEVlIwUEJBUURBZ0VHTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCCkFRQ3hUb0dFUXVFSUhwanJEQTIzakxOaVlVcmgxeEFwZlI4bWZxcmR4Ukp2NnN6ZTdIQks2QkRoelY2UURVQlcKRG1tWVNFMDlURkx0WGkyT1VWaGNsVkg3Qm90bVRseUEvTWlZRm5CS2x3ekE1S0VSdGRTU2l2V2tYL2xBQXkvYQpSTTNmZTVnR042RVROS3NvSEI0S1lLaU5DS2poVmR6RjdhajNkUktqanZHTmducEQ1R1hBd1AyY2NONXUxbXBjCkpPV2RzWlh4QmMyRkxyWmFaK1hBdWJWQ0V2amNJMSs0MzlMdEplNU1IdU94Q2UyS1k4N2lOU2gyaTl6cDAxc08KSVUxR0s0VHlCYU1Lc0Nhblh6ZmRyNGFoeXdvdUlVd3p3RDEzeTF6M1Q5RnRJRDltTUlxdlNPKzhOZFNDU3dINQpMUmJIa0xDdFZvMVF2alpDREV2eWFVSWsKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== server: https://172.31.50.188:16443 name: microk8s-cluster contexts: - context: cluster: microk8s-cluster user: kube-proxy name: microk8s current-context: microk8s kind: Config preferences: {} users: - name: kube-proxy user: token: RPY4OSU13Js323K4FDmXgVW4YWOdS65i ================================================ FILE: microk8s-resources/microk8s.default.yaml ================================================ # Default launch configuration for MicroK8s. --- version: 0.1.0 extraKubeletArgs: --cluster-domain: cluster.local --cluster-dns: 10.152.183.10 addons: - name: dns ================================================ FILE: microk8s-resources/wrappers/apiservice-kicker ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "api service kicker will not run on a cluster node" exit 0 fi if [ -e "${SNAP_DATA}/var/lock/low-memory-guard.lock" ] then echo "not starting api service kicker because of low memory guard lock" exit 0 fi group=$(get_microk8s_or_cis_group) restart_attempt=0 installed_registry_help=0 while true do if [ $restart_attempt -ge 5 ] then echo "Service kicker restarted the apiserver too quickly. Exiting." exit 1 fi # every 5 seconds sleep 5 if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] && [ ! -e "${SNAP_DATA}/var/lock/cis-hardening" ] && getent group ${group} >/dev/null 2>&1 then chmod -R ug+rwX ${SNAP_DATA}/var/kubernetes/backend || true chgrp ${group} -R ${SNAP_DATA}/var/kubernetes/backend || true chgrp ${group} ${SNAP_COMMON}/run/containerd.sock || true fi # certificate regeneration on: IP/csr change if not clustered, CA mismatch if clustered manage_certs() { if ! [ -e "${SNAP_DATA}/var/lock/no-cert-reissue" ] && ! grep -E "(--advertise-address|--bind-address)" $SNAP_DATA/args/kube-apiserver &> /dev/null && ip route | grep default &> /dev/null then echo "$(produce_certs)" else echo "$(ensure_server_ca)" fi } if snapctl services microk8s.daemon-kubelite | grep active &> /dev/null then certs_modified="$(manage_certs)" if [[ "$certs_modified" -eq "1" ]] && ! [ -e "${SNAP_DATA}/var/lock/join-in-progress" ] then echo "cert change detected. Restarting the cluster-agent" snapctl restart microk8s.daemon-cluster-agent echo "cert change detected. Reconfiguring the kube-apiserver" rm -rf .srl snapctl stop microk8s.daemon-kubelite remove_all_containers kill_all_container_shims snapctl restart microk8s.daemon-containerd snapctl start microk8s.daemon-kubelite start_all_containers restart_attempt=$[$restart_attempt+1] else restart_attempt=0 fi fi # Run reconcile hooks $SNAP/usr/bin/python3 $SNAP/scripts/run-lifecycle-hooks.py reconcile || true # If no local-registry-hosting documentation has been installed, # install a help guide that points to the microk8s registry docs. # # Wait until the apiserver is up and is successfully responding to # namespace checks before we check for the registry configmap. if snapctl services microk8s.daemon-kubelite | grep active &> /dev/null then if [ $installed_registry_help -eq 0 ] && "$SNAP/kubectl" "--kubeconfig=$SNAP_DATA/credentials/client.config" get namespace kube-public &> /dev/null then if ! "$SNAP/kubectl" "--kubeconfig=$SNAP_DATA/credentials/client.config" get configmap local-registry-hosting -n kube-public &> /dev/null then use_manifest registry/registry-help apply fi installed_registry_help=1 fi fi done ================================================ FILE: microk8s-resources/wrappers/git.wrapper ================================================ #!/usr/bin/env bash . $SNAP/actions/common/utils.sh use_snap_env export GIT_EXEC_PATH="$SNAP/usr/lib/git-core" export GIT_TEMPLATE_DIR="$SNAP/usr/share/git-core/templates" export GIT_CONFIG_NOSYSTEM=1 HOME="${SNAP_DATA}/args/git" "$SNAP/usr/bin/git" "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-add-node.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s add-node on the master." exit 1 fi if echo "$*" | grep -q -- 'help'; then # Call add_token.py help ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/add_token.py --help exit 0 fi exit_if_not_root exit_if_no_permissions subject=$("${SNAP}/openssl.wrapper" x509 -sha256 -days 365 -noout -subject -in "$SNAP_DATA/certs/ca.crt") if [[ $subject == *"127.0.0.1"* ]]; then echo "Clustering requires a fresh MicroK8s installation. Reinstall with:" echo "sudo snap remove microk8s" install_message="sudo snap install microk8s" if ! is_strict then install_message="${install_message} --classic" fi echo "$install_message" exit 1 fi exit_if_stopped if [ ! -f "$SNAP_DATA/credentials/cluster-tokens.txt" ]; then touch $SNAP_DATA/credentials/cluster-tokens.txt fi group=$(get_microk8s_or_cis_group) if getent group ${group} >/dev/null 2>&1 then chgrp ${group} $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true chmod ug+rw $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true chmod o-rwX $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true fi # Use python's built-in (3.6+) secrets generator to produce the token. ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/add_token.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-addons.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/addons.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-config.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s config on the master." exit 0 fi USE_LOOPBACK=false PARSED=$(getopt --options=lho: --longoptions=use-loopback,help,output: --name "$@" -- "$@") eval set -- "$PARSED" while true; do case "$1" in -l|--use-loopback) USE_LOOPBACK=true shift ;; -h|--help) echo "Usage: microk8s config [OPTIONS]" echo echo "Retrieve the client config, similar to microk8s kubectl config view --raw" echo echo "Options:" echo " -h, --help Show this help" echo " -l, --use-loopback Report the cluster address using the loopback address" echo " (127.0.0.1) rather than the default interface address" exit 0 ;; --) shift break ;; *) echo "microk8s config: invalid option -- $1" exit 1 esac done exit_if_no_permissions if [[ "$USE_LOOPBACK" == "true" ]]; then cat "$SNAP_DATA/credentials/client.config" "$SNAP/bin/echo" else IP_ADDR="$(get_default_ip)" "$SNAP/bin/sed" -e "s/127.0.0.1/$IP_ADDR/" "$SNAP_DATA/credentials/client.config" "$SNAP/bin/echo" fi ================================================ FILE: microk8s-resources/wrappers/microk8s-ctr.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_no_permissions export CONTAINERD_SNAPSHOTTER=$(snapshotter) if ! [ -e $SNAP_DATA/args/ctr ] then echo "Arguments file $SNAP_DATA/args/ctr is missing." exit 1 fi declare -a args="($(cat $SNAP_DATA/args/ctr))" run_with_sudo "${SNAP}/bin/ctr" "${args[@]}" "$@" ================================================ FILE: microk8s-resources/wrappers/microk8s-dashboard-proxy.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_not_root exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dashboard_proxy.py ================================================ FILE: microk8s-resources/wrappers/microk8s-dbctl.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_not_root exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dbctl.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-disable.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env # avoid AppArmor denial in strict mode when running under sudo without -H if is_strict then cd "$SNAP" fi exit_if_not_root exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/disable.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-enable.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env # avoid AppArmor denial in strict mode when running under sudo without -H if is_strict then cd "$SNAP" fi exit_if_not_root exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/enable.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-helm.wrapper ================================================ #!/usr/bin/env bash source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s helm on the master." exit 0 fi exit_if_stopped "${SNAP}/bin/helm" --kubeconfig="${SNAP_DATA}"/credentials/client.config "$@" ================================================ FILE: microk8s-resources/wrappers/microk8s-helm3.wrapper ================================================ #!/usr/bin/env bash source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s helm on the master." exit 0 fi exit_if_stopped "${SNAP}/bin/helm" --kubeconfig="${SNAP_DATA}"/credentials/client.config "$@" ================================================ FILE: microk8s-resources/wrappers/microk8s-images.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/images.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-istioctl.wrapper ================================================ #!/usr/bin/env bash set -eu . $SNAP/actions/common/utils.sh use_snap_env if [ ! -f "${SNAP_DATA}/bin/istioctl" ]; then echo "Istio not available, try enabling is with 'microk8s enable istio'" exit 0 fi source $SNAP/actions/common/utils.sh if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s istioctl on the master." exit 0 fi if echo "$*" | grep -v -q -- '--kubeconfig'; then exit_if_no_permissions fi ARCH=$(arch) if ! [ "${ARCH}" = "amd64" ] then echo "Istio is not available for ${ARCH}" else exit_if_stopped "${SNAP_DATA}/bin/istioctl" --kubeconfig=${SNAP_DATA}/credentials/client.config "$@" fi ================================================ FILE: microk8s-resources/wrappers/microk8s-join.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -d "$SNAP_COMMON/default-storage" ] then echo "WARNING: Hostpath storage is enabled and is not suitable for multi node clusters." echo "" fi exit_if_no_permissions run_with_sudo ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/join.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-kubectl.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s kubectl on the master." exit 0 fi exit_if_stopped if echo "$*" | grep -v -q -- '--kubeconfig'; then exit_if_no_permissions fi if [ -e $SNAP_DATA/args/kubectl-env ] then source $SNAP_DATA/args/kubectl-env fi if is_strict then export EDITOR="${SNAP}/bin/nano" fi declare -a args="($(cat $SNAP_DATA/args/kubectl))" if [ -n "${args[@]-}" ] then "${SNAP}/kubectl" "${args[@]}" "$@" else "${SNAP}/kubectl" "$@" fi ================================================ FILE: microk8s-resources/wrappers/microk8s-leave.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_stopped exit_if_not_root exit_if_no_permissions if ! [ -e ${SNAP_DATA}/var/lock/no-cert-reissue ] || [[ ! -e ${SNAP_DATA}/var/lock/clustered.lock && ! -e ${SNAP_DATA}/args/k8s-dqlite ]] then echo "This MicroK8s deployment is not acting as a node in a cluster." exit 1 fi run_with_sudo preserve_env ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/leave.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-linkerd.wrapper ================================================ #!/usr/bin/env bash set -eu . $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s linkerd on the master." exit 0 fi if [ ! -f "${SNAP_DATA}/bin/linkerd" ]; then echo "Linkerd not available, try enabling Linkerd. 'microk8s enable linkerd' or 'microk8s enable linkerd:--proxy-auto-inject' " exit 0 fi if echo "$*" | grep -v -q -- '--kubeconfig'; then exit_if_no_permissions fi exit_if_stopped "${SNAP_DATA}/bin/linkerd" --kubeconfig="${SNAP_DATA}"/credentials/client.config "$@" ================================================ FILE: microk8s-resources/wrappers/microk8s-refresh-certs.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s refresh-certs command on the master" echo "and then return to this node to perform a microk8s leave and re-join." exit 0 fi exit_if_not_root ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/refresh_certs.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-remove-node.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_no_permissions if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster. Please use microk8s leave." exit 1 fi if [ "$#" -eq 0 ]; then echo "Please provide the node you want to remove." exit 1 fi if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] && [ "$#" -eq 2 ] && ! [ "$2" == "--force" ] ; then echo "Please provide the node and the optional --force flag." exit 1 fi ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/remove_node.py "$@" ================================================ FILE: microk8s-resources/wrappers/microk8s-reset.wrapper ================================================ #!/usr/bin/env bash set -eu . $SNAP/actions/common/utils.sh use_snap_env ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/reset.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-start.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster." echo "Use 'snap start microk8s' to start services on this node." exit 0 fi exit_if_not_root exit_if_no_permissions PARSED=$(getopt --options=lho: --longoptions=help,output:,disable-low-memory-guard --name "$@" -- "$@") eval set -- "$PARSED" while true; do case "$1" in --disable-low-memory-guard) rm "${SNAP_DATA}/var/lock/low-memory-guard.lock" || true shift ;; -h|--help) echo "Usage: microk8s start [OPTIONS]" echo echo "Start Kubernetes services" echo echo "Options:" echo " -h, --help Show this help" echo " --disable-low-memory-guard Start MicroK8s in machines with RAM < 512MB" exit 0 ;; --) shift break ;; *) echo "microk8s start: invalid option -- $1" exit 1 esac done exit_if_low_memory_guard if ! run_with_sudo preserve_env snapctl start ${SNAP_NAME} --enable then echo 'Failed to start microk8s services. Check snapd logs with "journalctl -u snapd.service"' exit 1 else start_all_containers if run_with_sudo test -e ${SNAP_DATA}/var/lock/stopped.lock then # Mark the api server as starting run_with_sudo rm -f ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null fi fi wait_for_node ================================================ FILE: microk8s-resources/wrappers/microk8s-status.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_no_permissions if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster." echo "Please use the control plane node." exit 0 fi exit_if_low_memory_guard ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/status.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s-stop.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "This MicroK8s deployment is acting as a node in a cluster." echo "Use 'snap stop microk8s' to stop services on this node." exit 0 fi exit_if_not_root exit_if_no_permissions FORCE=false PARSED=$(getopt --options=lho: --longoptions=force,help,output: --name "$@" -- "$@") eval set -- "$PARSED" while true; do case "$1" in -h|--help) echo "Usage: microk8s stop [OPTIONS]" echo echo "Stop Kubernetes services" echo echo "Options:" echo " -h, --help Show this help" exit 0 ;; --) shift break ;; *) echo "microk8s stop: invalid option -- $1" exit 1 esac done prefix_cmd="run_with_sudo snap" if is_strict then prefix_cmd="snapctl" fi $prefix_cmd stop microk8s.daemon-kubelite --disable stop_status=$? if ! [ $stop_status -eq 0 ] then echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"' exit 1 else remove_all_containers kill_all_container_shims $prefix_cmd stop microk8s --disable run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock fi ================================================ FILE: microk8s-resources/wrappers/microk8s-version.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env exit_if_no_permissions ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/version.py "${@}" ================================================ FILE: microk8s-resources/wrappers/microk8s.wrapper ================================================ #!/usr/bin/env bash set -e source $SNAP/actions/common/utils.sh use_snap_env function help() { echo "Available subcommands are:" for i in ${SNAP}/microk8s-*.wrapper; do echo -e "\t$(basename ${i} | sed 's/microk8s-//g' | sed 's/.wrapper//g')" done echo -e "\tinspect" plugins="$(find ${SNAP_COMMON}/plugins/* 2> /dev/null)" if [ ! -z "$plugins" ]; then echo "Available subcommands from addons are:" for i in "${SNAP_COMMON}/plugins/"*; do echo -e "\t$(basename "${i}")" done fi } if [ -z "$1" ]; then help exit 1 fi readonly APP="$1" shift if [ -f "${SNAP}/microk8s-${APP}.wrapper" ]; then "${SNAP}/microk8s-${APP}.wrapper" "$@" readonly EXIT="$?" elif [ "${APP}" == "inspect" ]; then run_with_sudo preserve_env ${SNAP}/inspect.sh "$@" readonly EXIT="$?" elif [ "${APP}" == "help" ] || [ "${APP}" == "--help" ] || [ "$APP" == "-h" ]; then help readonly EXIT="0" elif [ -f "${SNAP_COMMON}/plugins/${APP}" ]; then "${SNAP_COMMON}/plugins/${APP}" "$@" readonly EXIT="$?" else echo "'${APP}' is not a valid MicroK8s subcommand." help readonly EXIT="1" fi exit ${EXIT} ================================================ FILE: microk8s-resources/wrappers/openssl.wrapper ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env export OPENSSL_CONF="${SNAP}/etc/ssl/openssl.cnf" "${SNAP}/usr/bin/openssl" "${@}" ================================================ FILE: microk8s-resources/wrappers/run-apiserver-proxy-with-args ================================================ #!/usr/bin/env bash set -ex source $SNAP/actions/common/utils.sh use_snap_env if ! [ -e "$SNAP_DATA/var/lock/clustered.lock" ]; then echo "Not a worker node, exiting" exit 0 fi if ! [ -e "$SNAP_DATA/args/traefik" ] then exit 0 fi if ! [ -e "$SNAP_DATA/args/apiserver-proxy" ] then exit 0 fi sed 's@${SNAP}@'"${SNAP}"'@g;s@${SNAP_DATA}@'"${SNAP_DATA}"'@g' $SNAP_DATA/args/traefik/traefik-template.yaml > $SNAP_DATA/args/traefik/traefik.yaml # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/apiserver-proxy))" exec "$SNAP/bin/cluster-agent" apiserver-proxy "${args[@]}" ================================================ FILE: microk8s-resources/wrappers/run-cluster-agent-with-args ================================================ #!/usr/bin/env bash if [ -z "$HOME" ] then mkdir -p $SNAP_DATA/var/tmp/ export HOME=$SNAP_DATA/var/tmp/ fi set -eu . $SNAP/actions/common/utils.sh use_snap_env # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/cluster-agent))" ${SNAP}/bin/cluster-agent cluster-agent "${args[@]}" ================================================ FILE: microk8s-resources/wrappers/run-containerd-with-args ================================================ #!/usr/bin/env bash set -ex source $SNAP/actions/common/utils.sh use_snap_env # Re-exec outside of apparmor confinement if ! is_strict && [ -d /sys/kernel/security/apparmor ]; then status_file=/proc/self/attr/apparmor/current # Fallback for older kernels, check the legacy interfaces # https://www.kernel.org/doc/html/latest/admin-guide/LSM/index.html if [ ! -f $status_file ]; then status_file=/proc/self/attr/current fi if [ "$(cat $status_file)" != "unconfined" ]; then exec aa-exec -p unconfined -- "$0" "$@" fi fi # Why we put the /snap/microk8s/current in the path? # containerd-shims need to call runc. They inherit their PATH from containerd. # As the snap refreshes runc changes location, eg moves from # /snap/microk8s/123/usr/bin/runc to /snap/microk8s/124/usr/runc. # containerd-shims need to look for runc in /snap/microk8s/current/usr/bin/runc SNAP_CURRENT=`echo "${SNAP}" | sed -e "s,${SNAP_REVISION},current,"` CURRENT_PATH="$SNAP_CURRENT/usr/sbin:$SNAP_CURRENT/usr/bin:$SNAP_CURRENT/sbin:$SNAP_CURRENT/bin" export PATH="$CURRENT_PATH:$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" if is_strict then apparmor_parser -r $SNAP/containerd-profile else if [ -d "/etc/apparmor.d" ]; then echo "Using a default profile template" cp ${SNAP}/containerd-profile /etc/apparmor.d/cri-containerd.apparmor.d echo "Reloading AppArmor profiles" if ! service apparmor reload then echo "AppArmor profiles loading failed. AppArmor may be unavailable on this host." fi fi fi app=containerd RUNTIME="runc" RUNTIME_TYPE="io.containerd.runc.v1" SNAPSHOTTER=$(snapshotter) if mount -t cgroup2 | grep -q 'type cgroup2'; then RUNTIME_TYPE="io.containerd.runc.v2" fi sed 's@${SNAP}@'"${SNAP}"'@g;s@${SNAP_DATA}@'"${SNAP_DATA}"'@g;s@${SNAPSHOTTER}@'"${SNAPSHOTTER}"'@g;s@${RUNTIME}@'"${RUNTIME}"'@g' $SNAP_DATA/args/containerd-template.toml > $SNAP_DATA/args/containerd.toml sed -i 's@${RUNTIME_TYPE}@'"${RUNTIME_TYPE}"'@g' $SNAP_DATA/args/containerd.toml run_flanneld="$(is_service_expected_to_start flanneld)" if [ "${run_flanneld}" == "1" ] then sed 's@${SNAP}@'"${SNAP}"'@g;s@${SNAP_DATA}@'"${SNAP_DATA}"'@g;s@${SNAP_COMMON}@'"${SNAP_COMMON}"'@g' $SNAP_DATA/args/flannel-template.conflist > $SNAP_DATA/args/cni-network/flannel.conflist fi # clean leftover container state if we just booted if (is_strict && is_first_boot_on_strict) || (! is_strict && is_first_boot "${SNAP_COMMON}/run/containerd") then rm -rf "${SNAP_COMMON}/run/containerd" || true fi mkdir -p "${SNAP_COMMON}/run/containerd" if ! is_strict then mark_boot_time "${SNAP_COMMON}/run/containerd" fi # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/$app))" set -a . "${SNAP_DATA}/args/${app}-env" set +a # check if there is a default route available if ! default_route_exists then echo "WARNING: No default route exists. MicroK8s might not work properly." echo "Refer to https://microk8s.io/docs for instructions on air-gap deployments." fi # NOTE(neoaggelos): See https://github.com/canonical/microk8s/issues/423 for context wait_for_default_route # Set the path to the Cilium socket correctly for CNI export CILIUM_SOCK="${SNAP_DATA}/var/run/cilium/cilium.sock" exec "$SNAP/bin/$app" "${args[@]}" ================================================ FILE: microk8s-resources/wrappers/run-etcd-with-args ================================================ #!/usr/bin/env bash set -e source $SNAP/actions/common/utils.sh use_snap_env exit_if_service_not_expected_to_start etcd if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then echo "etcd will not run on a cluster node" exit 0 fi # etcd will not start if the socket already exists. if [ -S "${SNAP_DATA}/etcd.socket:2379" ]; then rm "${SNAP_DATA}/etcd.socket:2379" fi ARCH=$(arch) if ! [ "$ARCH" = "amd64" ]; then export ETCD_UNSUPPORTED_ARCH="$ARCH" fi export DEFAULT_INTERFACE_IP_ADDR="$(get_default_ip)" # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/etcd))" exec "$SNAP/etcd" "${args[@]}" ================================================ FILE: microk8s-resources/wrappers/run-flanneld-with-args ================================================ #!/usr/bin/env bash set -e source $SNAP/actions/common/utils.sh use_snap_env exit_if_service_not_expected_to_start flanneld # Allow some slack for containerd and etcd to start # so we avoid this edge case: https://forum.snapcraft.io/t/restarting-services-from-configure-hook-race-condition/2513/13 sleep 5 n=0 until [ $n -ge 10 ] do test -e "$SNAP_DATA/args/flannel-network-mgr-config" && test -e "$SNAP_DATA/args/flanneld" && break echo "Waiting for flannled configuration to appear. (attempt $n)" n=$[$n+1] sleep 2 done # TODO rewrite for snaps etcd_endpoints="$(cat $SNAP_DATA/args/flanneld | grep "etcd-endpoints" | tr "=" " "| awk '{print $2}')" cert_file="$(cat $SNAP_DATA/args/flanneld | grep "etcd-certfile" | tr "=" " "| awk '{print $2}')" cert_file="$(eval echo $cert_file)" key_file="$(cat $SNAP_DATA/args/flanneld | grep "etcd-keyfile" | tr "=" " "| awk '{print $2}')" key_file="$(eval echo $key_file)" ca_file="$(cat $SNAP_DATA/args/flanneld | grep "etcd-cafile" | tr "=" " "| awk '{print $2}')" ca_file="$(eval echo $ca_file)" export ETCDCTL_API=3 # TODO get this from a file data="$(cat $SNAP_DATA/args/flannel-network-mgr-config)" # Prepare etcd configuration for flannel, iff an etcd endpoint is set. # Skip this part if an alternate data store is used (e.g. Kubernetes). if [ ! -z "$etcd_endpoints" ]; then if ! "${SNAP}/etcdctl" --endpoints "${etcd_endpoints}" --cert "${cert_file}" --key "${key_file}" --cacert "${ca_file}" del "/coreos.com/network/config"; then echo "/coreos.com/network/config is not in etcd. Probably a first time run." fi "${SNAP}/etcdctl" --endpoints "${etcd_endpoints}" --cert "${cert_file}" --key "${key_file}" --cacert "${ca_file}" put "/coreos.com/network/config" "$data" fi set -a if [ -e "${SNAP_DATA}/args/flanneld-env" ] then . "${SNAP_DATA}/args/flanneld-env" fi set +a # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/flanneld))" exec "$SNAP_DATA/opt/cni/bin/flanneld" "${args[@]}" ================================================ FILE: microk8s-resources/wrappers/run-k8s-dqlite-with-args ================================================ #!/usr/bin/env bash set -ex source $SNAP/actions/common/utils.sh use_snap_env exit_if_service_not_expected_to_start k8s-dqlite if [ -e "${SNAP_DATA}/var/lock/low-memory-guard.lock" ] then echo "not starting dqlite because of low memory guard lock" exit 0 fi app=k8s-dqlite if ! [ -e "$SNAP_DATA/args/${app}" ] then exit 0 fi # We add some delay so that systemd really retries the restarts sleep 6 if [ ! -e "${SNAP_DATA}/var/lock/skip-aio-tune.lock" ] then increase_sysctl_parameter "fs.aio-max-nr" "1048576" fi if [ ! -e "${SNAP_DATA}/var/lock/skip-inotify-tune.lock" ] then increase_sysctl_parameter "fs.inotify.max_user_instances" "1024" increase_sysctl_parameter "fs.inotify.max_user_watches" "1048576" fi set -a if [ -e "${SNAP_DATA}/args/${app}-env" ] then . "${SNAP_DATA}/args/${app}-env" fi set +a declare -a args="($(cat $SNAP_DATA/args/$app))" exec "$SNAP/bin/$app" "${args[@]}" ================================================ FILE: microk8s-resources/wrappers/run-kubelite-with-args ================================================ #!/usr/bin/env bash set -ex source $SNAP/actions/common/utils.sh use_snap_env app=kubelite if ! [ -e ${SNAP_DATA}/var/lock/lite.lock ] then echo "${app} will not run" exit 0 fi if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then refresh_opt_in_local_config "start-control-plane" "false" kubelite elif [ -e ${SNAP_DATA}/var/lock/low-memory-guard.lock ] then echo "${app} will not run, memory guard is enabled" exit 0 else refresh_opt_in_local_config "start-control-plane" "true" kubelite fi if [ -e ${SNAP_DATA}/var/lock/stopped.lock ] then # Mark the cluster as starting. This is needed in case you # microk8s stop and then snap start microk8s rm -f ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null fi ## API server configuration # Check if we advertise an address. If we do we do not need to wait for a default network interface. if ! grep -E "(--advertise-address|--bind-address)" $SNAP_DATA/args/kube-apiserver &> /dev/null then # we keep this command to cleanup the file, as it is no longer needed. rm -f ${SNAP_DATA}/external_ip.txt # check if there is a default route available if ! default_route_exists then echo "WARNING: No default route exists. MicroK8s might not work properly." echo "Refer to https://microk8s.io/docs for instructions on air-gap deployments." fi wait_for_default_route fi if [ -e ${SNAP_DATA}/args/ha-conf ] then storage_param="$(get_opt_in_config '--storage-dir' 'k8s-dqlite')" storage_dir="$(eval echo $storage_param)" if $(grep -qE "^failure-domain" "${SNAP_DATA}/args/ha-conf"); then val="$(get_opt_in_config 'failure-domain' 'ha-conf')" echo "$val" > $storage_dir/failure-domain fi fi # Sanitise arguments if [ -e $SNAP_DATA/var/lock/no-arg-sanitisation ] then echo "Skipping argument sanitisation." else echo "Sanitise arguments." sanitise_args_kubeapi_server sanitise_args_kubelet sanitise_args_kube_proxy sanitise_args_kube_scheduler sanitise_args_kube_controller_manager fi ## Kubelet configuration pod_cidr="$(cat $SNAP_DATA/args/kube-proxy | grep "cluster-cidr" | tr "=" " "| gawk '{print $2}')" if [ -z "$pod_cidr" ] then pod_cidr="$(cat $SNAP_DATA/args/kubelet | grep "pod-cidr" | tr "=" " "| gawk '{print $2}')" if [ -z "$pod_cidr" ] then pod_cidr="$(jq .Network $SNAP_DATA/args/flannel-network-mgr-config | tr -d '\"')" fi fi if ! [ -z "$pod_cidr" ] then if ! iptables -C FORWARD -s "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT then # The point of "|| true" is that if for any reason this is failing we "fallback" # to the previous manual approach of pointing people to the troubleshooting guide. iptables -t filter -A FORWARD -s "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables -t filter -A FORWARD -d "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables-nft -t filter -A FORWARD -s "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables-nft -t filter -A FORWARD -d "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true fi fi # Configure host resolv.conf file with non-loopback upstream nameservers resolv_conf="$(cat $SNAP_DATA/args/kubelet | grep -- "--resolv-conf" | tr "=" " " | gawk '{print $2}')" if [ -z "$resolv_conf" ] then host_resolv_conf="$("$SNAP/usr/bin/python3" "$SNAP/scripts/find-resolv-conf.py")" if [ ! -z "$host_resolv_conf" ]; then refresh_opt_in_local_config "resolv-conf" "${host_resolv_conf}" "kubelet" echo "Configured kubelet to use ${host_resolv_conf} for DNS configuration" fi fi #UFW configuration if ! is_strict && (ufw version &> /dev/null) then if ufw status | grep -q "Status: active" && ! [ -e ${SNAP_DATA}/var/lock/skip.ufw ] then # These succeed regardless of whether the rule exists already or not echo "Found enabled UFW: adding rules to allow in/out traffic on 'cali+' and 'vxlan.calico' devices" if ! ufw allow in on vxlan.calico || ! ufw allow out on vxlan.calico || ! ufw allow in on cali+ || ! ufw allow out on cali+ then echo "Failed to update UFW rules. You may want to set them manually." fi fi fi if ! is_strict && (systemctl show snap.microk8s.daemon-kubelite.service -p NRestarts | grep -qv "NRestarts=0") && (grep -qv cpuset /sys/fs/cgroup/cgroup.subtree_control) && [ ! -e /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d/delegate.conf ] then mkdir -p /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d tee /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d/delegate.conf > /dev/null < /dev/null then socket=$(grep -e "--address " $SNAP_DATA/args/containerd | gawk '{print $2}') # socket_file may be of the form ${SNAP_DATA}/containerd.sock # we need to replace any variables socket_file_expand=$(eval echo ${socket}) # wait up until 20 seconds for the docker socket to appear n=0 until [ $n -ge 10 ] do test -S "${socket_file_expand}" && break echo "Waiting for containerd socket ${socket_file_expand} to appear. (attempt $n)" n=$[$n+1] sleep 2 done fi # Handle non-unified cgroups https://github.com/canonical/microk8s/issues/519 if [ -e /proc/$$/cgroup ] && [[ $(gawk -F '[:]' '(/cpu/ && !/cpuset/) || /memory/ {print $3}' /proc/$$/cgroup | uniq | wc -l) -eq "2" ]] && ! grep -e "runtime-cgroups" $SNAP_DATA/args/kubelet &> /dev/null && ! grep -e "kubelet-cgroups" $SNAP_DATA/args/kubelet &> /dev/null && [ -e /sys/fs/cgroup/systemd/system.slice ] then refresh_opt_in_local_config "runtime-cgroups" "/systemd/system.slice" kubelet refresh_opt_in_local_config "kubelet-cgroups" "/systemd/system.slice" kubelet fi if [ -L /var/lib/kubelet ] then echo "\`/var/lib/kubelet\` is a symbolic link" ls -l /var/lib/kubelet else echo "\`/var/lib/kubelet\` already exists. CSI add-ons have to point to $SNAP_COMMON for kubelet." fi ## Kube-proxy configuration if [ -e "$SNAP_DATA/args/cni-network/cni.yaml" ] then ipvs="ipv4 ipv6" for ipv in $ipvs do if [ -e "/proc/sys/net/$ipv/conf/all/forwarding" ] && ! grep -e "1" "/proc/sys/net/$ipv/conf/all/forwarding" then echo "Enable ip forwarding for $ipv" echo "1" > "/proc/sys/net/$ipv/conf/all/forwarding" fi done fi if [ -f "${SNAP_DATA}/var/lock/host-access-enabled" ] && ! ip link show lo:microk8s &> /dev/null then IP_ADDRESS=$(<"${SNAP_DATA}/var/lock/host-access-enabled") if ! "${SNAP}/sbin/ip" addr add "${IP_ADDRESS}" dev lo label lo:microk8s then echo "Failed to enable host-access" else echo "Host-access enabled [${IP_ADDRESS}]" fi fi # kube-proxy reads some values related to the 'nf_conntrack' kernel # module from procfs on startup, so we must ensure it is loaded: if ! [ -f /proc/sys/net/netfilter/nf_conntrack_max ] then if /sbin/modprobe nf_conntrack || modprobe nf_conntrack then echo "Successfully loaded nf_conntrack module." else echo -n "Failed to load nf_conntrack kernel module. " echo "ProxyServer will fail to start until it's loaded." fi fi # on lxc containers do not try to change the conntrack configuration # see https://github.com/canonical/microk8s/issues/1438 if grep -E lxc /proc/1/environ && ! grep -E "conntrack-max-per-core" $SNAP_DATA/args/kube-proxy then refresh_opt_in_local_config "conntrack-max-per-core" "0" kube-proxy fi if ! [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ] then # NOTE(neoaggelos): https://github.com/canonical/microk8s/issues/3085 # Attempt to use modprobe from the host, otherwise fallback to the one # provided with the snap. if /sbin/modprobe br_netfilter || modprobe br_netfilter then echo "Successfully loaded br_netfilter module." else echo "Failed to load br_netfilter. Calico might not work properly." fi fi if [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ] && grep 0 /proc/sys/net/bridge/bridge-nf-call-iptables then echo "1" > /proc/sys/net/bridge/bridge-nf-call-iptables fi declare -a args="($(cat $SNAP_DATA/args/$app))" exec "$SNAP/$app" "${args[@]}" ================================================ FILE: pyproject.toml ================================================ [tool.black] line-length = 100 target-version = ['py35'] ================================================ FILE: scripts/calico/upgrade.py ================================================ #!/usr/bin/python3 import os import yaml import shutil import sys def get_calico_node_spec(cni_file): """ Extract the section of the calico node container return: The container section of the calico node, None otherwise """ try: with open(cni_file, "r", encoding="utf8") as f: for doc in yaml.safe_load_all(f): if doc and doc["kind"] == "DaemonSet" and doc["metadata"]["name"] == "calico-node": # Reach for the containers if ( doc["spec"] and doc["spec"]["template"] and doc["spec"]["template"]["spec"] and doc["spec"]["template"]["spec"]["containers"] ): containers = doc["spec"]["template"]["spec"]["containers"] for c in containers: if c["name"] == "calico-node": return c except (yaml.YAMLError, TypeError) as e: print(e, file=sys.stderr) return None return None def is_calico_cni_manifest(cni_file): """ Check if this is a Calico CNI manifest return: True if the provided manifest is a Calico one, False otherwise """ try: with open(cni_file, "r", encoding="utf8") as f: for doc in yaml.safe_load_all(f): if doc and doc["kind"] == "DaemonSet" and doc["metadata"]["name"] == "calico-node": return True except (yaml.YAMLError, TypeError) as e: print(e, file=sys.stderr) return False return False def get_installed_version_of_calico(cni_file): """ Extract the Calico version in the provided CNI manifest return: A string with the version, None otherwise """ try: c = get_calico_node_spec(cni_file) if c: parts = c["image"].split(":") return parts[-1] else: return None except (yaml.YAMLError, TypeError) as e: print(e, file=sys.stderr) return None def get_calicos_autodetection_method(cni_file): """ Extract the IP autodetection method return: A string with the IP autodetection method, None otherwise """ try: c = get_calico_node_spec(cni_file) if c: methods = [i["value"] for i in c["env"] if i["name"] == "IP_AUTODETECTION_METHOD"] if len(methods) > 0: return methods[0] return None except (yaml.YAMLError, TypeError) as e: print(e, file=sys.stderr) return None def patch_manifest(cni_file, autodetection): """ Patch the CNI manifest with the IP autodetection method provided """ yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str def repr_str(dumper, data): if "\n" in data: return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") return dumper.org_represent_str(data) yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper) try: with open(cni_file, "r", encoding="utf8") as f: to_remove = [] docs = list(yaml.safe_load_all(f)) for doc in docs: if not doc: to_remove.append(doc) if doc and doc["kind"] == "DaemonSet" and doc["metadata"]["name"] == "calico-node": # Reach for the containers containers = doc["spec"]["template"]["spec"]["containers"] for c in containers: if c["name"] == "calico-node": env = c["env"] for variable in env: if variable["name"] == "IP_AUTODETECTION_METHOD": variable["value"] = autodetection # remove empty yaml documents for d in to_remove: docs.remove(d) with open(cni_file, "w", encoding="utf8") as fout: yaml.safe_dump_all(docs, fout) except (yaml.YAMLError, TypeError) as e: print(e, file=sys.stderr) def backup_old_cni(cni_file): """ Creating a backup of the provided file """ backup_cni_file = f"{cni_file}.backup" shutil.copyfile(cni_file, backup_cni_file) def try_upgrade(cni_file, new_cni_file, cni_no_manage=None): """ Perform the upgrade if possible. return: True if the CNI needs to be reloaded """ # If cni auto management is disabled by lock file do nothing if cni_no_manage is not None and os.path.exists(cni_no_manage): return False # If cni files are not in place do nothing if not (os.path.exists(cni_file) and os.path.exists(new_cni_file)): return False # If the current cni.yaml is not calico do nothing if not is_calico_cni_manifest(cni_file): return False # If the current cni.yaml is not from 3.21 do nothing # s390x will be filtered out because it is in 3.15 current_version = get_installed_version_of_calico(cni_file) if "3.21" not in current_version: return False backup_old_cni(cni_file) autodetection = get_calicos_autodetection_method(cni_file) shutil.copyfile(new_cni_file, cni_file) if autodetection and "can-reach" in autodetection: patch_manifest(cni_file, autodetection) return True def mark_apply_needed(lock_file): """ Remove the lock file provided so the apiserver kicker will re apply the CNI manifest """ try: os.remove(lock_file) except OSError: pass def main(): """ Run the upgrade. :return: None """ cni_reapply_lock_file = os.path.expandvars("${SNAP_DATA}/var/lock/cni-loaded") cni_file = os.path.expandvars("${SNAP_DATA}/args/cni-network/cni.yaml") cni_no_manage = os.path.expandvars("${SNAP_DATA}/var/lock/no-manage-calico") new_cni_file = os.path.expandvars( "${SNAP}/upgrade-scripts/000-switch-to-calico/resources/calico.yaml" ) if try_upgrade(cni_file, new_cni_file, cni_no_manage): # we mark the CNI needs to be updated so the api service kicker will take over mark_apply_needed(cni_reapply_lock_file) if __name__ == "__main__": main() ================================================ FILE: scripts/find-resolv-conf.py ================================================ #!/usr/bin/env python3 import ipaddress import re import click # https://regex101.com/r/RWZH94/1 NAMESERVER_REGEX = r"^\s*nameserver\s+(\S*)\s*$" DEFAULT_RESOLV_CONFS = [ "/etc/resolv.conf", "/run/systemd/resolve/resolv.conf", ] def safe_is_non_loopback_address(address: str): """ Return true if the given address is a valid non loopback address. Returns false if the address is loopback or invalid """ try: return not ipaddress.ip_address(address).is_loopback except ValueError: # NOTE(neoaggelos): https://github.com/canonical/microk8s/issues/4327 # Python 3.8 fails with scoped IPv6 address, e.g. "fe80::5054:ff:fe00:b61d%2" # Try to remove the scope suffix, and accept if value is an IPv6 address if "%" not in address: return False try: ip = ipaddress.ip_address(address[: address.find("%")]) return ip.version == 6 and not ip.is_loopback except (ValueError, IndexError): return False def find_resolv_conf_with_non_loopback_address(resolv_confs: list): """ Given a list of resolv.conf file paths, return the first one that contains non-loopback upstream nameservers. """ for path in resolv_confs: try: with open(path) as fin: contents = fin.read() nameservers = re.findall(NAMESERVER_REGEX, contents, re.MULTILINE) if nameservers and all(map(safe_is_non_loopback_address, nameservers)): return path except (OSError, ValueError): # ignore invalid resolv.conf files pass @click.command("find-resolv-conf") @click.argument("resolv_confs", nargs=-1) def main(resolv_confs): """ find-resolv-conf looks in the system for a resolv.conf file that contains non-loopback upstream nameservers. If there are any, the first one found is printed to stdout. Paths to resolv.conf files may be given as arguments. By default, known system locations defined in DEFAULT_RESOLV_CONFS are checked. If no resolv.conf file with non-loopback nameservers is found, nothing is printed to stdout. """ path = find_resolv_conf_with_non_loopback_address(resolv_confs or DEFAULT_RESOLV_CONFS) if path: print(path) if __name__ == "__main__": main() ================================================ FILE: scripts/generate-cni.sh ================================================ #!/usr/bin/env bash set -x source $SNAP/actions/common/utils.sh use_snap_env if [ -e "${SNAP_DATA}/args/cni-env" ]; then source "${SNAP_DATA}/args/cni-env" fi CNI_YAML="${SNAP_DATA}/args/cni-network/cni.yaml" CALICO_RESOURCES="${SNAP}/upgrade-scripts/000-switch-to-calico/resources" CLUSTER_CIDR="" SERVICE_CIDR="" function handle_calico { # Start with the default CNI and patch it based on the user needs cp "${CALICO_RESOURCES}/calico.yaml" "$CNI_YAML" # IPv4 configuration if test "x${IPv4_SUPPORT}" = "xtrue"; then # Update the calico cni.yaml with IPv4 specific configurations sed -i '/"type": "calico-ipam"/i \ "assign_ipv4": "true",' "$CNI_YAML" sed -i 's%10.1.0.0/16%'"${IPv4_CLUSTER_CIDR}"'%g' "$CNI_YAML" fi # IPv6 configuration if test "x${IPv6_SUPPORT}" = "xtrue"; then sed -i '/"type": "calico-ipam"/i \ "assign_ipv6": "true",' "$CNI_YAML" sed -i '/FELIX_IPV6SUPPORT/{n;s/.*/\ value: "true"/}' "$CNI_YAML" sed -i '/CALICO_IPV6POOL_VXLAN/{n;s/.*/\ value: "Always"/}' "$CNI_YAML" sed -i '/Enable or Disable VXLAN on the default IPv6 IP pool./a \ - name: IP6' "$CNI_YAML" sed -i '/- name: IP6/a \ value: "autodetect"' "$CNI_YAML" sed -i '/Enable or Disable VXLAN on the default IPv6 IP pool./a \ - name: CALICO_IPV6POOL_CIDR' "$CNI_YAML" sed -i '/CALICO_IPV6POOL_CIDR/a \ value: '"${IPv6_CLUSTER_CIDR}" "$CNI_YAML" sed -i '/Enable or Disable VXLAN on the default IPv6 IP pool./a \ - name: IP6_AUTODETECTION_METHOD' "$CNI_YAML" sed -i '/IP6_AUTODETECTION_METHOD/a \ value: "first-found"' "$CNI_YAML" fi # Other configuration if [ ! -z "${CALICO_VETH_MTU}" ]; then sed -i 's,veth_mtu: "0",veth_mtu: "'"${CALICO_VETH_MTU}"'",' "$CNI_YAML" fi } function validate_configuration { if test "x${IPv4_SUPPORT}" = "xtrue"; then if test "x${IPv4_CLUSTER_CIDR}" = "x"; then echo "Failed: IPv4_SUPPORT is true, but no IPv4_CLUSTER_CIDR was set" exit 1 fi if test "x${IPv4_SERVICE_CIDR}" = "x"; then echo "Failed: IPv4_SUPPORT is true, but no IPv4_SERVICE_CIDR was set" exit 1 fi fi if test "x${IPv6_SUPPORT}" = "xtrue"; then if test "x${IPv6_CLUSTER_CIDR}" = "x"; then echo "Failed: IPv6_SUPPORT is true, but no IPv6_CLUSTER_CIDR was set" exit 1 fi if test "x${IPv6_SERVICE_CIDR}" = "x"; then echo "Failed: IPv6_SUPPORT is true, but no IPv6_SERVICE_CIDR was set" exit 1 fi fi if test x"${IPv4_SUPPORT}" = "xtrue" && test x"${IPv6_SUPPORT}" = "xtrue"; then CLUSTER_CIDR="${IPv4_CLUSTER_CIDR},${IPv6_CLUSTER_CIDR}" SERVICE_CIDR="${IPv4_SERVICE_CIDR},${IPv6_SERVICE_CIDR}" elif test x"${IPv4_SUPPORT}" = "xtrue"; then CLUSTER_CIDR="${IPv4_CLUSTER_CIDR}" SERVICE_CIDR="${IPv4_SERVICE_CIDR}" elif test x"${IPv6_SUPPORT}" = "xtrue"; then CLUSTER_CIDR="${IPv6_CLUSTER_CIDR}" SERVICE_CIDR="${IPv6_SERVICE_CIDR}" else echo "Failed: At least one of IPv4_SUPPORT or IPv6_SUPPORT must be true" exit 1 fi } function setup_service_arguments { # Setup arguments in the microk8s services refresh_opt_in_local_config service-cluster-ip-range "${SERVICE_CIDR}" kube-controller-manager refresh_opt_in_local_config cluster-cidr "${CLUSTER_CIDR}" kube-controller-manager refresh_opt_in_local_config service-cluster-ip-range "${SERVICE_CIDR}" kube-apiserver refresh_opt_in_local_config cluster-cidr "${CLUSTER_CIDR}" kube-proxy } # Common setup, validation and setup service arguments validate_configuration setup_service_arguments # Do setup of the chosen CNI case "${CNI}" in calico) handle_calico ;; *) echo "CNI must be set to 'calico'" exit 1 ;; esac ================================================ FILE: scripts/inspect.sh ================================================ #!/usr/bin/env bash INSPECT_DUMP=${SNAP_DATA}/inspection-report RETURN_CODE=0 JOURNALCTL_LIMIT=100000 source $SNAP/actions/common/utils.sh function print_help { # Print the help message printf -- 'This script will inspect your microk8s installation. It will report any issue it finds,\n'; printf -- 'and create a tarball of logs and traces which can be attached to an issue filed against\n'; printf -- 'the microk8s project.\n'; } function check_service { # Check the service passed as the first argument is up and running and collect its logs. local service=$1 mkdir -p $INSPECT_DUMP/$service status="inactive" if is_strict then journalctl -n $JOURNALCTL_LIMIT -u snap.$service &> $INSPECT_DUMP/$service/journal.log snapctl services $service &> $INSPECT_DUMP/$service/snapctl.log if ! snapctl services $service | grep inactive &> /dev/null then status="active" fi else journalctl -n $JOURNALCTL_LIMIT -u $service &> $INSPECT_DUMP/$service/journal.log systemctl status $service &> $INSPECT_DUMP/$service/systemctl.log if systemctl status $service &> /dev/null then status="active" fi fi if [ "$status" == "active" ] then printf -- ' Service %s is running\n' "$service" else printf -- '\033[31m FAIL: \033[0m Service %s is not running\n' "$service" printf -- 'For more details look at: sudo journalctl -u %s\n' "$service" RETURN_CODE=1 fi } function check_apparmor { # Collect apparmor info. mkdir -p $INSPECT_DUMP/apparmor if is_strict then journalctl -k &> $INSPECT_DUMP/apparmor/dmesg else if [ -f /etc/apparmor.d/containerd ] then cp /etc/apparmor.d/containerd $INSPECT_DUMP/apparmor/ fi dmesg &> $INSPECT_DUMP/apparmor/dmesg aa-status &> $INSPECT_DUMP/apparmor/aa-status fi } function store_args { # Collect the services arguments. printf -- ' Copy service arguments to the final report tarball\n' cp -r ${SNAP_DATA}/args $INSPECT_DUMP } function store_network { # Collect network setup. printf -- ' Copy network configuration to the final report tarball\n' mkdir -p $INSPECT_DUMP/network ip addr &> $INSPECT_DUMP/network/ip-addr ip route &> $INSPECT_DUMP/network/ip-route iptables -t nat -L -n -v &> $INSPECT_DUMP/network/iptables iptables -S &> $INSPECT_DUMP/network/iptables-S iptables -L &> $INSPECT_DUMP/network/iptables-L if ! is_strict then ss -pln &> $INSPECT_DUMP/network/ss fi } function store_sys { # Generate sys directory mkdir -p $INSPECT_DUMP/sys # collect the processes running printf -- ' Copy processes list to the final report tarball\n' ps -ef > $INSPECT_DUMP/sys/ps # Store disk usage information printf -- ' Copy disk usage information to the final report tarball\n' df -h | grep ^/ &> $INSPECT_DUMP/sys/disk_usage # remove the grep to also include virtual in-memory filesystems # Store memory usage information printf -- ' Copy memory usage information to the final report tarball\n' free -m &> $INSPECT_DUMP/sys/memory_usage # Store server's uptime. printf -- ' Copy server uptime to the final report tarball\n' uptime &> $INSPECT_DUMP/sys/uptime # Store openssl information. printf -- ' Copy openSSL information to the final report tarball\n' openssl version -v -d -e &> $INSPECT_DUMP/sys/openssl if ! is_strict then printf -- ' Copy snap list to the final report tarball\n' snap version > $INSPECT_DUMP/sys/snap-version snap list > $INSPECT_DUMP/sys/snap-list # Stores VM name (or none, if we are not on a VM) printf -- ' Copy VM name (or none) to the final report tarball\n' systemd-detect-virt &> $INSPECT_DUMP/sys/vm_name # Store the current linux distro. printf -- ' Copy current linux distribution to the final report tarball\n' lsb_release -a &> $INSPECT_DUMP/sys/lsb_release # Store the aio request limits and current number # Returns sysctl: permission denied on key 'fs.aio-max-nr' on strict printf -- ' Copy asnycio usage and limits to the final report tarball\n' sysctl fs.aio-max-nr &> $INSPECT_DUMP/sys/aio-max-nr sysctl fs.aio-nr &> $INSPECT_DUMP/sys/aio-nr # Store the max inotify parameters printf -- ' Copy inotify max_user_instances and max_user_watches to the final report tarball\n' sysctl fs.inotify.max_user_instances &> $INSPECT_DUMP/sys/inotify_max_user_instances sysctl fs.inotify.max_user_watches &> $INSPECT_DUMP/sys/inotify_max_user_watches fi } function store_kubernetes_info { # Collect some in-k8s details printf -- ' Inspect kubernetes cluster\n' mkdir -p $INSPECT_DUMP/k8s if is_strict then export KUBECONFIG=$SNAP_DATA/credentials/client.config $SNAP/kubectl version > $INSPECT_DUMP/k8s/version 2>&1 $SNAP/kubectl cluster-info > $INSPECT_DUMP/k8s/cluster-info 2>&1 $SNAP/kubectl cluster-info dump -A > $INSPECT_DUMP/k8s/cluster-info-dump 2>&1 $SNAP/kubectl get all --all-namespaces -o wide > $INSPECT_DUMP/k8s/get-all 2>&1 $SNAP/kubectl get pv > $INSPECT_DUMP/k8s/get-pv 2>&1 $SNAP/kubectl get pvc --all-namespaces > $INSPECT_DUMP/k8s/get-pvc 2>&1 else run_with_sudo /snap/bin/microk8s kubectl version 2>&1 | sudo tee $INSPECT_DUMP/k8s/version > /dev/null run_with_sudo /snap/bin/microk8s kubectl cluster-info 2>&1 | sudo tee $INSPECT_DUMP/k8s/cluster-info > /dev/null run_with_sudo /snap/bin/microk8s kubectl cluster-info dump -A 2>&1 | sudo tee $INSPECT_DUMP/k8s/cluster-info-dump > /dev/null run_with_sudo /snap/bin/microk8s kubectl get all --all-namespaces -o wide 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-all > /dev/null run_with_sudo /snap/bin/microk8s kubectl get pv 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-pv > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found run_with_sudo /snap/bin/microk8s kubectl get pvc --all-namespaces 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-pvc > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found fi # Collect bill of materials cp $SNAP/bom.json $INSPECT_DUMP/bom.json } function check_storage_addon { export KUBECONFIG=$SNAP_DATA/credentials/client.config image="$($SNAP/kubectl get deploy -n kube-system hostpath-provisioner -o jsonpath='{.spec.template.spec.containers[0].image}' 2>&1)" case "$image" in cdkbot/hostpath-provisioner-amd64:1.0.0|cdkbot/hostpath-provisioner-arm64:1.0.0) printf -- '\n' printf -- '\033[0;33mWARNING: \033[0m You are using an outdated version of the hostpath-provisioner.\n' printf -- 'Existing PersistentVolumes are not affected, but you may be unable to create new ones.\n' printf -- "Image currently in use: ${image}\n" printf -- '\n' printf -- 'Consider updating the hostpath-provisioner with:\n' printf -- ' microk8s disable hostpath-storage\n' printf -- ' microk8s enable hostpath-storage\n' printf -- '\n' ;; esac } function store_dqlite_info { # Collect some dqlite details printf -- ' Inspect dqlite\n' mkdir -p $INSPECT_DUMP/dqlite run_with_sudo preserve_env cp ${SNAP_DATA}/var/kubernetes/backend/cluster.yaml $INSPECT_DUMP/dqlite/ run_with_sudo preserve_env cp ${SNAP_DATA}/var/kubernetes/backend/info.yaml $INSPECT_DUMP/dqlite/ run_with_sudo preserve_env ls -lh ${SNAP_DATA}/var/kubernetes/backend/ 2>&1 > $INSPECT_DUMP/dqlite/list.out } function suggest_fixes { # Propose fixes printf '\n' # This if is verbose but I hope is clear status="inactive" if is_strict then if ! snapctl services microk8s.daemon-kubelite | grep inactive &> /dev/null then status="active" fi else if systemctl status snap.microk8s.daemon-kubelite &> /dev/null then status="active" fi fi if [ "$status" == "inactive" ] then if lsof -Pi :16443 -sTCP:LISTEN -t &> /dev/null then printf -- '\033[0;33m WARNING: \033[0m Port 16443 seems to be in use by another application.\n' fi fi if iptables -L 2>&1 | grep FORWARD | grep DROP &> /dev/null then printf -- '\033[0;33m WARNING: \033[0m IPtables FORWARD policy is DROP. ' printf -- 'Consider enabling traffic forwarding with: sudo iptables -P FORWARD ACCEPT \n' printf -- 'The change can be made persistent with: sudo apt-get install iptables-persistent\n' fi if ! is_strict && [ /snap/core22/current/usr/bin/which ufw &> /dev/null ] then if ufw status | grep -q "Status: active" then header='\033[0;33m WARNING: \033[0m Firewall is enabled. Consider allowing pod traffic with: \n' content='' if ! echo $ufw | grep -q vxlan.calico then content+=' sudo ufw allow in on vxlan.calico && sudo ufw allow out on vxlan.calico\n' fi if ! echo $ufw | grep 'cali+' &> /dev/null then content+=' sudo ufw allow in on cali+ && sudo ufw allow out on cali+\n' fi if [[ ! -z "$content" ]] then echo printing printf -- "$header" printf -- "$content" fi fi fi # check for selinux. if enabled, print warning. if getenforce 2>&1 | grep 'Enabled' > /dev/null then printf -- '\033[0;33m WARNING: \033[0m SElinux is enabled. Consider disabling it.\n' fi # check for docker # if docker is installed if [ -d "/etc/docker/" ] && ! [ -z "$(which dockerd)" ]; then # if docker/daemon.json file doesn't exist print prompt to create it and mark the registry as insecure if [ ! -f "/etc/docker/daemon.json" ]; then printf -- '\033[0;33mWARNING: \033[0m Docker is installed. \n' printf -- 'File "/etc/docker/daemon.json" does not exist. \n' printf -- 'You should create it and add the following lines: \n' printf -- '{\n' printf -- ' "insecure-registries" : ["localhost:32000"] \n' printf -- '}\n' printf -- 'and then restart docker with: sudo systemctl restart docker\n' # else if the file docker/daemon.json exists else # if it doesn't include the registry as insecure, prompt to add the following lines if ! grep -qs localhost:32000 /etc/docker/daemon.json then printf -- '\033[0;33mWARNING: \033[0m Docker is installed. \n' printf -- 'Add the following lines to /etc/docker/daemon.json: \n' printf -- '{\n' printf -- ' "insecure-registries" : ["localhost:32000"] \n' printf -- '}\n' printf -- 'and then restart docker with: sudo systemctl restart docker\n' fi fi fi if ! grep -q 'cgroup/memory' /proc/self/mountinfo; then if ! grep -q 'cgroup2' /proc/self/mountinfo; then if ! mount -t cgroup2 | grep -q 'type cgroup2'; then printf -- '\033[0;33mWARNING: \033[0m The memory cgroup is not enabled. \n' printf -- 'The cluster may not be functioning properly. Please ensure cgroups are enabled \n' printf -- 'See for example: https://microk8s.io/docs/install-alternatives#heading--arm \n' fi fi fi # Fedora Specific Checks if ! is_strict && fedora_release then # Check if appropriate cgroup libraries for Fedora are installed if ! rpm -q libcgroup &> /dev/null then printf -- '\033[31m FAIL: \033[0m libcgroup v1 is not installed. Please install it\n' printf -- '\twith: dnf install libcgroup libcgroup-tools \n' fi # check if cgroups v1 is supported if [ ! -d "/sys/fs/cgroup/memory" ] &> /dev/null then printf -- '\033[31m FAIL: \033[0m Cgroup v1 seems not to be enabled. Please enable it \n' printf -- '\tby executing the following command and reboot: \n' printf -- '\tgrubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=0" \n' fi fi # Debian 9 checks if ! is_strict && debian9_release then # Check if snapctl is fresh if ! [ -L "/usr/bin/snapctl" ] then printf -- '\033[0;33mWARNING: \033[0m On Debian 9 the snapctl binary if outdated. Replace it with: \n' printf -- '\t sudo snap install core \n' printf -- '\t sudo mv /usr/bin/snapctl /usr/bin/snapctl.old \n' printf -- '\t sudo ln -s /snap/core/current/usr/bin/snapctl /usr/bin/snapctl \n' fi fi # LXD Specific Checks if ! is_strict && cat /proc/1/environ | grep "container=lxc" &> /dev/null then # make sure the /dev/kmsg is available, indicating a potential missing profile if [ ! -c "/dev/kmsg" ] # kmsg is a character device then printf -- '\033[0;33mWARNING: \033[0m the lxc profile for MicroK8s might be missing. \n' printf -- '\t Refer to this help document to get MicroK8s working in with LXD: \n' printf -- '\t https://microk8s.io/docs/lxd \n' fi fi # node name nodename="$(hostname)" if [[ "$nodename" =~ [A-Z|_] ]] && ! grep -e "hostname-override" "${SNAP_DATA}/args/kubelet" &> /dev/null then printf -- "\033[0;33mWARNING: \033[0m This machine's hostname contains capital letters and/or underscores. \n" printf -- "\t This is not a valid name for a Kubernetes node, causing node registration to fail.\n" printf -- "\t Please change the machine's hostname or refer to the documentation for more details: \n" printf -- "\t https://microk8s.io/docs/troubleshooting#heading--common-issues \n" fi if grep Raspberry /proc/cpuinfo -q && [ -e /etc/os-release ] && grep impish /etc/os-release -q && ! dpkg -l | grep linux-modules-extra-raspi -q then printf -- "\033[0;33mWARNING: \033[0m On Raspberry Pi consider installing the linux-modules-extra-raspi package with: \n" printf -- "\t 'sudo apt install linux-modules-extra-raspi' and reboot.\n" fi if ! [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ] then printf -- "\033[0;33mWARNING: \033[0m Please ensure br_netfilter kernel module is loaded on the host. \n" fi if ! ( ip route; ip -6 route ) | grep "^default" &>/dev/null then printf -- " \033[0;33mWARNING: \033[0m No default route exists. MicroK8s might not work properly. Refer to https://microk8s.io/docs for instructions on air-gap deployments.\n\n " fi if ! is_strict then AIO_MAX_NR=$(sysctl -n fs.aio-max-nr) AIO_NR=$(sysctl -n fs.aio-nr) if [ "$AIO_NR" -ge "$AIO_MAX_NR" ]; then printf -- "\033[0;33mWARNING: \033[0m Available asyncio requests are exhausted. This might lead to dqlite being unresponsive. \n" printf -- "\t Increase the limit and restart the k8s-dqlite service with: \n" printf -- "\t \t echo fs.aio.max-nr=1048576 | sudo tee -a /etc/sysctl.conf\n" printf -- "\t \t sudo sysctl --system\n" printf -- "\t \t sudo snap restart microk8s.daemon-k8s-dqlite\n" fi MAX_USER_INSTANCES=$(sysctl -n fs.inotify.max_user_instances) if [ "$MAX_USER_INSTANCES" -lt 1024 ]; then printf -- "\033[0;33mWARNING: \033[0m Maximum number of inotify user instances is less than the recommended value of 1024. \n" printf -- "\t Increase the limit with: \n" printf -- "\t \t echo fs.inotify.max_user_instances=1024 | sudo tee -a /etc/sysctl.conf\n" printf -- "\t \t sudo sysctl --system\n" fi MAX_USER_WATCHES=$(sysctl -n fs.inotify.max_user_watches) if [ "$MAX_USER_WATCHES" -lt 1048576 ]; then printf -- "\033[0;33mWARNING: \033[0m Maximum number of inotify user watches is less than the recommended value of 1048576. \n" printf -- "\t Increase the limit with: \n" printf -- "\t \t echo fs.inotify.max_user_watches=1048576 | sudo tee -a /etc/sysctl.conf\n" printf -- "\t \t sudo sysctl --system\n" fi fi } function fedora_release { local RELEASE=`cat /etc/os-release | grep "^NAME=" | cut -f2 -d=` if [ "${RELEASE}" == "Fedora" ] then return 0 else return 1 fi } function debian9_release { if [ -e /etc/debian_version ] && grep "9\." /etc/debian_version -q then return 0 else return 1 fi } function build_report_tarball { # Tar and gz the report local now_is=$(date +"%Y%m%d_%H%M%S") tar -C ${SNAP_DATA} -cf ${SNAP_DATA}/inspection-report-${now_is}.tar inspection-report &> /dev/null gzip ${SNAP_DATA}/inspection-report-${now_is}.tar printf -- ' Report tarball is at %s/inspection-report-%s.tar.gz\n' "${SNAP_DATA}" "${now_is}" } function check_certificates { exp_date_str="$(openssl x509 -enddate -noout -in "${SNAP_DATA}/certs/ca.crt" | cut -d= -f 2)" exp_date_secs="$(date -d "$exp_date_str" +%s)" now_secs=$(date +%s) difference=$(($exp_date_secs-$now_secs)) days=$(($difference/(3600*24))) if [ "3" -ge $days ]; then printf -- '\033[0;33mWARNING: \033[0m This deployments certificates will expire in $days days. \n' printf -- 'Either redeploy MicroK8s or attempt a refresh with "microk8s refresh-certs"\n' fi } function check_memory { MEMORY=`cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` if [ $MEMORY -le 524288 ] then printf -- "\033[0;33mWARNING: \033[0m This system has ${MEMORY} bytes of RAM available.\n" printf -- "It may not be enough to run the Kubernetes control plane services.\n" printf -- "Consider joining as a worker-only to a cluster.\n" fi } function check_low_memory_guard { if [ -e "${SNAP_DATA}/var/lock/low-memory-guard.lock" ] then printf -- '\033[0;33mWARNING: \033[0m The low memory guard is enabled.\n' printf -- 'This is to protect the server from running out of memory.\n' printf -- 'Consider joining as a worker-only to a cluster.\n' printf -- '\n' printf -- 'Alternatively, to disable the low memory guard, start MicroK8s with:\n' printf -- '\n' printf -- ' microk8s start --disable-low-memory-guard\n' fi } function check_hostname { HOST=$(hostname) if echo "${HOST}" | grep -q "[A-Z]" 2> /dev/null then printf -- "\033[0;33mWARNING: \033[0m The hostname of this server is '${HOST}'.\n" printf -- "Having uppercase letters in the hostname may cause issues with RBAC.\n" printf -- "Consider changing the hostname to only have lowercase letters with:\n" printf -- "\n" printf -- " hostnamectl set-hostname $(echo ${HOST} | tr '[:upper:]' '[:lower:]')\n" fi } if [[ (${#@} -ne 0) && (("$*" == "--help") || ("$*" == "-h")) ]]; then print_help exit 0; fi; exit_if_not_root rm -rf ${SNAP_DATA}/inspection-report mkdir -p ${SNAP_DATA}/inspection-report printf -- 'Inspecting system\n' check_memory check_low_memory_guard check_hostname printf -- 'Inspecting Certificates\n' check_certificates printf -- 'Inspecting services\n' svc_cluster_agent="microk8s.daemon-cluster-agent" svc_containerd="microk8s.daemon-containerd" svc_kubelite="microk8s.daemon-kubelite" svc_flanneld="microk8s.daemon-flanneld" svc_etcd="microk8s.daemon-etcd" svc_dqlite="microk8s.daemon-k8s-dqlite" svc_api_server_proxy="microk8s.daemon-apiserver-proxy" svc_api_server_kicker="microk8s.daemon-apiserver-kicker" if ! is_strict then svc_cluster_agent="snap.${svc_cluster_agent}" svc_containerd="snap.${svc_containerd}" svc_kubelite="snap.${svc_kubelite}" svc_flanneld="snap.${svc_flanneld}" svc_etcd="snap.${svc_etcd}" svc_dqlite="snap.${svc_dqlite}" svc_api_server_proxy="snap.${svc_api_server_proxy}" svc_api_server_kicker="snap.${svc_api_server_kicker}" fi check_service $svc_cluster_agent check_service $svc_containerd check_service $svc_kubelite if ! [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then check_service $svc_flanneld check_service $svc_etcd else if ! [ -e "${SNAP_DATA}/var/lock/no-k8s-dqlite" ] then # workers do not run dqlite check_service $svc_dqlite fi fi if ! [ -e "${SNAP_DATA}/var/lock/no-apiserver-proxy" ] then check_service $svc_api_server_proxy fi if ! [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then check_service $svc_api_server_kicker fi store_args printf -- 'Inspecting AppArmor configuration\n' check_apparmor printf -- 'Gathering system information\n' store_sys store_network printf -- 'Inspecting kubernetes cluster\n' store_kubernetes_info check_storage_addon if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then printf -- 'Inspecting dqlite\n' store_dqlite_info fi suggest_fixes printf -- 'Building the report tarball\n' build_report_tarball exit $RETURN_CODE ================================================ FILE: scripts/kill-host-pods.py ================================================ #!/usr/bin/env python3 import json import subprocess import os from pathlib import Path import click CTR = os.path.expandvars("$SNAP/microk8s-ctr.wrapper") SNAP = os.getenv("SNAP") SNAP_DATA = os.getenv("SNAP_DATA") SNAP_DATA_BASE = os.path.abspath(f"{SNAP_DATA}/..") SNAP_DATA_CURRENT = os.path.abspath(f"{SNAP_DATA_BASE}/current") SNAP_COMMON = os.path.abspath(f"{SNAP_DATA_BASE}/common") KUBECTL = [f"{SNAP}/kubectl", f"--kubeconfig={SNAP_DATA}/credentials/kubelet.config"] def post_filter_has_known_containers(pod, containers: list) -> bool: """ Return true if any of the container IDs on the pod match the list of containers passed on the second argument. """ for container in pod["status"]["containerStatuses"] or []: try: _, container_id = container["containerID"].split("containerd://", 2) if container_id in containers: return True except (KeyError, ValueError, TypeError, AttributeError): continue return False def post_filter_has_snap_data_mounts(pod) -> bool: """ Return true if a pod definition contains any volumes that mount paths from `/var/snap/microk8s/current/...`, e.g. CNI pods """ for volume in pod["spec"].get("volumes", []): hostpath_volume = volume.get("hostPath", {}) host_path = hostpath_volume.get("path", "") if not host_path: continue if host_path.startswith(SNAP_DATA_CURRENT): return True # handle the case where a symlink to a SNAP_DATA path is used try: resolved = Path(host_path).resolve().as_posix() if resolved.startswith(SNAP_DATA_BASE) and not resolved.startswith(SNAP_COMMON): return True except OSError: pass return False def post_filter_has_owner(pod: dict): """ Return true if a pod definition has an ownerReference (i.e. is it managed by a Deployment or a DaemonSet) """ owner_references = pod["metadata"].get("ownerReferences") or [] return len(owner_references) > 0 @click.command("kill-host-pods") @click.argument("selector", nargs=-1) @click.option("--dry-run", is_flag=True, default=False) @click.option("--with-snap-data-mounts", is_flag=True, default=False) @click.option("--with-owner", is_flag=True, default=False) def main(selector: list, dry_run: bool, with_snap_data_mounts: bool, with_owner: bool): """ Delete pods running on the local node based on Kubernetes selectors. Example usage: $ ./kill-host-pods.py -- -n kube-system -l k8s-app=calico-node $ ./kill-host-pods.py --with-snap-data-mounts -- -n kube-system $ ./kill-host-pods.py --with-snap-data-mounts -- -A """ containers = subprocess.check_output([CTR, "container", "ls", "-q"]).decode().split("\n") out = subprocess.check_output([*KUBECTL, "get", "pod", "-o", "json", *selector]) pods = json.loads(out) for pod in pods["items"]: if not post_filter_has_known_containers(pod, containers): continue if with_snap_data_mounts and not post_filter_has_snap_data_mounts(pod): continue if with_owner and not post_filter_has_owner(pod): continue meta = pod["metadata"] cmd = [*KUBECTL, "delete", "pod", "-n", meta["namespace"], meta["name"]] if dry_run: cmd = ["echo", *cmd] subprocess.check_call(cmd) if __name__ == "__main__": main() ================================================ FILE: scripts/run-lifecycle-hooks.py ================================================ #!/usr/bin/env python3 from pathlib import Path import os import click import subprocess SNAP_COMMON = Path(os.getenv("SNAP_COMMON")) TIMEOUT = 120 @click.command("run-lifecycle-hooks") @click.argument("hook") def main(hook: str): hooks_dir = SNAP_COMMON / "hooks" / f"{hook}.d" hooks = os.listdir(hooks_dir) for hook in sorted(hooks): try: if not (os.stat(hooks_dir / hook).st_mode & 0o100): # ignore non-executable files continue subprocess.run([hooks_dir / hook], timeout=TIMEOUT) except (subprocess.CalledProcessError, OSError): pass if __name__ == "__main__": main() ================================================ FILE: scripts/wrappers/add_token.py ================================================ import json import yaml import os import sys import time import argparse import subprocess from common.cluster.utils import is_node_running_dqlite, TOKEN_ΜΙΝ_LEN try: from secrets import token_hex except ImportError: from os import urandom def token_hex(nbytes=None): return urandom(nbytes).hex() cluster_tokens_file = os.path.expandvars("${SNAP_DATA}/credentials/cluster-tokens.txt") utils_sh_file = os.path.expandvars("${SNAP}/actions/common/utils.sh") token_with_expiry = "{}|{}\n" token_without_expiry = "{}\n" def add_token_with_expiry(token, file, ttl): """ This method will add a token to the token file with or without expiry Expiry time is in seconds. Format of the item in the file: | :param str token: The token to add to the file :param str file: The file name for which the token will be written to :param ttl: How long the token should last before expiry, represented in seconds. """ with open(file, "a+") as fp: if ttl != -1: expiry = int(round(time.time())) + ttl fp.write(token_with_expiry.format(token, expiry)) else: fp.write(token_without_expiry.format(token)) def run_util(*args, debug=False): env = os.environ.copy() prog = ["bash", utils_sh_file] prog.extend(args) if debug: print("\033[;1;32m+ %s\033[;0;0m" % " ".join(prog)) result = subprocess.run( prog, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) try: result.check_returncode() except subprocess.CalledProcessError: print("Failed to call utility function.") sys.exit(1) return result.stdout.decode("utf-8").strip() def get_network_info(): """ Obtain machine IP address(es) and cluster agent port. :return: tuple of default IP, all IPs, and cluster agent port """ default_ip = run_util("get_default_ip") all_ips = run_util("get_ips").split(" ") port = run_util("cluster_agent_port") return (default_ip, all_ips, port) def print_pretty(token, check): default_ip, all_ips, port = get_network_info() print("From the node you wish to join to this cluster, run the following:") print(f"microk8s join {default_ip}:{port}/{token}/{check}\n") if is_node_running_dqlite(): print( "Use the '--worker' flag to join a node as a worker not running the control plane, eg:" ) print(f"microk8s join {default_ip}:{port}/{token}/{check} --worker\n") print( "If the node you are adding is not reachable through the default interface you can use one of the following:" ) for ip in all_ips: print(f"microk8s join {ip}:{port}/{token}/{check}") def get_output_dict(token, check): _, all_ips, port = get_network_info() info = { "token": f"{token}/{check}", "urls": [f"{ip}:{port}/{token}/{check}" for ip in all_ips], } return info def print_json(token, check): info = get_output_dict(token, check) print(json.dumps(info, indent=2)) def print_yaml(token, check): info = get_output_dict(token, check) print(yaml.dump(info, indent=2)) def print_short(token, check): default_ip, all_ips, port = get_network_info() print(f"microk8s join {default_ip}:{port}/{token}/{check}") for ip in all_ips: if ip != default_ip: print(f"microk8s join {ip}:{port}/{token}/{check}") if __name__ == "__main__": # initiate the parser with a description parser = argparse.ArgumentParser( description="Produce a connection string for a node to join the cluster.", prog="microk8s add-node", ) parser.add_argument( "--token-ttl", "-l", help="Specify how long the token is valid, before it expires. " 'Value of "-1" indicates that the token is usable only once ' "(i.e. after joining a node, the token becomes invalid)", type=int, default="-1", ) parser.add_argument( "--token", "-t", help="Specify the bootstrap token to add, must be 32 characters long. " "Auto generates when empty.", ) parser.add_argument( "--format", help="Format the output of the token in pretty, short, token, or token-check", default="pretty", choices={"pretty", "short", "token", "token-check", "json", "yaml"}, ) # read arguments from the command line args = parser.parse_args() ttl = args.token_ttl if args.token is not None: token = args.token else: token = token_hex(16) if len(token) < TOKEN_ΜΙΝ_LEN: print("Invalid token size. It must be 32 characters long.") exit(1) add_token_with_expiry(token, cluster_tokens_file, ttl) check = run_util("server_cert_check") if args.format == "pretty": print_pretty(token, check) elif args.format == "short": print_short(token, check) elif args.format == "token-check": print(f"{token}/{check}") elif args.format == "json": print_json(token, check) elif args.format == "yaml": print_yaml(token, check) else: print(token) ================================================ FILE: scripts/wrappers/addons.py ================================================ #!/usr/bin/python3 import json import os import shutil import subprocess import sys from pathlib import Path from typing import List import click import jsonschema import yaml from common.utils import get_current_arch, snap_common, snap, exit_if_no_root from common.cluster.utils import get_group GIT = os.path.expandvars("$SNAP/git.wrapper") addons = click.Group() repository = click.Group("repo") addons.add_command(repository) class RepoValidationError(Exception): @property def message(self) -> str: raise NotImplementedError() class AddonsYamlNotFoundError(RepoValidationError): def __init__(self, repo_name: str): self.repo_name = repo_name @property def message(self) -> str: return f"Error: repository {self.repo_name} does not contain an addons.yaml file" class AddonsYamlFormatError(RepoValidationError): def __init__(self, message): self._message = message @property def message(self) -> str: return self._message class MissingHookError(RepoValidationError): def __init__(self, hook_name: str, addon: str): self.hook_name = hook_name self.addon = addon @property def message(self) -> str: return f"Missing {self.hook_name} hook for {self.addon} addon" class WrongHookPermissionsError(RepoValidationError): def __init__(self, hook_name: str, addon: str): self.hook_name = hook_name self.addon = addon @property def message(self) -> str: return f"{self.hook_name} hook for {self.addon} addon needs execute permissions" def validate_addons_repo(repo_dir: Path) -> None: """ Runs some checks on an addons repository. Inner validations raise SystemExit if any of the validations fail. """ validate_addons_file(repo_dir) validate_hooks(repo_dir) def validate_addons_file(repo_dir: Path) -> None: """ Checks that the addons.yaml file exists and that it has the appropriate format. """ contents = load_addons_yaml(repo_dir) schema = { "type": "object", "properties": { "microk8s-addons": { "type": "object", "properties": { "addons": { "type": "array", "items": { "type": "object", "properties": { "name": {"type": "string"}, "description": {"type": "string"}, "version": {"type": "string"}, "check_status": {"type": "string"}, "supported_architectures": { "type": "array", "items": { "type": "string", "enum": ["amd64", "arm64", "s390x", "ppc64le"], }, }, }, "required": [ "name", "description", "version", "check_status", "supported_architectures", ], }, } }, "required": ["addons"], } }, "required": ["microk8s-addons"], } try: jsonschema.validate(contents, schema=schema) except jsonschema.ValidationError as err: message = f"Invalid addons.yaml file: {err.message}" raise AddonsYamlFormatError(message) def load_addons_yaml(repo_dir: Path): addons_yaml = repo_dir / "addons.yaml" try: with open(addons_yaml, mode="r") as f: return yaml.safe_load(f.read()) except FileNotFoundError: raise AddonsYamlNotFoundError(repo_dir.name) except yaml.YAMLError as err: message = f"Yaml format error in addons.yaml file: {err.context} {err.problem}" raise AddonsYamlFormatError(message) def validate_hooks(repo_dir: Path) -> None: """ Check that, for each registered addon, the enable and disable hooks are present in the repository, and that they have execute permissions. """ for addon in get_addons_list(repo_dir): addon_folder = repo_dir / "addons" / addon for hook_name in ("enable", "disable"): hook = addon_folder / hook_name if not hook.exists(): raise MissingHookError(hook_name, addon) if not os.access(hook, os.X_OK): raise WrongHookPermissionsError(hook_name, addon) def get_addons_list(repo_dir: Path) -> List[str]: contents = load_addons_yaml(repo_dir) return [addon["name"] for addon in contents["microk8s-addons"]["addons"]] def pull_and_validate(name: str, repo_dir: Path): commit_before_pull = git_current_commit(repo_dir) subprocess.check_call([GIT, "pull"], cwd=repo_dir) try: validate_addons_repo(repo_dir) except RepoValidationError as err: click.echo(err.message, err=True) click.echo(f"Rolling back repository {name}") git_rollback(commit_before_pull, repo_dir) sys.exit(1) def clone_and_validate(remote_url: str, repo_dir: Path): if repo_dir.exists(): shutil.rmtree(repo_dir) subprocess.check_call([GIT, "clone", remote_url, repo_dir]) subprocess.check_call(["chgrp", get_group(), "-R", repo_dir]) try: validate_addons_repo(repo_dir) except RepoValidationError as err: click.echo(err.message, err=True) click.echo(f"Removing {repo_dir}") shutil.rmtree(repo_dir) sys.exit(1) @repository.command("add", help="Add a MicroK8s addons repository") @click.argument("name") @click.argument("repository") @click.option("--reference") @click.option("--force", is_flag=True, default=False) def add(name: str, repository: str, reference: str, force: bool): repo_dir = snap_common() / "addons" / name if repo_dir.exists(): if not force: click.echo("Error: repository '{}' already exists!".format(name), err=True) click.echo("Use the --force flag to overwrite it", err=True) sys.exit(1) click.echo("Removing {}".format(repo_dir)) shutil.rmtree(repo_dir) cmd = [GIT, "clone", repository, repo_dir] if reference is not None: cmd += ["-b", reference] subprocess.check_call(cmd) subprocess.check_call(["chgrp", get_group(), "-R", repo_dir]) try: validate_addons_repo(repo_dir) except RepoValidationError as err: click.echo(err.message, err=True) click.echo(f"Removing {repo_dir}") shutil.rmtree(repo_dir) sys.exit(1) @repository.command("remove", help="Remove a MicroK8s addons repository") @click.argument("name") def remove(name: str): repo_dir = snap_common() / "addons" / name if not repo_dir.exists(): click.echo("Error: repository '{}' does not exist".format(name), err=True) sys.exit(1) click.echo("Removing {}".format(repo_dir)) shutil.rmtree(repo_dir) @repository.command("update", help="Update a MicroK8s addons repository") @click.argument("name") @click.option("--skip-check-root/--no-skip-check-root", is_flag=True, default=False) def update(name: str, skip_check_root: bool): if not skip_check_root: exit_if_no_root() repo_dir = snap_common() / "addons" / name if not repo_dir.exists(): click.echo("Error: repository '{}' does not exist".format(name), err=True) sys.exit(1) if not (repo_dir / ".git").exists(): click.echo("Error: built-in repository '{}' cannot be updated".format(name), err=True) sys.exit(1) click.echo("Updating repository {}".format(name)) remote_url = ( subprocess.check_output( [GIT, "remote", "get-url", "origin"], cwd=repo_dir, stderr=subprocess.DEVNULL ) .decode() .strip() ) if remote_url.startswith(str(snap().parent)): # This is a repository that we have in the snap. # If the branch name we follow has not changed a simple git pull is enough # If the branch name changed we need to git repo add --force followed_branch_name = subprocess.check_output( [GIT, "rev-parse", "--abbrev-ref", "HEAD"], cwd=repo_dir, stderr=subprocess.DEVNULL ).decode() snapped_branch_name = subprocess.check_output( [GIT, "rev-parse", "--abbrev-ref", "HEAD"], cwd=remote_url, stderr=subprocess.DEVNULL ).decode() if followed_branch_name != snapped_branch_name: clone_and_validate(remote_url, repo_dir) else: pull_and_validate(name, repo_dir) else: pull_and_validate(name, repo_dir) class GettingGitCommitError(Exception): def __init__(self, exit_code, stderr): self.exit_code = exit_code self.stderr = stderr def git_current_commit(repository: Path) -> str: """ Returns the current commit hash of a given git repository """ cmd = [GIT, "rev-parse", "--verify", "HEAD"] process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repository) stdout, stderr = process.communicate() if process.returncode != 0: raise GettingGitCommitError(exit_code=process.returncode, stderr=stderr) return stdout def git_rollback(commit: str, repository: Path): """ Resets the git repository to a particular commit hash """ cmd = [GIT, "reset", "--hard", commit] subprocess.check_call(cmd, cwd=repository) @repository.command("list", help="List configured MicroK8s addons repositories") @click.option("--format", default="table", type=click.Choice(["json", "yaml", "table"])) def list(format: str): arch = get_current_arch() repositories = [] for dir in os.listdir(snap_common() / "addons"): try: repo_dir = snap_common() / "addons" / dir addons_yaml = repo_dir / "addons.yaml" with open(addons_yaml, "r") as fin: addons = yaml.safe_load(fin) count = 0 for addon in addons["microk8s-addons"]["addons"]: if arch in addon["supported_architectures"]: count += 1 source = "(built-in)" try: remote_url = subprocess.check_output( [GIT, "remote", "get-url", "origin"], cwd=repo_dir, stderr=subprocess.DEVNULL ).decode() revision = subprocess.check_output( [GIT, "rev-parse", "HEAD"], cwd=repo_dir, stderr=subprocess.DEVNULL ).decode()[:6] source = "{}@{}".format(remote_url.strip(), revision.strip()) except (subprocess.CalledProcessError, TypeError, ValueError): pass repositories.append( { "name": dir, "addons": count, "source": source, "description": addons["microk8s-addons"]["description"], } ) except Exception as e: click.echo("could not load addons from {}: {}".format(addons_yaml, e), err=True) if format == "json": click.echo(json.dumps(repositories)) elif format == "yaml": click.echo(yaml.safe_dump(repositories)) elif format == "table": click.echo(("{:10} {:>6} {}").format("REPO", "ADDONS", "SOURCE")) for repo in repositories: click.echo("{:10} {:>6} {}".format(repo["name"], repo["addons"], repo["source"])) if __name__ == "__main__": addons(prog_name="microk8s addons") ================================================ FILE: scripts/wrappers/common/__init__.py ================================================ ================================================ FILE: scripts/wrappers/common/cluster/__init__.py ================================================ ================================================ FILE: scripts/wrappers/common/cluster/utils.py ================================================ import base64 import datetime import ipaddress import json import os import random import re import shutil import socket import string import subprocess import time from pathlib import Path from subprocess import CalledProcessError, check_output import yaml FINGERPRINT_MIN_LEN = 12 TOKEN_ΜΙΝ_LEN = 32 class InvalidConnectionError(Exception): pass def is_strict(): snap_yaml = snap() / "meta/snap.yaml" with open(snap_yaml) as f: snap_meta = yaml.safe_load(f) return snap_meta["confinement"] == "strict" def get_group(): return "snap_microk8s" if is_strict() else "microk8s" def snap() -> Path: try: return Path(os.environ["SNAP"]) except KeyError: return Path("/snap/microk8s/current") def snap_data() -> Path: try: return Path(os.environ["SNAP_DATA"]) except KeyError: return Path("/var/snap/microk8s/current") def try_set_file_permissions(file): """ Try setting the ownership group and permission of the file :param file: full path and filename """ mode = 0o660 group = get_group() if os.path.exists(snap_data() / "var" / "lock" / "cis-hardening"): group = "root" mode = 0o600 os.chmod(file, mode) try: shutil.chown(file, group=group) except LookupError: # not setting the group means only the current user can access the file pass def remove_expired_token_from_file(file): """ Remove expired token from the valid tokens set :param file: the file to be removed from """ backup_file = "{}.backup".format(file) # That is a critical section. We need to protect it. # We are safe for now because flask serves one request at a time. with open(backup_file, "w") as back_fp: with open(file, "r") as fp: for _, line in enumerate(fp): if is_token_expired(line): continue back_fp.write("{}".format(line)) try_set_file_permissions(backup_file) shutil.copyfile(backup_file, file) def remove_token_from_file(token, file): """ Remove a token from the valid tokens set :param token: the token to be removed :param file: the file to be removed from """ backup_file = "{}.backup".format(file) # That is a critical section. We need to protect it. # We are safe for now because flask serves one request at a time. with open(backup_file, "w") as back_fp: with open(file, "r") as fp: for _, line in enumerate(fp): # Not considering cluster tokens with expiry in this method. if "|" not in line: if line.strip() == token: continue back_fp.write("{}".format(line)) try_set_file_permissions(backup_file) shutil.copyfile(backup_file, file) def is_token_expired(token_line): """ Checks if the token in the file is expired, when using the TTL based. :returns: True if the token is expired, otherwise False """ if "|" in token_line: expiry = token_line.strip().split("|")[1] if int(round(time.time())) > int(expiry): return True return False def get_callback_token(): """ Generate a token and store it in the callback token file :returns: the token """ snapdata_path = os.environ.get("SNAP_DATA") callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path) if os.path.exists(callback_token_file): with open(callback_token_file) as fp: token = fp.read() else: token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(64)) with open(callback_token_file, "w") as fp: fp.write("{}\n".format(token)) try_set_file_permissions(callback_token_file) return token def is_node_running_dqlite(): """ Check if we should use the dqlite joining process (join api version 2.0) :returns: True if dqlite is to be used, otherwise False """ ha_lock = os.path.expandvars("${SNAP_DATA}/var/lock/ha-cluster") return os.path.isfile(ha_lock) def is_node_dqlite_worker(): """ Check if this is a worker only node :returns: True if this is a worker node, otherwise False """ ha_lock = os.path.expandvars("${SNAP_DATA}/var/lock/ha-cluster") clustered_lock = os.path.expandvars("${SNAP_DATA}/var/lock/clustered.lock") no_apiserver_proxy_lock = os.path.expandvars("${SNAP_DATA}/var/lock/no-apiserver-proxy") return ( os.path.isfile(ha_lock) and os.path.isfile(clustered_lock) and not os.path.exists(no_apiserver_proxy_lock) ) def is_low_memory_guard_enabled(): """ Check if the low memory guard is enabled on this Node :returns: True if enabled, otherwise False """ lock = os.path.expandvars("${SNAP_DATA}/var/lock/low-memory-guard.lock") return os.path.isfile(lock) def get_dqlite_port(): """ What is the port dqlite listens on :return: the dqlite port """ # We get the dqlite port from the already existing deployment snapdata_path = os.environ.get("SNAP_DATA") cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path) dqlite_info = "{}/info.yaml".format(cluster_dir) port = 19001 if os.path.exists(dqlite_info): with open(dqlite_info) as f: data = yaml.safe_load(f) if "Address" in data: port = data["Address"].split(":")[1] return port def get_cluster_agent_port(): """ What is the cluster agent port :return: the port """ cluster_agent_port = "25000" snapdata_path = os.environ.get("SNAP_DATA") filename = "{}/args/cluster-agent".format(snapdata_path) with open(filename) as fp: for _, line in enumerate(fp): if line.startswith("--bind"): port_parse = line.split(" ") port_parse = port_parse[-1].split("=") port_parse = port_parse[-1].split(":") if len(port_parse) > 1: cluster_agent_port = port_parse[1].rstrip() return cluster_agent_port def get_cluster_cidr(): snapdata_path = os.environ.get("SNAP_DATA") filename = "{}/args/kube-proxy".format(snapdata_path) with open(filename) as fp: for _, line in enumerate(fp): if line.startswith("--cluster-cidr"): cidr_parse = line.split("=") if len(cidr_parse) > 1: return cidr_parse[1].rstrip() return "" def get_control_plane_nodes_internal_ips(): """ Return the internal IP of the nodes labeled running the control plane. :return: list of node internal IPs """ snap_path = os.environ.get("SNAP") control_plane_label = "node.kubernetes.io/microk8s-controlplane=microk8s-controlplane" nodes_info = subprocess.check_output( "{}/microk8s-kubectl.wrapper get no -o json -l {}".format( snap_path, control_plane_label ).split() ) info = json.loads(nodes_info.decode()) node_ips = [] for node_info in info["items"]: node_ip = get_internal_ip_from_get_node(node_info) node_ips.append(node_ip) return node_ips def get_internal_ip_from_get_node(node_info): """ Retrieves the InternalIp returned by kubectl get no -o json """ for status_addresses in node_info["status"]["addresses"]: if status_addresses["type"] == "InternalIP": return status_addresses["address"] def is_same_server(hostname, ip): """ Check if the hostname is the same as the current node's hostname """ try: hname, _, _ = socket.gethostbyaddr(ip) if hname == hostname: return True except socket.error: # Ignore any unresolvable IP by host, surely this is not from the same node. pass return False def apply_cni_manifest(timeout_insec=60): """ Apply the CNI yaml. If applying the manifest fails an exception is raised. :param timeout_insec: Try up to timeout seconds to apply the manifest. """ yaml = "{}/args/cni-network/cni.yaml".format(os.environ.get("SNAP_DATA")) snap_path = os.environ.get("SNAP") cmd = "{}/microk8s-kubectl.wrapper apply -f {}".format(snap_path, yaml) deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) while True: try: check_output(cmd.split()).strip().decode("utf8") break except CalledProcessError as err: output = err.output.strip().decode("utf8").replace("\\n", "\n") print("Applying {} failed with {}".format(yaml, output)) if datetime.datetime.now() > deadline: raise print("Retrying {}".format(cmd)) time.sleep(3) def cni_is_patched(): """ Detect if the cni.yaml manifest already has the hint for detecting nodes routing paths :return: True if calico knows where the rest of the nodes are. """ yaml = "{}/args/cni-network/cni.yaml".format(os.environ.get("SNAP_DATA")) with open(yaml) as f: if "can-reach" in f.read(): return True else: return False def cni_yaml_exists(): """ Detect if the cni.yaml manifest exists. :return: True if calico cni.yaml exists. """ yaml = "{}/args/cni-network/cni.yaml".format(os.environ.get("SNAP_DATA")) return os.path.exists(yaml) def patch_cni(ip): """ Patch the cni.yaml manifest with the proper hint on where the rest of the nodes are :param ip: The IP another k8s node has. """ cni_yaml = "{}/args/cni-network/cni.yaml".format(os.environ.get("SNAP_DATA")) backup_file = "{}.backup".format(cni_yaml) with open(backup_file, "w") as back_fp: with open(cni_yaml, "r") as fp: for _, line in enumerate(fp): if "first-found" in line: line = line.replace("first-found", "can-reach={}".format(ip)) back_fp.write("{}".format(line)) try_set_file_permissions(backup_file) shutil.copyfile(backup_file, cni_yaml) def try_initialise_cni_autodetect_for_clustering(ip, apply_cni=True): """ Try to initialise the calico route autodetection based on the IP provided, see https://docs.projectcalico.org/networking/ip-autodetection. If the cni manifest got changed by default it gets reapplied. :param ip: The IP another k8s node has. :param apply_cni: Should we apply the the manifest """ if not cni_yaml_exists() or cni_is_patched(): return True patch_cni(ip) if apply_cni: apply_cni_manifest() def is_kubelite(): """ Do we run kubelite? """ snap_data = os.environ.get("SNAP_DATA") if not snap_data: snap_data = "/var/snap/microk8s/current/" kubelite_lock = "{}/var/lock/lite.lock".format(snap_data) return os.path.exists(kubelite_lock) def service(operation, service_name): """ Restart a service. Handle case where kubelite is enabled. :param service_name: The service name :param operation: Operation to perform on the service """ if service_name in ["apiserver", "proxy", "kubelet", "scheduler", "controller-manager"]: daemon = "microk8s.daemon-kubelite" else: daemon = "microk8s.daemon-{}".format(service_name) subprocess.check_call(["snapctl", operation, daemon]) def mark_no_cert_reissue(): """ Mark a node as being part of a cluster that should not re-issue certs on network changes """ snap_data = os.environ.get("SNAP_DATA") lock_file = "{}/var/lock/no-cert-reissue".format(snap_data) open(lock_file, "a").close() os.chmod(lock_file, 0o700) def unmark_no_cert_reissue(): """ Unmark a node as being part of a cluster. The node should now re-issue certs on network changes """ snap_data = os.environ.get("SNAP_DATA") lock_file = "{}/var/lock/no-cert-reissue".format(snap_data) if os.path.exists(lock_file): os.unlink(lock_file) def restart_all_services(): """ Restart all services """ snap_path = os.environ.get("SNAP") waits = 10 while waits > 0: try: subprocess.check_call( "{}/microk8s-stop.wrapper".format(snap_path).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) break except subprocess.CalledProcessError: time.sleep(5) waits -= 1 waits = 10 while waits > 0: try: subprocess.check_call( "{}/microk8s-start.wrapper".format(snap_path).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) break except subprocess.CalledProcessError: time.sleep(5) waits -= 1 def get_token(name, tokens_file="known_tokens.csv"): """ Get token from known_tokens file :param name: the name of the node :param tokens_file: the file where the tokens should go :returns: the token or None(if name doesn't exist) """ snapdata_path = os.environ.get("SNAP_DATA") file = "{}/credentials/{}".format(snapdata_path, tokens_file) with open(file) as fp: for line in fp: if name in line: parts = line.split(",") return parts[0].rstrip() return None def get_arg(key, file): """ Get an argument from a file :param key: argument name :param file: the arguments file :return: value """ snapdata_path = os.environ.get("SNAP_DATA") filename = "{}/args/{}".format(snapdata_path, file) with open(filename, "r+") as fp: for _, line in enumerate(fp): if line.startswith(key): parts = re.split(r" |=", line) return parts[-1] return None def set_arg(key, value, file): """ Set an argument to a file :param key: argument name :param value: value :param file: the arguments file """ snapdata_path = os.environ.get("SNAP_DATA") filename = "{}/args/{}".format(snapdata_path, file) filename_remote = "{}/args/{}.remote".format(snapdata_path, file) done = False with open(filename_remote, "w+") as back_fp: with open(filename, "r+") as fp: for _, line in enumerate(fp): if line.startswith(key): done = True if value is not None: back_fp.write("{}={}\n".format(key, value)) else: back_fp.write("{}".format(line)) if not done and value is not None: back_fp.write("{}={}\n".format(key, value)) shutil.copyfile(filename, "{}.backup".format(filename)) try_set_file_permissions("{}.backup".format(filename)) shutil.copyfile(filename_remote, filename) try_set_file_permissions(filename) os.remove(filename_remote) def is_token_auth_enabled(): """ Return True if token auth is enabled """ if get_arg("--token-auth-file", "kube-apiserver"): return True else: return False def enable_token_auth(token): """ Turn on token auth and inject the admin token :param token: the admin token """ snapdata_path = os.environ.get("SNAP_DATA") file = "{}/credentials/known_tokens.csv".format(snapdata_path) with open(file, "w") as fp: fp.write(f'{token},admin,admin,"system:masters"\n') try_set_file_permissions(file) set_arg("--token-auth-file", "${SNAP_DATA}/credentials/known_tokens.csv", "kube-apiserver") def ca_one_line(ca): """ The CA in one line :param ca: the ca :return: one line """ return base64.b64encode(ca.encode("utf-8")).decode("utf-8") def rebuild_x509_auth_client_configs(): """ Recreate all the client configs """ if is_token_auth_enabled(): set_arg("--token-auth-file", None, "kube-apiserver") subprocess.check_call( [f"{snap()}/actions/common/utils.sh", "create_user_certs_and_configs"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) def get_valid_connection_parts(connection): """ Ensure that connection has a valid format :/[/] :param connection: the connection string :return: connection parts :raise: InvalidConnectionError: if connection string is not valid """ connection_parts = connection.split("/") if len(connection_parts) not in [2, 3]: raise InvalidConnectionError( "Expected format: :/[/]" ) master_ep = connection_parts[0].split(":") if len(master_ep) != 2: raise InvalidConnectionError( "Expected format: :/[/]" ) try: ipaddress.ip_address(master_ep[0]) except ValueError: raise InvalidConnectionError("Invalid master IP") try: port = int(master_ep[1]) if port < 1 or port > 65535: raise InvalidConnectionError("Master PORT not in range 1:65535") except ValueError: raise InvalidConnectionError("Master PORT not a number") if len(connection_parts[1]) < TOKEN_ΜΙΝ_LEN: raise InvalidConnectionError(f"Cluster token size should be at least {TOKEN_ΜΙΝ_LEN} bytes") if len(connection_parts) == 3 and len(connection_parts[2]) < FINGERPRINT_MIN_LEN: raise InvalidConnectionError(f"Fingerprint should be at least {FINGERPRINT_MIN_LEN} bytes") return connection_parts ================================================ FILE: scripts/wrappers/common/utils.py ================================================ import fcntl import getpass import json import os import platform import subprocess import sys import time from pathlib import Path import logging import click import yaml from common.cluster.utils import ( try_set_file_permissions, is_strict, ) LOG = logging.getLogger(__name__) KUBECTL = os.path.expandvars("$SNAP/microk8s-kubectl.wrapper") def get_current_arch(): # architecture mapping arch_mapping = { "aarch64": "arm64", "armv7l": "armhf", "x86_64": "amd64", "s390x": "s390x", "ppc64le": "ppc64le", "ppc64el": "ppc64le", } return arch_mapping[platform.machine()] def snap() -> Path: try: return Path(os.environ["SNAP"]) except KeyError: return Path("/snap/microk8s/current") def snap_data() -> Path: try: return Path(os.environ["SNAP_DATA"]) except KeyError: return Path("/var/snap/microk8s/current") def snap_common() -> Path: try: return Path(os.environ["SNAP_COMMON"]) except KeyError: return Path("/var/snap/microk8s/common") def run(*args, die=True): # Add wrappers to $PATH env = os.environ.copy() env["PATH"] += ":%s" % os.environ["SNAP"] result = subprocess.run( args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) try: result.check_returncode() except subprocess.CalledProcessError as err: if die: if result.stderr: print(result.stderr.decode("utf-8")) print(err) sys.exit(1) else: raise return result.stdout.decode("utf-8") def is_cluster_ready(with_ready_node=True): try: return "service/kubernetes" in kubectl_get("all") and ( not with_ready_node or " Ready " in kubectl_get("nodes") ) except Exception: return False def is_ha_enabled(): ha_lock = os.path.expandvars("${SNAP_DATA}/var/lock/ha-cluster") return os.path.isfile(ha_lock) def get_dqlite_info(): cluster_dir = os.path.expandvars("${SNAP_DATA}/var/kubernetes/backend") snap_path = os.environ.get("SNAP") info = [] if not is_ha_enabled(): return info waits = 10 while waits > 0: try: with open("{}/info.yaml".format(cluster_dir), mode="r") as f: data = yaml.safe_load(f) out = subprocess.check_output( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s .cluster".format( snappath=snap_path, dbdir=cluster_dir ).split(), timeout=4, stderr=subprocess.DEVNULL, ) if data["Address"] in out.decode(): break else: time.sleep(5) waits -= 1 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): time.sleep(2) waits -= 1 if waits == 0: return info nodes = json.loads(out.decode()) for n in nodes: if n["Role"] == 0: info.append((n["Address"], "voter")) if n["Role"] == 1: info.append((n["Address"], "standby")) if n["Role"] == 2: info.append((n["Address"], "spare")) return info def get_etcd_info(): kube_apiserver_args = os.path.expandvars("${SNAP_DATA}/args/kube-apiserver") with open(kube_apiserver_args, "r") as f: kube_apiserver_args_content = f.read() etcd_endpoints = [] for line in kube_apiserver_args_content.split("\n"): if "etcd-servers" in line: server_url = get_server_urls(line) # list all etcd endpointss for endpoint in server_url.split(","): if "//" in endpoint: ip_port = endpoint.split("//")[1] etcd_endpoints.append(ip_port) break return etcd_endpoints def get_server_urls(args): server_url = None parts = args.split("=") if len(parts) == 2: # Argument has an equals sign, e.g. "--etcd-servers=http://10.0.0.1:2379" server_url = parts[1] elif len(parts) == 1: # Argument has a space, e.g. "--etcd-servers http://10.0.0.1:2379" server_url = args.split("--etcd-servers")[1].strip() return server_url def is_external_etcd(): kube_apiserver_args = os.path.expandvars("${SNAP_DATA}/args/kube-apiserver") with open(kube_apiserver_args, "r") as f: kube_apiserver_args_content = f.read() for line in kube_apiserver_args_content.split("\n"): if "var/kubernetes/backend/kine.sock" in line: return False return True def is_cluster_locked(): if (snap_data() / "var/lock/clustered.lock").exists(): click.echo("This MicroK8s deployment is acting as a node in a cluster.") click.echo("Please use the master node.") sys.exit(1) def wait_for_ready(timeout, with_ready_node=True): start_time = time.time() end_time = start_time + timeout while True: if is_cluster_ready(with_ready_node=with_ready_node): return True elif timeout and time.time() > end_time: return False else: time.sleep(2) def exit_if_no_root(): """ Exit if the user is not root """ if not os.geteuid() == 0: click.echo( "Elevated permissions is needed for this operation. Please run this command with sudo." ) exit(50) def exit_if_stopped(): stoppedLockFile = os.path.expandvars("${SNAP_DATA}/var/lock/stopped.lock") if os.path.isfile(stoppedLockFile): print("microk8s is not running, try microk8s start") exit(0) def exit_if_no_permission(): user = getpass.getuser() # test if we can access the default kubeconfig client_config_file = os.path.expandvars("${SNAP_DATA}/credentials/client.config") if not os.access(client_config_file, os.R_OK): print("Insufficient permissions to access MicroK8s.") print( "You can either try again with sudo or add the user {} to the 'microk8s' group:".format( user ) ) print("") print(" sudo usermod -a -G microk8s {}".format(user)) print(" sudo chown -R $USER ~/.kube") print("") print( "After this, reload the user groups either via a reboot or by running 'newgrp microk8s'." ) exit(1) def ensure_started(): if (snap_data() / "var/lock/stopped.lock").exists(): click.echo("microk8s is not running, try microk8s start", err=True) sys.exit(1) def kubectl_get(cmd, namespace="--all-namespaces"): if namespace == "--all-namespaces": return run(KUBECTL, "get", cmd, "--all-namespaces", die=False) else: return run(KUBECTL, "get", cmd, "-n", namespace, die=False) def kubectl_get_clusterroles(): return run( KUBECTL, "get", "clusterroles", "--show-kind", "--no-headers", die=False, ) def is_community_addon(arch, addon_name): """ Check if an addon is part of the community repo. :param arch: architecture of the addon we are looking for :param addon_name: name of the addon we are looking for :return: True if the addon is in the community repo """ try: addons_yaml = f"{os.environ['SNAP']}/addons/community/addons.yaml" with open(addons_yaml, "r") as fin: addons = yaml.safe_load(fin) for addon in addons["microk8s-addons"]["addons"]: if arch in addon["supported_architectures"]: if addon_name == addon["name"]: return True except Exception: LOG.exception("could not load addons from %s", addons_yaml) return False def get_available_addons(arch): available = [] strict = is_strict() for dir in os.listdir(snap_common() / "addons"): try: addons_yaml = snap_common() / "addons" / dir / "addons.yaml" with open(addons_yaml, "r") as fin: addons = yaml.safe_load(fin) for addon in addons["microk8s-addons"]["addons"]: if arch not in addon["supported_architectures"]: continue if "confinement" in addon: if strict and "strict" not in addon["confinement"]: continue if not strict and "classic" not in addon["confinement"]: continue available.append({**addon, "repository": dir}) except Exception: LOG.exception("could not load addons from %s", addons_yaml) available = sorted(available, key=lambda k: (k["repository"], k["name"])) return available def get_addon_by_name(addons, name): filtered_addon = [] parts = name.split("/") if len(parts) == 1: repo_name, addon_name = None, parts[0] elif len(parts) == 2: repo_name, addon_name = parts[0], parts[1] else: # just fallback to the addon name repo_name, addon_name = None, name for addon in addons: if addon_name == addon["name"] and (repo_name == addon["repository"] or not repo_name): filtered_addon.append(addon) return filtered_addon def is_service_expected_to_start(service): """ Check if a service is supposed to start :param service: the service name :return: True if the service is meant to start """ lock_path = os.path.expandvars("${SNAP_DATA}/var/lock") lock = "{}/{}".format(lock_path, service) return os.path.exists(lock_path) and not os.path.isfile(lock) def set_service_expected_to_start(service, start=True): """ Check if a service is not expected to start. :param service: the service name :param start: should the service start or not """ lock_path = os.path.expandvars("${SNAP_DATA}/var/lock") lock = "{}/{}".format(lock_path, service) if start: os.remove(lock) else: fd = os.open(lock, os.O_CREAT, mode=0o700) os.close(fd) def check_help_flag(addons: list) -> bool: """Checks to see if a help message needs to be printed for an addon. Not all addons check for help flags themselves. Until they do, intercept calls to print help text and print out a generic message to that effect. """ addon = addons[0] if any(help_arg in addons for help_arg in ("-h", "--help")): print("Addon %s does not yet have a help message." % addon) print("For more information about it, visit https://microk8s.io/docs/addons") return True return False def parse_xable_addon_args(addon_args: list, available_addons: list): """ Parse the list of addons passed into the microk8s enable or disable commands. Further, it will infer the repository name for addons when possible. If any errors are encountered, we print them to stderr and exit. :param addon_args: The parameters passed to the microk8s enable command :param available_addons: List of available addons as (repo_name, addon_name) tuples Handles the following cases: - microk8s enable foo bar:--baz # enable many addons, inline arguments - microk8s enable bar --baz # enable one addon, unix style command line arguments :return: a list of (repo_name, addon_name, args) tuples """ # Backwards compatibility with enabling multiple addons at once, e.g. # `microk8s.enable foo bar:"baz"` available_addon_names = [addon_name for (_, addon_name) in available_addons] available_addon_names += [ "/".join([repo_name, addon_name]) for (repo_name, addon_name) in available_addons ] addon_names = [arg.split(":")[0] for arg in addon_args] if set(addon_names) < set(available_addon_names): return [parse_xable_single_arg(addon_arg, available_addons) for addon_arg in addon_args] # The new way of xabling addons, that allows for unix-style argument passing, # such as `microk8s.enable foo --bar`. repo_name, addon_name, args = parse_xable_single_arg(addon_args[0], available_addons) if args and addon_args[1:]: click.echo( "Can't pass string arguments and flag arguments simultaneously!\n" "Enable or disable addons with only one argument style at a time:\n" "\n" " microk8s enable foo:'bar'\n" "or\n" " microk8s enable foo --bar\n" ) sys.exit(1) return [(repo_name, addon_name, addon_args[1:])] def parse_xable_single_arg(addon_arg: str, available_addons: list): """ Parse an addon arg of the following form: `(repo_name/)addon_name(:args)` It will automatically infer the repository name if not specified. If multiple repositories are found for the addon, we print an error and exit. :param addon_arg: A parameter passed to the microk8s enable command :param available_addons: List of available addons as (repo_name, addon_name) tuples :return: a (repo_name, addon_name, args) tuple """ addon_name, *args = addon_arg.split(":") parts = addon_name.split("/") if len(parts) == 2: return (parts[0], parts[1], args) elif len(parts) == 1: matching_repos = [repo for (repo, addon) in available_addons if addon == addon_name] if len(matching_repos) == 0: click.echo("Addon {} was not found in any repository".format(addon_name), err=True) if is_community_addon(get_current_arch(), addon_name): click.echo( "To use the community maintained flavor enable the respective repository:" ) click.echo("") click.echo(" microk8s enable community") click.echo("") sys.exit(1) elif len(matching_repos) == 1: click.echo( "Infer repository {} for addon {}".format(matching_repos[0], addon_name), err=True ) return (matching_repos[0], addon_name, args) else: click.echo( "Addon {} exists in more than repository. Please explicitly specify\n" "the repository using any of:\n".format(addon_name), err=True, ) for repo in matching_repos: click.echo(" {}/{}".format(repo, addon_name), err=True) click.echo("", err=True) sys.exit(1) else: click.echo("Invalid addon name {}".format(addon_name)) sys.exit(1) def xable(action: str, addon_args: list): if os.getenv("MICROK8S_ADDONS_SKIP_LOCK") == "1": unprotected_xable(action, addon_args) else: protected_xable(action, addon_args) def protected_xable(action: str, addon_args: list): """ Get an exclusive lock file and then perform enable/disable of addons. Ensure that the lock file is always unlocked on exit. """ lock_file_path = snap_data() / "var/lock/.microk8s-addon-lock" with open(lock_file_path, "w") as f: # set file permissions so non-root users do not fail try: try_set_file_permissions(lock_file_path) except OSError: pass try: fcntl.lockf(f, fcntl.LOCK_EX) # NOTE(neoaggelos): We now have the lock, ensure any recursive # invocations will not deadlock. One example is addons that # enable other addons as requirements. # # See the relevant check in xable(). os.environ["MICROK8S_ADDONS_SKIP_LOCK"] = "1" unprotected_xable(action, addon_args) finally: fcntl.lockf(f, fcntl.LOCK_UN) def unprotected_xable(action: str, addon_args: list): """Enables or disables the given addons. Collated into a single function since the logic is identical other than the script names. :param action: "enable" or "disable" :param addons: List of addons to enable. Each addon may be prefixed with `repository/` to specify which addon repository it will be sourced from. """ available_addons_info = get_available_addons(get_current_arch()) enabled_addons_info, disabled_addons_info = get_status(available_addons_info, True) if action == "enable": xabled_addons_info = enabled_addons_info elif action == "disable": xabled_addons_info = disabled_addons_info else: click.echo("Invalid action {}. Only enable and disable are supported".format(action)) sys.exit(1) # available_addons is a list of (repo_name, addon_name) tuples for all available addons available_addons = [(addon["repository"], addon["name"]) for addon in available_addons_info] # xabled_addons is a list (repo_name, addon_name) tuples of already xabled addons xabled_addons = [(addon["repository"], addon["name"]) for addon in xabled_addons_info] addons = parse_xable_addon_args(addon_args, available_addons) if len(addons) > 1: click.echo( "WARNING: Do not enable or disable multiple addons in one command.\n" " This form of chained operations on addons will be DEPRECATED in the future.\n" f" Please, {action} one addon at a time: 'microk8s {action} '" ) for repo_name, addon_name, args in addons: if (repo_name, addon_name) not in available_addons: click.echo("Addon {}/{} not found".format(repo_name, addon_name)) continue if (repo_name, addon_name) in xabled_addons: click.echo("Addon {}/{} is already {}d".format(repo_name, addon_name, action)) continue wait_for_ready(timeout=30, with_ready_node=False) p = subprocess.run( [snap_common() / "addons" / repo_name / "addons" / addon_name / action, *args] ) if p.returncode: sys.exit(p.returncode) wait_for_ready(timeout=30, with_ready_node=False) def is_enabled(addon, item): if addon in item: return True else: filepath = os.path.expandvars(addon) return os.path.isfile(filepath) def get_status(available_addons, isReady): enabled = [] disabled = [] if isReady: # 'all' does not include ingress kube_output = kubectl_get("all,ingress,ingressclass") cluster_output = kubectl_get_clusterroles() kube_output = kube_output + cluster_output for addon in available_addons: found = False for row in kube_output.split("\n"): if is_enabled(addon["check_status"], row): enabled.append(addon) found = True break if not found: disabled.append(addon) return enabled, disabled def is_within_directory(directory, target): abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) prefix = os.path.commonprefix([abs_directory, abs_target]) return prefix == abs_directory def safe_extract(tar, path=".", members=None, *, numeric_owner=False): for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") tar.extractall(path, members, numeric_owner=numeric_owner) ================================================ FILE: scripts/wrappers/dashboard_proxy.py ================================================ #!/usr/bin/python3 import base64 import os from subprocess import check_output, run, CalledProcessError import time import click MICROK8S_ENABLE = os.path.expandvars("$SNAP/microk8s-enable.wrapper") KUBECTL = os.path.expandvars("$SNAP/microk8s-kubectl.wrapper") SECRET_YAML = """ apiVersion: v1 kind: Secret metadata: name: microk8s-dashboard-token namespace: kube-system annotations: kubernetes.io/service-account.name: "default" type: kubernetes.io/service-account-token """ def get_token(secret): """ Get a token to be used """ token = None print("Trying to get token from {}".format(secret)) for attempt in range(3): print("Waiting for secret token (attempt {})".format(attempt)) command = [ KUBECTL, "-n", "kube-system", "get", "secret", secret, "-o", "jsonpath={.data.token}", ] try: output = check_output(command) except CalledProcessError: time.sleep(5) continue if output: token = base64.b64decode(output).decode() break time.sleep(5) return token @click.command(context_settings={"help_option_names": ["-h", "--help"]}) def dashboard_proxy(): """ Enable the dashboard add-on and configures port-forwarding to allow accessing the dashboard from the local machine. """ print("Checking if Dashboard is running.") command = [MICROK8S_ENABLE, "dashboard"] output = check_output(command) ns = "kubernetes-dashboard" deploy = "kubernetes-dashboard-kong" service = "kubernetes-dashboard-kong-proxy" namespaces = check_output([KUBECTL, "get", "ns"]) if b"kubernetes-dashboard" not in namespaces: # NOTE(Hue): Backwards compatibility for installs in kube-system ns ns = "kube-system" deploy = "kubernetes-dashboard" service = "kubernetes-dashboard" if b"Addon dashboard is already enabled." not in output: print("Waiting for Dashboard to come up.") command = [ KUBECTL, "-n", ns, "wait", "--timeout=240s", "deployment", deploy, "--for", "condition=available", ] check_output(command) # Get token created by the dashboard enable script token = get_token("microk8s-dashboard-token") if not token: # Create a token. This is needed in case the enable script is pre-1.25. print("Create token for accessing the dashboard") run([KUBECTL, "apply", "-f", "-"], input=SECRET_YAML.encode("ascii")) token = get_token("microk8s-dashboard-token") print("Dashboard will be available at https://127.0.0.1:10443") print("Use the following token to login:") print(token) command = [ KUBECTL, "port-forward", "-n", ns, f"service/{service}", "10443:443", "--address", "0.0.0.0", ] try: check_output(command) except KeyboardInterrupt: exit(0) if __name__ == "__main__": dashboard_proxy() ================================================ FILE: scripts/wrappers/dbctl.py ================================================ #!/usr/bin/python3 import os import argparse import tempfile import datetime import subprocess import tarfile import os.path from common.utils import ( exit_if_no_permission, is_cluster_locked, is_ha_enabled, snap_data, safe_extract, ) def get_kine_endpoint(): """ Return the default kine endpoint """ return "unix://{}/var/kubernetes/backend/kine.sock:12379".format(snap_data()) def kine_exists(): """ Check the existence of the kine socket :return: True if the kine socket exists """ kine_socket = get_kine_endpoint() kine_socket_path = kine_socket.replace("unix://", "") return os.path.exists(kine_socket_path) def generate_backup_name(): """ Generate a filename based on the current time and date :return: a generated filename """ now = datetime.datetime.now() return "backup-{}".format(now.strftime("%Y-%m-%d-%H-%M-%S")) def run_command(command): """ Run a command while printing the output :param command: the command to run :return: the return code of the command """ process = subprocess.Popen(command.split(), stdout=subprocess.PIPE) while True: output = process.stdout.readline() if (not output or output == "") and process.poll() is not None: break if output: print(output.decode().strip()) rc = process.poll() return rc def backup(fname=None, debug=False): """ Backup the database to a provided file :param fname_tar: the tar file :param debug: show debug output """ snap_path = os.environ.get("SNAP") kine_ep = get_kine_endpoint() if not fname: fname = generate_backup_name() if fname.endswith(".tar.gz"): fname = fname[:-7] fname_tar = "{}.tar.gz".format(fname) with tempfile.TemporaryDirectory() as tmpdirname: backup_cmd = ( "{}/bin/k8s-dqlite migrator --endpoint {} --mode backup-dqlite --db-dir {}".format( snap_path, kine_ep, "{}/{}".format(tmpdirname, fname) ) ) if debug: backup_cmd = "{} {}".format(backup_cmd, "--debug") try: rc = run_command(backup_cmd) if rc > 0: print("Backup process failed. {}".format(rc)) exit(1) with tarfile.open(fname_tar, "w:gz") as tar: tar.add( "{}/{}".format(tmpdirname, fname), arcname=os.path.basename("{}/{}".format(tmpdirname, fname)), ) print("The backup is: {}".format(fname_tar)) except subprocess.CalledProcessError as e: print("Backup process failed. {}".format(e)) exit(2) def restore(fname_tar, debug=False): """ Restore the database from the provided file :param fname_tar: the tar file :param debug: show debug output """ snap_path = os.environ.get("SNAP") kine_ep = get_kine_endpoint() with tempfile.TemporaryDirectory() as tmpdirname: with tarfile.open(fname_tar, "r:gz") as tar: safe_extract(tar, path=tmpdirname) if fname_tar.endswith(".tar.gz"): fname = fname_tar[:-7] else: fname = fname_tar fname = os.path.basename(fname) restore_cmd = ( "{}/bin/k8s-dqlite migrator --endpoint {} --mode restore-to-dqlite --db-dir {}".format( snap_path, kine_ep, "{}/{}".format(tmpdirname, fname) ) ) if debug: restore_cmd = "{} {}".format(restore_cmd, "--debug") try: rc = run_command(restore_cmd) if rc > 0: print("Restore process failed. {}".format(rc)) exit(3) except subprocess.CalledProcessError as e: print("Restore process failed. {}".format(e)) exit(4) if __name__ == "__main__": exit_if_no_permission() is_cluster_locked() if not kine_exists() or not is_ha_enabled(): print("Please ensure the kubernetes apiserver is running and HA is enabled.") exit(10) # initiate the parser with a description parser = argparse.ArgumentParser( description="backup and restore the Kubernetes datastore.", prog="microk8s dbctl" ) parser.add_argument("--debug", action="store_true", help="print debug output") commands = parser.add_subparsers(title="commands", help="backup and restore operations") restore_parser = commands.add_parser("restore") restore_parser.add_argument("backup-file", help="name of file with the backup") backup_parser = commands.add_parser("backup") backup_parser.add_argument("-o", metavar="backup-file", help="output filename") args = parser.parse_args() if "backup-file" in args: fname = vars(args)["backup-file"] print("Restoring from {}".format(fname)) restore(fname, args.debug) elif "o" in args: print("Backing up the datastore") backup(vars(args)["o"], args.debug) else: parser.print_help() ================================================ FILE: scripts/wrappers/disable.py ================================================ #!/usr/bin/env python3 import click from common.utils import ( ensure_started, exit_if_no_permission, is_cluster_locked, check_help_flag, wait_for_ready, xable, ) @click.command( context_settings={ "ignore_unknown_options": True, "help_option_names": ["-h", "--help"], }, ) @click.argument("addons", nargs=-1, required=True) def disable(addons): """Disable one or more MicroK8s addons. For a list of available addons, run `microk8s status`. To see help for individual addons, run: microk8s disable ADDON -- --help """ if check_help_flag(addons): return is_cluster_locked() exit_if_no_permission() ensure_started() wait_for_ready(timeout=30, with_ready_node=False) xable("disable", addons) if __name__ == "__main__": disable(prog_name="microk8s disable") ================================================ FILE: scripts/wrappers/distributed_op.py ================================================ #!/usr/bin/python3 import getopt import subprocess import requests import urllib3 import os import sys import json import socket import time from common.cluster.utils import ( get_callback_token, get_cluster_agent_port, is_node_running_dqlite, get_internal_ip_from_get_node, is_same_server, ) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) CLUSTER_API_V1 = "cluster/api/v1.0" CLUSTER_API_V2 = "cluster/api/v2.0" snapdata_path = os.environ.get("SNAP_DATA") snap_path = os.environ.get("SNAP") callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path) callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path) KUBECTL = "{}/microk8s-kubectl.wrapper".format(snap_path) MICROK8S_STATUS = "{}/microk8s-status.wrapper".format(snap_path) def get_cluster_agent_endpoints(include_self=False): """ Get a list of all cluster agent endpoints and their callback token. :param include_self: If true, include the current node in the list. :return: [("node1:25000", "token1"), ("node2:25000", "token2"), ...] """ nodes = [] if is_node_running_dqlite(): hostname = socket.gethostname() token = get_callback_token() for attempt in range(10): try: stdout = subprocess.check_output([KUBECTL, "get", "node", "-o", "json"]) break except subprocess.CalledProcessError as e: print("Failed to list nodes (try {}): {}".format(attempt + 1, e), file=sys.stderr) if attempt == 9: raise e time.sleep(3) info = json.loads(stdout) for node_info in info["items"]: node_ip = get_internal_ip_from_get_node(node_info) if not include_self and is_same_server(hostname, node_ip): continue nodes.append(("{}:25000".format(node_ip), token.rstrip())) else: if include_self: token = get_callback_token() port = get_cluster_agent_port() nodes.append(("127.0.0.1:{}".format(port), token.rstrip())) try: with open(callback_tokens_file, "r+") as fin: for line in fin: node_ep, token = line.split() host = node_ep.split(":")[0] try: subprocess.check_call( [KUBECTL, "get", "node", host], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) nodes.append((node_ep, token.rstrip())) except subprocess.CalledProcessError: print("Node {} not present".format(host)) except OSError: pass return nodes def do_configure_op(remote_op): """ Perform a /configure operation on all remote nodes :param remote_op: the operation json string """ try: endpoints = get_cluster_agent_endpoints(include_self=False) except subprocess.CalledProcessError as e: print("Could not query for nodes") raise SystemExit(e) for node_ep, token in endpoints: try: remote_op["callback"] = token.rstrip() # TODO: handle ssl verification res = requests.post( "https://{}/{}/configure".format(node_ep, CLUSTER_API_V1), json=remote_op, verify=False, ) if res.status_code != 200: print( "Failed to perform a {} on node {} {}".format( remote_op["action_str"], node_ep, res.status_code ) ) except requests.exceptions.RequestException as e: print("Failed to reach node.") raise SystemExit(e) def do_image_import(image_data): """ Perform a /image/import operation on all nodes :param image_data: Raw bytes of the OCI image tar file """ try: endpoints = get_cluster_agent_endpoints(include_self=True) except subprocess.CalledProcessError as e: print("Could not query for nodes") raise SystemExit(e) for node_ep, token in endpoints: try: print("Pushing OCI images to {}".format(node_ep)) res = requests.post( "https://{}/{}/image/import".format(node_ep, CLUSTER_API_V2), data=image_data, headers={ "x-microk8s-callback-token": token, }, verify=False, ) if res.status_code != 200: print("Failed to import images on {}: {}".format(node_ep, res.content.decode())) except requests.exceptions.RequestException as e: print("Failed to reach {}: {}".format(node_ep, e)) def restart(service): """ Restart service on all nodes :param service: the service name """ print("Restarting nodes.") remote_op = { "action_str": "restart {}".format(service), "service": [{"name": service, "restart": "yes"}], } do_configure_op(remote_op) def update_argument(service, key, value): """ Configure an argument on all nodes :param service: the service we configure :param key: the argument we configure :param value: the value we set """ print("Adding argument {} to nodes.".format(key)) remote_op = { "action_str": "change of argument {} to {}".format(key, value), "service": [{"name": service, "arguments_update": [{key: value}]}], } do_configure_op(remote_op) def remove_argument(service, key): """ Drop an argument from all nodes :param service: the service we configure :param key: the argument we configure """ print("Removing argument {} from nodes.".format(key)) remote_op = { "action_str": "removal of argument {}".format(key), "service": [{"name": service, "arguments_remove": [key]}], } do_configure_op(remote_op) def set_addon(addon, state): """ Enable or disable an add-on across all nodes :param addon: the add-on name :param state: 'enable' or 'disable' """ if state not in ("enable", "disable"): raise ValueError( "Wrong value '{}' for state. Must be one of 'enable' or 'disable'".format(state) ) else: print("Setting add-on {} to {} on nodes.".format(addon, state)) remote_op = { "action_str": "set of {} to {}".format(addon, state), "addon": [{"name": addon, state: "true"}], } do_configure_op(remote_op) def usage(): print("usage: dist_refresh_opt [OPERATION] [SERVICE] (ARGUMENT) (value)") print("OPERATION is one of restart, update_argument, remove_argument, set_addon") if __name__ == "__main__": if is_node_running_dqlite() and not os.path.isfile(callback_token_file): # print("Single node cluster.") exit(0) if not is_node_running_dqlite() and not os.path.isfile(callback_tokens_file): print("No callback tokens file.") exit(1) try: opts, args = getopt.getopt(sys.argv[1:], "h", ["help"]) except getopt.GetoptError as err: # print help information and exit: print(err) # will print something like "option -a not recognized" usage() sys.exit(2) for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() else: assert False, "unhandled option" operation = args[0] service = args[1] if operation == "restart": restart(service) if operation == "update_argument": update_argument(service, args[2], args[3]) if operation == "remove_argument": remove_argument(service, args[2]) if operation == "set_addon": set_addon(service, args[2]) exit(0) ================================================ FILE: scripts/wrappers/enable.py ================================================ #!/usr/bin/env python3 import click from common.utils import ( ensure_started, exit_if_no_permission, is_cluster_locked, check_help_flag, wait_for_ready, xable, ) @click.command( context_settings={"ignore_unknown_options": True, "help_option_names": ["-h", "--help"]}, ) @click.argument("addons", nargs=-1, required=True) def enable(addons) -> None: """ Enable a MicroK8s addon. For a list of available addons, run `microk8s status`. To see help for individual addons, run: microk8s enable ADDON -- --help """ if check_help_flag(addons): return is_cluster_locked() exit_if_no_permission() ensure_started() wait_for_ready(timeout=30, with_ready_node=False) xable("enable", addons) if __name__ == "__main__": enable(prog_name="microk8s enable") ================================================ FILE: scripts/wrappers/images.py ================================================ #!/usr/bin/python3 import os import subprocess import sys from typing import List import click from distributed_op import do_image_import CTR = "{}/microk8s-ctr.wrapper".format(os.getenv("SNAP")) images = click.Group() @images.command("import", help="Import OCI images into the MicroK8s cluster") @click.argument("image", default="-") def import_images(image: str): if image == "-": image_data = sys.stdin.buffer.read() else: try: with open(image, "rb") as fin: image_data = fin.read() except OSError as e: click.echo("Error: failed to read {}: {}".format(image, e), err=True) sys.exit(1) do_image_import(image_data) def get_all_ctr_images(): """ Return list of all OCI images from containerd. """ images = subprocess.check_output([CTR, "image", "ls", "--quiet"]).decode().split("\n") # drop the sha256-digest aliases return [tag for tag in images if tag and not tag.startswith("sha256:")] @images.command("export-local", help="Export OCI images from the current MicroK8s node") @click.argument("output", default="-") @click.argument("images", nargs=-1) def export_images(output: str, images: List[str]): if not images: images = get_all_ctr_images() for image in images: click.echo("Checking {}".format(image), err=True) try: subprocess.check_call([CTR, "image", "export", "-", image], stdout=subprocess.DEVNULL) except subprocess.CalledProcessError: subprocess.check_call( [CTR, "content", "fetch", "--all-platforms", image], stdout=sys.stderr ) subprocess.check_call([CTR, "image", "export", output, *images]) if __name__ == "__main__": images(prog_name="microk8s images") ================================================ FILE: scripts/wrappers/join.py ================================================ #!/usr/bin/python3 import hashlib import http import json import os import random import shutil import socket import ssl import string import subprocess import sys import time import ipaddress import click import requests import urllib3 import yaml from common.cluster.utils import ( ca_one_line, enable_token_auth, get_cluster_agent_port, get_cluster_cidr, get_token, get_valid_connection_parts, is_low_memory_guard_enabled, is_node_running_dqlite, is_token_auth_enabled, mark_no_cert_reissue, rebuild_x509_auth_client_configs, service, set_arg, try_initialise_cni_autodetect_for_clustering, try_set_file_permissions, snap, snap_data, FINGERPRINT_MIN_LEN, InvalidConnectionError, ) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) CLUSTER_API = "cluster/api/v1.0" snapdata_path = os.environ.get("SNAP_DATA") snap_path = os.environ.get("SNAP") ca_cert_file_via_env = "${SNAP_DATA}/certs/ca.remote.crt" ca_cert_file = "{}/certs/ca.remote.crt".format(snapdata_path) callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path) callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path) server_cert_file_via_env = "${SNAP_DATA}/certs/server.remote.crt" server_cert_file = "{}/certs/server.remote.crt".format(snapdata_path) CLUSTER_API_V2 = "cluster/api/v2.0" cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path) cluster_backup_dir = "{}/var/kubernetes/backend.backup".format(snapdata_path) cluster_cert_file = "{}/cluster.crt".format(cluster_dir) cluster_key_file = "{}/cluster.key".format(cluster_dir) def get_traefik_port(): """ Return the port Traefik listens to. Try read the port from the Traefik configuration or return the default value """ config_file = "{}/args/traefik/traefik-template.yaml".format(snapdata_path) with open(config_file) as f: data = yaml.load(f, Loader=yaml.FullLoader) if ( "entryPoints" in data and "apiserver" in data["entryPoints"] and "address" in data["entryPoints"]["apiserver"] ): port = data["entryPoints"]["apiserver"]["address"] port = port.replace(":", "") return port else: return "16443" def join_request(conn, api_version, req_data, master_ip, verify_peer, fingerprint): json_params = json.dumps(req_data) headers = {"Content-type": "application/json", "Accept": "application/json"} try: if verify_peer and fingerprint: if len(fingerprint) < FINGERPRINT_MIN_LEN: print( "Joining cluster failed. Fingerprint too short." " Use '--skip-verify' to skip server certificate check." ) exit(4) # Do the peer certificate verification der_cert_bin = conn.sock.getpeercert(True) peer_cert_hash = hashlib.sha256(der_cert_bin).hexdigest() if not peer_cert_hash.startswith(fingerprint): print( "Joining cluster failed. Could not verify the identity of {}." " Use '--skip-verify' to skip server certificate check.".format(master_ip) ) exit(4) conn.request("POST", "/{}/join".format(api_version), json_params, headers) response = conn.getresponse() if not response.status == 200: message = extract_error(response) print("{} ({}).".format(message, response.status)) exit(6) body = response.read() return json.loads(body) except http.client.HTTPException as e: print("Please ensure the master node is reachable. {}".format(e)) exit(1) except ssl.SSLError as e: print("Peer node verification failed ({}).".format(e)) exit(4) def extract_error(response): message = "Connection failed." try: resp = response.read().decode() if resp: res_data = json.loads(resp) if "error" in res_data: message = "{} {}".format(message, res_data["error"]) except ValueError: pass return message def get_connection_info( master_ip, master_port, token, callback_token=None, cluster_type="etcd", verify_peer=False, fingerprint=None, worker=False, ): """ Contact the master and get all connection information :param master_ip: the master IP :param master_port: the master port :param token: the token to contact the master with :param callback_token: callback token for etcd based clusters :param cluster_type: the type of cluster we want to join, etcd or dqlite :param verify_peer: flag indicating if we should verify peers certificate :param fingerprint: the certificate fingerprint we expect from the peer :param worker: this is a worker only node :return: the json response of the master """ cluster_agent_port = get_cluster_agent_port() try: context = ssl._create_unverified_context() conn = http.client.HTTPSConnection("{}:{}".format(master_ip, master_port), context=context) conn.connect() if cluster_type == "dqlite": req_data = { "token": token, "hostname": socket.gethostname().lower(), "port": cluster_agent_port, "worker": worker, "can_handle_x509_auth": True, "can_handle_custom_etcd": True, } return join_request(conn, CLUSTER_API_V2, req_data, master_ip, verify_peer, fingerprint) else: req_data = { "token": token, "hostname": socket.gethostname().lower(), "port": cluster_agent_port, "callback": callback_token, "can_handle_x509_auth": True, } return join_request( conn, CLUSTER_API, req_data, master_ip, verify_peer=False, fingerprint=None ) except http.client.HTTPException as e: print("Connecting to cluster failed with {}.".format(e)) exit(5) except ssl.SSLError as e: print("Peer node verification failed with {}.".format(e)) exit(4) def get_etcd_client_cert(master_ip, master_port, token): """ Get a signed cert to access etcd :param master_ip: master ip :param master_port: master port :param token: token to contact the master with """ cer_req_file = "{}/certs/server.remote.csr".format(snapdata_path) cmd_cert = ( "{snap}/openssl.wrapper req -new -sha256 -key {snapdata}/certs/server.key -out {csr} " "-config {snapdata}/certs/csr.conf".format( snap=snap_path, snapdata=snapdata_path, csr=cer_req_file ) ) subprocess.check_call(cmd_cert.split()) with open(cer_req_file) as fp: csr = fp.read() req_data = {"token": token, "request": csr} # TODO: enable ssl verification signed = requests.post( "https://{}:{}/{}/sign-cert".format(master_ip, master_port, CLUSTER_API), json=req_data, verify=False, ) if signed.status_code != 200: print("Failed to sign certificate. {}".format(signed.json()["error"])) exit(1) info = signed.json() with open(server_cert_file, "w") as cert_fp: cert_fp.write(info["certificate"]) try_set_file_permissions(server_cert_file) def get_client_cert(master_ip, master_port, fname: str, token: str, subject: str, with_sans: bool): """ Get a signed cert signed by a remote cluster-agent. See https://kubernetes.io/docs/reference/access-authn-authz/authentication/#x509-client-certs :param master_ip: master ip :param master_port: master port :param fname: file name prefix for the certificate :param token: token to contact the master with :param subject: the subject of the certificate :param with_sans: whether to include hostname and node IPs as subject alternate names """ cert_crt = (snap_data() / "certs" / fname).with_suffix(".crt") cert_key = (snap_data() / "certs" / fname).with_suffix(".key") # generate csr script = "generate_csr_with_sans" if with_sans else "generate_csr" p = subprocess.run( [f"{snap()}/actions/common/utils.sh", script, subject, cert_key], check=True, capture_output=True, ) csr = p.stdout.decode() req_data = {"token": token, "request": csr} # TODO: enable ssl verification signed = requests.post( "https://{}:{}/{}/sign-cert".format(master_ip, master_port, CLUSTER_API), json=req_data, verify=False, ) if signed.status_code != 200: error = "Failed to sign {} certificate ({}).".format(fname, signed.status_code) try: if "error" in signed.json(): error = "{} {}".format(error, format(signed.json()["error"])) except ValueError: print("Make sure the cluster you connect to supports joining worker nodes.") print(error) exit(1) info = signed.json() cert_crt.write_text(info["certificate"]) try_set_file_permissions(cert_crt) def update_flannel(etcd, master_ip, master_port, token): """ Configure flannel :param etcd: etcd endpoint :param master_ip: master ip :param master_port: master port :param token: token to contact the master with """ get_etcd_client_cert(master_ip, master_port, token) etcd = etcd.replace("0.0.0.0", master_ip) set_arg("--etcd-endpoints", etcd, "flanneld") set_arg("--etcd-cafile", ca_cert_file_via_env, "flanneld") set_arg("--etcd-certfile", server_cert_file_via_env, "flanneld") set_arg("--etcd-keyfile", "${SNAP_DATA}/certs/server.key", "flanneld") service("restart", "flanneld") def create_kubeconfig(token, ca, master_ip, api_port, filename, user): """ Create a kubeconfig file. The file in stored under credentials named after the user :param token: the token to be in the kubeconfig :param ca: the ca :param master_ip: the master node IP :param api_port: the API server port :param filename: the name of the config file :param user: the user to use al login """ snap_path = os.environ.get("SNAP") config_template = "{}/{}".format(snap_path, "kubelet.config.template") config = "{}/credentials/{}".format(snapdata_path, filename) shutil.copyfile(config, "{}.backup".format(config)) try_set_file_permissions("{}.backup".format(config)) ca_line = ca_one_line(ca) with open(config_template, "r") as tfp: with open(config, "w+") as fp: config_txt = tfp.read() config_txt = config_txt.replace("CADATA", ca_line) config_txt = config_txt.replace("NAME", user) config_txt = config_txt.replace("TOKEN", token) config_txt = config_txt.replace("127.0.0.1", master_ip) config_txt = config_txt.replace("16443", api_port) fp.write(config_txt) try_set_file_permissions(config) def update_kubeproxy(token, ca, master_ip, api_port): """ Configure the kube-proxy :param token: the token to be in the kubeconfig :param ca: the ca :param master_ip: the master node IP :param api_port: the API server port """ create_kubeconfig(token, ca, master_ip, api_port, "proxy.config", "kubeproxy") set_arg("--master", None, "kube-proxy") set_arg("--hostname-override", None, "kube-proxy") service("restart", "proxy") def update_cert_auth_kubeproxy(token, master_ip, master_port): """ Configure the kube-proxy :param token: the token to be in the kubeconfig :param ca: the ca :param master_ip: the master node IP :param master_port: the master node port where the cluster agent listens """ proxy_token = "{}-proxy".format(token) get_client_cert(master_ip, master_port, "proxy", proxy_token, "/CN=system:kube-proxy", False) set_arg("--master", None, "kube-proxy") set_arg("--hostname-override", None, "kube-proxy") def update_kubeproxy_cidr(cidr): if cidr is not None: set_arg("--cluster-cidr", cidr, "kube-proxy") service("restart", "proxy") def update_cert_auth_kubelet(token, master_ip, master_port): """ Configure the kubelet :param token: the token to be in the kubeconfig :param ca: the ca :param master_ip: the master node IP :param master_port: the master node port where the cluster agent listens """ kubelet_token = "{}-kubelet".format(token) subject = f"/CN=system:node:{socket.gethostname().lower()}/O=system:nodes" get_client_cert(master_ip, master_port, "kubelet", kubelet_token, subject, True) set_arg("--client-ca-file", "${SNAP_DATA}/certs/ca.remote.crt", "kubelet") set_arg( "--node-labels", "microk8s.io/cluster=true,node.kubernetes.io/microk8s-worker=microk8s-worker", "kubelet", ) def update_kubelet(token, ca, master_ip, api_port): """ Configure the kubelet :param token: the token to be in the kubeconfig :param ca: the ca :param master_ip: the master node IP :param api_port: the API server port """ create_kubeconfig(token, ca, master_ip, api_port, "kubelet.config", "kubelet") set_arg("--client-ca-file", "${SNAP_DATA}/certs/ca.remote.crt", "kubelet") set_arg( "--node-labels", "microk8s.io/cluster=true,node.kubernetes.io/microk8s-worker=microk8s-worker", "kubelet", ) service("restart", "kubelet") def update_apiserver(api_authz_mode, apiserver_port): """ Configure the API server :param api_authz_mode: the authorization mode to be used :param apiserver_port: the apiserver port """ if api_authz_mode: set_arg("--authorization-mode", api_authz_mode, "kube-apiserver") if apiserver_port: set_arg("--secure-port", apiserver_port, "kube-apiserver") service("restart", "apiserver") def store_remote_ca(ca): """ Store the remote ca :param ca: the CA """ with open(ca_cert_file, "w+") as fp: fp.write(ca) try_set_file_permissions(ca_cert_file) def mark_worker_node(): """ Mark a node as being part of a cluster not running the control plane by creating a var/lock/clustered.lock """ locks = ["clustered.lock", "no-k8s-dqlite"] for lock in locks: lock_file = "{}/var/lock/{}".format(snapdata_path, lock) open(lock_file, "a").close() os.chmod(lock_file, 0o700) services = ["kubelite", "etcd", "apiserver-kicker", "apiserver-proxy", "k8s-dqlite"] for s in services: service("restart", s) def generate_callback_token(): """ Generate a token and store it in the callback token file :return: the token """ token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(64)) with open(callback_token_file, "w") as fp: fp.write("{}\n".format(token)) try_set_file_permissions(callback_token_file) return token def store_base_kubelet_args(args_string): """ Create a kubelet args file from the set of args provided :param args_string: the arguments provided """ args_file = "{}/args/kubelet".format(snapdata_path) with open(args_file, "w") as fp: fp.write(args_string) try_set_file_permissions(args_file) def update_kubelet_node_ip(args_string, hostname_override): """ Update the kubelet --node-ip argument if it was set on the node that we join. :param args_string: the kubelet arguments :param hostname_override: the source IP address used by the node when joining """ if "--node-ip" in args_string: set_arg("--node-ip", hostname_override, "kubelet") def update_kubelet_hostname_override(args_string): """ Remove the kubelet --hostname-override argument if it was set on the node that we join. :param args_string: the kubelet arguments """ if "--hostname-override" in args_string: set_arg("--hostname-override", None, "kubelet") def replace_admin_token(token): """ Replaces the admin token in the known tokens :param token: the admin token """ file = "{}/credentials/known_tokens.csv".format(snapdata_path) backup_file = "{}.backup".format(file) # That is a critical section. We need to protect it. with open(backup_file, "w") as back_fp: with open(file, "r") as fp: for _, line in enumerate(fp): if 'admin,admin,"system:masters"' in line: continue back_fp.write("{}".format(line)) back_fp.write('{},admin,admin,"system:masters"\n'.format(token)) try_set_file_permissions(backup_file) shutil.copyfile(backup_file, file) def store_cert(filename, payload): """ Store a certificate :param filename: where to store the certificate :param payload: certificate payload """ file_with_path = "{}/certs/{}".format(snapdata_path, filename) if os.path.exists(file_with_path): backup_file_with_path = "{}.backup".format(file_with_path) shutil.copyfile(file_with_path, backup_file_with_path) try_set_file_permissions(backup_file_with_path) with open(file_with_path, "w+") as fp: fp.write(payload) try_set_file_permissions(file_with_path) def store_cluster_certs(cluster_cert, cluster_key): """ Store the dqlite cluster certs :param cluster_cert: the cluster certificate :param cluster_key: the cluster certificate key """ with open(cluster_cert_file, "w+") as fp: fp.write(cluster_cert) try_set_file_permissions(cluster_cert_file) with open(cluster_key_file, "w+") as fp: fp.write(cluster_key) try_set_file_permissions(cluster_key_file) def create_admin_kubeconfig(ca, ha_admin_token=None): """ Create a kubeconfig file. The file in stored under credentials named after the admin :param ca: the ca :param ha_admin_token: the ha_cluster_token """ if not ha_admin_token: token = get_token("admin", "basic_auth.csv") if not token: print("Error, could not locate admin token. Joining cluster failed.") exit(2) else: token = ha_admin_token assert token is not None config_template = "{}/{}".format(snap_path, "client.config.template") config = "{}/credentials/client.config".format(snapdata_path) shutil.copyfile(config, "{}.backup".format(config)) try_set_file_permissions("{}.backup".format(config)) ca_line = ca_one_line(ca) with open(config_template, "r") as tfp: with open(config, "w+") as fp: for _, config_txt in enumerate(tfp): if config_txt.strip().startswith("username:"): continue else: config_txt = config_txt.replace("CADATA", ca_line) config_txt = config_txt.replace("NAME", "admin") config_txt = config_txt.replace("AUTHTYPE", "token") config_txt = config_txt.replace("PASSWORD", token) fp.write(config_txt) try_set_file_permissions(config) def store_callback_token(token): """ Store the callback token :param token: the callback token """ callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path) with open(callback_token_file, "w") as fp: fp.write(token) try_set_file_permissions(callback_token_file) def update_dqlite(cluster_cert, cluster_key, voters, host): """ Configure the dqlite cluster :param cluster_cert: the dqlite cluster cert :param cluster_key: the dqlite cluster key :param voters: the dqlite voters :param host: the hostname others see of this node """ service("stop", "apiserver") service("stop", "k8s-dqlite") # will allow for apiservice-kicker to generate new certificates @5s loop rate time.sleep(10) shutil.rmtree(cluster_backup_dir, ignore_errors=True) shutil.move(cluster_dir, cluster_backup_dir) os.mkdir(cluster_dir) store_cluster_certs(cluster_cert, cluster_key) # We get the dqlite port from the already existing deployment port = 19001 try: with open("{}/info.yaml".format(cluster_backup_dir)) as f: data = yaml.safe_load(f) if "Address" in data: port = data["Address"].rsplit(":")[-1] except OSError: pass # If host is an IPv6 address, wrap it in square brackets try: if ipaddress.ip_address(host).version == 6: host = "[{}]".format(host) except ValueError: pass init_data = {"Cluster": voters, "Address": "{}:{}".format(host, port)} with open("{}/init.yaml".format(cluster_dir), "w") as f: yaml.dump(init_data, f) service("start", "k8s-dqlite") waits = 10 print("Waiting for this node to finish joining the cluster.", end=" ", flush=True) while waits > 0: try: out = subprocess.check_output( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s .cluster".format( snappath=snap_path, dbdir=cluster_dir ).split(), timeout=4, stderr=subprocess.STDOUT, ) if host in out.decode(): break else: print(".", end=" ", flush=True) time.sleep(5) waits -= 1 except (subprocess.CalledProcessError, subprocess.TimeoutExpired): print("..", end=" ", flush=True) time.sleep(2) waits -= 1 print(" ") # start kube-apiserver after dqlite comes up service("start", "apiserver") def join_dqlite(connection_parts, verify=False, worker=False): """ Configure node to join a dqlite cluster. :param connection_parts: connection string parts """ token = connection_parts[1] master_ep = connection_parts[0].rsplit(":", 1) master_ip = master_ep[0] master_port = master_ep[1] fingerprint = None if len(connection_parts) > 2: fingerprint = connection_parts[2] else: # we do not have a fingerprint, do not attempt to verify the remote cert verify = False print("Contacting cluster at {}".format(master_ip)) info = get_connection_info( master_ip, master_port, token, cluster_type="dqlite", verify_peer=verify, fingerprint=fingerprint, worker=worker, ) # Get the cluster_cidr from kube-proxy args cluster_cidr = get_cluster_cidr() if "cluster_cidr" in info and info["cluster_cidr"] != cluster_cidr: print( "WARNING: Joining a cluster that has a different CIDR. " "The kube-proxy CIDR configuration will be overwritten." ) print( f"Cluster CIDR: {info['cluster_cidr']} -- Node CIDR: {cluster_cidr}(will be overwritten)" ) update_kubeproxy_cidr(info["cluster_cidr"]) if worker: join_dqlite_worker_node(info, master_ip, master_port, token) else: join_dqlite_master_node(info, master_ip) def update_apiserver_proxy(master_ip, api_port): """ Update the apiserver-proxy configuration """ lock_path = os.path.expandvars("${SNAP_DATA}/var/lock") lock = "{}/no-apiserver-proxy".format(lock_path) if os.path.exists(lock): os.remove(lock) # add the initial control plane endpoint addresses = [{"address": "{}:{}".format(master_ip, api_port)}] traefik_providers = os.path.expandvars("${SNAP_DATA}/args/traefik/provider-template.yaml") traefik_providers_out = os.path.expandvars("${SNAP_DATA}/args/traefik/provider.yaml") with open(traefik_providers) as f: p = yaml.safe_load(f) p["tcp"]["services"]["kube-apiserver"]["loadBalancer"]["servers"] = addresses with open(traefik_providers_out, "w") as out_file: yaml.dump(p, out_file) try_set_file_permissions(traefik_providers_out) service("restart", "apiserver-proxy") def rebuild_token_based_auth_configs(info): # We need to make sure token-auth is enabled in this node too. if not is_token_auth_enabled(): enable_token_auth(info["admin_token"]) else: replace_admin_token(info["admin_token"]) subprocess.check_call( [f"{snap_path}/actions/common/utils.sh", "create_user_certs_and_configs"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) def print_worker_usage(): """ Print Worker usage """ print("") print("The node has joined the cluster and will appear in the nodes list in a few seconds.") print("") print("This worker node gets automatically configured with the API server endpoints.") print( "If the API servers are behind a loadbalancer please set the '--refresh-interval' to '0s' in:" ) print(" /var/snap/microk8s/current/args/apiserver-proxy") print("and replace the API server endpoints with the one provided by the loadbalancer in:") print(" /var/snap/microk8s/current/args/traefik/provider.yaml") print("") def join_dqlite_worker_node(info, master_ip, master_port, token): """ Join this node as a worker to a cluster running dqlite. :param info: dictionary with the connection information :param master_ip: the IP of the master node we contacted to connect to the cluster :param master_port: the port of the mester node we contacted to connect to the cluster :param token: the token to pass to the master in order to authenticate with it """ hostname_override = info["hostname_override"] if info["ca_key"] is not None: print( "Joining process failed. Make sure the cluster you connect to supports joining worker nodes." ) exit(1) store_remote_ca(info["ca"]) update_apiserver(info.get("api_authz_mode"), info.get("apiport")) store_base_kubelet_args(info["kubelet_args"]) update_kubelet_node_ip(info["kubelet_args"], hostname_override) update_kubelet_hostname_override(info["kubelet_args"]) update_cert_auth_kubeproxy(token, master_ip, master_port) update_cert_auth_kubelet(token, master_ip, master_port) subprocess.check_call( [f"{snap()}/actions/common/utils.sh", "create_worker_kubeconfigs"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, ) store_callback_token(info["callback_token"]) update_apiserver_proxy(master_ip, info["apiport"]) mark_worker_node() mark_no_cert_reissue() print_worker_usage() def join_dqlite_master_node(info, master_ip): """ Join this node to a cluster running dqlite. :param info: dictionary with the connection information :param master_ip: the IP of the master node we contacted to connect to the cluster """ # The cluster we want to join may be either token-auth based or x509-auth based. # The way to identify the cluster type is to look for the "admin_token" in the info # we got back from the cluster we try to join. # In the case of token-auth we need to: # - create the known_tokens.csv file (if it does not exist) with the admin token # - turn on token-auth on kube-apiserver # - create the token based admin kubeconfig # - recreate the kubelet, proxy, scheduler, controller kubeconfigs for the new ca # - restart kubelite # In the case of x509-auth we need to: # - recreate the admin/client, kubelet, proxy, scheduler, controller kubeconfigs for the new ca # - restart kubelite hostname_override = info["hostname_override"] store_cert("ca.crt", info["ca"]) store_cert("ca.key", info["ca_key"]) store_cert("serviceaccount.key", info["service_account_key"]) if "admin_token" in info: # We try to join a cluster where token-auth is in place. rebuild_token_based_auth_configs(info) else: # We are joining a x509-auth based cluster rebuild_x509_auth_client_configs() update_apiserver(info.get("api_authz_mode"), info.get("apiport")) store_base_kubelet_args(info["kubelet_args"]) update_kubelet_node_ip(info["kubelet_args"], hostname_override) update_kubelet_hostname_override(info["kubelet_args"]) store_callback_token(info["callback_token"]) if "etcd_servers" in info: set_arg("--etcd-servers", info["etcd_servers"], "kube-apiserver") if info.get("etcd_ca"): store_cert("remote-etcd-ca.crt", info["etcd_ca"]) set_arg("--etcd-cafile", "${SNAP_DATA}/certs/remote-etcd-ca.crt", "kube-apiserver") if info.get("etcd_cert"): store_cert("remote-etcd.crt", info["etcd_cert"]) set_arg("--etcd-certfile", "${SNAP_DATA}/certs/remote-etcd.crt", "kube-apiserver") if info.get("etcd_key"): store_cert("remote-etcd.key", info["etcd_key"]) set_arg("--etcd-keyfile", "${SNAP_DATA}/certs/remote-etcd.key", "kube-apiserver") mark_no_dqlite() service("restart", "k8s-dqlite") service("restart", "apiserver") else: update_dqlite(info["cluster_cert"], info["cluster_key"], info["voters"], hostname_override) # We want to update the local CNI yaml but we do not want to apply it. # The cni is applied already in the cluster we join try_initialise_cni_autodetect_for_clustering(master_ip, apply_cni=False) mark_no_cert_reissue() def join_etcd(connection_parts, verify=True): """ Configure node to join an etcd cluster. :param connection_parts: connection string parts """ token = connection_parts[1] master_ep = connection_parts[0].rsplit(":", 1) master_ip = master_ep[0] master_port = master_ep[1] callback_token = generate_callback_token() info = get_connection_info(master_ip, master_port, token, callback_token=callback_token) # check api authn mode from response. default is fallback to Token api_authn_mode = info.get("api_authn_mode", "Token") if api_authn_mode not in ["Cert", "Token"]: print("Error: Unknown API auth mode '{api_authn_mode}' received from control plane node.") print("Please update this MicroK8s node to the latest version before joining.") exit(7) # Get the cluster_cidr from kube-proxy args cluster_cidr = get_cluster_cidr() if "cluster_cidr" in info and info["cluster_cidr"] != cluster_cidr: print( "WARNING: Joining a cluster that has a different CIDR. " "The kube-proxy CIDR configuration will be overwritten." ) print( f"Cluster CIDR: {info['cluster_cidr']} -- Node CIDR: {cluster_cidr}(will be overwritten)" ) update_kubeproxy_cidr(info["cluster_cidr"]) store_base_kubelet_args(info["kubelet_args"]) update_kubelet_hostname_override(info["kubelet_args"]) hostname_override = None if "hostname_override" in info: hostname_override = info["hostname_override"] update_kubelet_node_ip(info["kubelet_args"], hostname_override) store_remote_ca(info["ca"]) update_flannel(info["etcd"], master_ip, master_port, token) if api_authn_mode == "Token": update_kubeproxy(info["kubeproxy"], info["ca"], master_ip, info["apiport"]) update_kubelet(info["kubelet"], info["ca"], master_ip, info["apiport"]) elif api_authn_mode == "Cert": update_cert_auth_kubeproxy(info["kubeproxy"], master_ip, master_port) update_cert_auth_kubelet(info["kubelet"], master_ip, master_port) subprocess.check_call( [ f"{snap()}/actions/common/utils.sh", "create_worker_kubeconfigs", master_ip, info["apiport"], ], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, ) else: assert False, "this should never happen" mark_worker_node() mark_no_cert_reissue() def mark_join_in_progress(): """ Mark node as currently being in the process of joining a cluster. """ lock_file = "{}/var/lock/join-in-progress".format(snapdata_path) open(lock_file, "a").close() os.chmod(lock_file, 0o700) def unmark_join_in_progress(): """ Unmark as joining cluster; join operation has finished. """ lock_file = "{}/var/lock/join-in-progress".format(snapdata_path) if os.path.exists(lock_file): os.unlink(lock_file) def mark_no_dqlite(): """ Mark node to not run k8s-dqlite service. """ lock_file = "{}/var/lock/no-k8s-dqlite".format(snapdata_path) open(lock_file, "a").close() os.chmod(lock_file, 0o700) @click.command( context_settings={"ignore_unknown_options": True, "help_option_names": ["-h", "--help"]} ) @click.argument("connection", required=True) @click.option( "--worker", "worker", default=False, flag_value="as-worker", help="Join as a worker only node." ) @click.option( "--controlplane", "worker", flag_value="as-master", help="Join running the control plane on HA clusters. (default)", ) @click.option( "--skip-verify", is_flag=True, required=False, default=False, help="Skip the certificate verification of the node we are joining to. (default: false)", ) @click.option( "--disable-low-memory-guard", is_flag=True, required=False, default=False, help="Disable the low memory guard. (default: false)", ) def join(connection, worker, skip_verify, disable_low_memory_guard): """ Join the node to a cluster CONNECTION: the cluster connection endpoint in format :/ """ try: connection_parts = get_valid_connection_parts(connection) except InvalidConnectionError as err: print("Invalid connection:", err) sys.exit(1) verify = not skip_verify if is_low_memory_guard_enabled() and disable_low_memory_guard: os.remove(os.path.expandvars("$SNAP_DATA/var/lock/low-memory-guard.lock")) if is_low_memory_guard_enabled() and not worker: print( """ This node does not have enough RAM to host the Kubernetes control plane services and join the database quorum. You may consider joining this node as a worker instead: microk8s join {connection} --worker If you would still like to join the cluster as a control plane node, use: microk8s join {connection} --disable-low-memory-guard """.format( connection=connection ) ) sys.exit(1) mark_join_in_progress() if is_node_running_dqlite(): join_dqlite(connection_parts, verify, worker) else: join_etcd(connection_parts, verify) unmark_join_in_progress() print("Successfully joined the cluster.") sys.exit(0) if __name__ == "__main__": join(prog_name="microk8s join") ================================================ FILE: scripts/wrappers/leave.py ================================================ #!/usr/bin/python3 import json import os import shutil import socket import subprocess import sys import time import click import netifaces import yaml from common.cluster.utils import ( is_node_running_dqlite, service, unmark_no_cert_reissue, restart_all_services, is_node_dqlite_worker, rebuild_x509_auth_client_configs, ) snapdata_path = os.environ.get("SNAP_DATA") snap_path = os.environ.get("SNAP") ca_cert_file = "{}/certs/ca.remote.crt".format(snapdata_path) callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path) server_cert_file = "{}/certs/server.remote.crt".format(snapdata_path) cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path) cluster_backup_dir = "{}/var/kubernetes/backend.backup".format(snapdata_path) def reset_current_dqlite_worker_installation(): """ Take a node out of a cluster """ print("Configuring services.", flush=True) disable_apiserver_proxy() os.remove(ca_cert_file) service("stop", "apiserver") service("stop", "k8s-dqlite") rebuild_x509_auth_client_configs() print("Generating new cluster certificates.", flush=True) reinit_cluster() for config_file in ["kubelet", "kube-proxy"]: shutil.copyfile( "{}/default-args/{}".format(snap_path, config_file), "{}/args/{}".format(snapdata_path, config_file), ) unmark_no_cert_reissue() unmark_worker_node() restart_all_services() apply_cni() def disable_apiserver_proxy(): """ Stop apiserver-proxy """ lock_path = os.path.expandvars("${SNAP_DATA}/var/lock") lock = "{}/no-apiserver-proxy".format(lock_path) if not os.path.exists(lock): open(lock, "a").close() service("stop", "apiserver-proxy") def unmark_worker_node(): """ Unmark a node as being part of a cluster not running the control plane by deleting a var/lock/clustered.lock """ locks = ["clustered.lock", "no-k8s-dqlite"] for lock in locks: lock_file = "{}/var/lock/{}".format(snapdata_path, lock) if not os.path.isfile(lock_file): print("Not in clustering mode.") exit(2) os.remove(lock_file) def reset_current_etcd_installation(): """ Take a node out of a cluster """ lock_file = "{}/var/lock/clustered.lock".format(snapdata_path) if not os.path.isfile(lock_file): print("Not in clustering mode.") exit(2) os.remove(lock_file) os.remove(ca_cert_file) os.remove(callback_token_file) os.remove(server_cert_file) for config_file in ["kubelet", "flanneld", "kube-proxy"]: shutil.copyfile( "{}/default-args/{}".format(snap_path, config_file), "{}/args/{}".format(snapdata_path, config_file), ) for user in ["proxy", "kubelet"]: config = "{}/credentials/{}.config".format(snapdata_path, user) shutil.copyfile("{}.backup".format(config), config) subprocess.check_call("{}/microk8s-stop.wrapper".format(snap_path).split()) waits = 10 while waits > 0: try: subprocess.check_call("{}/microk8s-start.wrapper".format(snap_path).split()) break except subprocess.CalledProcessError: print("Services not ready to start. Waiting...") time.sleep(5) waits -= 1 unmark_no_cert_reissue() def reset_current_dqlite_installation(): """ Take a node out of a dqlite cluster """ if is_leader_without_successor(): print( "This node currently holds the only copy of the Kubernetes " "database so it cannot leave the cluster." ) print( "To remove this node you can either first remove all other " "nodes with 'microk8s remove-node' or" ) print("form a highly available cluster by adding at least three nodes.") exit(3) # We need to: # 1. Stop the apiserver # 2. Send a DELETE request to any member of the dqlite cluster # 3. wipe out the existing installation my_ep, other_ep = get_dqlite_endpoints() service("stop", "apiserver") service("stop", "k8s-dqlite") time.sleep(10) delete_dqlite_node(my_ep, other_ep) print("Generating new cluster certificates.", flush=True) reinit_cluster() rebuild_x509_auth_client_configs() service("start", "k8s-dqlite") service("start", "apiserver") apply_cni() unmark_no_cert_reissue() restart_all_services() def apply_cni(): waits = 10 # type: int print("Waiting for node to start.", end=" ", flush=True) time.sleep(10) while waits > 0: try: subprocess.check_call( "{}/microk8s-kubectl.wrapper get service/kubernetes".format(snap_path).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) subprocess.check_call( "{}/microk8s-kubectl.wrapper apply -f {}/args/cni-network/cni.yaml".format( snap_path, snapdata_path ).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) break except subprocess.CalledProcessError: print(".", end=" ", flush=True) time.sleep(5) waits -= 1 print(" ") def reinit_cluster(): shutil.rmtree(cluster_dir, ignore_errors=True) os.mkdir(cluster_dir) if os.path.isfile("{}/cluster.crt".format(cluster_backup_dir)): # reuse the certificates we had before the cluster formation shutil.copy( "{}/cluster.crt".format(cluster_backup_dir), "{}/cluster.crt".format(cluster_dir) ) shutil.copy( "{}/cluster.key".format(cluster_backup_dir), "{}/cluster.key".format(cluster_dir) ) else: # This node never joined a cluster. A cluster was formed around it. hostname = socket.gethostname() # type: str ip = "127.0.0.1" # type: str shutil.copy( "{}/certs/csr-dqlite.conf.template".format(snap_path), "{}/var/tmp/csr-dqlite.conf".format(snapdata_path), ) subprocess.check_call( "{}/bin/sed -i s/HOSTNAME/{}/g {}/var/tmp/csr-dqlite.conf".format( snap_path, hostname, snapdata_path ).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) subprocess.check_call( "{}/bin/sed -i s/HOSTIP/{}/g {}/var/tmp/csr-dqlite.conf".format( snap_path, ip, snapdata_path ).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) subprocess.check_call( "{0}/openssl.wrapper req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes " "-keyout {1}/var/kubernetes/backend/cluster.key " "-out {1}/var/kubernetes/backend/cluster.crt " "-subj /CN=k8s -config {1}/var/tmp/csr-dqlite.conf -extensions v3_ext".format( snap_path, snapdata_path ).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) # We reset to the default port and address init_data = {"Address": "127.0.0.1:19001"} with open("{}/init.yaml".format(cluster_dir), "w") as f: yaml.dump(init_data, f) def is_leader_without_successor(): """Checks if the current node is safe to be removed. Check if this node acts as a leader to a cluster with more than one nodes where there is no other node to take over the leadership. :return: True if this node is the leader without a successor. """ out = subprocess.check_output( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s .cluster".format( snappath=snap_path, dbdir=cluster_dir ).split() ) voters = 0 data = json.loads(out.decode()) ep_addresses = [] for ep in data: ep_addresses.append((ep["Address"], ep["Role"])) # Role == 0 means we are voters if ep["Role"] == 0: voters += 1 local_ips = [] for interface in netifaces.interfaces(): if netifaces.AF_INET not in netifaces.ifaddresses(interface): continue for link in netifaces.ifaddresses(interface)[netifaces.AF_INET]: local_ips.append(link["addr"]) is_voter = False for ep in ep_addresses: for ip in local_ips: if "{}:".format(ip) in ep[0]: # ep[1] == ep[Role] == 0 means we are voters if ep[1] == 0: is_voter = True if voters == 1 and is_voter and len(ep_addresses) > 1: # We have one voter in the cluster and the current node is the only voter # and there are other nodes that depend on this node. return True else: return False def get_dqlite_endpoints(): """ Return the endpoints the current node has on dqlite and the endpoints of the rest of the nodes. :return: two lists with the endpoints """ out = subprocess.check_output( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s .cluster".format( snappath=snap_path, dbdir=cluster_dir ).split() ) data = json.loads(out.decode()) ep_addresses = [] for ep in data: ep_addresses.append(ep["Address"]) local_ips = [] for interface in netifaces.interfaces(): if netifaces.AF_INET not in netifaces.ifaddresses(interface): continue for link in netifaces.ifaddresses(interface)[netifaces.AF_INET]: local_ips.append(link["addr"]) my_ep = [] other_ep = [] for ep in ep_addresses: found = False for ip in local_ips: if "{}:".format(ip) in ep: my_ep.append(ep) found = True if not found: other_ep.append(ep) return my_ep, other_ep def delete_dqlite_node(delete_node, dqlite_ep): if len(delete_node) > 0 and "127.0.0.1" not in delete_node[0]: for ep in dqlite_ep: try: cmd = ( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s".format( snappath=snap_path, dbdir=cluster_dir ).split() ) cmd.append(".remove {}".format(delete_node[0])) subprocess.check_output(cmd) break except Exception as err: print("Contacting node {} failed. Error:".format(ep)) print(repr(err)) exit(2) @click.command(context_settings={"help_option_names": ["-h", "--help"]}) def leave(): """ The node will depart from the cluster it is in. """ if is_node_running_dqlite(): if is_node_dqlite_worker(): reset_current_dqlite_worker_installation() else: reset_current_dqlite_installation() else: reset_current_etcd_installation() sys.exit(0) if __name__ == "__main__": leave(prog_name="microk8s leave") ================================================ FILE: scripts/wrappers/refresh_certs.py ================================================ #!/usr/bin/python3 import click import os import subprocess from dateutil.parser import parse import datetime from common.utils import ( exit_if_no_root, ) from common.cluster.utils import ( is_token_auth_enabled, rebuild_x509_auth_client_configs, ) snapdata_path = os.environ.get("SNAP_DATA") snap_path = os.environ.get("SNAP") backup_dir = "{}/certs-backup/".format(snapdata_path) certs = { "ca.crt": "CA", "server.crt": "server", "front-proxy-client.crt": "front proxy client", } def check_certificate(): """ Print the days until the current certificate expires """ try: for file in certs.keys(): cmd = "{}/openssl.wrapper x509 -enddate -noout -in {}/certs/{}".format( snap_path, snapdata_path, file ) cert_expire = subprocess.check_output(cmd.split()) cert_expire_date = cert_expire.decode().split("=") date = parse(cert_expire_date[1]) diff = date - datetime.datetime.now(datetime.timezone.utc) click.echo("The {} certificate will expire in {} days.".format(certs[file], diff.days)) except subprocess.CalledProcessError as e: click.echo("Failed to get certificate info. {}".format(e)) exit(4) def undo_refresh(): """ Revert last certificate operation """ if not os.path.exists(backup_dir): click.echo("No previous backup found") exit(1) try: subprocess.check_call("cp -r {}/certs {}/".format(backup_dir, snapdata_path).split()) subprocess.check_call("cp -r {}/credentials {}".format(backup_dir, snapdata_path).split()) except subprocess.CalledProcessError: click.echo("Failed to recover certificates") exit(4) restart() def restart(service="all"): """Restart microk8s services""" if service == "all": click.echo("Restarting, please wait.") try: subprocess.check_call("{}/microk8s-stop.wrapper".format(snap_path).split()) except subprocess.CalledProcessError: pass try: subprocess.check_call("{}/microk8s-start.wrapper".format(snap_path).split()) except subprocess.CalledProcessError: click.echo("Failed to start MicroK8s after reverting the certificates") exit(4) else: click.echo("Restarting service {}.".format(service)) try: subprocess.check_call("snapctl restart microk8s.daemon-{}".format(service).split()) except subprocess.CalledProcessError: click.echo("Failed to restart service microk8s.daemon-{}.".format(service)) exit(4) def update_configs(): """ Update all kubeconfig files used by the client and the services """ if is_token_auth_enabled(): p = subprocess.Popen( ["bash", "-c", ". {}/actions/common/utils.sh; update_configs".format(snap_path)] ) p.communicate() else: rebuild_x509_auth_client_configs() restart("kubelite") restart("cluster-agent") def take_backup(): """ Backup the current certificates and credentials """ try: subprocess.check_call("mkdir -p {}".format(backup_dir).split()) subprocess.check_call("cp -r {}/certs {}".format(snapdata_path, backup_dir).split()) subprocess.check_call("cp -r {}/credentials {}".format(snapdata_path, backup_dir).split()) except subprocess.CalledProcessError as e: click.echo("Failed to backup the current CA. {}".format(e)) exit(10) def reproduce_all_root_ca_certs(): """ Produce the CA and the rest of the needed certificates (eg service, front-proxy) """ subprocess.check_call("rm -rf {}/certs/ca.crt".format(snapdata_path).split()) subprocess.check_call("rm -rf {}/certs/csr.conf".format(snapdata_path).split()) p = subprocess.Popen( ["bash", "-c", ". {}/actions/common/utils.sh; produce_certs".format(snap_path)] ) p.communicate() subprocess.check_call("rm -rf .slr".split()) click.echo("Creating new kubeconfig file") update_configs() msg = """ The CA certificates have been replaced. Kubernetes will restart the pods of your workloads. Any worker nodes you may have in your cluster need to be removed and \ re-joined to become aware of the new CA. """ click.echo(msg) def reproduce_front_proxy_client_cert(): """ Produce the front proxy client certificate """ subprocess.check_call("rm -rf {}/certs/front-proxy-client.crt".format(snapdata_path).split()) subprocess.check_call( [ "bash", "-c", ". {}/actions/common/utils.sh; refresh_csr_conf; gen_proxy_client_cert".format( snap_path ), ] ) subprocess.check_call("rm -rf .slr".split()) restart("kubelite") def reproduce_server_cert(): """ Produce the server certificate """ subprocess.check_call("rm -rf {}/certs/server.crt".format(snapdata_path).split()) subprocess.check_call( [ "bash", "-c", ". {}/actions/common/utils.sh; refresh_csr_conf; gen_server_cert".format(snap_path), ] ) subprocess.check_call("rm -rf .slr".split()) restart("kubelite") restart("cluster-agent") def refresh_cert(cert): """ Refresh the selected certificate with an autogenerated CA """ click.echo("Taking a backup of the current certificates under {}".format(backup_dir)) take_backup() try: click.echo("Creating new certificates") if cert == "ca.crt": reproduce_all_root_ca_certs() elif cert == "server.crt": reproduce_server_cert() elif cert == "front-proxy-client.crt": reproduce_front_proxy_client_cert() else: # this should never happen click.echo("Unknown certificate to refresh") except subprocess.CalledProcessError: click.echo("Failed to produce new certificates. Reverting.") undo_refresh() exit(20) def install_certs(ca_dir): """ Recreate service certificate and front proxy using a user provided CA :param ca_dir: path to the ca.crt and ca.key """ subprocess.check_call("cp {}/ca.crt {}/certs/".format(ca_dir, snapdata_path).split()) subprocess.check_call("cp {}/ca.key {}/certs/".format(ca_dir, snapdata_path).split()) p = subprocess.Popen( ["bash", "-c", ". {}/actions/common/utils.sh; gen_server_cert".format(snap_path)] ) p.communicate() def validate_certificates(ca_dir): """ Perform some basic testing of the user provided CA :param ca_dir: path to the ca.crt and ca.key """ if not os.path.isfile("{}/ca.crt".format(ca_dir)) or not os.path.isfile( "{}/ca.key".format(ca_dir) ): click.echo("Could not find ca.crt and ca.key files in {}".format(ca_dir)) exit(30) try: cmd = "{}/openssl.wrapper rsa -in {}/ca.key -check -noout -out /dev/null".format( snap_path, ca_dir ) subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.CalledProcessError as e: click.echo("CA private key is invalid. {}".format(e)) exit(31) try: cmd = "{}/openssl.wrapper x509 -in {}/ca.crt -text -noout -out /dev/null".format( snap_path, ca_dir ) subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.CalledProcessError as e: click.echo("CA certificate is invalid. {}".format(e)) exit(32) def install_ca(ca_dir): """ Install the user provided CA :param ca_dir: path to the user provided CA files """ click.echo("Validating provided certificates") validate_certificates(ca_dir) click.echo("Taking a backup of the current certificates under {}".format(backup_dir)) take_backup() click.echo("Installing provided certificates") try: install_certs(ca_dir) except subprocess.CalledProcessError: click.echo("Failed to produce new certificates. Reverting.") undo_refresh() exit(20) click.echo("Creating new kubeconfig file") update_configs() msg = """ The CA certificates have been replaced. Kubernetes will restart the pods of your workloads. Any worker nodes you may have in your cluster need to be removed and re-joined to become aware of the new CA. """ click.echo(msg) @click.command( name="refresh-certs", help="Autogenerate a certificate or replace the root CA with the one found in CA_DIR.\n" "Omit the CA_DIR and use the --cert flag to autogenerate a new certificate.", ) @click.argument("ca_dir", required=False, default=None, type=click.Path(exists=True)) @click.option("-u", "--undo", is_flag=True, default=False, help="Revert the last refresh performed") @click.option( "-c", "--check", is_flag=True, default=False, help="Check the expiration time of the installed CA", ) @click.option( "-e", "--cert", type=click.Choice(certs.keys()), help="The certificate to be autogenerated", ) @click.option( "-h", "--help", is_flag=True, default=False, ) def refresh_certs(ca_dir, undo, check, cert, help): if help: show_help() exit(0) if ca_dir is not None and (undo or check or cert): click.echo("Please do not set any options in combination with the CA_DIR.") exit(1) operations_selected = 0 for op in [undo, check, cert]: if op: operations_selected += 1 if operations_selected > 1: click.echo("Please select only one of the options -c, -u or -e.") exit(2) # Operations here will need root privileges as some of the credentials # and certificates are used by system services. exit_if_no_root() if check: check_certificate() exit(0) if undo: undo_refresh() exit(0) if not ca_dir: if not cert: click.echo("Please use the '--cert' flag to select the certificate you need refreshed.") click.echo("") click.echo("Available certificate options:") click.echo("'server.crt': refreshes the server certificate") click.echo("'front-proxy-client.crt': refreshes the front proxy client certificate") click.echo("'ca.crt': refreshes the root CA and all certificates created from it.") click.echo( " Warning: refreshing the root CA requires nodes to leave and re-join the cluster" ) exit(3) else: refresh_cert(cert) else: install_ca(ca_dir) def show_help(): msg = """Usage: microk8s refresh-certs [OPTIONS] [CA_DIR] Replace the CA certificates with the ca.crt and ca.key found in CA_DIR. Omit the CA_DIR argument and use the '--cert' flag to auto-generate a new CA or any other certificate. Options: -c, --check Check the expiration time of the installed certificates -e, --cert The certificate to be autogenerated, must be one of {} -u, --undo Revert the last refresh performed -h, --help Show this message and exit.""" click.echo(msg.format(list(certs.keys()))) if __name__ == "__main__": refresh_certs() ================================================ FILE: scripts/wrappers/remove_node.py ================================================ #!/usr/bin/python3 import json import os import shutil import subprocess import sys import click import netifaces from ipaddress import ip_address from common.cluster.utils import ( try_set_file_permissions, is_node_running_dqlite, is_token_auth_enabled, ) snap_path = os.environ.get("SNAP") snapdata_path = os.environ.get("SNAP_DATA") callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path) cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path) def remove_dqlite_node(node, force=False): try: # If node is an IP address, find the node name. is_node_ip = True try: ip_address(node) except ValueError: is_node_ip = False if is_node_ip: node_info = subprocess.check_output( "{}/microk8s-kubectl.wrapper get no -o json".format(snap_path).split() ) info = json.loads(node_info.decode()) found = False for n in info["items"]: if found: break for a in n["status"]["addresses"]: if a["type"] == "InternalIP" and a["address"] == node: node = n["metadata"]["name"] found = True break # Make sure this node exists node_info = subprocess.check_output( "{}/microk8s-kubectl.wrapper get no {} -o json".format(snap_path, node).split() ) info = json.loads(node_info.decode()) node_address = None for a in info["status"]["addresses"]: if a["type"] == "InternalIP": node_address = a["address"] break if not node_address: print("Node {} is not part of the cluster.".format(node)) exit(1) node_ep = None my_ep, other_ep = get_dqlite_endpoints() for ep in other_ep: if ep.startswith("{}:".format(node_address)): node_ep = ep if node_ep and force: delete_dqlite_node([node_ep], my_ep) elif node_ep and not force: print( "Removal failed. Node {} is registered with dqlite. " "Please, run first 'microk8s leave' on the departing node. \n" "If the node is not available anymore and will never attempt to join the cluster " "in the future use the '--force' flag \n" "to unregister the node while removing it.".format(node) ) exit(1) except subprocess.CalledProcessError: print("Node {} does not exist in Kubernetes.".format(node)) if force: print("Attempting to remove {} from dqlite.".format(node)) # Make sure we do not have the node in dqlite. # We assume the IP is provided to denote the my_ep, other_ep = get_dqlite_endpoints() for ep in other_ep: if ep.startswith("{}:".format(node)): print("Removing node entry found in dqlite.") delete_dqlite_node([ep], my_ep) exit(1) remove_node(node) def remove_node(node): try: # Make sure this node exists subprocess.check_call( "{}/microk8s-kubectl.wrapper get no {}".format(snap_path, node).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) except subprocess.CalledProcessError: print("Node {} does not exist.".format(node)) exit(1) if is_token_auth_enabled(): remove_kubelet_token(node) remove_callback_token(node) subprocess.check_call( "{}/microk8s-kubectl.wrapper delete no {}".format(snap_path, node).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) def remove_kubelet_token(node): """ Remove a token for a node in the known tokens :param node: the name of the node """ file = "{}/credentials/known_tokens.csv".format(snapdata_path) backup_file = "{}.backup".format(file) token = "system:node:{}".format(node) # That is a critical section. We need to protect it. with open(backup_file, "w") as back_fp: with open(file, "r") as fp: for _, line in enumerate(fp): if token in line: continue back_fp.write("{}".format(line)) try_set_file_permissions(backup_file) shutil.copyfile(backup_file, file) def get_dqlite_endpoints(): """ Return the endpoints the current node has on dqlite and the endpoints of the rest of the nodes. :return: two lists with the endpoints """ out = subprocess.check_output( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s .cluster".format( snappath=snap_path, dbdir=cluster_dir ).split() ) data = json.loads(out.decode()) ep_addresses = [] for ep in data: ep_addresses.append(ep["Address"]) local_ips = [] for interface in netifaces.interfaces(): if netifaces.AF_INET not in netifaces.ifaddresses(interface): continue for link in netifaces.ifaddresses(interface)[netifaces.AF_INET]: local_ips.append(link["addr"]) my_ep = [] other_ep = [] for ep in ep_addresses: found = False for ip in local_ips: if "{}:".format(ip) in ep: my_ep.append(ep) found = True if not found: other_ep.append(ep) return my_ep, other_ep def delete_dqlite_node(delete_node, dqlite_ep): if len(delete_node) > 0 and "127.0.0.1" not in delete_node[0]: for ep in dqlite_ep: try: cmd = ( "{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt " "-k {dbdir}/cluster.key -f json k8s".format( snappath=snap_path, dbdir=cluster_dir ).split() ) cmd.append(".remove {}".format(delete_node[0])) subprocess.check_output(cmd) break except Exception as err: print("Contacting node {} failed. Error:".format(ep)) print(repr(err)) exit(2) def remove_callback_token(node): """ Remove a callback token :param node: the node """ tmp_file = "{}.tmp".format(callback_tokens_file) if not os.path.isfile(callback_tokens_file): open(callback_tokens_file, "a+") os.chmod(callback_tokens_file, 0o600) with open(tmp_file, "w") as backup_fp: os.chmod(tmp_file, 0o600) with open(callback_tokens_file, "r+") as callback_fp: # Entries are of the format: 'node_hostname:agent_port token' # We need to get the node_hostname part for line in callback_fp: parts = line.split(":") if parts[0] == node: continue else: backup_fp.write(line) try_set_file_permissions(tmp_file) shutil.move(tmp_file, callback_tokens_file) @click.command() @click.argument("node", required=True) @click.option( "--force", is_flag=True, required=False, default=False, help="Force the node removal operation. (default: false)", ) def reset(node, force): """ Remove a node from the cluster """ if is_node_running_dqlite(): remove_dqlite_node(node, force) else: remove_node(node) sys.exit(0) if __name__ == "__main__": reset(prog_name="microk8s remove-node") ================================================ FILE: scripts/wrappers/reset.py ================================================ #!/usr/bin/python3 import click import os import subprocess import shutil import sys import time from common.utils import ( get_available_addons, get_current_arch, get_status, snap_data, wait_for_ready, snap_common, is_cluster_locked, exit_if_no_permission, ensure_started, exit_if_no_root, ) KUBECTL = os.path.expandvars("$SNAP/microk8s-kubectl.wrapper") def exit_if_multinode(): """ Exit if we cannot get the list of nodes or if we are in a multinode cluster """ cmd = [KUBECTL, "get", "no", "-o", "name"] res = run_silently(cmd) if not res: print("Failed to query the cluster nodes.") sys.exit(1) nodes = res.split() if len(nodes) > 1: print( "This is a multi-node MicroK8s deployment. Reset is applicable for single node clusters." ) print("Please remove all joined nodes before calling reset.") sys.exit(0) def disable_addon(repo, addon, args=None): """ Try to disable an addon. Ignore any errors and/or silence any output. """ if args is None: args = [] wait_for_ready(timeout=30) script = snap_common() / "addons" / repo / "addons" / addon / "disable" if not script.exists(): return try: subprocess.run([script, *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except (FileNotFoundError, PermissionError, subprocess.CalledProcessError) as e: print(f"Ignoring error: {e}", file=sys.stderr) wait_for_ready(timeout=30) def disable_addons(destroy_storage): """ Iterate over all addons and disable the enabled ones. """ print("Disabling all addons") available_addons_info = get_available_addons(get_current_arch()) enabled, disabled = get_status(available_addons_info, True) for addon in available_addons_info: # Do not disable HA if addon["name"] == "ha-cluster": continue # Do not disable the community repository if addon["name"] == "community": continue print(f"Disabling addon : {addon['repository']}/{addon['name']}") # Do not disable disabled addons if addon in disabled: continue if (addon["name"] == "hostpath-storage" or addon["name"] == "storage") and destroy_storage: disable_addon(addon["repository"], f"{addon['name']}", ["destroy-storage"]) else: disable_addon(addon["repository"], addon["name"]) print("All addons are disabled.") def cni(operation="apply"): """ Apply of delete the CNI manifest of our cluster if it exists. Silence any output. """ cni_yaml = f"{snap_data()}/args/cni-network/cni.yaml" if os.path.exists(cni_yaml): if operation == "apply": print("Setting up the CNI") else: print("Deleting the CNI") subprocess.run( [KUBECTL, operation, "-f", cni_yaml], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) def clean_cluster(): """ Clean up the cluster by: 1. Delete any resources left 2. Restart so the cluster resets 3. Delete any locks and addon binaries. """ cmd = [KUBECTL, "get", "ns", "-o=name"] res = run_silently(cmd) nss = [] if res: nss = res.split() resources = ["replicationcontrollers", "daemonsets", "deployments", "statefulsets"] for ns in nss: ns_name = ns.split("/")[-1] print(f"Cleaning resources in namespace {ns_name}") for rs in resources: # we remove first resources that are automatically recreated so we do not risk race conditions # during which a deployment for example is recreated while any tokens are missing cmd = [KUBECTL, "delete", "--all", rs, "-n", ns_name, "--timeout=60s"] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) remove_extra_resources(ns_name) remove_crds() remove_priority_classes() remove_storage_classes() remove_non_namespaced_resources() for ns in nss: ns_name = ns.split("/")[-1] if ns_name in ["default", "kube-public", "kube-system", "kube-node-lease"]: continue print(f"Removing {ns}") cmd = [KUBECTL, "delete", ns, "--timeout=60s"] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) restart_cluster() remove_binaries() reset_cert_reissue() def remove_storage_classes(): """ Remove storage classes. Silence any output. """ print("Removing StorageClasses") cmd = [KUBECTL, "get", "storageclasses", "-o=name"] res = run_silently(cmd) classes = [] if res: classes = res.split() for cs in classes: if "microk8s-hostpath" in cs: continue cmd = [KUBECTL, "delete", cs, "--timeout=60s"] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def remove_crds(): print("Removing CRDs") cmd = [ KUBECTL, "delete", "--all", "customresourcedefinitions.apiextensions.k8s.io", "--timeout=60s", ] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def remove_non_namespaced_resources(): resources = [ "MutatingWebhookConfigurations", "ValidatingAdmissionPolicies", "ValidatingAdmissionPolicyBindings", "ValidatingWebhookConfigurations", "CertificateSigningRequests", ] for r in resources: print(f"Removing {r}") cmd = [KUBECTL, "delete", "--all", r, "--timeout=60s"] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def remove_priority_classes(): """ Remove priority classes. Silence any output. """ print("Removing PriorityClasses") cmd = [KUBECTL, "get", "priorityclasses", "-o=name"] res = run_silently(cmd) classes = [] if res: classes = res.split() for cs in classes: if "system-cluster-critical" in cs or "system-node-critical" in cs: continue cmd = [KUBECTL, "delete", cs, "--timeout=60s"] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def reset_cert_reissue(): """ Remove the certificate no refresh lock. """ lock = snap_data() / "var" / "lock" / "no-cert-reissue" if lock.exists(): cmd = f"rm -rf {lock}" subprocess.run(cmd.split()) def remove_binaries(): """ Remove binaries pulled in by addons. """ bins_dir = snap_data() / "bin" if bins_dir.exists(): shutil.rmtree(bins_dir) def restart_cluster(): """ Restart a cluster by calling the stop and start wrappers. """ print("Restarting cluster") cmd = [f"{os.environ['SNAP']}/microk8s-stop.wrapper"] subprocess.run(cmd) time.sleep(5) cmd = [f"{os.environ['SNAP']}/microk8s-start.wrapper"] subprocess.run(cmd) wait_for_ready(timeout=30) ensure_started() def remove_extra_resources(ns_name): # Remove all resource types except the standard k8s apiservices themselves cmd = [KUBECTL, "api-resources", "-o", "name", "--verbs=delete", "--namespaced=true"] res = run_silently(cmd) if not res: return extra_resources = res.split() for rs in extra_resources: if rs.startswith("apiservices"): continue cmd = [KUBECTL, "delete", "--all", rs, "-n", ns_name, "--timeout=3s"] subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def run_silently(cmd): result = subprocess.run( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) try: result.check_returncode() except subprocess.CalledProcessError: return None return result.stdout.decode("utf-8") def preflight_check(): """ Preliminary checks to see if we can proceed with cluster reset """ exit_if_no_root() exit_if_no_permission() is_cluster_locked() ensure_started() wait_for_ready(timeout=30) exit_if_multinode() @click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option( "--destroy-storage", is_flag=True, required=False, default=True, help="Also destroy storage. (default: true)", ) def reset(destroy_storage): """ Return the MicroK8s node to the default initial state. This process may take some time. All addons will be disabled and the configuration will be reinitialized. """ preflight_check() disable_addons(destroy_storage) cni("delete") clean_cluster() cni("apply") if __name__ == "__main__": reset(prog_name="microk8s reset") ================================================ FILE: scripts/wrappers/status.py ================================================ #!/usr/bin/python3 import argparse import sys from common.utils import ( exit_if_no_permission, exit_if_stopped, is_cluster_locked, is_ha_enabled, get_dqlite_info, wait_for_ready, is_cluster_ready, get_available_addons, get_current_arch, get_addon_by_name, get_status, get_etcd_info, is_external_etcd, ) def print_short(isReady, enabled_addons, disabled_addons): if isReady: print("microk8s is running") print("addons:") if enabled_addons and len(enabled_addons) > 0: for enabled in enabled_addons: print("{}/{}: enabled".format(enabled["repository"], enabled["name"])) if disabled_addons and len(disabled_addons) > 0: for disabled in disabled_addons: print("{}/{}: disabled".format(disabled["repository"], disabled["name"])) else: print("microk8s is not running. Use microk8s inspect for a deeper inspection.") def print_pretty(isReady, enabled_addons, disabled_addons): console_formatter = "{:>3} {:<20} # ({}) {}" if isReady: print("microk8s is running") etcd_endpoints = get_etcd_info() if not is_ha_enabled(): print("high-availability: no") if etcd_endpoints: print("{:>2}{}".format("", "datastore endpoints:")) for endpoint in etcd_endpoints: print("{:>4}{}".format("", endpoint)) elif not is_external_etcd(): info = get_dqlite_info() if ha_cluster_formed(info): print("high-availability: yes") else: print("high-availability: no") masters = "none" standby = "none" for node in info: if node[1] == "voter": if masters == "none": masters = "{}".format(node[0]) else: masters = "{} {}".format(masters, node[0]) if node[1] == "standby": if standby == "none": standby = "{}".format(node[0]) else: standby = "{} {}".format(standby, node[0]) print("{:>2}{} {}".format("", "datastore master nodes:", masters)) print("{:>2}{} {}".format("", "datastore standby nodes:", standby)) elif etcd_endpoints: print("{:>2}{}".format("", "datastore endpoints:")) for endpoint in etcd_endpoints: print("{:>4}{}".format("", endpoint)) print("addons:") if enabled_addons and len(enabled_addons) > 0: print("{:>2}{}".format("", "enabled:")) for enabled in enabled_addons: print( console_formatter.format( "", enabled["name"], enabled["repository"], enabled["description"] ) ) if disabled_addons and len(disabled_addons) > 0: print("{:>2}{}".format("", "disabled:")) for disabled in disabled_addons: print( console_formatter.format( "", disabled["name"], disabled["repository"], disabled["description"] ) ) else: print("microk8s is not running. Use microk8s inspect for a deeper inspection.") def print_short_yaml(isReady, enabled_addons, disabled_addons): print("microk8s:") print("{:>2}{} {}".format("", "running:", isReady)) if isReady: print("addons:") for enabled in enabled_addons: print(" {}/{}: enabled".format(enabled["repository"], enabled["name"])) for disabled in disabled_addons: print(" {}/{}: disabled".format(disabled["repository"], disabled["name"])) else: print( "{:>2}{} {}".format( "", "message:", "microk8s is not running. Use microk8s inspect for a deeper inspection.", ) ) def print_yaml(isReady, enabled_addons, disabled_addons): print("microk8s:") print("{:>2}{} {}".format("", "running:", isReady)) print("{:>2}".format("high-availability:")) ha_enabled = is_ha_enabled() print("{:>2}{} {}".format("", "enabled:", ha_enabled)) if ha_enabled: info = get_dqlite_info() print("{:>2}{}".format("", "nodes:")) for node in info: print("{:>6}address: {:<1}".format("- ", node[0])) print("{:>6}role: {:<1}".format("", node[1])) if isReady: print("{:>2}".format("addons:")) for enabled in enabled_addons: print("{:>4}name: {:<1}".format("- ", enabled["name"])) print("{:>4}repository: {:<1}".format("", enabled["repository"])) print("{:>4}description: {:<1}".format("", enabled["description"])) print("{:>4}version: {:<1}".format("", enabled["version"])) print("{:>4}status: enabled".format("")) for disabled in disabled_addons: print("{:>4}name: {:<1}".format("- ", disabled["name"])) print("{:>4}repository: {:<1}".format("", disabled["repository"])) print("{:>4}description: {:<1}".format("", disabled["description"])) print("{:>4}version: {:<1}".format("", disabled["version"])) print("{:>4}status: disabled".format("")) else: print( "{:>2}{} {}".format( "", "message:", "microk8s is not running. Use microk8s inspect for a deeper inspection.", ) ) def print_addon_status(enabled): if len(enabled) > 0: print("enabled") else: print("disabled") def ha_cluster_formed(info): voters = 0 for node in info: if node[1] == "voter": voters += 1 ha_formed = False if voters > 2: ha_formed = True return ha_formed if __name__ == "__main__": exit_if_no_permission() exit_if_stopped() is_cluster_locked() # initiate the parser with a description parser = argparse.ArgumentParser( description="Microk8s cluster status check.", prog="microk8s status" ) parser.add_argument( "--format", help="print cluster and addon status, output can be in yaml, pretty or short", default="pretty", choices={"pretty", "yaml", "short"}, ) parser.add_argument( "-w", "--wait-ready", action="store_true", help="wait until the cluster is in ready state" ) parser.add_argument( "-t", "--timeout", help="specify a timeout in seconds when waiting for the cluster to be ready.", type=int, default=0, ) parser.add_argument("-a", "--addon", help="check the status of an addon.", default="all") parser.add_argument( "--yaml", action="store_true", help="DEPRECATED, use '--format yaml' instead" ) # read arguments from the command line args = parser.parse_args() wait_ready = args.wait_ready timeout = args.timeout yaml_short = args.yaml if wait_ready: is_ready = wait_for_ready(timeout) else: is_ready = is_cluster_ready() available_addons = get_available_addons(get_current_arch()) if args.addon != "all": available_addons = get_addon_by_name(available_addons, args.addon) enabled, disabled = get_status(available_addons, is_ready) if args.addon != "all": print_addon_status(enabled) else: if args.format == "yaml": print_yaml(is_ready, enabled, disabled) elif args.format == "short": print_short(is_ready, enabled, disabled) else: if yaml_short: print_short_yaml(is_ready, enabled, disabled) else: print_pretty(is_ready, enabled, disabled) if not is_ready: sys.exit(1) ================================================ FILE: scripts/wrappers/upgrade.py ================================================ #!/usr/bin/python3 import os import argparse import subprocess import requests import urllib3 from common.utils import exit_if_no_permission, is_cluster_locked urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) CLUSTER_API = "cluster/api/v1.0" snapdata_path = os.environ.get("SNAP_DATA") snap_path = os.environ.get("SNAP") def upgrade_master(upgrade, phase): """ Upgrade the master node :param upgrade: which upgrade to call :param phase: prepare, commit or rollback :return: """ try: upgrade_script = "{}/upgrade-scripts/{}/{}-master.sh".format(snap_path, upgrade, phase) if os.path.isfile(upgrade_script): print("Running {}-upgrade script".format(phase)) out = subprocess.check_output(upgrade_script) print(out) except subprocess.CalledProcessError as e: print("{}-upgrade step failed".format(phase)) raise e def node_upgrade(upgrade, phase, node_ep, token): """ Upgrade a node :param upgrade: which upgrade to call :param phase: prepare, commit or rollback :param node_ep: the node endpoint the nodes cluster agent listens at :param token: the callback token to access the node :return: """ try: upgrade_script = "{}/upgrade-scripts/{}/{}-node.sh".format(snap_path, upgrade, phase) if os.path.isfile(upgrade_script): remote_op = {"callback": token, "phase": phase, "upgrade": upgrade} # TODO: handle ssl verification res = requests.post( "https://{}/{}/upgrade".format(node_ep, CLUSTER_API), json=remote_op, verify=False ) if res.status_code != 200: print("Failed to perform a {} on node {}".format(remote_op["upgrade"], node_ep)) raise Exception("Failed to {} on {}".format(phase, node_ep)) except subprocess.CalledProcessError as e: print("{} upgrade step failed on {}".format(phase, node_ep)) raise e def rollback(upgrade): """ The rollback method that oversees the rollback of the cluster :param upgrade: which upgrade to call """ # We should get the nodes without checking their existence from the API server node_info = get_nodes_info(safe=False) upgrade_log_file = "{}/var/log/upgrades/{}.log".format(snapdata_path, upgrade) with open(upgrade_log_file, "r") as log: for line in log: parts = line.split(" ") node_type = parts[0] phase = parts[1] if node_type == "node": node_ep = parts[2].rstrip() else: node_ep = "localhost" if phase == "commit": print("Rolling back {} on {}".format(phase, node_ep)) if node_type == "node": tokens = [t for ep, t in node_info if node_ep.startswith(ep)] if len(tokens) != 0: token = tokens[0] node_upgrade(upgrade, "rollback", node_ep, token) else: upgrade_master(upgrade, "rollback") def run_upgrade(upgrade): """ The upgrade method that oversees the upgrade of the cluster :param upgrade: which upgrade to call """ node_info = get_nodes_info() log_dir = "{}/var/log/upgrades".format(snapdata_path) upgrade_log_file = "{}/{}.log".format(log_dir, upgrade) try: os.makedirs(log_dir, exist_ok=True) with open(upgrade_log_file, "w") as log: log.writelines(["master prepare"]) upgrade_master(upgrade, "prepare") log.flush() for node_ep, token in node_info: log.writelines(["\nnode prepare {}".format(node_ep)]) node_upgrade(upgrade, "prepare", node_ep, token) log.flush() for node_ep, token in node_info: log.writelines(["\nnode commit {}".format(node_ep)]) node_upgrade(upgrade, "commit", node_ep, token) log.flush() log.writelines(["\nmaster commit"]) upgrade_master(upgrade, "commit") log.flush() except Exception as e: print("Error in upgrading. Error: {}".format(e)) log.close() rollback(upgrade) exit(2) def get_nodes_info(safe=True): """ Get the list of node endpoints and tokens in the cluster :return: """ callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path) node_info = [] if safe: try: nodes = subprocess.check_output( "{}/microk8s-kubectl.wrapper get no".format(snap_path).split() ) if os.path.isfile(callback_tokens_file): with open(callback_tokens_file, "r+") as fp: for _, line in enumerate(fp): parts = line.split() node_ep = parts[0] host = node_ep.split(":")[0] if host not in nodes.decode(): print("Node {} not present".format(host)) continue node_info.append((parts[0], parts[1])) except subprocess.CalledProcessError: print("Error in gathering cluster node information. Upgrade aborted.") exit(1) else: if os.path.isfile(callback_tokens_file): with open(callback_tokens_file, "r+") as fp: for _, line in enumerate(fp): parts = line.split() node_info.append((parts[0], parts[1])) return node_info def list_upgrades(): """ List all available upgrades """ upgrades_dir = "{}/upgrade-scripts/".format(snap_path) upgrades = [ dI for dI in os.listdir(upgrades_dir) if os.path.isdir(os.path.join(upgrades_dir, dI)) ] for u in upgrades: print(u) if __name__ == "__main__": exit_if_no_permission() is_cluster_locked() # initiate the parser with a description parser = argparse.ArgumentParser(description="MicroK8s supervised upgrades.", prog="upgrade") parser.add_argument( "-l", "--list", help="list available upgrades", nargs="?", const=True, type=bool ) parser.add_argument( "-r", "--run", help="run a specific upgrade script", nargs="?", type=str, default=None ) parser.add_argument( "-u", "--undo", help="rollback a specific upgrade", nargs="?", type=str, default=None ) args = parser.parse_args() run = args.run ls = args.list undo = args.undo if ls: list_upgrades() elif run: run_upgrade(run) elif undo: rollback(undo) else: print("Unknown option") exit(1) exit(0) ================================================ FILE: scripts/wrappers/version.py ================================================ #!/usr/bin/env python3 from os import getenv def get_snap_version() -> str: return getenv("SNAP_VERSION", "(unknown)") def get_snap_revision() -> str: return getenv("SNAP_REVISION", "(unknown)") def print_versions() -> None: version = get_snap_version() revision = get_snap_revision() print(f"MicroK8s {version} revision {revision}") if __name__ == "__main__": print_versions() ================================================ FILE: snap/hooks/configure ================================================ #!/usr/bin/env bash set -eux source $SNAP/actions/common/utils.sh use_snap_env # Make sure either the install hook has run or we are refreshing an already existing snap as indicated # by the existence of certificates. if [ ! -f "${SNAP_DATA}/var/lock/installed.lock" ] && [ ! -f ${SNAP_DATA}/certs/csr.conf.template ] then exit 0 fi if is_strict then echo "Checking snap interfaces..." check_snap_interfaces # Check for interfaces but do not start until this script has run. else # In classic we do not make use of the status flag. Here we clean any "blocked" message that may have # come from a failed strict deployment snapctl set-health okay fi need_api_restart=false need_cluster_agent_restart=false need_proxy_restart=false need_kubelet_restart=false need_controller_restart=false need_scheduler_restart=false # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box. if ! [ -e /var/lib/kubelet ] && ln -s $SNAP_COMMON/var/lib/kubelet /var/lib/kubelet; then echo "/var/lib/kubelet linked to $SNAP_COMMON" fi SNAP_DATA_CURRENT=`echo "${SNAP_DATA}" | sed -e "s,${SNAP_REVISION},current,"` # Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration. if ! [ -e /var/lib/calico ]; then if ln -s $SNAP_DATA_CURRENT/var/lib/calico /var/lib/calico; then echo "/var/lib/calico linked to $SNAP_DATA_CURRENT/var/lib/calico" fi fi # Try to symlink standard CNI Kubernetes directories. if ! [ -e /etc/cni/net.d ]; then if mkdir -p /etc/cni && ln -s $SNAP_DATA_CURRENT/args/cni-network /etc/cni/net.d; then echo "/etc/cni/net.d linked to $SNAP_DATA_CURRENT/args/cni-network" fi fi if ! [ -e /opt/cni/bin ]; then if mkdir -p /opt/cni && ln -s $SNAP_DATA_CURRENT/opt/cni/bin /opt/cni/bin; then echo "/opt/cni/bin linked to $SNAP_DATA_CURRENT/opt/cni/bin" fi fi # If the configurations directory is missing from SNAP_COMMON, we are upgrading from an older MicroK8s version. if [ ! -d "${SNAP_COMMON}/etc/launcher" ] then mkdir -p "${SNAP_COMMON}/etc/launcher" fi # snap set microk8s config="$(cat config.yaml)" config="$(snapctl get config || true)" if [ ! -z "${config}" ] then # Only write config file if not already applied. applied_config="$(cat ${SNAP_COMMON}/etc/launcher/snap-set.yaml.applied || true)" if [ -z "${applied_config}" ] || [ "${config}" != "${applied_config}" ] then echo "${config}" > "${SNAP_COMMON}/etc/launcher/snap-set.yaml" fi fi # If the addons directory is missing from SNAP_COMMON, then we are upgrading from an older MicroK8s version. if [ ! -d "${SNAP_COMMON}/addons" ] then mkdir -p ${SNAP_COMMON}/addons snap_current=`echo "${SNAP}" | sed -e "s,${SNAP_REVISION},current,"` for addon in $(cat "${SNAP}/addons/.auto-add"); do "${SNAP}/git.wrapper" clone "${snap_current}/addons/${addon}" "${SNAP_COMMON}/addons/${addon}" done fi if [ ! -d "${SNAP_COMMON}/plugins" ] then mkdir -p ${SNAP_COMMON}/plugins fi #Allow the ability to add external IPs to the csr, by moving the csr.conf.template to SNAP_DATA # TODO(neoaggelos): investigate if this is needed if [ ! -f ${SNAP_DATA}/certs/csr.conf.template ] then cp ${SNAP}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.template fi # Enable the aggregation layer if ! grep "requestheader-client-ca-file" ${SNAP_DATA}/args/kube-apiserver then echo "Patching requestheader-client-ca-file argument" # Add a new line at the end echo "" >> ${SNAP_DATA}/args/kube-apiserver echo "--requestheader-client-ca-file=\${SNAP_DATA}/certs/front-proxy-ca.crt" >> ${SNAP_DATA}/args/kube-apiserver need_api_restart=true fi # Enable the aggregation layer (continue) if ! grep -E -- '--(requestheader-allowed-names|requestheader-extra-headers-prefix|requestheader-group-headers|requestheader-username-headers|proxy-client-cert-file|proxy-client-key-file)=' ${SNAP_DATA}/args/kube-apiserver then echo "Enabling Enable the aggregation layer" echo "" >> ${SNAP_DATA}/args/kube-apiserver echo '--requestheader-allowed-names=front-proxy-client' >> ${SNAP_DATA}/args/kube-apiserver echo '--requestheader-extra-headers-prefix=X-Remote-Extra-' >> ${SNAP_DATA}/args/kube-apiserver echo '--requestheader-group-headers=X-Remote-Group' >> ${SNAP_DATA}/args/kube-apiserver echo '--requestheader-username-headers=X-Remote-User' >> ${SNAP_DATA}/args/kube-apiserver echo '--proxy-client-cert-file=${SNAP_DATA}/certs/front-proxy-client.crt' >> ${SNAP_DATA}/args/kube-apiserver echo '--proxy-client-key-file=${SNAP_DATA}/certs/front-proxy-client.key' >> ${SNAP_DATA}/args/kube-apiserver need_api_restart=true fi # Patch for issue: https://github.com/canonical/microk8s/issues/121 if grep -e "requestheader-client-ca-file=/var/snap/microk8s/.../certs/ca.crt" ${SNAP_DATA}/args/kube-apiserver then "$SNAP/bin/sed" -i 's@requestheader-client-ca-file=/var/snap/microk8s/.../certs/ca.crt@requestheader-client-ca-file=\${SNAP_DATA}/certs/ca.crt@g' ${SNAP_DATA}/args/kube-apiserver fi # Patch for issue: https://github.com/canonical/microk8s/issues/721 if grep -F 'requestheader-client-ca-file=${SNAP_DATA}/certs/ca.crt' ${SNAP_DATA}/args/kube-apiserver then "$SNAP/bin/sed" -i 's@requestheader-client-ca-file=${SNAP_DATA}/certs/ca.crt@requestheader-client-ca-file=${SNAP_DATA}/certs/front-proxy-ca.crt@g' ${SNAP_DATA}/args/kube-apiserver fi # Create the locks directory mkdir -p ${SNAP_DATA}/var/lock/ # Create tmp directory mkdir -p ${SNAP_DATA}/tmp/ # This will allow us to refresh the snap to the more secure version. # We need to make sure the client certificate used in microk8s kubectl is available under $SNAP_DATA # TODO(neoaggelos): investigate whether this is needed if [ ! -f ${SNAP_DATA}/credentials/client.config ] then echo "Patching client config location" mkdir -p ${SNAP_DATA}/credentials/ cp ${SNAP}/client.config ${SNAP_DATA}/credentials/ fi # copy kubectl-env if [ ! -e ${SNAP_DATA}/args/kubectl-env ] && grep -e "\-\-kubeconfig=\${SNAP_DATA}/credentials/client.config" ${SNAP_DATA}/args/kubectl then echo "Making sure we have kubectl environment file" cp ${SNAP}/default-args/kubectl-env ${SNAP_DATA}/args/kubectl-env skip_opt_in_config kubeconfig kubectl fi # copy kubectl if [ ! -e ${SNAP_DATA}/args/kubectl ] then echo "Making sure we have kubectl arguments file" cp ${SNAP}/default-args/kubectl ${SNAP_DATA}/args/kubectl fi # copy traefik if [ ! -e ${SNAP_DATA}/args/traefik ] then echo "Making sure we have traefik configuration" cp -r ${SNAP}/default-args/traefik ${SNAP_DATA}/args/ fi # copy apiserver-proxy if [ ! -e ${SNAP_DATA}/args/apiserver-proxy ] then echo "Making sure we have apiserver-proxy configuration" cp -r ${SNAP}/default-args/apiserver-proxy ${SNAP_DATA}/args/ fi # (1.24 -> 1.25) migrate from traefik to apiserver-proxy if [ -e ${SNAP_DATA}/var/lock/no-traefik ] then touch ${SNAP_DATA}/var/lock/no-apiserver-proxy fi # Upgrading to containerd if [ ! -e ${SNAP_DATA}/args/containerd ] || grep -e "\-\-docker unix://\${SNAP_DATA}/docker.sock" ${SNAP_DATA}/args/kubelet then echo "Making sure we have containerd file" cp ${SNAP_DATA}/args/containerd ${SNAP_DATA}/args/containerd.backup || true cp ${SNAP}/default-args/containerd ${SNAP_DATA}/args/containerd cp ${SNAP_DATA}/args/containerd-template.toml ${SNAP_DATA}/args/containerd-template.toml.backup || true cp ${SNAP}/default-args/containerd-template.toml ${SNAP_DATA}/args/containerd-template.toml cp ${SNAP_DATA}/args/containerd-env ${SNAP_DATA}/args/containerd-env.backup || true cp ${SNAP}/default-args/containerd-env ${SNAP_DATA}/args/containerd-env cp -r ${SNAP}/default-args/cni-network ${SNAP_DATA}/args/ cp ${SNAP}/default-args/ctr ${SNAP_DATA}/args/ctr refresh_opt_in_config container-runtime remote kubelet refresh_opt_in_config container-runtime-endpoint \${SNAP_COMMON}/run/containerd.sock kubelet skip_opt_in_config docker-root kubelet skip_opt_in_config docker kubelet skip_opt_in_config docker-endpoint kubelet snapctl restart ${SNAP_NAME}.daemon-containerd need_kubelet_restart=true if [ -e ${SNAP_DATA}/args/dockerd ] && grep -e "default-runtime=nvidia" ${SNAP_DATA}/args/dockerd then # Deployment used to run docker with nvidia enabled we need to enable nvidia on containerd # Allow for kubelet and containerd to restart sleep 10 ${SNAP}/microk8s-enable.wrapper gpu fi fi # Install default-hooks. # # Updated hooks may contain fixes, so we'll always replace existing files. # Custom hooks are expected to use separate files so that they won't get # overwritten during upgrades. mkdir -p ${SNAP_COMMON}/hooks cp -r --preserve=mode ${SNAP}/default-hooks/* ${SNAP_COMMON}/hooks/ # Make sure the server certificate includes the IP we are using if [ "$(produce_certs)" == "1" ] then rm -rf .srl need_api_restart=true need_proxy_restart=true need_cluster_agent_restart=true fi # Make containerd stream server listen to localhost if [ -e ${SNAP_DATA}/args/containerd-template.toml ] && grep -e "stream_server_address = \"\"" ${SNAP_DATA}/args/containerd-template.toml then "$SNAP/bin/sed" -i 's@stream_server_address = ""@stream_server_address = "127.0.0.1"@g' ${SNAP_DATA}/args/containerd-template.toml if grep -e "stream_server_port = \"10010\"" ${SNAP_DATA}/args/containerd-template.toml then "$SNAP/bin/sed" -i 's@stream_server_port = "10010"@stream_server_port = "0"@g' ${SNAP_DATA}/args/containerd-template.toml fi snapctl restart ${SNAP_NAME}.daemon-containerd need_kubelet_restart=true fi # With v1.15 allow-privileged is removed from kubelet if grep -e "\-\-allow-privileged" ${SNAP_DATA}/args/kubelet then echo "Patching 1.15 allow-privileged" "${SNAP}/bin/sed" -i '/allow-privileged/d' ${SNAP_DATA}/args/kubelet need_kubelet_restart=true fi # Add option to support kata containers if [ -e "${SNAP_DATA}/args/containerd-env" ] && ! grep -e "KATA_PATH" ${SNAP_DATA}/args/containerd-env then echo "" >> "${SNAP_DATA}/args/containerd-env" echo "# You can set the of the kata containers runtime here." >> "${SNAP_DATA}/args/containerd-env" echo "#" >> "${SNAP_DATA}/args/containerd-env" echo "# KATA_PATH=" >> "${SNAP_DATA}/args/containerd-env" echo "#" >> "${SNAP_DATA}/args/containerd-env" echo "PATH=\$PATH:\$KATA_PATH" >> "${SNAP_DATA}/args/containerd-env" fi # Add option to support kata containers if [ -e "${SNAP_DATA}/args/containerd-template.toml" ] && ! grep -e "io.containerd.kata.v2" ${SNAP_DATA}/args/containerd-template.toml then KATA_HANDLER_BEFORE='\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime\]' KATA_HANDLER_AFTER=' [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options] BinaryName = "kata-runtime" ' CD_TOML="${SNAP_DATA}/args/containerd-template.toml" CD_TOML_TMP="${SNAP_DATA}/args/containerd-template.toml.tmp" "$SNAP/usr/bin/gawk" -v kata="${KATA_HANDLER_AFTER}" '/'${KATA_HANDLER_BEFORE}'/{print kata }1' ${CD_TOML} > ${CD_TOML_TMP} mv ${CD_TOML_TMP} ${CD_TOML} fi for dir in ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks do chmod -R ug+rwX ${dir} chmod -R o-rwX ${dir} done group=$(get_microk8s_or_cis_group) # Try to create the snap_microk8s group. Do not fail the installation if something goes wrong if ! getent group ${group} >/dev/null 2>&1 then groupadd ${group} || true fi if getent group ${group} >/dev/null 2>&1 && ! [ -e "${SNAP_DATA}/var/lock/cis-hardening" ] then chgrp ${group} -R ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true fi if [ -e "${SNAP_DATA}/var/lock/cis-hardening" ] then chmod -R g-wr ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true chmod -R o-wr ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true if ! is_strict && [ -e /etc/systemd/system/snap.microk8s.daemon-kubelite.service ] then chmod -R g-wr /etc/systemd/system/snap.microk8s.daemon-kubelite.service chmod -R o-wr /etc/systemd/system/snap.microk8s.daemon-kubelite.service fi fi if ! is_strict then try_copy_users_to_snap_microk8s fi # as only one cni bin dir can be used we will use the one in SNAP_DATA but have links to # the real CNI plugins we distribute in SNAP mkdir -p "${SNAP_DATA}/opt/cni/bin/" ( cd "${SNAP}/opt/cni/bin/" MY_SNAP_DIR=$(dirname "${SNAP}") for i in *; do ln -sf "${MY_SNAP_DIR}/current/opt/cni/bin/$i" "${SNAP_DATA}/opt/cni/bin/${i}"; done ) if ! [ -e "${SNAP_DATA}/opt/cni/bin/flanneld" ] then # cover situation where cilium was installed prior to this update if [ -f "${SNAP_DATA}/opt/cni/bin/loopback" ] && [ -f "${SNAP}/opt/cni/bin/loopback" ]; then rm -f "${SNAP_DATA}/opt/cni/bin/loopback" fi fi if ! [ -f "${SNAP_DATA}/args/flanneld" ] then mkdir -p ${SNAP_DATA}/args/cni-network/ cp -r ${SNAP}/default-args/cni-network/flannel.conflist ${SNAP_DATA}/args/cni-network/ cp ${SNAP}/default-args/flanneld ${SNAP_DATA}/args/ cp ${SNAP}/default-args/flannel-template.conflist ${SNAP_DATA}/args/ cp ${SNAP}/default-args/flannel-network-mgr-config ${SNAP_DATA}/args/ snapctl restart ${SNAP_NAME}.daemon-etcd snapctl restart ${SNAP_NAME}.daemon-containerd snapctl restart ${SNAP_NAME}.daemon-flanneld fi if grep -e "etcd.socket:2379" ${SNAP_DATA}/args/etcd then echo "Using a port for etcd" # TODO: Do something smart in selecting a port refresh_opt_in_config advertise-client-urls https://\${DEFAULT_INTERFACE_IP_ADDR}:12379 etcd refresh_opt_in_config listen-client-urls https://0.0.0.0:12379 etcd refresh_opt_in_config client-cert-auth true etcd refresh_opt_in_config trusted-ca-file \${SNAP_DATA}/certs/ca.crt etcd refresh_opt_in_config cert-file \${SNAP_DATA}/certs/server.crt etcd refresh_opt_in_config key-file \${SNAP_DATA}/certs/server.key etcd snapctl restart ${SNAP_NAME}.daemon-etcd refresh_opt_in_config etcd-servers https://127.0.0.1:12379 kube-apiserver refresh_opt_in_config etcd-cafile \${SNAP_DATA}/certs/ca.crt kube-apiserver refresh_opt_in_config etcd-certfile \${SNAP_DATA}/certs/server.crt kube-apiserver refresh_opt_in_config etcd-keyfile \${SNAP_DATA}/certs/server.key kube-apiserver need_api_restart=true fi if ! grep -e "service-account-issuer" ${SNAP_DATA}/args/kube-apiserver then echo "--service-account-issuer='https://kubernetes.default.svc'" >> ${SNAP_DATA}/args/kube-apiserver need_api_restart=true fi if ! grep -e "service-account-signing-key-file" ${SNAP_DATA}/args/kube-apiserver then echo '--service-account-signing-key-file=${SNAP_DATA}/certs/serviceaccount.key' >> ${SNAP_DATA}/args/kube-apiserver need_api_restart=true fi # RemoveSelfLink feature flag is removed after 1.24 if grep -e "feature-gates=RemoveSelfLink" ${SNAP_DATA}/args/kube-apiserver then "${SNAP}/bin/sed" -i '/feature-gates=RemoveSelfLink/d' "$SNAP_DATA/args/kube-apiserver" need_api_restart=true fi if remove_docker_specific_args then need_kubelet_restart=true fi # scheduler --address flag is removed after 1.24 if grep -e "--address=" ${SNAP_DATA}/args/kube-scheduler then "${SNAP}/bin/sed" -i '/--address=/d' "$SNAP_DATA/args/kube-scheduler" fi # controller-manager --address flag is removed after 1.24 if grep -e "--address=" ${SNAP_DATA}/args/kube-controller-manager then "${SNAP}/bin/sed" -i '/--address=/d' "$SNAP_DATA/args/kube-controller-manager" need_api_restart=true fi # etcd --enable-v2 flag is removed after 3.6 if grep -e "--enable-v2=" ${SNAP_DATA}/args/etcd then "${SNAP}/bin/sed" -i '/--enable-v2=/d' "$SNAP_DATA/args/etcd" snapctl restart ${SNAP_NAME}.daemon-etcd fi if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] then if grep -e "\-\-etcd-cafile /var/snap/microk8s/.*/ca.remote.crt" ${SNAP_DATA}/args/flanneld then skip_opt_in_config etcd-cafile flanneld refresh_opt_in_config etcd-cafile \${SNAP_DATA}/certs/ca.remote.crt flanneld fi if grep -e "\-\-etcd-certfile /var/snap/microk8s/.*/server.remote.crt" ${SNAP_DATA}/args/flanneld then skip_opt_in_config etcd-certfile flanneld refresh_opt_in_config etcd-certfile \${SNAP_DATA}/certs/server.remote.crt flanneld fi fi # This patches flanneld conf template by adding cniversion if it does not exist. if [ -e ${SNAP_DATA}/args/flannel-template.conflist ] && ! grep -e "cniVersion" ${SNAP_DATA}/args/flannel-template.conflist then "$SNAP/bin/sed" -i 's@"name": "microk8s-flannel-network",@"name": "microk8s-flannel-network",\n "cniVersion": "0.3.1",@g' ${SNAP_DATA}/args/flannel-template.conflist snapctl restart ${SNAP_NAME}.daemon-flanneld snapctl restart ${SNAP_NAME}.daemon-containerd fi if [ ! -f ${SNAP_DATA}/args/cluster-agent ] then cp ${SNAP}/default-args/cluster-agent ${SNAP_DATA}/args/cluster-agent fi if ! grep -e "\-\-timeout" ${SNAP_DATA}/args/cluster-agent then refresh_opt_in_config timeout 240 cluster-agent snapctl restart ${SNAP_NAME}.daemon-containerd fi if ! grep -e "\-\-ip-masq" ${SNAP_DATA}/args/flanneld then refresh_opt_in_config ip-masq true flanneld snapctl restart ${SNAP_NAME}.daemon-flanneld fi if grep -e "\-\-cluster-cidr=10.152.183.0/24" ${SNAP_DATA}/args/kube-proxy then refresh_opt_in_config cluster-cidr 10.1.0.0/16 kube-proxy need_proxy_restart=true fi if [ -e ${SNAP_DATA}/var/lock/stopped.lock ] then snapctl stop ${SNAP_NAME}.daemon-kubelite fi # Enable kubelite if ! [ -e ${SNAP_DATA}/var/lock/lite.lock ] then touch "${SNAP_DATA}/var/lock/lite.lock" if ! [ -e ${SNAP_DATA}/args/kubelite ] then cp ${SNAP}/default-args/kubelite ${SNAP_DATA}/args/kubelite fi date if [ -e ${SNAP_DATA}/var/lock/stopped.lock ] then snapctl stop ${SNAP_NAME}.daemon-kubelite else snapctl start ${SNAP_NAME}.daemon-kubelite fi fi if ! [ -e ${SNAP_DATA}/args/kubelite ] then cp ${SNAP}/default-args/kubelite ${SNAP_DATA}/args/kubelite need_api_restart=true fi # Removed --insecure-port argument if grep -e "\-\-insecure\-port" ${SNAP_DATA}/args/kube-apiserver then $SNAP/bin/sed -i '/\-\-insecure\-port/d' ${SNAP_DATA}/args/kube-apiserver need_api_restart=true fi # Are we using etcd or some other non-dqlite datastore? if ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] && ! grep -e "\-\-storage-backend=dqlite" ${SNAP_DATA}/args/kube-apiserver then set_service_not_expected_to_start k8s-dqlite snapctl stop ${SNAP_NAME}.daemon-k8s-dqlite fi # Configure the API sever to talk to the external dqlite if [ -e ${SNAP}/default-args/k8s-dqlite ] && ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] && grep -e "\-\-storage-backend=dqlite" ${SNAP_DATA}/args/kube-apiserver then echo "Reconfiguring the API server for dqlite" cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite cp ${SNAP}/default-args/k8s-dqlite-env ${SNAP_DATA}/args/k8s-dqlite-env need_api_restart=true snapctl stop ${SNAP_NAME}.daemon-kubelite refresh_opt_in_local_config etcd-servers unix://\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver $SNAP/bin/sed -i '/\-\-storage\-backend=dqlite/d' ${SNAP_DATA}/args/kube-apiserver storage_dir="$(get_opt_in_config '--storage-dir' 'kube-apiserver')" if ! [ -z $storage_dir ] then refresh_opt_in_local_config storage-dir "$storage_dir" k8s-dqlite fi $SNAP/bin/sed -i '/\-\-storage\-dir/d' ${SNAP_DATA}/args/kube-apiserver snapctl restart ${SNAP_NAME}.daemon-k8s-dqlite fi # Disable the features misbehaving with dqlite if grep -e '^\-\-etcd\-servers=.*kine.sock:12379' ${SNAP_DATA}/args/kube-apiserver then if grep -e '^\-\-feature\-gates' ${SNAP_DATA}/args/kube-apiserver then for feature in ListFromCacheSnapshot SizeBasedListCostEstimate DetectCacheInconsistency do if ! grep -e "$feature" "$SNAP_DATA/args/kube-apiserver" then "${SNAP}/bin/sed" -i '/^--feature-gates=/ s/$/,'$feature'=false/' "$SNAP_DATA/args/kube-apiserver" fi done else echo '--feature-gates=ListFromCacheSnapshot=false,SizeBasedListCostEstimate=false,DetectCacheInconsistency=false' >> ${SNAP_DATA}/args/kube-apiserver fi need_api_restart=true fi # Fix hard-coded snap revision numbers in worker node services of existing clusters # https://github.com/canonical/microk8s/pull/3554 for svc in kubelet proxy; do cfg="${SNAP_DATA}/credentials/${svc}.config" if [ -e ${cfg} ]; then sed -i 's,/var/snap/microk8s/[x0-9]*/,/var/snap/microk8s/current/,' "${cfg}" || true fi done # (1.26) Removed --log-dir argument from kubelet if grep -e "\-\-log\-dir" "${SNAP_DATA}/args/kubelet" then "${SNAP}/bin/sed" -i '/\-\-log\-dir/d' "${SNAP_DATA}/args/kubelet" need_api_restart=true fi # Refresh calico if needed refresh_calico_if_needed # Refresh apiserver proxy snapctl restart "${SNAP_NAME}.daemon-apiserver-proxy" if is_strict then enable_snap fi # if we are refreshing in a no-flanneld we need to restart the CNI pods because they mount parts of $SNAP_DATA if [ -e "${SNAP_DATA}/var/lock/no-flanneld" ] then touch "${SNAP_DATA}/var/lock/snapdata-mounts-need-reload" fi # Restart reconfigured services if ${need_api_restart} || ${need_proxy_restart} || ${need_controller_restart} || ${need_kubelet_restart} then if [ -e ${SNAP_DATA}/var/lock/lite.lock ] then snapctl restart ${SNAP_NAME}.daemon-kubelite else echo "Unable to restart service" exit 1 fi fi if ${need_cluster_agent_restart} then snapctl restart ${SNAP_NAME}.daemon-cluster-agent fi ================================================ FILE: snap/hooks/connect-plug-network-control ================================================ #!/usr/bin/env bash set -eu for link in cni0 cilium_vxlan do if $SNAP/sbin/ip link show ${link} then $SNAP/sbin/ip link delete ${link} fi done ${SNAP}/meta/hooks/configure ================================================ FILE: snap/hooks/disconnect-plug-network-control ================================================ #!/usr/bin/env bash set -eu for link in cni0 cilium_vxlan do if $SNAP/sbin/ip link show ${link} then $SNAP/sbin/ip link delete ${link} fi done ================================================ FILE: snap/hooks/install ================================================ #!/usr/bin/env bash set -eux source $SNAP/actions/common/utils.sh if [ -f "${SNAP_DATA}/var/lock/installed.lock" ] then exit 0 fi use_snap_env # LXD Specific Checks if ! is_strict && cat /proc/1/environ | grep "container=lxc" &> /dev/null then # make sure the /dev/kmsg is available, indicating a potential missing profile if [ ! -c "/dev/kmsg" ] # kmsg is a character device then printf -- '\e[1;31mERROR: \033[0m the lxc profile for MicroK8s might be missing. \n' printf -- '\t Refer to this help document to get MicroK8s working in with LXD: \n' printf -- '\t https://microk8s.io/docs/lxd \n' exit 1 fi fi cp -r --preserve=mode ${SNAP}/default-args ${SNAP_DATA}/args mv ${SNAP_DATA}/args/certs.d/localhost__32000 ${SNAP_DATA}/args/certs.d/localhost:32000 SNAP_DATA_CURRENT=`echo "${SNAP_DATA}" | sed -e "s,${SNAP_REVISION},current,"` # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box. if ! [ -e /var/lib/kubelet ] && ln -s $SNAP_COMMON/var/lib/kubelet /var/lib/kubelet; then echo "/var/lib/kubelet linked to $SNAP_COMMON" fi # Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration. if ! [ -e /var/lib/calico ]; then if ln -s $SNAP_DATA_CURRENT/var/lib/calico /var/lib/calico; then echo "/var/lib/calico linked to $SNAP_DATA_CURRENT/var/lib/calico" fi fi # Try to symlink standard CNI Kubernetes directories. if ! [ -e /etc/cni/net.d ]; then if mkdir -p /etc/cni && ln -s $SNAP_DATA_CURRENT/args/cni-network /etc/cni/net.d; then echo "/etc/cni/net.d linked to $SNAP_DATA_CURRENT/args/cni-network" fi fi if ! [ -e /opt/cni/bin ]; then if mkdir -p /opt/cni && ln -s $SNAP_DATA_CURRENT/opt/cni/bin /opt/cni/bin; then echo "/opt/cni/bin linked to $SNAP_DATA_CURRENT/opt/cni/bin" fi fi # Create the credentials directory mkdir -p ${SNAP_DATA}/credentials # Create the certificates mkdir ${SNAP_DATA}/certs # Allow the ability to add external IPs to the csr, by moving the csr.conf.template to SNAP_DATA cp ${SNAP}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.template # Copy initial configuration from well-known paths. # This is done prior to any other initialization. # # The following paths are checked in order for a config file. # Only the first file found will be used, the rest will be ignored. # # - /etc/microk8s.yaml <-- user-defined config file # - $SNAP_USER_COMMON/.microk8s.yaml <-- user-defined config file, typically in /root/snap/microk8s/common/.microk8s.yaml # - $SNAP_COMMON/.microk8s.yaml <-- user-defined config file, typically in /var/snap/microk8s/common/.microk8s.yaml # - $SNAP/microk8s.default.yaml <-- default config file bundled with the snap # # If the pre-init step fails, the snap installation will fail, as it is considered to be a user error. mkdir -p "${SNAP_COMMON}/etc/launcher" for config_file in "/etc/microk8s.yaml" "$SNAP_USER_COMMON/.microk8s.yaml" "$SNAP_COMMON/.microk8s.yaml" "$SNAP/microk8s.default.yaml"; do if [ -f "${config_file}" ]; then echo "Found config file ${config_file}, will use to initialize cluster." if cp "${config_file}" "${SNAP_COMMON}/etc/launcher/install.yaml"; then "${SNAP}/bin/cluster-agent" init --pre-init --config-file "${SNAP_COMMON}/etc/launcher/install.yaml" break fi fi done # Side-load images from well-known paths. # This is done prior to any other initialization. # # Any *.tar files that are found in these directories will be loaded into containerd after it starts. # # - $SNAP_USER_COMMON/sideload/*.tar <-- typically /root/snap/microk8s/common/sideload/images.tar # - $SNAP_COMMON/sideload/*.tar <-- typically /var/snap/microk8s/common/sideload/images.tar # - $SNAP/sideload/*.tar <-- typically empty, reserved for future use-cases mkdir -p "${SNAP_COMMON}/etc/sideload" for source_dir in "${SNAP_USER_COMMON}" "${SNAP_COMMON}" "${SNAP}"; do cp "${source_dir}/sideload/"*.tar "${SNAP_COMMON}/etc/sideload/" || true done # Produce cluster certificates produce_certs rm -rf .srl create_user_certs_and_configs # Install default-hooks cp -r --preserve=mode ${SNAP}/default-hooks ${SNAP_COMMON}/hooks for dir in ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_COMMON}/hooks do chmod -R ug+rwX ${dir} chmod -R o-rwX ${dir} done if is_strict && snapctl is-connected k8s-kubelet then snapctl restart microk8s.daemon-containerd fi init_cluster mkdir -p "${SNAP_DATA}/var/lock" set_service_not_expected_to_start etcd set_service_not_expected_to_start flanneld set_service_not_expected_to_start apiserver-proxy touch "${SNAP_DATA}/var/lock/ha-cluster" touch "${SNAP_DATA}/var/lock/lite.lock" RESOURCES="${SNAP}/upgrade-scripts/000-switch-to-calico/resources" BACKUP_DIR="${SNAP_DATA}/var/tmp/upgrades/000-switch-to-calico" mkdir -p "${BACKUP_DIR}" mkdir -p "${BACKUP_DIR}/args/cni-network/" cp "${SNAP_DATA}"/args/cni-network/* "${BACKUP_DIR}/args/cni-network/" 2>/dev/null || true rm -rf "${SNAP_DATA}"/args/cni-network/* ${SNAP}/scripts/generate-cni.sh mkdir -p "$SNAP_DATA/opt/cni/bin/" cp -R "$SNAP"/opt/cni/bin/* "$SNAP_DATA"/opt/cni/bin/ # Low memory guard. Enable by default when system RAM is less than 512MB. MEMORY=`cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` if [ $MEMORY -le 524288 ] then touch ${SNAP_DATA}/var/lock/low-memory-guard.lock fi # copy git config mkdir -p ${SNAP_DATA}/args/git cp ${SNAP}/default-args/git/.gitconfig ${SNAP_DATA}/args/git/.gitconfig touch "${SNAP_DATA}/var/lock/installed.lock" ================================================ FILE: snap/hooks/post-refresh ================================================ #!/usr/bin/env bash . $SNAP/actions/common/utils.sh use_snap_env # Run post-refresh hooks $SNAP/usr/bin/python3 $SNAP/scripts/run-lifecycle-hooks.py post-refresh || true ================================================ FILE: snap/hooks/remove ================================================ #!/usr/bin/env bash set -eu source $SNAP/actions/common/utils.sh use_snap_env snapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true # Temporarily start containerd so we can stop and kill all the microk8s containers. snapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true # wait for containerd to start. sleep 5 # Remove any lingering containers and shims. remove_all_containers kill_all_container_shims # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box. if test -L /var/lib/kubelet; then unlink /var/lib/kubelet || true fi # Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration. if test -L /var/lib/calico; then unlink /var/lib/calico || true fi # Try to symlink standard CNI Kubernetes directories. if test -L /etc/cni/net.d; then unlink /etc/cni/net.d || true fi if test -L /opt/cni/bin; then unlink /opt/cni/bin || true fi pod_cidr="$(cat $SNAP_DATA/args/kube-proxy | grep "cluster-cidr" | tr "=" " "| gawk '{print $2}')" if [ -z "$pod_cidr" ] then pod_cidr="$(cat $SNAP_DATA/args/kubelet | grep "pod-cidr" | tr "=" " "| gawk '{print $2}')" if [ -z "$pod_cidr" ] then pod_cidr="$(jq .Network $SNAP_DATA/args/flannel-network-mgr-config | tr -d '\"')" fi fi if ! [ -z "$pod_cidr" ] then iptables -D FORWARD -s "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables -D FORWARD -d "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables-nft -D FORWARD -s "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables-nft -D FORWARD -d "$pod_cidr" -m comment --comment "generated for MicroK8s pods" -j ACCEPT || true iptables-legacy -t nat -F CNI-HOSTPORT-DNAT || true fi snapctl stop ${SNAP_NAME}.daemon-containerd 2>&1 || true # wait for containerd to stop its processes or we will be getting a umount error # because the mount points are busy sleep 10 if ! is_strict then # remove custom sysctl parameters rm -f /etc/sysctl.d/10-microk8s.conf sysctl --system rm -rf /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d || true fi # Clean the container location so we do not snapshot it. rm -rf ${SNAP_COMMON}/var/lib/containerd/* || true rm -rf ${SNAP_COMMON}/run/containerd/* || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/kubelet/pods | cut -d ' ' -f 2 | xargs umount -l) || true # in case this is a pre root-dir fix deployment (cat /proc/mounts | grep ${SNAP_COMMON}/pods | cut -d ' ' -f 2 | xargs umount -l) || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/containerd | cut -d ' ' -f 2 | xargs umount -l) || true (cat /proc/mounts | grep ${SNAP_COMMON}/run/containerd | cut -d ' ' -f 2 | xargs umount) || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/docker | cut -d ' ' -f 2 | xargs umount -l) || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/run/docker | cut -d ' ' -f 2 | xargs umount) || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/kubelet | cut -d ' ' -f 2 | xargs umount) || true # Run remove hooks $SNAP/usr/bin/python3 $SNAP/scripts/run-lifecycle-hooks.py remove || true ================================================ FILE: snap/snapcraft.yaml ================================================ name: microk8s adopt-info: kubernetes-version summary: Kubernetes for workstations and appliances description: |- MicroK8s is a small, fast, secure, single node Kubernetes that installs on just about any Linux box. Use it for offline development, prototyping, testing, or use it on a VM as a small, cheap, reliable k8s for CI/CD. It's also a great k8s for appliances - develop your IoT apps for k8s and deploy them to MicroK8s on your boxes. license: Apache-2.0 grade: stable confinement: classic base: core22 assumes: [snapd2.52] environment: REAL_PATH: $PATH REAL_LD_LIBRARY_PATH: $LD_LIBRARY_PATH REAL_PYTHONPATH: $PYTHONPATH SNAPCRAFT_ARCH_TRIPLET: $CRAFT_ARCH_TRIPLET parts: build-deps: plugin: nil build-snaps: - go/1.24/stable build-attributes: [enable-patchelf] build-packages: - sudo - autoconf - automake - autopoint - autotools-dev - bison - btrfs-progs - libbtrfs-dev - build-essential - curl - flex - git - libjansson-dev - liblz4-dev - libnetfilter-conntrack-dev - libnetfilter-conntrack3 - libnfnetlink-dev - libseccomp-dev - libtool - libuv1-dev - pkg-config - rsync - tcl k8s-dqlite: after: [build-deps] source: build-scripts/components/k8s-dqlite build-attributes: [enable-patchelf] plugin: nil override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh k8s-dqlite etcd: after: [build-deps] plugin: nil source: build-scripts/components/etcd build-attributes: [enable-patchelf] override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh etcd cni: after: [build-deps] plugin: nil source: build-scripts/components/cni build-attributes: [enable-patchelf] override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh cni flannel-cni-plugin: after: [build-deps] plugin: nil source: build-scripts/components/flannel-cni-plugin build-attributes: [enable-patchelf] override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh flannel-cni-plugin flanneld: after: [build-deps] plugin: nil source: build-scripts/components/flanneld build-attributes: [enable-patchelf] override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh flanneld kubernetes: after: [build-deps] plugin: nil source: build-scripts/components/kubernetes build-attributes: [enable-patchelf] override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh kubernetes kubernetes-version: plugin: nil source: build-scripts/components/kubernetes override-build: snapcraftctl set-version "$(./version.sh)" helm: after: [build-deps] plugin: nil source: build-scripts/components/helm override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh helm libmnl: after: [build-deps] plugin: autotools source: https://www.netfilter.org/pub/libmnl/libmnl-1.0.5.tar.bz2 build-attributes: [enable-patchelf] prime: - -usr/local/include libnftnl: after: [libmnl] plugin: autotools source: https://www.netfilter.org/projects/libnftnl/files/libnftnl-1.1.8.tar.bz2 build-attributes: [enable-patchelf] build-environment: - LIBMNL_LIBS: $CRAFT_STAGE/usr/lib prime: - -usr/local/include iptables: after: [libnftnl] source: https://www.netfilter.org/projects/iptables/files/iptables-1.8.6.tar.bz2 build-attributes: [enable-patchelf] plugin: autotools build-environment: - LIBMNL_LIBS: $CRAFT_STAGE/usr/lib - LIBNFTNL_LIBS: $CRAFT_STAGE/usr/lib autotools-configure-parameters: - "--prefix=/usr" - "--exec-prefix=/" - "--disable-shared" - "--enable-static" stage: - -usr - -lib/pkgconfig - -bin/iptables-xml containerd: after: [runc] plugin: nil source: build-scripts/components/containerd override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh containerd build-attributes: [no-patchelf] runc: after: [iptables, build-deps] source: build-scripts/components/runc build-attributes: [no-patchelf] plugin: nil override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh runc bash-utils: plugin: nil build-attributes: [enable-patchelf] stage-packages: - conntrack - coreutils - curl - diffutils - ethtool - gawk - git - grep - hostname - iproute2 - jq - kmod - libatm1 - libnss-resolve - libnss-myhostname - libnss-mymachines - members - nano - net-tools - openssl - procps - python3-rados - python3-rbd - sed - tar - ufw - util-linux - zfsutils-linux - erofs-utils stage: - -etc/bash_completion.d - -etc/cron.d - -etc/depmod.d - -etc/ldap - -etc/logrotate.d - -etc/init.d - -etc/perl - -etc/rsyslog.d - -etc/sudoers.d - -lib/systemd/system - -usr/bin/perl* - -usr/include - -usr/lib/*/*perl* - -usr/share/bash-completion - -usr/share/doc - -usr/share/doc-base - -usr/share/info - -usr/share/initramfs-tools - -usr/share/lintian - -usr/share/man - -usr/share/nano - -usr/share/perl - -usr/share/perl5 - -usr/share/zsh cluster-agent: after: [build-deps] plugin: nil source: build-scripts/components/cluster-agent build-attributes: [enable-patchelf] override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh cluster-agent microk8s-addons: plugin: nil source: build-scripts/addons override-build: ./repositories.sh "${CRAFT_PART_INSTALL}" microk8s-scripts: plugin: nil source: scripts/ override-build: | if [ -d "${CRAFT_PART_INSTALL}" ]; then rm -rf "${CRAFT_PART_INSTALL}/*" fi cp -r . "${CRAFT_PART_INSTALL}/scripts" cp inspect.sh "${CRAFT_PART_INSTALL}/inspect.sh" microk8s-upgrade-scripts: plugin: nil source: upgrade-scripts/ override-build: | if [ -d "${CRAFT_PART_INSTALL}" ]; then rm -rf "${CRAFT_PART_INSTALL}/*" fi cp -r . "${CRAFT_PART_INSTALL}/upgrade-scripts" microk8s: plugin: nil source: microk8s-resources/ override-build: | if [ -d "${CRAFT_PART_INSTALL}" ]; then rm -rf "${CRAFT_PART_INSTALL}/*" fi cp microk8s.default.yaml "${CRAFT_PART_INSTALL}/microk8s.default.yaml" cp -r default-args "${CRAFT_PART_INSTALL}/default-args" cp -r default-hooks "${CRAFT_PART_INSTALL}/default-hooks" cp -r certs "${CRAFT_PART_INSTALL}/certs" cp containerd-profile "${CRAFT_PART_INSTALL}/containerd-profile" cp client.config "${CRAFT_PART_INSTALL}/client.config" cp client.config.template "${CRAFT_PART_INSTALL}/client.config.template" cp kubelet.config.template "${CRAFT_PART_INSTALL}/kubelet.config.template" cp client-x509.config.template "${CRAFT_PART_INSTALL}/client-x509.config.template" cp -r wrappers/* "${CRAFT_PART_INSTALL}/" cp -r actions/ "${CRAFT_PART_INSTALL}/actions" microk8s-completion: after: [build-deps] plugin: nil source: build-scripts/components/microk8s-completion override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh microk8s-completion python-runtime: after: [build-deps] build-attributes: [enable-patchelf] plugin: nil source: build-scripts/components/python override-build: | pip3 install -r requirements.txt build-packages: - python3-dev build-environment: - C_INCLUDE_PATH: /usr/include/python3.10 stage-packages: - libpython3-stdlib - libpython3.10-stdlib - libpython3.10-minimal - python3-pip - python3-setuptools - python3-wheel - python3-venv - python3-minimal - python3-distutils - python3-pkg-resources - python3.10-minimal - python3-openssl - python3-requests - python3-click - python3-dateutil stage: - -usr/lib/python3.11 - -usr/share/doc - -usr/share/lintian - -usr/share/man - -usr/share/python-wheels bom: after: - cluster-agent - cni - containerd - etcd - flannel-cni-plugin - flanneld - helm - k8s-dqlite - kubernetes - microk8s-addons - python-runtime - runc plugin: nil source: . build-packages: - python3-yaml override-build: | ./build-scripts/generate-bom.py > "${CRAFT_PART_INSTALL}/bom.json" apps: microk8s: command: microk8s.wrapper completer: microk8s.bash daemon-etcd: command: run-etcd-with-args daemon: simple daemon-flanneld: command: run-flanneld-with-args daemon: simple daemon-containerd: command: run-containerd-with-args daemon: notify # when stopped send only sigterm # https://forum.snapcraft.io/t/process-lifecycle-on-snap-refresh/140/37 stop-mode: sigterm restart-condition: always start-timeout: 5m daemon-kubelite: command: run-kubelite-with-args daemon: simple after: [daemon-containerd] daemon-apiserver-kicker: command: apiservice-kicker daemon: simple daemon-apiserver-proxy: command: run-apiserver-proxy-with-args daemon: simple daemon-cluster-agent: command: run-cluster-agent-with-args daemon: simple daemon-k8s-dqlite: command: run-k8s-dqlite-with-args daemon: simple dashboard-proxy: command: microk8s-dashboard-proxy.wrapper kubectl: command: microk8s-kubectl.wrapper completer: kubectl.bash add-node: command: microk8s-add-node.wrapper addons: command: microk8s-addons.wrapper refresh-certs: command: microk8s-refresh-certs.wrapper images: command: microk8s-images.wrapper join: command: microk8s-join.wrapper remove-node: command: microk8s-remove-node.wrapper leave: command: microk8s-leave.wrapper ctr: command: microk8s-ctr.wrapper inspect: command: microk8s.wrapper inspect enable: command: microk8s-enable.wrapper disable: command: microk8s-disable.wrapper start: command: microk8s-start.wrapper stop: command: microk8s-stop.wrapper status: command: microk8s-status.wrapper config: command: microk8s-config.wrapper reset: command: microk8s-reset.wrapper istioctl: command: microk8s-istioctl.wrapper linkerd: command: microk8s-linkerd.wrapper helm: command: microk8s-helm.wrapper completer: helm.bash helm3: command: microk8s-helm3.wrapper completer: helm3.bash dbctl: command: microk8s-dbctl.wrapper version: command: microk8s-version.wrapper ================================================ FILE: tests/libs/addons-upgrade.sh ================================================ #!/usr/bin/env bash function setup_addons_upgrade_tests() { local NAME=$1 local DISTRO=$2 local PROXY=$3 create_machine "$NAME" "$DISTRO" "$PROXY" } function run_addons_upgrade_tests() { local NAME=$1 local FROM_CHANNEL=$2 local TO_CHANNEL=$3 # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774 lxc exec "$NAME" -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /root/tests/test-upgrade.py" } function post_addons_upgrade_tests() { local NAME=$1 lxc delete "$NAME" --force } TEMP=$(getopt -o "lh" \ --long lib-mode,help,node-name:,distro:,from-channel:,to-channel:,proxy: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" NAME="${NAME-"machine-$RANDOM"}" DISTRO="${DISTRO-}" FROM_CHANNEL="${FROM_CHANNEL-}" TO_CHANNEL="${TO_CHANNEL-}" PROXY="${PROXY-}" LIBRARY_MODE=false while true; do case "$1" in -l | --lib-mode ) LIBRARY_MODE=true; shift ;; --node-name ) NAME="$2"; shift 2 ;; --distro ) DISTRO="$2"; shift 2 ;; --from-channel ) FROM_CHANNEL="$2"; shift 2 ;; --to-channel ) TO_CHANNEL="$2"; shift 2 ;; --proxy ) PROXY="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...]" echo " --node-name Name to be used for LXD containers" echo " Can also be set by using NAME environment variable" echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" echo " Can also be set by using DISTRO environment variable" echo " --from-channel Channel to upgrade from to the channel under testing Eg. latest/beta" echo " Can also be set by using FROM_CHANNEL environment variable" echo " --to-channel Channel to be tested Eg. latest/edge" echo " Can also be set by using TO_CHANNEL environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo " -l, --lib-mode Make the script act like a library Eg. true / false" echo exit ;; -- ) shift; break ;; * ) break ;; esac done if [ "$LIBRARY_MODE" == "false" ]; then setup_addons_upgrade_tests "$NAME" "$DISTRO" "$PROXY" run_addons_upgrade_tests "$NAME" "$FROM_CHANNEL" "$TO_CHANNEL" post_addons_upgrade_tests "$NAME" fi ================================================ FILE: tests/libs/addons.sh ================================================ #!/usr/bin/env bash function setup_addons_tests() { local NAME=$1 local DISTRO=$2 local PROXY=$3 local TO_CHANNEL=$4 create_machine "$NAME" "$DISTRO" "$PROXY" if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] then lxc file push "${TO_CHANNEL}" "$NAME"/tmp/microk8s_latest_amd64.snap lxc exec "$NAME" -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic else lxc exec "$NAME" -- snap install microk8s --channel="${TO_CHANNEL}" --classic fi } function run_smoke_test() { local NAME=$1 lxc exec "$NAME" -- /root/tests/smoke-test.sh lxc exec "$NAME" -- script -e -c "pytest -s /root/tests/test-cluster-agent.py" } function run_core_addons_tests() { local NAME=$1 # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774 lxc exec "$NAME" -- script -e -c "pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py" } function run_community_addons_tests() { local NAME=$1 lxc exec "$NAME" -- microk8s enable community lxc exec "$NAME" -- script -e -c "pytest -s /var/snap/microk8s/common/addons/community/tests/" } function run_gpu_addon_test() { if [ -f "/var/snap/microk8s/common/addons/core/tests/test-addons.py" ] && grep test_gpu /var/snap/microk8s/common/addons/core/tests/test-addons.py -q then timeout 3600 pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py -k test_gpu fi } function post_addons_tests() { local NAME=$1 lxc exec "$NAME" -- microk8s reset lxc delete "$NAME" --force } TEMP=$(getopt -o "l,h" \ --long help,lib-mode,node-name:,distro:,channel:,proxy: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" NAME="${NAME-"machine-$RANDOM"}" DISTRO="${DISTRO-}" TO_CHANNEL="${TO_CHANNEL-}" PROXY="${PROXY-}" LIBRARY_MODE=false while true; do case "$1" in -l | --lib-mode ) LIBRARY_MODE=true; shift ;; --node-name ) NAME="$2"; shift 2 ;; --distro ) DISTRO="$2"; shift 2 ;; --channel ) TO_CHANNEL="$2"; shift 2 ;; --proxy ) PROXY="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...]" echo " --node-name Name to be used for LXD containers" echo " Can also be set by using NAME environment variable" echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" echo " Can also be set by using DISTRO environment variable" echo " --channel Channel to be tested Eg. latest/edge" echo " Can also be set by using TO_CHANNEL environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo " -l, --lib-mode Make the script act like a library Eg. true / false" echo exit ;; -- ) shift; break ;; * ) break ;; esac done if [ "$LIBRARY_MODE" == "false" ]; then setup_addons_tests "$NAME" "$DISTRO" "$PROXY" "$TO_CHANNEL" run_smoke_test "$NAME" run_core_addons_tests "$NAME" DISABLE_COMMUNITY_TESTS="${DISABLE_COMMUNITY_TESTS:-0}" if [ "x${DISABLE_COMMUNITY_TESTS}" != "x1" ]; then run_community_addons_tests "$NAME" fi run_gpu_addon_test post_addons_tests "$NAME" fi ================================================ FILE: tests/libs/airgap.sh ================================================ #!/usr/bin/env bash source tests/libs/utils.sh function airgap_wait_for_pods() { container="$1" lxc exec "$container" -- bash -c " while ! microk8s kubectl wait -n kube-system ds/calico-node --for=jsonpath='{.status.numberReady}'=1; do echo waiting for calico sleep 3 done while ! microk8s kubectl wait -n kube-system deploy/hostpath-provisioner --for=jsonpath='{.status.readyReplicas}'=1; do echo waiting for hostpath provisioner sleep 3 done while ! microk8s kubectl wait -n kube-system deploy/coredns --for=jsonpath='{.status.readyReplicas}'=1; do echo waiting for coredns sleep 3 done " } function setup_airgap_registry_mirror() { local NAME=$1 local DISTRO=$2 local PROXY=$3 local TO_CHANNEL=$4 create_machine "$NAME" "$DISTRO" "$PROXY" lxc exec "$NAME" -- bash -c " mkdir -p /root/snap/microk8s/common echo ' --- version: 0.1.0 # pre-configure DNS args to save time from unnecessary kubelet restarts extraKubeletArgs: --cluster-dns: 10.152.183.10 --cluster-domain: cluster.local addons: - name: dns - name: storage - name: registry ' > /root/snap/microk8s/common/.microk8s.yaml " if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] then lxc file push "${TO_CHANNEL}" "$NAME"/var/tmp/microk8s_latest_amd64.snap while ! lxc exec "$NAME" -- bash -c "snap install snapd"; do echo retry install snapd sleep 1 done while ! lxc exec "$NAME" -- bash -c "snap install core22"; do echo retry install core22 sleep 1 done while ! lxc exec "$NAME" -- bash -c "snap install /var/tmp/microk8s_latest_amd64.snap --dangerous --classic"; do echo retry snap install sleep 1 done else lxc exec "$NAME" -- snap install microk8s --channel="${TO_CHANNEL}" --classic fi } function wait_airgap_registry() { local NAME=$1 airgap_wait_for_pods "$NAME" lxc exec "$NAME" -- bash -c ' while ! curl --silent 127.0.0.1:32000/v2/_catalog; do echo waiting for registry sleep 2 done ' } function push_images_to_registry() { local NAME=$1 lxc exec "$NAME" -- bash -c ' for image in $(microk8s ctr image ls -q | grep -v "sha256:"); do mirror=$(echo $image | sed '"'s,\(docker.io\|k8s.gcr.io\|registry.k8s.io\|quay.io\|public.ecr.aws\),${NAME}:32000,g'"') sudo microk8s ctr image convert ${image} ${mirror} sudo microk8s ctr image push --plain-http ${mirror} done ' } function setup_airgapped_microk8s() { local NAME=$1 local DISTRO=$2 local PROXY=$3 local TO_CHANNEL=$4 create_machine "$NAME" "$DISTRO" "$PROXY" if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] then lxc file push "${TO_CHANNEL}" "$NAME"/var/tmp/microk8s.snap else lxc exec "$NAME" -- snap download microk8s --channel="${TO_CHANNEL}" --target-directory /var/tmp --basename microk8s fi while ! lxc exec "$NAME" -- bash -c "snap install snapd"; do echo retry install snapd sleep 1 done while ! lxc exec "$NAME" -- bash -c "snap install core22"; do echo retry install core22 sleep 1 done lxc exec "$NAME" -- bash -c " echo ' network: version: 2 ethernets: eth0: dhcp4-overrides: { use-routes: false } routes: [{ to: 0.0.0.0/0, scope: link }] ' > /etc/netplan/70-airgap.yaml netplan apply " if lxc exec "$NAME" -- bash -c "ping -c1 1.1.1.1"; then echo "machine for airgap test has internet access when it should not" exit 1 fi lxc exec "$NAME" -- bash -c ' mkdir -p /root/snap/microk8s/common echo " --- version: 0.1.0 # pre-configure DNS args to save time from unnecessary kubelet restarts extraKubeletArgs: --cluster-dns: 10.152.183.10 --cluster-domain: cluster.local containerdRegistryConfigs: docker.io: | [host.\"http://'"${REGISTRY_NAME}"':32000\"] capabilities = [\"pull\", \"resolve\"] registry.k8s.io: | [host.\"http://'"${REGISTRY_NAME}"':32000\"] capabilities = [\"pull\", \"resolve\"] quay.io: | [host.\"http://'"${REGISTRY_NAME}"':32000\"] capabilities = [\"pull\", \"resolve\"] k8s.gcr.io: | [host.\"http://'"${REGISTRY_NAME}"':32000\"] capabilities = [\"pull\", \"resolve\"] public.ecr.aws: | [host.\"http://'"${REGISTRY_NAME}"':32000\"] capabilities = [\"pull\", \"resolve\"] addons: - name: dns - name: storage - name: registry " > /root/snap/microk8s/common/.microk8s.yaml while ! snap install /var/tmp/microk8s.snap --dangerous --classic; do sleep 1 done ' } function test_airgapped_microk8s() { local NAME=$1 lxc exec "$NAME" -- bash -c 'sudo microk8s enable hostpath-storage dns' airgap_wait_for_pods "$NAME" } function post_airgap_tests() { local REGISTRY_NAME=$1 local AIRGAPPED_NAME=$2 lxc rm "$REGISTRY_NAME" --force lxc rm "$AIRGAPPED_NAME" --force } TEMP=$(getopt -o "lh" \ --long help,lib-mode,registry-name:,node-name:,distro:,channel:,proxy: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" REGISTRY_NAME="${REGISTRY_NAME:-"registry-$RANDOM"}" AIRGAPPED_NAME="${AIRGAPPED_NAME:-"machine-$RANDOM"}" DISTRO="${DISTRO:-}" TO_CHANNEL="${TO_CHANNEL:-}" PROXY="${PROXY:-}" LIBRARY_MODE=false while true; do case "$1" in -l | --lib-mode ) LIBRARY_MODE=true; shift ;; --registry-name ) REGISTRY_NAME="$2"; shift 2 ;; --node-name ) AIRGAPPED_NAME="$2"; shift 2 ;; --distro ) DISTRO="$2"; shift 2 ;; --channel ) TO_CHANNEL="$2"; shift 2 ;; --proxy ) PROXY="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...]" echo " --registry-name Name to be used for registry LXD containers" echo " Can also be set by using REGISTRY_NAME environment variable" echo " --node-name Name to be used for LXD containers" echo " Can also be set by using AIRGAPPED_NAME environment variable" echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" echo " Can also be set by using DISTRO environment variable" echo " --channel Channel to be tested Eg. latest/edge" echo " Can also be set by using TO_CHANNEL environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo " -l, --lib-mode Make the script act like a library Eg. true / false" echo exit ;; -- ) shift; break ;; * ) break ;; esac done if [ "$LIBRARY_MODE" == "false" ]; then echo "1/5 -- Install registry mirror" setup_airgap_registry_mirror "$REGISTRY_NAME" "$DISTRO" "$PROXY" "$TO_CHANNEL" echo "2/5 -- Wait for MicroK8s instance with registry to come up" wait_airgap_registry "$REGISTRY_NAME" echo "3/5 -- Push images to registry mirror" push_images_to_registry "$REGISTRY_NAME" echo "4/5 -- Install MicroK8s on an airgap environment (using registry mirror)" setup_airgapped_microk8s "$AIRGAPPED_NAME" "$DISTRO" "$PROXY" "$TO_CHANNEL" echo "5/5 -- Wait for airgapped MicroK8s to come up" airgap_wait_for_pods "$AIRGAPPED_NAME" echo "Cleaning up" post_airgap_tests "$REGISTRY_NAME" "$AIRGAPPED_NAME" fi ================================================ FILE: tests/libs/clustering.sh ================================================ #!/usr/bin/env bash function run_clustering_tests() { # Test clustering. This test will create lxc containers or multipass VMs # therefore we do not need to run it inside a VM/container TRY_ATTEMPT=0 while ! (timeout 3600 pytest -s tests/test-cluster.py) && ! [ ${TRY_ATTEMPT} -eq 3 ] do TRY_ATTEMPT=$((TRY_ATTEMPT+1)) sleep 1 done if [ ${TRY_ATTEMPT} -eq 3 ] then echo "Test clusterring took longer than expected" exit 1 fi } TEMP=$(getopt -o "lh" \ --long help,lib-mode,channel:,backend:,lxd-profile: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" BACKEND="${BACKEND-"lxc"}" CHANNEL_TO_TEST="${CHANNEL_TO_TEST-}" LXC_PROFILE="${LXC_PROFILE-}" LIBRARY_MODE=false while true; do case "$1" in -l | --lib-mode ) LIBRARY_MODE=true; shift ;; --backend ) BACKEND="$2"; shift 2 ;; --lxd-profile ) LXC_PROFILE="$2"; shift 2 ;; --channel ) CHANNEL_TO_TEST="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...]" echo " --backend Backend to be used for clustering tests Eg. lxc" echo " Can also be set by using BACKEND environment variable" echo " --lxd-profile Profile to be used for lxc backend Eg. tests/lxc/microk8s.profile" echo " Can also be set by using LXC_PROFILE environment variable" echo " --channel Channel to be tested Eg. latest/edge" echo " Can also be set by using CHANNEL_TO_TEST environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo " -l, --lib-mode Make the script act like a library Eg. true / false" echo exit ;; -- ) shift; break ;; * ) break ;; esac done if [ "$LIBRARY_MODE" == "false" ]; then run_clustering_tests fi ================================================ FILE: tests/libs/spread.sh ================================================ #!/usr/bin/env bash set -ex source tests/libs/utils.sh function run_spread_tests() { local NAME=$1 local DISTRO=$2 local PROXY=$3 local TO_CHANNEL=$4 create_machine "$NAME" "$DISTRO" "$PROXY" if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] then lxc file push "${TO_CHANNEL}" "$NAME"/tmp/microk8s_latest_amd64.snap for i in {1..5}; do lxc exec "$NAME" -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic && break || sleep 5; done else lxc exec "$NAME" -- snap install microk8s --channel="${TO_CHANNEL}" --classic fi lxc exec "$NAME" -- /snap/bin/microk8s stop lxc exec "$NAME" -- sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a \ \ disable_apparmor=true' /var/snap/microk8s/current/args/containerd-template.toml lxc exec "$NAME" -- /snap/bin/microk8s start lxc exec "$NAME" -- /snap/bin/microk8s status --wait-ready --timeout 300 sleep 45 lxc exec "$NAME" -- /snap/bin/microk8s kubectl wait pod --all --for=condition=Ready -A --timeout=300s lxc exec "$NAME" -- script -e -c "pytest -s /root/tests/test-simple.py" } TEMP=$(getopt -o "l,h" \ --long help,lib-mode,node-name:,distro:,channel:,proxy: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" NAME="${NAME-"machine-$RANDOM"}" DISTRO="${DISTRO-}" TO_CHANNEL="${TO_CHANNEL-}" PROXY="${PROXY-}" LIBRARY_MODE=false while true; do case "$1" in -l | --lib-mode ) LIBRARY_MODE=true; shift ;; --node-name ) NAME="$2"; shift 2 ;; --distro ) DISTRO="$2"; shift 2 ;; --channel ) TO_CHANNEL="$2"; shift 2 ;; --proxy ) PROXY="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...]" echo " --node-name Name to be used for LXD containers" echo " Can also be set by using NAME environment variable" echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" echo " Can also be set by using DISTRO environment variable" echo " --channel Channel to be tested Eg. latest/edge" echo " Can also be set by using TO_CHANNEL environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo " -l, --lib-mode Make the script act like a library Eg. true / false" echo exit ;; -- ) shift; break ;; * ) break ;; esac done if [ "$LIBRARY_MODE" == "false" ]; then run_spread_tests "$NAME" "$DISTRO" "$PROXY" "$TO_CHANNEL" fi ================================================ FILE: tests/libs/upgrade-path.sh ================================================ #!/usr/bin/env bash function setup_upgrade_path_tests() { # Test upgrade-path local NAME=$1 local DISTRO=$2 local PROXY=$3 create_machine "$NAME" "$DISTRO" "$PROXY" } function run_upgrade_path_tests() { local NAME=$1 local FROM_CHANNEL=$2 local TO_CHANNEL=$3 # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774 if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] then lxc file push "${TO_CHANNEL}" "$NAME"/tmp/microk8s_latest_amd64.snap lxc exec "$NAME" -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=/tmp/microk8s_latest_amd64.snap pytest -s /root/tests/test-upgrade-path.py" else lxc exec "$NAME" -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /root/tests/test-upgrade-path.py" fi } function post_upgrade_path_test() { local NAME=$1 lxc delete "$NAME" --force } TEMP=$(getopt -o "lh" \ --long help,lib-mode,node-name:,distro:,from-channel:,to-channel:,proxy: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" NAME="${NAME-"machine-$RANDOM"}" DISTRO="${DISTRO-}" FROM_CHANNEL="${FROM_CHANNEL-}" TO_CHANNEL="${TO_CHANNEL-}" PROXY="${PROXY-}" LIBRARY_MODE=false while true; do case "$1" in -l | --lib-mode ) LIBRARY_MODE=true; shift ;; --node-name ) NAME="$2"; shift 2 ;; --distro ) DISTRO="$2"; shift 2 ;; --from-channel ) FROM_CHANNEL="$2"; shift 2 ;; --to-channel ) TO_CHANNEL="$2"; shift 2 ;; --proxy ) PROXY="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...]" echo " --node-name Name to be used for LXD containers" echo " Can also be set by using NAME environment variable" echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" echo " Can also be set by using DISTRO environment variable" echo " --from-channel Channel to upgrade from to the channel under testing Eg. latest/beta" echo " Can also be set by using FROM_CHANNEL environment variable" echo " --to-channel Channel to be tested Eg. latest/edge" echo " Can also be set by using TO_CHANNEL environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo " -l, --lib-mode Make the script act like a library Eg. true / false" echo exit ;; -- ) shift; break ;; * ) break ;; esac done if [ "$LIBRARY_MODE" == "false" ]; then setup_upgrade_path_tests "$NAME" "$DISTRO" "$PROXY" run_upgrade_path_tests "$NAME" "$FROM_CHANNEL" "$TO_CHANNEL" post_upgrade_path_test "$NAME" fi ================================================ FILE: tests/libs/utils.sh ================================================ #!/usr/bin/env bash function create_machine() { local NAME=$1 local DISTRO=$2 local PROXY=$3 if ! lxc profile show microk8s then lxc profile copy default microk8s fi lxc profile edit microk8s < tests/lxc/microk8s.profile lxc launch -p default -p microk8s "$DISTRO" "$NAME" # Allow for the machine to boot and get an IP sleep 20 # CentOS 8,9 variants(rocky, alma) don't ship with tar, such a dirty hack... lxc exec "$NAME" -- /bin/bash -c "yum install tar -y || true" tar cf - ./tests | lxc exec "$NAME" -- tar xvf - -C /root DISTRO_DEPS_TMP="${DISTRO//:/_}" DISTRO_DEPS="${DISTRO_DEPS_TMP////-}" lxc exec "$NAME" -- /bin/bash "/root/tests/lxc/install-deps/$DISTRO_DEPS" lxc exec "$NAME" -- reboot sleep 20 trap 'lxc delete '"${NAME}"' --force || true' EXIT if [ ! -z "${PROXY}" ] then lxc exec "$NAME" -- /bin/bash -c "echo HTTPS_PROXY=$PROXY >> /etc/environment" lxc exec "$NAME" -- /bin/bash -c "echo https_proxy=$PROXY >> /etc/environment" lxc exec "$NAME" -- reboot sleep 20 fi } function setup_tests() { DISTRO="${1-$DISTRO}" FROM_CHANNEL="${2-$FROM_CHANNEL}" TO_CHANNEL="${3-$TO_CHANNEL}" PROXY="${4-$PROXY}" export DEBIAN_FRONTEND=noninteractive apt-get install python3-pip -y pip3 install -U pytest requests pyyaml sh psutil apt-get install jq -y snap install kubectl --classic export ARCH=$(uname -m) export LXC_PROFILE="tests/lxc/microk8s.profile" export BACKEND="lxc" export CHANNEL_TO_TEST=${TO_CHANNEL} } ================================================ FILE: tests/lxc/install-deps/images_almalinux-8 ================================================ #!/usr/bin/env bash dnf install epel-release -y dnf upgrade -y yum install sudo -y yum install fuse squashfuse -y yum install snapd -y systemctl enable --now snapd.socket ln -s /var/lib/snapd/snap /snap yum install python3-pip -y yum install docker -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_archlinux ================================================ #!/usr/bin/env bash pacman -S --noconfirm --needed base-devel pacman -S --noconfirm --needed git useradd user echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers mkdir /home/user chown user:user /home/user cd /tmp su user -c "git clone https://aur.archlinux.org/snapd.git" su user -c "cd snapd; makepkg -si --noconfirm" sudo systemctl enable --now snapd.socket sudo ln -s /var/lib/snapd/snap /snap su user -c "git clone https://aur.archlinux.org/squashfuse-git.git" su user -c "cd squashfuse-git; makepkg -si --noconfirm" pacman -S --noconfirm --needed fuse3 echo "PATH=$PATH:/snap/bin" >> /etc/profile pacman -S --noconfirm python-pip pacman -S --noconfirm python pacman -S --noconfirm docker sudo systemctl enable --now docker.service echo "127.0.0.1 localhost" | sudo tee -a /etc/hosts pip3 install pytest==8.3.4 requests pyyaml # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_centos-7 ================================================ #!/usr/bin/env bash yum install epel-release -y yum install sudo -y yum install snapd -y systemctl enable --now snapd.socket ln -s /var/lib/snapd/snap /snap yum install python3-pip -y yum install docker -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_centos-8-Stream ================================================ #!/usr/bin/env bash dnf install epel-release -y dnf upgrade -y yum install sudo -y yum install fuse squashfuse -y yum install snapd -y systemctl enable --now snapd.socket ln -s /var/lib/snapd/snap /snap yum install python3-pip -y yum install docker -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_debian-10 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core && break # substitute your command here n=$[$n+1] sleep 10 done n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_debian-11 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core && break # substitute your command here n=$[$n+1] sleep 10 done n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_debian-12 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y pip3 install pytest==8.3.4 requests pyyaml sh psutil --break-system-packages # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_fedora-37 ================================================ #!/usr/bin/env bash yum install epel-release -y yum install sudo -y yum install fuse squashfuse -y yum install snapd -y systemctl enable --now snapd.socket ln -s /var/lib/snapd/snap /snap yum install python3-pip -y yum install docker -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_fedora-38 ================================================ #!/usr/bin/env bash yum install epel-release -y yum install sudo -y yum install fuse squashfuse -y yum install snapd -y systemctl enable --now snapd.socket ln -s /var/lib/snapd/snap /snap yum install python3-pip -y yum install docker -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/images_rockylinux-8 ================================================ #!/usr/bin/env bash dnf install epel-release -y dnf upgrade -y yum install sudo -y yum install fuse squashfuse -y yum install snapd -y systemctl enable --now snapd.socket ln -s /var/lib/snapd/snap /snap yum install python3-pip -y yum install docker -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # wait for the snapd seeding to take place! n=0 until [ $n -ge 7 ] do sudo snap install core22 && break # substitute your command here n=$[$n+1] sleep 10 done ================================================ FILE: tests/lxc/install-deps/ubuntu_16.04 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install python3-pip docker.io -y pip3 install importlib-metadata==2.1.0 pytest requests pyyaml sh # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails snap install core22 | true ================================================ FILE: tests/lxc/install-deps/ubuntu_18.04 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install python3-pip docker.io -y # In Ubuntu 18.04 for arm64 on LXC, "pip3 install -U pyyaml" breaks netplan pip3 install pytest==8.3.4 requests pyyaml sh psutil # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails snap install core22 | true ================================================ FILE: tests/lxc/install-deps/ubuntu_20.04 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install xdelta3 -y apt-get install python3-pip docker.io -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails snap install core22 | true ================================================ FILE: tests/lxc/install-deps/ubuntu_22.04 ================================================ #!/usr/bin/env bash export $(grep -v '^#' /etc/environment | xargs) export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install python3-pip docker.io -y pip3 install pytest==8.3.4 requests pyyaml sh psutil # Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698 # if core is to be installed by microk8s it fails snap install core22 | true ================================================ FILE: tests/lxc/microk8s-zfs.profile ================================================ name: microk8s config: boot.autostart: "true" linux.kernel_modules: ip_vs,ip_vs_rr,ip_vs_wrr,ip_vs_sh,ip_tables,ip6_tables,netlink_diag,nf_nat,overlay,br_netfilter raw.lxc: | lxc.apparmor.profile=unconfined lxc.mount.auto=proc:rw sys:rw cgroup:rw lxc.cgroup.devices.allow=a lxc.cap.drop= security.nesting: "true" security.privileged: "true" description: "" devices: aadisable: path: /sys/module/nf_conntrack/parameters/hashsize source: /sys/module/nf_conntrack/parameters/hashsize type: disk aadisable2: path: /dev/zfs source: /dev/zfs type: disk aadisable3: path: /dev/kmsg source: /dev/kmsg type: unix-char aadisable4: path: /sys/fs/bpf source: /sys/fs/bpf type: disk aadisable5: path: /proc/sys/net/netfilter/nf_conntrack_max source: /proc/sys/net/netfilter/nf_conntrack_max type: disk ================================================ FILE: tests/lxc/microk8s.profile ================================================ name: microk8s config: boot.autostart: "true" linux.kernel_modules: ip_vs,ip_vs_rr,ip_vs_wrr,ip_vs_sh,ip_tables,ip6_tables,netlink_diag,nf_nat,overlay,br_netfilter raw.lxc: | lxc.apparmor.profile=unconfined lxc.mount.auto=proc:rw sys:rw cgroup:rw lxc.cgroup.devices.allow=a lxc.cap.drop= security.nesting: "true" security.privileged: "true" description: "" devices: aadisable: path: /sys/module/nf_conntrack/parameters/hashsize source: /sys/module/nf_conntrack/parameters/hashsize type: disk aadisable2: path: /dev/kmsg source: /dev/kmsg type: unix-char aadisable3: path: /sys/fs/bpf source: /sys/fs/bpf type: disk aadisable4: path: /proc/sys/net/netfilter/nf_conntrack_max source: /proc/sys/net/netfilter/nf_conntrack_max type: disk ================================================ FILE: tests/requirements.txt ================================================ kubernetes jinja2 pylxd pytest pytest-xdist pyyaml sh jsonschema>=4.0.0 pdbpp psutil netifaces requests ================================================ FILE: tests/smoke-test.sh ================================================ #!/bin/bash set -eux n=0 until [ $n -ge 10 ] do (sudo /snap/bin/microk8s kubectl get all --all-namespaces | grep -z "service/kubernetes") && break n=$[$n+1] if [ $n -ge 10 ]; then exit 1 fi sleep 20 done n=0 until [ $n -ge 3 ] do (sudo /snap/bin/microk8s kubectl get no | grep -z "Ready") && exit 0 n=$[$n+1] sleep 20 done sudo /snap/bin/microk8s kubectl -n kube-system rollout status deployment.apps/calico-kube-controllers exit 1 ================================================ FILE: tests/templates/bbox-local.yaml ================================================ apiVersion: v1 kind: Pod metadata: name: busybox namespace: default spec: containers: - name: busybox image: localhost:32000/my-busybox command: - sleep - "3600" imagePullPolicy: IfNotPresent restartPolicy: Always ================================================ FILE: tests/templates/dual-stack.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: nginxdualstack spec: selector: matchLabels: run: nginxdualstack replicas: 1 template: metadata: labels: run: nginxdualstack spec: containers: - name: nginxdualstack image: cdkbot/nginxdualstack:1.0.0 ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx6 labels: run: nginxdualstack spec: type: NodePort ipFamilies: - IPv6 ipFamilyPolicy: RequireDualStack ports: - port: 80 protocol: TCP selector: run: nginxdualstack ================================================ FILE: tests/templates/ingress.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null labels: app: microbot name: microbot spec: replicas: 1 selector: matchLabels: app: microbot strategy: {} template: metadata: creationTimestamp: null labels: app: microbot spec: containers: - image: cdkbot/microbot-$ARCH imagePullPolicy: "" name: microbot ports: - containerPort: 80 livenessProbe: httpGet: path: / port: 80 initialDelaySeconds: 5 timeoutSeconds: 30 resources: {} restartPolicy: Always serviceAccountName: "" status: {} --- apiVersion: v1 kind: Service metadata: name: microbot labels: app: microbot spec: ports: - port: 80 protocol: TCP targetPort: 80 selector: app: microbot --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: microbot-ingress-nip spec: rules: - host: microbot.127.0.0.1.nip.io http: paths: - path: / pathType: Prefix backend: service: name: microbot port: number: 80 ================================================ FILE: tests/templates/nginx-pod.yaml ================================================ apiVersion: v1 kind: Pod metadata: labels: app: nginx name: nginx namespace: default spec: containers: - name: nginx image: nginx:latest restartPolicy: Always ================================================ FILE: tests/templates/pvc.yaml ================================================ kind: PersistentVolumeClaim apiVersion: v1 metadata: name: myclaim spec: accessModes: - ReadWriteOnce volumeMode: Filesystem resources: requests: storage: 1Gi --- kind: Pod apiVersion: v1 metadata: name: hostpath-test-pod spec: containers: - name: hostpath-test-container image: busybox command: ["/bin/sh", "-c", "while true; do date >> /mnt/dates; sleep 2; done"] volumeMounts: - name: hostpath-volume mountPath: "/mnt" restartPolicy: "Never" volumes: - name: hostpath-volume persistentVolumeClaim: claimName: myclaim ================================================ FILE: tests/templates/registry-sc.yaml ================================================ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: registry-test-sc provisioner: microk8s.io/hostpath reclaimPolicy: Delete volumeBindingMode: Immediate ================================================ FILE: tests/templates/simple-deploy.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx:1.14.2 ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-service spec: selector: app: nginx ports: - protocol: TCP port: 80 targetPort: 80 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx-ingress spec: rules: - http: paths: - path: / pathType: Prefix backend: service: name: nginx-service port: number: 80 ================================================ FILE: tests/test-cluster-agent.py ================================================ import requests class TestClusterAgent(object): """ Validates the cluster agent in MicroK8s """ def test_cluster_agent_health(self): """ Query the cluster agent health endpoint to verify it is up and healthy. """ response = requests.get( "https://127.0.0.1:25000/health", verify="/var/snap/microk8s/current/certs/ca.crt" ) assert response.status_code == 200 and response.json()["status"] == "OK" ================================================ FILE: tests/test-cluster.py ================================================ import string import random import time import pytest import os import datetime import requests import signal import subprocess from os import path from pathlib import Path from utils import ( is_strict, is_ipv6_configured, ) # Provide a list of VMs you want to reuse. VMs should have already microk8s installed. # the test will attempt a refresh to the channel requested for testing # reuse_vms = ['vm-ldzcjb', 'vm-nfpgea', 'vm-pkgbtw'] reuse_vms = None # Channel we want to test. A full path to a local snap can be used for local builds channel_to_test = os.environ.get("CHANNEL_TO_TEST", "latest/stable") backend = os.environ.get("BACKEND", None) profile = os.environ.get("LXC_PROFILE", "lxc/microk8s.profile") snap_data = os.environ.get("SNAP_DATA", "/var/snap/microk8s/current") distro = os.environ.get("DISTRO", "ubuntu:22.04") TEMPLATES = Path(__file__).absolute().parent / "templates" class VM: """ This class abstracts the backend we are using. It could be either multipass or lxc. """ launch_config = """--- version: 0.1.0 extraCNIEnv: IPv4_SUPPORT: true IPv4_CLUSTER_CIDR: 10.3.0.0/16 IPv4_SERVICE_CIDR: 10.153.183.0/24 IPv6_SUPPORT: true IPv6_CLUSTER_CIDR: fd02::/64 IPv6_SERVICE_CIDR: fd99::/108 extraSANs: - 10.153.183.1""" def __init__(self, backend=None, attach_vm=None): """Detect the available backends and instantiate a VM. If `attach_vm` is provided we just make sure the right MicroK8s is deployed. :param backend: either multipass of lxc :param attach_vm: the name of the VM we want to reuse """ rnd_letters = "".join(random.choice(string.ascii_lowercase) for i in range(6)) self.backend = backend self.vm_name = "vm-{}".format(rnd_letters) self.attached = False if attach_vm: self.attached = True self.vm_name = attach_vm def setup(self, channel_or_snap): """ Setup the VM with the right snap. :param channel_or_snap: the snap channel or the path to the local snap build """ if (path.exists("/snap/bin/multipass") and not self.backend) or self.backend == "multipass": print("Creating mulitpass VM") self.backend = "multipass" self._setup_multipass(channel_or_snap) elif (path.exists("/snap/bin/lxc") and not self.backend) or self.backend == "lxc": print("Creating lxc VM") self.backend = "lxc" self._setup_lxc(channel_or_snap) else: raise Exception("Need to install multipass or lxc") def _setup_lxc(self, channel_or_snap): if not self.attached: profiles = subprocess.check_output("/snap/bin/lxc profile list".split()) if "microk8s" not in profiles.decode(): subprocess.check_call("/snap/bin/lxc profile copy default microk8s".split()) with open(profile, "r+") as fp: profile_string = fp.read() process = subprocess.Popen( "/snap/bin/lxc profile edit microk8s".split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) process.stdin.write(profile_string.encode()) process.stdin.close() subprocess.check_call( "/snap/bin/lxc launch -p default -p microk8s {} {}".format( distro, self.vm_name ).split() ) time.sleep(20) if is_ipv6_configured(): self._load_launch_configuration_lxc() if channel_or_snap.startswith("/"): self._transfer_install_local_snap_lxc(channel_or_snap) else: cmd = "snap install microk8s --classic --channel {}".format(channel_or_snap) time.sleep(20) print("About to run {}".format(cmd)) output = "" attempt = 0 while attempt < 3: try: output = self.run(cmd) break except ChildProcessError: time.sleep(10) attempt += 1 print(output.decode()) else: if is_ipv6_configured(): self._load_launch_configuration_lxc() if channel_or_snap.startswith("/"): self._transfer_install_local_snap_lxc(channel_or_snap) else: cmd = "/snap/bin/lxc exec {} -- ".format(self.vm_name).split() cmd.append("sudo snap refresh microk8s --channel {}".format(channel_or_snap)) subprocess.check_call(cmd) def _load_launch_configuration_lxc(self): # Set launch configurations before installing microk8s print("Setting launch configurations") self.run("mkdir -p /var/snap/microk8s/common/") file_path = "microk8s.yaml" print(self.launch_config) with open(file_path, "w") as file: file.write(self.launch_config) # Copy the file to the VM cmd = "lxc file push {} {}/var/snap/microk8s/common/.microk8s.yaml".format( file_path, self.vm_name ).split() subprocess.check_output(cmd) os.remove(file_path) def _transfer_install_local_snap_lxc(self, channel_or_snap): try: print("Installing snap from {}".format(channel_or_snap)) cmd_prefix = "/snap/bin/lxc exec {} -- script -e -c".format(self.vm_name).split() cmd = ["rm -rf /var/tmp/microk8s.snap"] subprocess.check_output(cmd_prefix + cmd) cmd = "lxc file push {} {}/var/tmp/microk8s.snap".format( channel_or_snap, self.vm_name ).split() subprocess.check_output(cmd) cmd = ["snap install /var/tmp/microk8s.snap --dangerous --classic"] subprocess.check_output(cmd_prefix + cmd) time.sleep(20) except subprocess.CalledProcessError as e: print(e.output.decode()) raise def _setup_multipass(self, channel_or_snap): if not self.attached: version = distro.split(":") subprocess.check_call( "/snap/bin/multipass launch {} -n {} -m 2G".format(version[1], self.vm_name).split() ) if is_ipv6_configured(): self._load_launch_configuration_multipass() if channel_or_snap.startswith("/"): self._transfer_install_local_snap_multipass(channel_or_snap) else: subprocess.check_call( "/snap/bin/multipass exec {} -- sudo " "snap install microk8s --classic --channel {}".format( self.vm_name, channel_or_snap ).split() ) else: if is_ipv6_configured(): self._load_launch_configuration_multipass() if channel_or_snap.startswith("/"): self._transfer_install_local_snap_multipass(channel_or_snap) else: subprocess.check_call( "/snap/bin/multipass exec {} -- sudo " "snap refresh microk8s --channel {}".format( self.vm_name, channel_or_snap ).split() ) def _load_launch_configuration_multipass(self): # Set launch configurations before installing microk8s print("Setting launch configurations") self.run("mkdir -p /var/snap/microk8s/common/") self.run("chmod 777 /var/snap/microk8s/common/") file_path = "microk8s.yaml" print(self.launch_config) with open(file_path, "w") as file: file.write(self.launch_config) # Copy the file to the VM subprocess.check_call( "/snap/bin/multipass transfer {} {}:/var/snap/microk8s/common/.microk8s.yaml".format( file_path, self.vm_name ).split() ) os.remove(file_path) def _transfer_install_local_snap_multipass(self, channel_or_snap): print("Installing snap from {}".format(channel_or_snap)) subprocess.check_call( "/snap/bin/multipass transfer {} {}:/var/tmp/microk8s.snap".format( channel_or_snap, self.vm_name ).split() ) subprocess.check_call( "/snap/bin/multipass exec {} -- sudo " "snap install /var/tmp/microk8s.snap --classic --dangerous".format(self.vm_name).split() ) def run(self, cmd): """ Run a command :param cmd: the command we are running. :return: the output of the command """ if self.backend == "multipass": output = subprocess.check_output( "/snap/bin/multipass exec {} -- sudo " "{}".format(self.vm_name, cmd).split() ) return output elif self.backend == "lxc": cmd_prefix = "/snap/bin/lxc exec {} -- ".format(self.vm_name) with subprocess.Popen( cmd_prefix + cmd, shell=True, stdout=subprocess.PIPE, preexec_fn=os.setsid ) as process: try: output = process.communicate(timeout=300)[0] if process.returncode != 0: raise ChildProcessError("Failed to run command") except subprocess.TimeoutExpired: os.killpg(process.pid, signal.SIGKILL) # send signal to the process group print("Process timed out") output = process.communicate()[0] return output else: raise Exception("Not implemented for backend {}".format(self.backend)) def transfer_file(self, file_path, remote_path): """ Transfer a file to the VM. """ print("Transferring {} to {}".format(file_path, remote_path)) if self.backend == "multipass": subprocess.check_call( "/snap/bin/multipass transfer {} {}:{} ".format( file_path, self.vm_name, remote_path ).split() ) elif self.backend == "lxc": subprocess.check_call( "/snap/bin/lxc file push {} {}{}".format( file_path, self.vm_name, remote_path ).split() ) def release(self): """ Release a VM. """ print("Destroying VM in {}".format(self.backend)) if self.backend == "multipass": subprocess.check_call("/snap/bin/multipass stop {}".format(self.vm_name).split()) subprocess.check_call("/snap/bin/multipass delete {}".format(self.vm_name).split()) elif self.backend == "lxc": subprocess.check_call("/snap/bin/lxc stop {}".format(self.vm_name).split()) subprocess.check_call("/snap/bin/lxc delete {}".format(self.vm_name).split()) class TestCluster(object): @pytest.fixture(autouse=True, scope="module") def setup_cluster(self): """ Provision VMs and for a cluster. :return: """ try: print("Setting up cluster") type(self).VM = [] if not reuse_vms: size = 3 for i in range(0, size): print("Creating machine {}".format(i)) vm = VM(backend) vm.setup(channel_to_test) print("Waiting for machine {}".format(i)) vm.run("/snap/bin/microk8s.status --wait-ready --timeout 120") self.VM.append(vm) else: for vm_name in reuse_vms: vm = VM(backend, vm_name) vm.setup(channel_to_test) self.VM.append(vm) # Form cluster vm_master = self.VM[0] connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no") for vm in self.VM: if vm.vm_name in connected_nodes.decode(): continue else: print("Adding machine {} to cluster".format(vm.vm_name)) add_node = vm_master.run("/snap/bin/microk8s.add-node") endpoint = [ep for ep in add_node.decode().split() if ":25000/" in ep] vm.run("/snap/bin/microk8s.join {}".format(endpoint[0])) # Wait for nodes to be ready print("Waiting for nodes to register") attempt = 0 while attempt < 10: try: connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no") if "NotReady" in connected_nodes.decode(): time.sleep(5) connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no") print(connected_nodes.decode()) break except ChildProcessError: time.sleep(10) attempt += 1 if attempt == 10: raise # Wait for CNI pods print("Waiting for cni") while True: ready_pods = 0 pods = vm_master.run("/snap/bin/microk8s.kubectl get po -n kube-system -o wide") for line in pods.decode().splitlines(): if "calico" in line and "Running" in line: ready_pods += 1 if ready_pods == (len(self.VM) + 1): print(pods.decode()) break time.sleep(5) yield finally: print("Cleanup up cluster") if not reuse_vms: for vm in self.VM: print("Releasing machine {} in {}".format(vm.vm_name, vm.backend)) vm.release() def test_calico_in_nodes(self): """ Test each node has a calico pod. """ print("Checking calico is in all nodes") pods = self.VM[0].run("/snap/bin/microk8s.kubectl get po -n kube-system -o wide") for vm in self.VM: if vm.vm_name not in pods.decode(): assert False print("Calico found in node {}".format(vm.vm_name)) @pytest.mark.skipif( is_strict(), reason="Skipping because calico interfaces are not removed on strict", ) def test_calico_interfaces_removed_on_snap_remove(self): """ Test that calico interfaces are not present on the node when the microk8s snap is removed. """ vm = VM(backend) vm.setup(channel_to_test) print("Waiting for machine {}".format(vm.vm_name)) vm.run("/snap/bin/microk8s.status --wait-ready --timeout 240") timeout = time.time() + 240 ready = False while time.time() <= timeout and not ready: try: pods = vm.run("/snap/bin/microk8s.kubectl get po -n kube-system -o wide") for line in pods.decode().splitlines(): if "calico" in line and "Running" in line: ready = True except ChildProcessError: print("Waiting for k8s pods to come up") time.sleep(5) assert ready vm.run("snap remove --purge microk8s") interfaces = vm.run("/sbin/ip a") assert "cali" not in interfaces.decode() vm.release() def test_nodes_in_ha(self): """ Test all nodes are seeing the database while removing nodes """ # All nodes see the same pods for vm in self.VM: pods = vm.run("/snap/bin/microk8s.kubectl get po -n kube-system -o wide") for other_vm in self.VM: if other_vm.vm_name not in pods.decode(): assert False print("All nodes see the same pods") attempt = 100 while True: assert attempt > 0 for vm in self.VM: status = vm.run("/snap/bin/microk8s.status") if "high-availability: yes" not in status.decode(): attempt += 1 continue break # remove a node print("Removing machine {}".format(self.VM[0].vm_name)) self.VM[0].run("/snap/bin/microk8s.leave") self.VM[1].run("/snap/bin/microk8s.remove-node {}".format(self.VM[0].vm_name)) # allow for some time for the leader to hand over leadership time.sleep(10) attempt = 100 while True: ready_pods = 0 pods = self.VM[1].run("/snap/bin/microk8s.kubectl get po -n kube-system -o wide") for line in pods.decode().splitlines(): if "calico" in line and "Running" in line: ready_pods += 1 if ready_pods == (len(self.VM)): print(pods.decode()) break attempt -= 1 if attempt <= 0: assert False time.sleep(5) print("Checking calico is on the nodes running") leftVMs = [self.VM[1], self.VM[2]] attempt = 100 while True: assert attempt > 0 for vm in leftVMs: status = vm.run("/snap/bin/microk8s.status") if "high-availability: no" not in status.decode(): attempt += 1 time.sleep(2) continue break for vm in leftVMs: pods = vm.run("/snap/bin/microk8s.kubectl get po -n kube-system -o wide") for other_vm in leftVMs: if other_vm.vm_name not in pods.decode(): time.sleep(2) assert False print("Remaining nodes see the same pods") print("Waiting for two ingress to appear") self.VM[1].run("/snap/bin/microk8s.enable ingress") # wait for two ingress to appear time.sleep(10) attempt = 100 while True: ready_pods = 0 pods = self.VM[1].run("/snap/bin/microk8s.kubectl get po -A -o wide") for line in pods.decode().splitlines(): if "ingress" in line and "Running" in line: ready_pods += 1 if ready_pods == (len(self.VM) - 1): print(pods.decode()) break attempt -= 1 if attempt <= 0: assert False time.sleep(5) print("Rejoin the node") add_node = self.VM[1].run("/snap/bin/microk8s.add-node") endpoint = [ep for ep in add_node.decode().split() if ":25000/" in ep] self.VM[0].run("/snap/bin/microk8s.join {}".format(endpoint[0])) print("Waiting for nodes to be ready") attempt = 0 while attempt < 10: try: connected_nodes = self.VM[0].run("/snap/bin/microk8s.kubectl get no") if "NotReady" in connected_nodes.decode(): time.sleep(5) continue print(connected_nodes.decode()) break except ChildProcessError: time.sleep(10) attempt += 1 if attempt == 10: raise attempt = 100 while True: assert attempt > 0 for vm in self.VM: status = vm.run("/snap/bin/microk8s.status") if "high-availability: yes" not in status.decode(): attempt += 1 time.sleep(2) continue break def test_worker_node(self): """ Test a worker node is setup """ print("Setting up a worker node") vm = VM(backend) vm.setup(channel_to_test) self.VM.append(vm) # Form cluster vm_master = self.VM[0] print("Adding machine {} to cluster".format(vm.vm_name)) add_node = vm_master.run("/snap/bin/microk8s.add-node") endpoint = [ep for ep in add_node.decode().split() if ":25000/" in ep] vm.run("/snap/bin/microk8s.join {} --worker".format(endpoint[0])) ep_parts = endpoint[0].split(":") master_ip = ep_parts[0] # Wait for nodes to be ready print("Waiting for node to register") attempt = 0 while attempt < 10: try: connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no") if ( "NotReady" in connected_nodes.decode() or vm.vm_name not in connected_nodes.decode() ): time.sleep(5) continue print(connected_nodes.decode()) break except ChildProcessError: time.sleep(10) attempt += 1 if attempt == 10: raise # Check that kubelet talks to the control plane node via the local proxy print("Checking the worker's configuration") provider = vm.run("cat /var/snap/microk8s/current/args/traefik/provider.yaml") assert master_ip in provider.decode() kubelet = vm.run("cat /var/snap/microk8s/current/credentials/kubelet.config") assert "127.0.0.1" in kubelet.decode() # Leave the worker node from the cluster print("Leaving the worker node {} from the cluster".format(vm.vm_name)) vm.run("/snap/bin/microk8s.leave") vm_master.run("/snap/bin/microk8s.remove-node {}".format(vm.vm_name)) # Wait for worker node to leave the cluster attempt = 0 while attempt < 10: try: connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no") print(connected_nodes.decode()) if "NotReady" in connected_nodes.decode() or vm.vm_name in connected_nodes.decode(): time.sleep(5) continue print(connected_nodes.decode()) break except ChildProcessError: time.sleep(10) attempt += 1 if attempt == 10: raise # Check that the worker node is Ready print("Checking that the worker node {} is working and Ready".format(vm.vm_name)) worker_node = vm.run("/snap/bin/microk8s status --wait-ready") print(worker_node.decode()) assert "microk8s is running" in worker_node.decode() # The removed node now isn't part of the cluster and will re-issue certificates. # This will interfere when testing `test_no_cert_reissue_in_nodes`. # Hence, we remove this machine from the VM list. print("Remove machine {}".format(vm.vm_name)) self.VM.remove(vm) vm.release() def test_no_cert_reissue_in_nodes(self): """ Test that each node has the cert no-reissue lock. """ print("Checking for the no re-issue lock") for vm in self.VM: lock_files = vm.run("ls /var/snap/microk8s/current/var/lock/") assert "no-cert-reissue" in lock_files.decode() @pytest.mark.skipif( # If the host system does not have IPv6 support or is not configured for IPv6, # it won't be able to create VMs with IPv6 connectivity. not is_ipv6_configured, reason="Skipping dual stack tests on VMs which are not lxc based and not dual-stack enabled", ) def test_dual_stack_cluster(self): vm = self.VM[0] # Deploy the test deployment and service manifest = TEMPLATES / "dual-stack.yaml" remote_path = "{}/tmp/dual-stack.yaml".format(snap_data) vm.transfer_file(manifest, remote_path) vm.run("ls -al {}".format(remote_path)) vm.run("/snap/bin/microk8s.kubectl apply -f {}".format(remote_path)) # Wait for the deployment to become ready print("Waiting for nginx deployment") while True: ready_pods = 0 pods = vm.run("/snap/bin/microk8s.kubectl get po -o wide") for line in pods.decode().splitlines(): if "nginxdualstack" in line and "Running" in line: ready_pods = 1 if ready_pods == 1: print(pods.decode()) break time.sleep(5) # ping the service attached with the deployment ep = ( "/snap/bin/microk8s.kubectl get service nginx6 " "-o jsonpath='{.spec.clusterIP}' --output='jsonpath=['{.spec.clusterIP}']'" ) ipv6_endpoint = vm.run(ep).decode() print("Pinging endpoint: http://{}/".format(ipv6_endpoint)) url = f"http://{ipv6_endpoint}/" attempt = 10 while attempt >= 0: try: resp = vm.run("curl {}".format(url)) if "Kubernetes IPv6 nginx" in resp.decode(): print(resp) break except subprocess.CalledProcessError as e: print("Error occurred during the request:", str(e)) raise attempt -= 1 time.sleep(2) class TestUpgradeCluster(object): @pytest.fixture(autouse=True, scope="module") def setup_old_versioned_cluster(self): """ Provision VMs of the previous version and form the cluster. """ try: print("Setting up cluster of an older version") if channel_to_test.startswith("latest") or "/" not in channel_to_test: attempt = 0 release_url = "https://dl.k8s.io/release/stable.txt" while attempt < 10: try: r = requests.get(release_url) if r.status_code == 200: last_stable_str = r.content.decode().strip() last_stable_str = last_stable_str.replace("v", "") last_stable_str = ".".join(last_stable_str.split(".")[:2]) break except TimeoutError: time.sleep(3) attempt += 1 if attempt == 10: raise track, *_ = channel_to_test.split("/") if track == "latest": track = last_stable_str # For eksd and stable tracks, we need a previous version on these tracks. # Eg, to test 1.24-eksd, we need 1.23-eksd track. major = track.split(".")[0] minor = track.split(".")[1].split("-")[0] branch = track.split("-")[1] if len(track.split("-")) > 1 else "" if minor == "0" and major >= "1": major = str(int(major) - 1) minor = "9" else: if "-" in track: minor = str(int(minor.split("-")[0]) - 1) else: minor = str(int(minor) - 1) branch = "-" + branch if branch != "" else "" older_version = major + "." + minor + branch + "/" + "stable" print("Old version is {}".format(older_version)) type(self).VM = [] if not reuse_vms: print("Creating machine") vm = VM(backend) vm.setup(older_version) print("Waiting for machine") vm.run("/snap/bin/microk8s.status --wait-ready --timeout 120") self.VM.append(vm) else: vm = VM(backend, reuse_vms[0]) vm.setup(older_version) self.VM.append(vm) vm_older_version = self.VM[0] # Wait for CNI pods print("Waiting for cni") while True: ready_pods = 0 attempt = 0 try: pods = vm_older_version.run( "/snap/bin/microk8s.kubectl get po -n kube-system -o wide" ) for line in pods.decode().splitlines(): if "calico" in line and "Running" in line: ready_pods += 1 if ready_pods == (len(self.VM) + 1): print(pods.decode()) break time.sleep(5) except ChildProcessError: time.sleep(10) attempt += 1 if attempt == 10: raise time.sleep(5) yield finally: print("Cleanup up cluster") if not reuse_vms: for vm in self.VM: print("Releasing machine {} in {}".format(vm.vm_name, vm.backend)) vm.release() @pytest.mark.skipif( is_strict() and backend == "lxc", reason="Skipping test of multi-version cluster on strict and lxc", ) def test_mixed_version_join(self): """ Test n versioned node joining a n-1 versioned cluster. """ print("Setting up an newer versioned node") vm = VM(backend) vm.setup(channel_to_test) self.VM.append(vm) # Form cluster vm_older_version = self.VM[0] print("Adding newer versioned machine {} to cluster".format(vm.vm_name)) add_node = vm_older_version.run("/snap/bin/microk8s.add-node") endpoint = [ep for ep in add_node.decode().split() if ":25000/" in ep] vm.run("/snap/bin/microk8s.join {}".format(endpoint[0])) # Wait for nodes to be ready print("Waiting for two node to be Ready") attempt = 0 timeout_insec = 300 deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) while attempt < 10: try: # Timeout after few minutes. if datetime.datetime.now() > deadline: raise TimeoutError( "Nodes not in Ready state after {} seconds.".format(timeout_insec) ) connected_nodes = vm_older_version.run("/snap/bin/microk8s.kubectl get no") num_nodes = connected_nodes.count(b" Ready") if num_nodes != 2: time.sleep(5) continue print(connected_nodes.decode()) break except ChildProcessError: time.sleep(10) attempt += 1 if attempt == 10: raise ================================================ FILE: tests/test-distro.sh ================================================ #!/usr/bin/env bash source tests/libs/utils.sh TEMP=$(getopt -o "h" \ --long help,distro:,from-channel:,to-channel:,proxy: \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi eval set -- "$TEMP" DISTRO="${DISTRO-}" FROM_CHANNEL="${FROM_CHANNEL-}" TO_CHANNEL="${TO_CHANNEL-}" PROXY="${PROXY-}" while true; do case "$1" in --distro ) DISTRO="$2"; shift 2 ;; --from-channel ) FROM_CHANNEL="$2"; shift 2 ;; --to-channel ) TO_CHANNEL="$2"; shift 2 ;; --proxy ) PROXY="$2"; shift 2 ;; -h | --help ) prog=$(basename -s.wrapper "$0") echo "Usage: $prog [options...] " echo " --distro Distro image to be used for LXD containers Eg. ubuntu:18.04" echo " Can also be set by using DISTRO environment variable" echo " --from-channel Channel to upgrade from to the channel under testing Eg. latest/beta" echo " Can also be set by using FROM_CHANNEL environment variable" echo " --to-channel Channel to be tested Eg. latest/edge" echo " Can also be set by using TO_CHANNEL environment variable" echo " --proxy Proxy url to be used by the nodes" echo " Can also be set by using PROXY environment variable" echo exit ;; -- ) shift; break ;; * ) break ;; esac done set -uex setup_tests "$@" DISABLE_AIRGAP_TESTS="${DISABLE_AIRGAP_TESTS:-0}" if [ "x${DISABLE_AIRGAP_TESTS}" != "x1" ]; then . tests/libs/airgap.sh fi . tests/libs/clustering.sh . tests/libs/addons-upgrade.sh . tests/libs/upgrade-path.sh . tests/libs/addons.sh ================================================ FILE: tests/test-simple.py ================================================ import subprocess import time import requests import os.path import utils class TestSimple(object): def test_microk8s_nodes_ready(self): # Get MicroK8s node status output = subprocess.check_output(["microk8s", "kubectl", "get", "nodes"]) # Split the output into lines and skip the header lines = output.decode("utf-8").split("\n")[1:] # Iterate through the lines to check the node status for line in lines: # Split the line by whitespace parts = line.split() # Check if the node status is "Ready" if len(parts) > 1 and parts[1] != "Ready": # If any node is not in "Ready" status, fail the test assert False, f"Node {parts[0]} is not in Ready status" def test_calico_cni_pods_running(self): # Get Calico CNI pod status output = subprocess.check_output( ["microk8s", "kubectl", "get", "pods", "-n", "kube-system"] ) # Split the output into lines and skip the header lines = output.decode("utf-8").split("\n")[1:] # Iterate through the lines to check the Calico CNI pod status for line in lines: # Split the line by whitespace parts = line.split() # Check if the pod is related to Calico CNI and not in the "Running" state if len(parts) >= 4 and parts[0].startswith("calico") and parts[2] != "Running": # If any Calico CNI pod is not in the "Running" state, fail the test assert False, f"Calico CNI pod {parts[0]} is not running" def test_nginx_ingress(self): # Create Ingress resource for the Nginx pod subprocess.run( ["microk8s", "kubectl", "apply", "-f", "tests/templates/simple-deploy.yaml"], check=True ) # Wait for the pod to be in a ready state subprocess.run( [ "microk8s", "kubectl", "rollout", "status", "deployment", "nginx-deployment", "-n", "default", "--timeout=90s", ], check=True, ) # Send a curl request to the Ingress IP if "core/ingress: enabled" not in subprocess.check_output( ["microk8s", "status", "--format", "short"] ).decode("utf-8"): output = subprocess.check_output( [ "microk8s", "kubectl", "get", "svc", "nginx-service", "-o", "jsonpath={.spec.clusterIP}", ] ) response = requests.get(f"http://{output.decode('utf-8')}:80", timeout=15) else: # Wait for ingress to be ready time.sleep(3) response = requests.get("http://127.0.0.1:80", timeout=15) subprocess.run( ["microk8s", "kubectl", "delete", "-f", "tests/templates/simple-deploy.yaml"], check=True, ) # Verify the HTTP status code is 200 assert response.status_code == 200 def test_microk8s_services_running(self): # Define the services to check for control plane and worker nodes node_services = [ "snap.microk8s.daemon-kubelite.service", "snap.microk8s.daemon-containerd.service", "snap.microk8s.daemon-cluster-agent.service", ] control_plane_services = [ "snap.microk8s.daemon-apiserver-kicker.service", "snap.microk8s.daemon-k8s-dqlite.service", ] worker_node_services = ["snap.microk8s.daemon-apiserver-proxy.service"] if os.path.exists("/var/snap/microk8s/current/var/lock/clustered.lock"): node_services += worker_node_services else: node_services += control_plane_services # Get the list of running systemd services output = subprocess.check_output(["systemctl", "list-units", "--state=running"]) # Split the output into lines and skip the header lines = output.decode("utf-8").split("\n")[1:] # Create sets to store the running services for control plane and worker nodes running_node_services = set() # Iterate through the lines to check the running services for line in lines: # Split the line by whitespace parts = line.split() # Check if the line contains a running service if len(parts) >= 2 and parts[0].endswith(".service"): service_name = parts[0] # Check if the running service is in the control plane services list if service_name in node_services: running_node_services.add(service_name) # Verify that all node services are running assert running_node_services == set(node_services), "Not all node services are running" def test_microk8s_stop_start(self): coredns_procs = utils._get_process("coredns") assert len(coredns_procs) > 0, "Expected to find a coredns process running." utils.run_until_success("/snap/bin/microk8s.stop", timeout_insec=180) new_coredns_procs = utils._get_process("coredns") assert len(new_coredns_procs) == 0, "coredns found still running after microk8s stop." utils.run_until_success("/snap/bin/microk8s.start", timeout_insec=180) new_coredns_procs = utils._get_process("coredns") assert len(new_coredns_procs) > 0, "Expected to find a new coredns process running." ================================================ FILE: tests/test-upgrade-path.py ================================================ import pytest import os import time import requests from utils import ( wait_for_installation, run_until_success, is_strict, ) upgrade_from = os.environ.get("UPGRADE_MICROK8S_FROM", "beta") # Have UPGRADE_MICROK8S_TO point to a file to upgrade to that file upgrade_to = os.environ.get("UPGRADE_MICROK8S_TO", "edge") class TestUpgradePath(object): """ Validates a microk8s upgrade path """ @pytest.mark.skipif( os.environ.get("UNDER_TIME_PRESSURE", "").lower() == "true", reason="Skipping refresh path test as we are under time pressure", ) def test_refresh_path(self): """ Deploy an old snap and try to refresh until the current one. """ start_channel = 24 if is_strict(): start_channel = 25 last_stable_minor = None if upgrade_from.startswith("latest") or "/" not in upgrade_from: attempt = 0 release_url = "https://dl.k8s.io/release/stable.txt" while attempt < 10 and not last_stable_minor: r = requests.get(release_url) if r.status_code == 200: last_stable_str = r.content.decode().strip() # We have "v1.18.4" and we need the "18" last_stable_parts = last_stable_str.split(".") last_stable_minor = int(last_stable_parts[1]) else: time.sleep(3) attempt += 1 else: channel_parts = upgrade_from.split(".") channel_parts = channel_parts[1].split("/") if is_strict(): channel_parts = channel_parts[0].split("-") print(channel_parts) last_stable_minor = int(channel_parts[0]) last_stable_minor -= 1 print("") print( "Testing refresh path from 1.{} to 1.{} and finally refresh to {}".format( "{}-strict".format(start_channel) if is_strict() else start_channel, "{}-strict".format(last_stable_minor) if is_strict() else last_stable_minor, upgrade_to, ) ) assert last_stable_minor is not None channel = "1.{}/stable".format( "{}-strict".format(start_channel) if is_strict() else start_channel ) print("Installing {}".format(channel)) cmd = "sudo snap install microk8s --channel={} {}".format( channel, "" if is_strict() else "--classic" ) run_until_success(cmd) wait_for_installation() channel_minor = start_channel channel_minor += 1 while channel_minor <= last_stable_minor: channel = "1.{}/stable".format( "{}-strict".format(channel_minor) if is_strict() else channel_minor ) print("Refreshing to {}".format(channel)) cmd = "sudo snap refresh microk8s --channel={} {}".format( channel, "" if is_strict() else "--classic" ) run_until_success(cmd) wait_for_installation() time.sleep(30) channel_minor += 1 print("Installing {}".format(upgrade_to)) if upgrade_to.endswith(".snap"): cmd = "sudo snap install {} --dangerous {}".format( upgrade_to, "" if is_strict() else "--classic" ) else: cmd = "sudo snap refresh microk8s --channel={}".format(upgrade_to) run_until_success(cmd, timeout_insec=600) # Allow for the refresh to be processed time.sleep(20) wait_for_installation(timeout_insec=1200) ================================================ FILE: tests/test-upgrade.py ================================================ import os import platform import time from validators import ( validate_dns_dashboard, validate_storage, validate_ingress, validate_registry, validate_forward, validate_metrics_server, validate_metallb_config, validate_dual_stack, ) from subprocess import check_call, CalledProcessError from utils import ( microk8s_enable, wait_for_pod_state, wait_for_installation, run_until_success, is_container, is_ipv6_configured, kubectl, _get_process, ) upgrade_from = os.environ.get("UPGRADE_MICROK8S_FROM", "beta") # Have UPGRADE_MICROK8S_TO point to a file to upgrade to that file upgrade_to = os.environ.get("UPGRADE_MICROK8S_TO", "edge") under_time_pressure = os.environ.get("UNDER_TIME_PRESSURE", "false").lower() class TestUpgrade(object): """ Validates a microk8s upgrade path """ def test_upgrade(self): """ Deploy, probe, upgrade, validate nothing broke. """ print("Testing upgrade from {} to {}".format(upgrade_from, upgrade_to)) if is_ipv6_configured: print("IPv6 is configured, will test dual stack") launch_config = """--- version: 0.1.0 extraCNIEnv: IPv4_SUPPORT: true IPv4_CLUSTER_CIDR: 10.3.0.0/16 IPv4_SERVICE_CIDR: 10.153.183.0/24 IPv6_SUPPORT: true IPv6_CLUSTER_CIDR: fd02::/64 IPv6_SERVICE_CIDR: fd99::/108 extraSANs: - 10.153.183.1""" lc_config_dir = "/var/snap/microk8s/common/" if not os.path.exists(lc_config_dir): os.makedirs(lc_config_dir) file_path = os.path.join(lc_config_dir, ".microk8s.yaml") with open(file_path, "w") as file: file.write(launch_config) cmd = "sudo snap install microk8s --classic --channel={}".format(upgrade_from) run_until_success(cmd) wait_for_installation() if is_ipv6_configured: kubectl("set env daemonset/calico-node -n kube-system IP=10.3.0.0/16 IP6=fd02::/64") # Run through the validators and # select those that were valid for the original snap test_matrix = {} try: enable = microk8s_enable("dns") wait_for_pod_state("", "kube-system", "running", label="k8s-app=kube-dns") assert "Nothing to do for" not in enable enable = microk8s_enable("dashboard") assert "Nothing to do for" not in enable validate_dns_dashboard() test_matrix["dns_dashboard"] = validate_dns_dashboard except CalledProcessError: print("Will not test dns-dashboard") try: enable = microk8s_enable("storage") assert "Nothing to do for" not in enable validate_storage() test_matrix["storage"] = validate_storage except CalledProcessError: print("Will not test storage") try: enable = microk8s_enable("ingress") assert "Nothing to do for" not in enable validate_ingress() test_matrix["ingress"] = validate_ingress except CalledProcessError: print("Will not test ingress") try: enable = microk8s_enable("registry") assert "Nothing to do for" not in enable validate_registry() test_matrix["registry"] = validate_registry except CalledProcessError: print("Will not test registry") try: validate_forward() test_matrix["forward"] = validate_forward except CalledProcessError: print("Will not test port forward") try: enable = microk8s_enable("metrics-server") assert "Nothing to do for" not in enable validate_metrics_server() test_matrix["metrics_server"] = validate_metrics_server except CalledProcessError: print("Will not test the metrics server") # AMD64 only tests if platform.machine() == "x86_64" and under_time_pressure == "false": try: ip_ranges = ( "192.168.0.105-192.168.0.105,192.168.0.110-192.168.0.111,192.168.1.240/28" ) enable = microk8s_enable("{}:{}".format("metallb", ip_ranges), timeout_insec=500) assert "MetalLB is enabled" in enable and "Nothing to do for" not in enable validate_metallb_config(ip_ranges) test_matrix["metallb"] = validate_metallb_config except CalledProcessError: print("Will not test the metallb addon") if is_ipv6_configured: try: validate_dual_stack() test_matrix["dual_stack"] = validate_dual_stack except CalledProcessError: print("Will not test the dual stack configuration") # Refresh the snap to the target if upgrade_to.endswith(".snap"): cmd = "sudo snap install {} --classic --dangerous".format(upgrade_to) else: cmd = "sudo snap refresh microk8s --channel={}".format(upgrade_to) run_until_success(cmd) # Allow for the refresh to be processed time.sleep(10) wait_for_installation() # Test any validations that were valid for the original snap for test, validation in test_matrix.items(): print("Testing {}".format(test)) validation() if not is_container(): # On lxc umount docker overlay is not permitted. check_call("sudo snap remove microk8s".split()) coredns_procs = _get_process("coredns") assert len(coredns_procs) == 0, "Expected to have 0 coredns processes running." ================================================ FILE: tests/unit/cluster/test_join.py ================================================ import os from shutil import rmtree from unittest import mock import join from click.testing import CliRunner from join import join as command def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(command, [help_arg]) assert result.exit_code == 0 assert "Join the node to a cluster" in result.output def test_command_errors_if_no_arguments(): runner = CliRunner() result = runner.invoke(command, []) assert result.exit_code != 0 assert "Error: Missing argument" in result.output @mock.patch("subprocess.check_call") @mock.patch("os.chown") @mock.patch("os.chmod") @mock.patch("subprocess.check_output") @mock.patch("time.sleep") def test_join_dqlite_master_node( mock_sleep, mock_subprocess_check_output, mock_chmod, mock_chown, mock_subprocess_check_call, tmp_path, ): """ Test the join operation. Create a directory layout and let the join operation work with it. """ snapcommon = tmp_path / "snapcommon" snapdata = tmp_path / "snapdata" / "rev-123" snapdatacurrnet = tmp_path / "snapdata" / "current" snap = tmp_path / "snap" args = snapdata / "args" certs = snapdata / "certs" credentials = snapdata / "credentials" dqlite = snapdata / "var" / "kubernetes" / "backend" join.snapdata_path = snapdata join.snap_path = snap join.cluster_dir = f"{snapdata}/var/kubernetes/backend" join.cluster_backup_dir = f"{snapdata}/var/kubernetes/backend.backup" join.cluster_cert_file = f"{join.cluster_dir}/cluster.crt" join.cluster_key_file = f"{join.cluster_dir}/cluster.key" os.environ["SNAP"] = str(snap) os.environ["SNAP_DATA"] = str(snapdata) os.environ["SNAP_COMMON"] = str(snapcommon) def create_dir_layout(tokens): for d in [ snapcommon, snapdata, dqlite, snap / "meta", args / "cni-network", snapdata / "var" / "lock", certs, credentials, ]: if os.path.exists(d): rmtree(d, ignore_errors=True) os.makedirs(d, exist_ok=True) if os.path.exists(snapdatacurrnet): os.remove(snapdatacurrnet) snapdatacurrnet os.symlink(snapdata, snapdatacurrnet) for cert in ["ca", "client", "controller", "proxy", "scheduler", "kubelet"]: with open(certs / f"{cert}.crt", "w") as ca: ca.write(f"{cert}_data") with open(certs / f"{cert}.key", "w") as ca: ca.write(f"{cert}_key_data") with open(snap / "meta" / "snap.yaml", "w") as f: f.write("confinement: classic") with open(certs / "serviceaccount.key", "w") as f: f.write("service_account_key_data") with open(args / "kube-apiserver", "w") as f: f.write("--some-argument") with open(dqlite / "info.yaml", "w") as f: f.write("Address: 127.0.0.1:19001\nID: 3297041220608546238\nRole: 0\n") with open(args / "cni-network" / "cni.yaml", "w") as f: f.write("can-reach\n") with open(snap / "client.config.template", "w") as f: f.write("token\nUSERNAME") with open(snap / "client-x509.config.template", "w") as f: f.write("x509\nUSERNAME") for config in [ "client.config", "controller.config", "proxy.config", "scheduler.config", "kubelet.config", ]: with open(credentials / config, "w") as f: f.write("kubeconfig") create_dir_layout(tokens=False) info = { "hostname_override": "no-host", "ca": "new_ca_bytes", "ca_key": "new_ca_key_bytes", "service_account_key": "srv_account_key_bytes", "kubelet_args": "--some-kubelet-arg", "callback_token": "123callback456", "cluster_cert": "dqlite_cert", "cluster_key": "dqlite_cert_key", "voters": "none", } join.join_dqlite_master_node(info, "123.123.123.123") # Assert we stored the CA and the dqlite certs mock_chmod.assert_any_call(str(certs / "ca.crt"), 0o660) mock_chmod.assert_any_call(str(certs / "ca.key"), 0o660) mock_chmod.assert_any_call(str(dqlite / "cluster.crt"), 0o660) mock_chmod.assert_any_call(str(dqlite / "cluster.key"), 0o660) # Assert we restart services mock_subprocess_check_call.assert_any_call("snapctl stop microk8s.daemon-kubelite".split()) mock_subprocess_check_call.assert_any_call("snapctl start microk8s.daemon-kubelite".split()) mock_subprocess_check_call.assert_any_call("snapctl stop microk8s.daemon-k8s-dqlite".split()) mock_subprocess_check_call.assert_any_call("snapctl start microk8s.daemon-k8s-dqlite".split()) # Assert we created admin kubeconfig from certificate mock_subprocess_check_call.assert_any_call( [f"{snap}/actions/common/utils.sh", "create_user_certs_and_configs"], stdout=-3, stderr=-3 ) mock_subprocess_check_call.reset_mock() # Check joining with tokens based create_dir_layout(tokens=True) mock_subprocess_check_call.reset_mock() info["admin_token"] = "some-token" join.join_dqlite_master_node(info, "123.123.123.123") mock_subprocess_check_call.assert_any_call( [f"{snap}/actions/common/utils.sh", "create_user_certs_and_configs"], stdout=-3, stderr=-3 ) ================================================ FILE: tests/unit/cluster/test_leave.py ================================================ from click.testing import CliRunner from leave import leave as command def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(command, [help_arg]) assert result.exit_code == 0 assert "The node will depart from the cluster it is in" in result.output ================================================ FILE: tests/unit/test_addons.py ================================================ import os import stat import shutil from contextlib import contextmanager from copy import deepcopy from pathlib import Path from unittest.mock import mock_open, patch, Mock import pytest import yaml from addons import ( AddonsYamlFormatError, AddonsYamlNotFoundError, MissingHookError, WrongHookPermissionsError, add, get_addons_list, load_addons_yaml, update, validate_addons_file, validate_addons_repo, ) from common.utils import parse_xable_addon_args, get_available_addons ADDONS = [ ("core", "addon1"), ("core", "addon2"), ("community", "addon3"), ("core", "conflict"), ("community", "conflict"), ] REPO_YAML = """ microk8s-addons: description: "List of all addons included in Microk8s." addons: - name: "dns" description: "CoreDNS" version: "1.9.0" check_status: "pod/coredns" supported_architectures: - amd64 - arm64 - s390x - name: "rbac" description: "Role-Based Access Control for authorisation" version: "" check_status: "clusterrole.rbac.authorization.k8s.io/cluster-admin" confinement: "strict" supported_architectures: - arm64 - amd64 - s390x - name: "dashboard" description: "The Kubernetes dashboard" version: "2.3.0" check_status: "pod/kubernetes-dashboard" confinement: "classic" supported_architectures: - arm64 - amd64 - s390x """ @pytest.mark.parametrize( "confinement, result", [(True, ["dns", "rbac"]), (False, ["dns", "dashboard"])] ) @patch("common.utils.is_strict") @patch("os.listdir") @patch("common.utils.snap_common") def test_get_available_addons(snap_common_mock, listdir_mock, strict_mock, confinement, result): strict_mock.return_value = confinement listdir_mock.return_value = ["/a/path/"] snap_common_mock.return_value = Path("/common_dir") with patch("common.utils.open", mock_open(read_data=REPO_YAML)): addons = get_available_addons("amd64") assert len(addons) == len(result) for addon in addons: assert addon["name"] in result @pytest.mark.parametrize( "args, result", [ (["addon1"], [("core", "addon1", [])]), (["core/conflict"], [("core", "conflict", [])]), (["community/conflict"], [("community", "conflict", [])]), (["community/conflict", "--with-arg"], [("community", "conflict", ["--with-arg"])]), (["addon1", "--with-arg"], [("core", "addon1", ["--with-arg"])]), ( ["addon1:arg1", "addon2:arg2", "addon3"], [ ("core", "addon1", ["arg1"]), ("core", "addon2", ["arg2"]), ("community", "addon3", []), ], ), ( ["addon1:arg1", "addon2:arg2", "community/conflict"], [ ("core", "addon1", ["arg1"]), ("core", "addon2", ["arg2"]), ("community", "conflict", []), ], ), ( ["core/addon1:arg1", "addon2:arg2", "community/conflict:arg3"], [ ("core", "addon1", ["arg1"]), ("core", "addon2", ["arg2"]), ("community", "conflict", ["arg3"]), ], ), ], ) def test_parse_addons_args(args, result): addons = parse_xable_addon_args(args, ADDONS) assert addons == result TEST_ADDON_NAME = "foo" VALID_ADDONS = { "microk8s-addons": { "addons": [ { "name": TEST_ADDON_NAME, "description": "bar", "version": "1.1.1", "check_status": "foobar", "supported_architectures": ["arm64", "amd64"], } ] } } INVALID_ADDONS = deepcopy(VALID_ADDONS) # Remove one of the required properties of an addon in the addons list INVALID_ADDONS["microk8s-addons"]["addons"][0].pop("check_status") @patch("addons.load_addons_yaml", return_value=VALID_ADDONS) def test_validate_addons_file(load_addons_mock): validate_addons_file(Mock()) @patch("addons.load_addons_yaml", return_value=INVALID_ADDONS) def test_validate_addons_file_raises_if_invalid_format(load_addons_mock): with pytest.raises(AddonsYamlFormatError) as err: validate_addons_file(Mock()) assert "Invalid addons.yaml file" in err.value.message def test_load_addons_raises_on_file_not_found(): # When addons.yaml file is not found with pytest.raises(AddonsYamlNotFoundError) as exc: load_addons_yaml(Path("/some/repo")) assert exc.value.message == "Error: repository repo does not contain an addons.yaml file" def test_load_addons_raises_on_invalid_yaml_contents(repo_dir): with patch("addons.open", mock_open(read_data="unbalanced blackets: ][")): with pytest.raises(AddonsYamlFormatError) as exc: load_addons_yaml(repo_dir) assert ( exc.value.message == "Yaml format error in addons.yaml file: while parsing a block node expected the node content, but found ']'" # noqa ) def test_get_addons_list(): with patch("addons.open", mock_open(read_data=yaml.dump(VALID_ADDONS))): assert get_addons_list(Path("/some/path")) == [TEST_ADDON_NAME] def test_validate_addons_repo(repo_dir): validate_addons_repo(repo_dir) def test_validate_addons_repo_raises_on_missing_enable_hook(addon_missing_enable_hook): with pytest.raises(MissingHookError) as err: validate_addons_repo(addon_missing_enable_hook) assert err.value.message == "Missing enable hook for foo addon" def test_validate_addons_repo_raises_on_missing_disable_hook(addon_missing_disable_hook): with pytest.raises(MissingHookError) as err: validate_addons_repo(addon_missing_disable_hook) assert err.value.message == "Missing disable hook for foo addon" def test_validate_addons_repo_raises_on_enable_not_executable(enable_not_executable): with pytest.raises(WrongHookPermissionsError) as err: validate_addons_repo(enable_not_executable) assert err.value.message == "enable hook for foo addon needs execute permissions" def test_validate_addons_repo_raises_on_disable_not_executable(disable_not_executable): with pytest.raises(WrongHookPermissionsError) as err: validate_addons_repo(disable_not_executable) assert err.value.message == "disable hook for foo addon needs execute permissions" @patch("addons.subprocess") @patch("addons.snap_common", return_value=Path("/tmp/")) @patch("addons.get_group", return_value="microk8s") @patch("addons.validate_addons_repo", side_effect=AddonsYamlFormatError("foo")) @patch("addons.shutil.rmtree") def test_add_removes_repo_on_validation_error( rm_mock, validate_addons_repo_mock, get_group_mock, snap_common_mock, subprocess_mock, ): with pytest.raises(SystemExit): add.callback("myrepo", "http://github.com/me/myrepo", None, False) repo_dir = Path("/tmp/addons/myrepo") validate_addons_repo_mock.assert_called_once_with(repo_dir) rm_mock.assert_called_once_with(repo_dir) @pytest.fixture(scope="function") def repo_dir(tmp_path): repo_dir = tmp_path / "myrepo" repo_dir.mkdir() addons_yaml = repo_dir / "addons.yaml" addons_yaml.write_text(yaml.dump(VALID_ADDONS)) addons = repo_dir / "addons" addons.mkdir() myaddon = addons / TEST_ADDON_NAME myaddon.mkdir() everyone_can_exec = stat.S_IXGRP | stat.S_IXUSR | stat.S_IEXEC enable = myaddon / "enable" enable.write_text("echo 1") enable.chmod(everyone_can_exec) disable = myaddon / "disable" disable.write_text("echo 0") disable.chmod(everyone_can_exec) yield repo_dir @pytest.fixture(scope="function") def invalid_addons_yaml(tmp_path): repo_dir = tmp_path / "invalid_repo" repo_dir.mkdir() addons_yaml = repo_dir / "addons.yaml" addons_yaml.write_text(yaml.dump(INVALID_ADDONS)) yield repo_dir @pytest.fixture(scope="function") def missing_addons_yaml(tmp_path): repo_dir = tmp_path / "invalid_repo" repo_dir.mkdir() yield repo_dir @pytest.fixture(scope="function") def addon_missing_enable_hook(repo_dir): addon_dir = repo_dir / "addons" / TEST_ADDON_NAME os.remove(addon_dir / "enable") yield repo_dir @pytest.fixture(scope="function") def addon_missing_disable_hook(repo_dir): addon_dir = repo_dir / "addons" / TEST_ADDON_NAME os.remove(addon_dir / "disable") yield repo_dir @pytest.fixture(scope="function") def enable_not_executable(repo_dir): addon_dir = repo_dir / "addons" / TEST_ADDON_NAME (addon_dir / "enable").chmod(stat.S_IREAD) yield repo_dir @pytest.fixture(scope="function") def disable_not_executable(repo_dir): addon_dir = repo_dir / "addons" / TEST_ADDON_NAME (addon_dir / "disable").chmod(stat.S_IREAD) yield repo_dir @contextmanager def create_test_repo(repo_name: str): # Create temporary dummy test git repository addons = Path("/tmp/addons") os.mkdir(Path(addons)) repo_dir = addons / repo_name os.mkdir(Path(repo_dir)) # Create the .git file with open(repo_dir / ".git", mode="w+"): pass yield repo_dir # Cleanup shutil.rmtree(Path(addons)) @patch("addons.git_rollback") @patch("addons.git_current_commit") @patch("addons.subprocess") @patch("addons.snap_common", return_value=Path("/tmp/")) @patch("addons.validate_addons_repo", side_effect=AddonsYamlFormatError("foo")) def test_update_rollbacks_repo_on_validation_error( validate_addons_repo_mock, snap_common_mock, subprocess_mock, git_current_commit_mock, git_rollback_mock, ): with create_test_repo("repo_to_update") as repo_dir: with pytest.raises(SystemExit): update.callback("repo_to_update", skip_check_root=True) validate_addons_repo_mock.assert_called_once_with(repo_dir) git_current_commit_mock.assert_called_once_with(repo_dir) git_rollback_mock.assert_called_once_with(git_current_commit_mock.return_value, repo_dir) ================================================ FILE: tests/unit/test_addtoken.py ================================================ from unittest import mock from add_token import print_short def test_single(capsys): with mock.patch( "add_token.get_network_info", return_value=["10.23.53.54", ["10.23.53.54"], "32"] ): print_short("t", "c") captured = capsys.readouterr() output = captured.out.strip() assert output == "microk8s join 10.23.53.54:32/t/c" def test_multiple(capsys): with mock.patch( "add_token.get_network_info", return_value=["d_ip", ["ip1", "ip2", "d_ip"], "4"] ): print_short("t", "c") captured = capsys.readouterr() all_outputs = captured.out.strip().split("\n") assert all_outputs == [ "microk8s join d_ip:4/t/c", "microk8s join ip1:4/t/c", "microk8s join ip2:4/t/c", ] ================================================ FILE: tests/unit/test_dashboard_proxy.py ================================================ from click.testing import CliRunner from dashboard_proxy import dashboard_proxy as command def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(command, [help_arg]) assert result.exit_code == 0 assert "Enable the dashboard add-on and configures port-forwarding" in result.output ================================================ FILE: tests/unit/test_disable.py ================================================ from click.testing import CliRunner from disable import disable as command from unittest.mock import patch def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(command, [help_arg]) assert result.exit_code == 0 assert "Disable one or more MicroK8s addons" in result.output def test_command_errors_if_no_arguments(): runner = CliRunner() result = runner.invoke(command, []) assert result.exit_code != 0 assert "Error: Missing argument" in result.output @patch("disable.xable") def test_command_shows_addon_help_message(xable_mock): runner = CliRunner() for help_flag in ("-h", "--help"): result = runner.invoke(command, ["dns", "--", help_flag]) assert result.output.startswith("Addon dns does not yet have a help message.") xable_mock.assert_not_called() ================================================ FILE: tests/unit/test_enable.py ================================================ from click.testing import CliRunner from enable import enable as command from unittest.mock import patch def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(command, [help_arg]) assert result.exit_code == 0 assert "Enable a MicroK8s addon" in result.output def test_command_errors_if_no_arguments(): runner = CliRunner() result = runner.invoke(command, []) assert result.exit_code != 0 assert "Error: Missing argument" in result.output @patch("enable.xable") def test_command_shows_addon_help_message(xable_mock): runner = CliRunner() for help_flag in ("-h", "--help"): result = runner.invoke(command, ["dns", "--", help_flag]) assert result.output.startswith("Addon dns does not yet have a help message.") xable_mock.assert_not_called() ================================================ FILE: tests/unit/test_refresh_certs.py ================================================ import os import pytest from unittest.mock import patch, call, Mock from click.testing import CliRunner from refresh_certs import ( refresh_certs, restart, reproduce_all_root_ca_certs, reproduce_front_proxy_client_cert, reproduce_server_cert, ) class TestRefreshCerts(object): @patch("subprocess.check_call") def test_restart(self, mock_check_call): restart() # We stop and start microk8s assert mock_check_call.call_count == 2 @patch("common.cluster.utils.get_arg") @patch("subprocess.Popen") @patch("subprocess.check_call") def test_reproduce_all_root_ca_certs(self, mock_check_call, mock_subproc_popen, mock_get_arg): process_mock = Mock() mock_subproc_popen.return_value = process_mock mock_get_arg.return_value = "known_tokens.csv" reproduce_all_root_ca_certs() """ Make sure we: 1. remove the ca.crt 2. call produce_certs 3. call update_configs that also restarts microk8s """ snapdata_path = os.environ.get("SNAP_DATA") assert ( call("rm -rf {}/certs/ca.crt".format(snapdata_path).split()) in mock_check_call.call_args_list ) assert mock_check_call.called assert mock_subproc_popen.called assert self.is_argument_in_call(mock_subproc_popen, "produce_certs") assert self.is_argument_in_call(mock_subproc_popen, "update_configs") @patch("refresh_certs.restart") @patch("subprocess.check_call") def test_reproduce_front_proxy_client_cert(self, mock_check_call, mock_restart): """ Make sure we: 1. remove the front-proxy-client.crt 2. call gen_proxy_client_cert 3. restart microk8s """ reproduce_front_proxy_client_cert() snapdata_path = os.environ.get("SNAP_DATA") cmd = "rm -rf {}/certs/front-proxy-client.crt".format(snapdata_path).split() assert call(cmd) in mock_check_call.call_args_list assert mock_check_call.called assert mock_restart.called assert self.is_argument_in_call(mock_check_call, "gen_proxy_client_cert") @patch("refresh_certs.restart") @patch("subprocess.check_call") def test_reproduce_server_cert(self, mock_check_call, mock_restart): """ Make sure we: 1. remove the server.crt 2. call gen_server_cert 3. restart microk8s """ reproduce_server_cert() snapdata_path = os.environ.get("SNAP_DATA") assert ( call("rm -rf {}/certs/server.crt".format(snapdata_path).split()) in mock_check_call.call_args_list ) assert mock_check_call.called assert mock_restart.called assert self.is_argument_in_call(mock_check_call, "gen_server_cert") def is_argument_in_call(self, mock_function, argument_substring): """Search for a substring in the list of arguments in all calls of a mocked function""" for calls in mock_function.call_args_list: # calls is the list of calls for arglist in calls.args: # list of arguments in call for arg in arglist: if argument_substring in arg: return True return False @pytest.mark.parametrize( "ca_dir,undo,check,cert,help,expected_output,expected_err_code", [ (None, None, True, "ca.crt", None, "Please select only one of the options", 2), (None, True, True, None, None, "Please select only one of the options", 2), (None, True, None, "ca.crt", None, "Please select only one of the options", 2), (None, True, None, "ca.crt", True, "Usage:", 0), ("/some/path", True, None, None, None, "does not exist", 2), ("/", True, None, None, None, "options in combination", 1), (None, True, None, "wrong_file", None, "Invalid value", 2), ], ) def test_refresh_cert_errors( self, ca_dir, undo, check, cert, help, expected_output, expected_err_code ): """ Test conditions under which the upgrade should not continue """ runner = CliRunner() args = [] if ca_dir: args.append(ca_dir) if undo: args.append("-u") if check: args.append("-c") if cert: args.append("-e") args.append(cert) if help: args.append("-h") result = runner.invoke(refresh_certs, args) assert expected_output in result.output assert result.exit_code == expected_err_code def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(refresh_certs, [help_arg]) assert result.exit_code == 0 assert "Replace the CA certificates with the ca.crt and ca.key" in result.output ================================================ FILE: tests/unit/test_reset.py ================================================ from click.testing import CliRunner from reset import reset as command def test_command_help_arguments(): runner = CliRunner() for help_arg in ("-h", "--help"): result = runner.invoke(command, [help_arg]) assert result.exit_code == 0 assert "Return the MicroK8s node to the default initial state" in result.output ================================================ FILE: tests/unit/test_upgrade_calico_cni.py ================================================ import os import shutil from calico.upgrade import ( try_upgrade, get_installed_version_of_calico, get_calicos_autodetection_method, mark_apply_needed, ) class TestCNIUpgrade(object): def setup_class(self): dirname, filename = os.path.split(os.path.abspath(__file__)) self._invalid_yaml = os.path.join(dirname, "yamls/invalid.yaml") self._calico_new_yaml = os.path.join(dirname, "yamls/calico-new.yaml") self._calico_old_yaml = os.path.join(dirname, "yamls/cni.yaml") self._calico_old_copy_yaml = os.path.join(dirname, "yamls/cni.yaml.copy") self._calico_old_copy_backup_yaml = os.path.join(dirname, "yamls/cni.yaml.copy.backup") self._lock_file = os.path.join(dirname, "yamls/lock_file") self._cni_no_manage_file = os.path.join(dirname, "yamls/cni_no_manage") def test_no_op(self): """ Test conditions under which the upgrade should not continue """ res = try_upgrade("foo", "bar") assert res is False res = try_upgrade(self._calico_new_yaml, self._calico_new_yaml) assert res is False res = try_upgrade(self._invalid_yaml, self._calico_new_yaml) assert res is False shutil.copyfile(self._invalid_yaml, self._cni_no_manage_file) res = try_upgrade(self._calico_new_yaml, self._calico_new_yaml, self._cni_no_manage_file) assert res is False os.remove(self._cni_no_manage_file) def test_get_version(self): """ Test extracting the Calico version """ res = get_installed_version_of_calico(self._calico_new_yaml) assert res == "v3.23.4" def test_get_autodetect_method(self): """ Test extracting the IP autodetection method """ res = get_calicos_autodetection_method(self._calico_new_yaml) assert res == "first-found" def test_patch(self): """ Test patching the manifest """ shutil.copyfile(self._calico_old_yaml, self._calico_old_copy_yaml) res = get_calicos_autodetection_method(self._calico_old_copy_yaml) assert res == "can-reach=192.168.1.43" res = get_calicos_autodetection_method(self._calico_new_yaml) assert res == "first-found" res = get_installed_version_of_calico(self._calico_old_copy_yaml) assert res == "v3.21.1" res = try_upgrade(self._calico_old_copy_yaml, self._calico_new_yaml) assert os.path.exists(self._calico_old_copy_backup_yaml) res = get_installed_version_of_calico(self._calico_old_copy_yaml) assert res == "v3.23.4" os.remove(self._calico_old_copy_yaml) os.remove(self._calico_old_copy_backup_yaml) def test_mark(self): """ Test marking the need for reapplying the manifest """ shutil.copyfile(self._calico_old_yaml, self._lock_file) mark_apply_needed(self._lock_file) assert not os.path.exists(self._lock_file) ================================================ FILE: tests/unit/test_version.py ================================================ from version import get_snap_version, get_snap_revision import os def test_get_snap_version(): os.environ["SNAP_VERSION"] = "v1.21.11" assert get_snap_version() == "v1.21.11" def test_get_snap_revision(): os.environ["SNAP_REVISION"] = "3086" assert get_snap_revision() == "3086" ================================================ FILE: tests/unit/yamls/calico-new.yaml ================================================ --- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap apiVersion: v1 metadata: name: calico-config namespace: kube-system data: # Typha is disabled. typha_service_name: "none" # Configure the backend to use. calico_backend: "vxlan" # Configure the MTU to use for workload interfaces and tunnels. # By default, MTU is auto-detected, and explicitly setting this field should not be required. # You can override auto-detection by providing a non-zero value. veth_mtu: "0" # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", "cniVersion": "0.3.1", "plugins": [ { "type": "calico", "nodename_file_optional": true, "log_level": "info", "log_file_path": "/var/log/calico/cni/cni.log", "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { "type": "calico-ipam" }, "policy": { "type": "k8s" }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { "type": "portmap", "snat": true, "capabilities": {"portMappings": true} }, { "type": "bandwidth", "capabilities": {"bandwidth": true} } ] } --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bgpconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPConfiguration listKind: BGPConfigurationList plural: bgpconfigurations singular: bgpconfiguration scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: BGPConfiguration contains the configuration for any BGP routing. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPConfigurationSpec contains the values of the BGP configuration. properties: asNumber: description: 'ASNumber is the default AS number used by a node. [Default: 64512]' format: int32 type: integer communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. items: description: Community contains standard or large community value and its name. properties: name: description: Name given to community value. type: string value: description: Value must be of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where, `aa` is an AS Number, `nn` and `mm` are per-AS identifier. pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ type: string type: object type: array listenPort: description: ListenPort is the port where BGP protocol should listen. Defaults to 179 maximum: 65535 minimum: 1 type: integer logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' type: boolean prefixAdvertisements: description: PrefixAdvertisements contains per-prefix advertisement configuration. items: description: PrefixAdvertisement configures advertisement properties for the specified CIDR. properties: cidr: description: CIDR for which properties should be advertised. type: string communities: description: Communities can be list of either community names already defined in `Specs.Communities` or community value of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and `mm` are per-AS identifier. items: type: string type: array type: object type: array serviceClusterIPs: description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them. items: description: ServiceClusterIPBlock represents a single allowed ClusterIP CIDR block. properties: cidr: type: string type: object type: array serviceExternalIPs: description: ServiceExternalIPs are the CIDR blocks for Kubernetes Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. items: description: ServiceExternalIPBlock represents a single allowed External IP CIDR block. properties: cidr: type: string type: object type: array serviceLoadBalancerIPs: description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress IPs will only be advertised if they are within one of these blocks. items: description: ServiceLoadBalancerIPBlock represents a single allowed LoadBalancer IP CIDR block. properties: cidr: type: string type: object type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bgppeers.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPPeer listKind: BGPPeerList plural: bgppeers singular: bgppeer scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPPeerSpec contains the specification for a BGPPeer resource. properties: asNumber: description: The AS Number of the peer. format: int32 type: integer keepOriginalNextHop: description: Option to keep the original nexthop field when routes are sent to a BGP Peer. Setting "true" configures the selected BGP Peers node to use the "next hop keep;" instead of "next hop self;"(default) in the specific branch of the Node on "bird.cfg". type: boolean maxRestartTime: description: Time to allow for software restart. When specified, this is configured as the graceful restart timeout. When not specified, the BIRD default of 120s is used. type: string node: description: The node name identifying the Calico node instance that is targeted by this peer. If this is not set, and no nodeSelector is specified, then this BGP peer selects all nodes in the cluster. type: string nodeSelector: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string password: description: Optional BGP password for the peerings generated by this BGPPeer resource. properties: secretKeyRef: description: Selects a key of a secret in the node pod's namespace. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object peerIP: description: The IP address of the peer followed by an optional port number to peer with. If port number is given, format should be `[]:port` or `:` for IPv4. If optional port number is not set, and this peer IP and ASNumber belongs to a calico/node with ListenPort set in BGPConfiguration, then we use that port to peer. type: string peerSelector: description: Selector for the remote nodes to peer with. When this is set, the PeerIP and ASNumber fields must be empty. For each peering between the local node and selected remote nodes, we configure an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The remote AS number comes from the remote node's NodeBGPSpec.ASNumber, or the global default if that is not set. type: string sourceAddress: description: Specifies whether and how to configure a source address for the peerings generated by this BGPPeer resource. Default value "UseNodeIP" means to configure the node IP as the source address. "None" means not to configure a source address. type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: blockaffinities.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BlockAffinity listKind: BlockAffinityList plural: blockaffinities singular: blockaffinity scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BlockAffinitySpec contains the specification for a BlockAffinity resource. properties: cidr: type: string deleted: description: Deleted indicates that this block affinity is being deleted. This field is a string for compatibility with older releases that mistakenly treat this field as a string. type: string node: type: string state: type: string required: - cidr - deleted - node - state type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (devel) creationTimestamp: null name: caliconodestatuses.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: CalicoNodeStatus listKind: CalicoNodeStatusList plural: caliconodestatuses singular: caliconodestatus scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus resource. properties: classes: description: Classes declares the types of information to monitor for this calico/node, and allows for selective status reporting about certain subsets of information. items: type: string type: array node: description: The node name identifies the Calico node instance for node status. type: string updatePeriodSeconds: description: UpdatePeriodSeconds is the period at which CalicoNodeStatus should be updated. Set to 0 to disable CalicoNodeStatus refresh. Maximum update period is one day. format: int32 type: integer type: object status: description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. No validation needed for status since it is updated by Calico. properties: agent: description: Agent holds agent status on the node. properties: birdV4: description: BIRDV4 represents the latest observed status of bird4. properties: lastBootTime: description: LastBootTime holds the value of lastBootTime from bird.ctl output. type: string lastReconfigurationTime: description: LastReconfigurationTime holds the value of lastReconfigTime from bird.ctl output. type: string routerID: description: Router ID used by bird. type: string state: description: The state of the BGP Daemon. type: string version: description: Version of the BGP daemon type: string type: object birdV6: description: BIRDV6 represents the latest observed status of bird6. properties: lastBootTime: description: LastBootTime holds the value of lastBootTime from bird.ctl output. type: string lastReconfigurationTime: description: LastReconfigurationTime holds the value of lastReconfigTime from bird.ctl output. type: string routerID: description: Router ID used by bird. type: string state: description: The state of the BGP Daemon. type: string version: description: Version of the BGP daemon type: string type: object type: object bgp: description: BGP holds node BGP status. properties: numberEstablishedV4: description: The total number of IPv4 established bgp sessions. type: integer numberEstablishedV6: description: The total number of IPv6 established bgp sessions. type: integer numberNotEstablishedV4: description: The total number of IPv4 non-established bgp sessions. type: integer numberNotEstablishedV6: description: The total number of IPv6 non-established bgp sessions. type: integer peersV4: description: PeersV4 represents IPv4 BGP peers status on the node. items: description: CalicoNodePeer contains the status of BGP peers on the node. properties: peerIP: description: IP address of the peer whose condition we are reporting. type: string since: description: Since the state or reason last changed. type: string state: description: State is the BGP session state. type: string type: description: Type indicates whether this peer is configured via the node-to-node mesh, or via en explicit global or per-node BGPPeer object. type: string type: object type: array peersV6: description: PeersV6 represents IPv6 BGP peers status on the node. items: description: CalicoNodePeer contains the status of BGP peers on the node. properties: peerIP: description: IP address of the peer whose condition we are reporting. type: string since: description: Since the state or reason last changed. type: string state: description: State is the BGP session state. type: string type: description: Type indicates whether this peer is configured via the node-to-node mesh, or via en explicit global or per-node BGPPeer object. type: string type: object type: array required: - numberEstablishedV4 - numberEstablishedV6 - numberNotEstablishedV4 - numberNotEstablishedV6 type: object lastUpdated: description: LastUpdated is a timestamp representing the server time when CalicoNodeStatus object last updated. It is represented in RFC3339 form and is in UTC. format: date-time nullable: true type: string routes: description: Routes reports routes known to the Calico BGP daemon on the node. properties: routesV4: description: RoutesV4 represents IPv4 routes on the node. items: description: CalicoNodeRoute contains the status of BGP routes on the node. properties: destination: description: Destination of the route. type: string gateway: description: Gateway for the destination. type: string interface: description: Interface for the destination type: string learnedFrom: description: LearnedFrom contains information regarding where this route originated. properties: peerIP: description: If sourceType is NodeMesh or BGPPeer, IP address of the router that sent us this route. type: string sourceType: description: Type of the source where a route is learned from. type: string type: object type: description: Type indicates if the route is being used for forwarding or not. type: string type: object type: array routesV6: description: RoutesV6 represents IPv6 routes on the node. items: description: CalicoNodeRoute contains the status of BGP routes on the node. properties: destination: description: Destination of the route. type: string gateway: description: Gateway for the destination. type: string interface: description: Interface for the destination type: string learnedFrom: description: LearnedFrom contains information regarding where this route originated. properties: peerIP: description: If sourceType is NodeMesh or BGPPeer, IP address of the router that sent us this route. type: string sourceType: description: Type of the source where a route is learned from. type: string type: object type: description: Type indicates if the route is being used for forwarding or not. type: string type: object type: array type: object type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clusterinformations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: ClusterInformation listKind: ClusterInformationList plural: clusterinformations singular: clusterinformation scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: ClusterInformation contains the cluster specific information. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: ClusterInformationSpec contains the values of describing the cluster. properties: calicoVersion: description: CalicoVersion is the version of Calico that the cluster is running type: string clusterGUID: description: ClusterGUID is the GUID of the cluster type: string clusterType: description: ClusterType describes the type of the cluster type: string datastoreReady: description: DatastoreReady is used during significant datastore migrations to signal to components such as Felix that it should wait before accessing the datastore. type: boolean variant: description: Variant declares which variant of Calico should be active. type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: felixconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: FelixConfiguration listKind: FelixConfigurationList plural: felixconfigurations singular: felixconfiguration scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: Felix Configuration contains the configuration for Felix. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: FelixConfigurationSpec contains the values of the Felix configuration. properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads [Default: false]' type: boolean awsSrcDstCheck: description: 'Set source-destination-check on AWS EC2 instances. Accepted value must be one of "DoNothing", "Enable" or "Disable". [Default: DoNothing]' enum: - DoNothing - Enable - Disable type: string bpfConnectTimeLoadBalancingEnabled: description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes. [Default: true]' type: boolean bpfDataIfacePattern: description: BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually named cali...). type: string bpfDisableUnprivileged: description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable unprivileged use of BPF. This ensures that unprivileged users cannot access Calico''s BPF maps and cannot insert their own BPF programs to interfere with Calico''s. [Default: true]' type: boolean bpfEnabled: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check. [Default: 0]' type: integer bpfExternalServiceMode: description: 'BPFExternalServiceMode in BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) are forwarded to remote workloads. If set to "Tunnel" then both request and response traffic is tunneled to the remote node. If set to "DSR", the request traffic is tunneled but the response traffic is sent directly from the remote node. In "DSR" mode, the remote node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' type: string bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls whether Felix's embedded kube-proxy accepts EndpointSlices or not. type: boolean bpfKubeProxyIptablesCleanupEnabled: description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s iptables chains. Should only be enabled if kube-proxy is not running. [Default: true]' type: boolean bpfKubeProxyMinSyncPeriod: description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the minimum time between updates to the dataplane for Felix''s embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by batching up more work. [Default: 1s]' type: string bpfLogLevel: description: 'BPFLogLevel controls the log level of the BPF programs when in BPF dataplane mode. One of "Off", "Info", or "Debug". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents Calico''s rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: type: string debugDisableLogDropping: type: boolean debugMemoryProfilePath: type: string debugSimulateCalcGraphHangAfter: type: string debugSimulateDataplaneHangAfter: type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an iptables "DROP" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables "INPUT" chain; Calico will insert its rules at the top of that chain, then "RETURN" packets to the "INPUT" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]' type: string deviceRouteProtocol: description: This defines the route protocol added to programmed device routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: description: This is the source address to use on programmed device routes. By default the source address is left blank, leaving the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean endpointReportingDelay: type: string endpointReportingEnabled: type: boolean externalNodesList: description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have the tunneled traffic be accepted at calico nodes. items: type: string type: array failsafeInboundHostPorts: description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For back-compatibility, if the protocol is not specified, it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value none. The default value allows ssh access and DHCP. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' items: description: ProtoPort is combination of protocol, port, and CIDR. Protocol and port must be specified. properties: net: type: string port: type: integer protocol: type: string required: - port - protocol type: object type: array failsafeOutboundHostPorts: description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For back-compatibility, if the protocol is not specified, it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value none. The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP and DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, udp:53, udp:67]' items: description: ProtoPort is combination of protocol, port, and CIDR. Protocol and port must be specified. properties: net: type: string port: type: integer protocol: type: string required: - port - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override the feature detection. Values are specified in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This is not recommended since it doesn''t provide better performance than iptables. [Default: false]' type: boolean healthEnabled: type: boolean healthHost: type: string healthPort: type: integer interfaceExclude: description: 'InterfaceExclude is a comma-separated list of interfaces that Felix should exclude when monitoring for host endpoints. The default value ensures that Felix ignores Kubernetes'' IPVS dummy interface, which is used internally by kube-proxy. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with ''/''. For example having values ''/^kube/,veth1'' will exclude all interfaces that begin with ''kube'' and also the interface ''veth1''. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker integrations set the ''cali'' value, and our OpenStack integration sets the ''tap'' value. [Default: cali]' type: string interfaceRefreshInterval: description: InterfaceRefreshInterval is the period at which Felix rescans local interfaces to verify their state. The rescan can be disabled by setting the interval to 0. type: string ipipEnabled: type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440]' type: integer ipsetsRefreshInterval: description: 'IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: 90s]' type: string iptablesBackend: description: IptablesBackend specifies which backend of iptables will be used. The default is legacy. type: string iptablesFilterAllowAction: type: string iptablesLockFilePath: description: 'IptablesLockFilePath is the location of the iptables lock file. You may need to change this if the lock file is not in its standard location (for example if you have mapped it into Felix''s container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: description: 'IptablesLockProbeInterval is the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU. [Default: 50ms]' type: string iptablesLockTimeout: description: 'IptablesLockTimeout is the time that Felix will wait for the iptables lock, or 0, to disable. To use this feature, Felix must share the iptables lock file with all other processes that also take the lock. When running Felix inside a container, this requires the /run directory of the host to be mounted into the calico/node or calico/felix container. [Default: 0s disabled]' type: string iptablesMangleAllowAction: type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xff000000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. This should only occur if another application on the system doesn''t respect the iptables lock. [Default: 1s]' type: string iptablesRefreshInterval: description: 'IptablesRefreshInterval is the period at which Felix re-checks the IP sets in the dataplane to ensure that no other process has accidentally broken Calico''s rules. Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. [Default: 10s]' type: string ipv6Support: type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. Felix uses these ranges to separate host and workload traffic. [Default: 30000:32767].' items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' type: string logPrefix: description: 'LogPrefix is the log prefix that Felix uses when rendering LOG rules. [Default: calico-packet]' type: string logSeverityFile: description: 'LogSeverityFile is the log severity above which logs are sent to the log file. [Default: Info]' type: string logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string logSeveritySys: description: 'LogSeveritySys is the log severity above which logs are sent to the syslog. Set to None for no logging to syslog. [Default: Info]' type: string maxIpsetSize: type: integer metadataAddr: description: 'MetadataAddr is the IP address or domain name of the server that can answer VM queries for cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in Ubuntu, nova-api-metadata). A value of none (case insensitive) means that Felix should not set up any NAT rule for the metadata path. [Default: 127.0.0.1]' type: string metadataPort: description: 'MetadataPort is the port of the metadata server. This, combined with global.MetadataAddr (if not ''None''), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed [Default: 8775].' type: integer mtuIfacePattern: description: MTUIfacePattern is a regular expression that controls which interfaces Felix should scan in order to calculate the host's MTU. This should not match workload interfaces (usually named cali...). type: string natOutgoingAddress: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (ie it uses the iptables MASQUERADE target) type: string natPortRange: anyOf: - type: integer - type: string description: NATPortRange specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the network stack is used. pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: type: string openstackRegion: description: 'OpenstackRegion is the name of the region that a particular Felix belongs to. In a multi-region Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, or in felix.cfg or the environment on each compute node), and must match the [calico] openstack_region value configured in neutron.conf on each node. [Default: Empty]' type: string policySyncPathPrefix: description: 'PolicySyncPathPrefix is used to by Felix to communicate policy changes to external services, like Application layer policy. [Default: Empty]' type: string prometheusGoMetricsEnabled: description: 'PrometheusGoMetricsEnabled disables Go runtime metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]' type: boolean prometheusMetricsEnabled: description: 'PrometheusMetricsEnabled enables the Prometheus metrics server in Felix if set to true. [Default: false]' type: boolean prometheusMetricsHost: description: 'PrometheusMetricsHost is the host that the Prometheus metrics server should bind to. [Default: empty]' type: string prometheusMetricsPort: description: 'PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. [Default: 9091]' type: integer prometheusProcessMetricsEnabled: description: 'PrometheusProcessMetricsEnabled disables process metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]' type: boolean prometheusWireGuardMetricsEnabled: description: 'PrometheusWireGuardMetricsEnabled disables wireguard metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: description: Whether or not to remove device routes that have not been programmed by Felix. Disabling this will allow external applications to also add device routes. This is enabled by default which means we will remove externally added routes. type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports its status into the datastore or 0 to disable. Must be non-zero in OpenStack deployments. [Default: 30s]' type: string reportingTTL: description: 'ReportingTTL is the time-to-live setting for process-wide status reports. [Default: 90s]' type: string routeRefreshInterval: description: 'RouteRefreshInterval is the period at which Felix re-checks the routes in the dataplane to ensure that no other process has accidentally broken Calico''s rules. Set to 0 to disable route refresh. [Default: 90s]' type: string routeSource: description: 'RouteSource configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: description: Calico programs additional Linux route tables for various purposes. RouteTableRange specifies the indices of the route tables that Calico should use. properties: max: type: integer min: type: integer required: - max - min type: object serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", in which case such routing loops continue to be allowed. [Default: Drop]' type: string sidecarAccelerationEnabled: description: 'SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false]' type: boolean usageReportingEnabled: description: 'UsageReportingEnabled reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage server. For example, if a significant security vulnerability has been discovered in the version of Calico being used. [Default: true]' type: boolean usageReportingInitialDelay: description: 'UsageReportingInitialDelay controls the minimum delay before Felix makes a report. [Default: 300s]' type: string usageReportingInterval: description: 'UsageReportingInterval controls the interval at which Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: type: boolean vxlanEnabled: type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440]' type: integer vxlanPort: type: integer vxlanVNI: type: integer wireguardEnabled: description: 'WireguardEnabled controls whether Wireguard is enabled. [Default: false]' type: boolean wireguardHostEncryptionEnabled: description: 'WireguardHostEncryptionEnabled controls whether Wireguard host-to-host encryption is enabled. [Default: false]' type: boolean wireguardInterfaceName: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' type: integer wireguardMTU: description: 'WireguardMTU controls the MTU on the Wireguard interface. See Configuring MTU [Default: 1420]' type: integer wireguardRoutingRulePriority: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' type: boolean xdpRefreshInterval: description: 'XDPRefreshInterval is the period at which Felix re-checks all XDP state to ensure that no other process has accidentally broken Calico''s BPF maps or attached programs. Set to 0 to disable XDP refresh. [Default: 90s]' type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: globalnetworkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: GlobalNetworkPolicy listKind: GlobalNetworkPolicyList plural: globalnetworkpolicies singular: globalnetworkpolicy scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: properties: applyOnForward: description: ApplyOnForward indicates to apply the rules in this policy on forward traffic. type: boolean doNotTrack: description: DoNotTrack indicates whether packets matched by the rules in this policy should go through the data plane's connection tracking, such as Linux conntrack. If True, the rules in this policy are applied before any data plane connection tracking, and packets allowed by this policy are marked as not to be tracked. type: boolean egress: description: The ordered set of egress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array ingress: description: The ordered set of ingress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array namespaceSelector: description: NamespaceSelector is an optional field for an expression used to select a pod based on namespaces. type: string order: description: Order is an optional field that specifies the order in which the policy is applied. Policies with higher "order" are applied after those with lower order. If the order is omitted, it may be considered to be "infinite" - i.e. the policy will be applied last. Policies with identical order will be applied in alphanumerical order based on the Policy "Name". type: number preDNAT: description: PreDNAT indicates to apply the rules in this policy before any DNAT. type: boolean selector: description: "The selector is an expression used to pick pick out the endpoints that the policy should be applied to. \n Selector expressions follow this syntax: \n \tlabel == \"string_literal\" \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" \ -> not equal; also matches if label is not present \tlabel in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name) -> True if that label is present \t! expr -> negation of expr \texpr && expr -> Short-circuit and \texpr || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() or the empty selector -> matches all endpoints. \n Label names are allowed to contain alphanumerics, -, _ and /. String literals are more permissive but they do not support escape characters. \n Examples (with made-up labels): \n \ttype == \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != \"dev\" \t! has(label_name)" type: string serviceAccountSelector: description: ServiceAccountSelector is an optional field for an expression used to select a pod based on service accounts. type: string types: description: "Types indicates whether this policy applies to ingress, or to egress, or to both. When not explicitly specified (and so the value on creation is empty or nil), Calico defaults Types according to what Ingress and Egress rules are present in the policy. The default is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are also no Ingress rules) \n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. \n When the policy is read back again, Types will always be one of these values, never empty or nil." items: description: PolicyType enumerates the possible values of the PolicySpec Types field. type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: globalnetworksets.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: GlobalNetworkSet listKind: GlobalNetworkSetList plural: globalnetworksets singular: globalnetworkset scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs that share labels to allow rules to refer to them via selectors. The labels of GlobalNetworkSet are not namespaced. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: GlobalNetworkSetSpec contains the specification for a NetworkSet resource. properties: nets: description: The list of IP networks that belong to this set. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: hostendpoints.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: HostEndpoint listKind: HostEndpointList plural: hostendpoints singular: hostendpoint scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: HostEndpointSpec contains the specification for a HostEndpoint resource. properties: expectedIPs: description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. If \"InterfaceName\" is not present, Calico will look for an interface matching any of the IPs in the list and apply policy to that. Note: \tWhen using the selector match criteria in an ingress or egress security Policy \tor Profile, Calico converts the selector into a set of IP addresses. For host \tendpoints, the ExpectedIPs field is used for that purpose. (If only the interface \tname is specified, Calico does not learn the IPs of the interface for use in match \tcriteria.)" items: type: string type: array interfaceName: description: "Either \"*\", or the name of a specific Linux interface to apply policy to; or empty. \"*\" indicates that this HostEndpoint governs all traffic to, from or through the default network namespace of the host named by the \"Node\" field; entering and leaving that namespace via any interface, including those from/to non-host-networked local workloads. \n If InterfaceName is not \"*\", this HostEndpoint only governs traffic that enters or leaves the host through the specific interface named by InterfaceName, or - when InterfaceName is empty - through the specific interface that has one of the IPs in ExpectedIPs. Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only external interfaces (such as \"eth0\") are supported here; it isn't possible for a HostEndpoint to protect traffic through a specific local workload interface. \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; initially just pre-DNAT policy. Please check Calico documentation for the latest position." type: string node: description: The node name identifying the Calico node instance. type: string ports: description: Ports contains the endpoint's named ports, which may be referenced in security policy rules. items: properties: name: type: string port: type: integer protocol: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true required: - name - port - protocol type: object type: array profiles: description: A list of identifiers of security Profile objects that apply to this endpoint. Each profile is applied in the order that they appear in this list. Profile rules are applied after the selector-based security policy. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamblocks.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMBlock listKind: IPAMBlockList plural: ipamblocks singular: ipamblock scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAMBlockSpec contains the specification for an IPAMBlock resource. properties: affinity: type: string allocations: items: type: integer # TODO: This nullable is manually added in. We should update controller-gen # to handle []*int properly itself. nullable: true type: array attributes: items: properties: handle_id: type: string secondary: additionalProperties: type: string type: object type: object type: array cidr: type: string deleted: type: boolean strictAffinity: type: boolean unallocated: items: type: integer type: array required: - allocations - attributes - cidr - strictAffinity - unallocated type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamconfigs.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMConfig listKind: IPAMConfigList plural: ipamconfigs singular: ipamconfig scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAMConfigSpec contains the specification for an IPAMConfig resource. properties: autoAllocateBlocks: type: boolean maxBlocksPerHost: description: MaxBlocksPerHost, if non-zero, is the max number of blocks that can be affine to each host. type: integer strictAffinity: type: boolean required: - autoAllocateBlocks - strictAffinity type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamhandles.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMHandle listKind: IPAMHandleList plural: ipamhandles singular: ipamhandle scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAMHandleSpec contains the specification for an IPAMHandle resource. properties: block: additionalProperties: type: integer type: object deleted: type: boolean handleID: type: string required: - block - handleID type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ippools.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPPool listKind: IPPoolList plural: ippools singular: ippool scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPPoolSpec contains the specification for an IPPool resource. properties: allowedUses: description: AllowedUse controls what the IP pool will be used for. If not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility items: type: string type: array blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 112 for IPv6. type: integer cidr: description: The pool CIDR. type: string disabled: description: When disabled is true, Calico IPAM will not assign addresses from this pool. type: boolean disableBGPExport: description: 'Disable exporting routes from this IP Pool’s CIDR over BGP. [Default: false]' type: boolean ipip: description: 'Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is for internal use only.' properties: enabled: description: When enabled is true, ipip tunneling will be used to deliver packets to destinations within this pool. type: boolean mode: description: The IPIP mode. This can be one of "always" or "cross-subnet". A mode of "always" will also use IPIP tunneling for routing to destination IP addresses within this pool. A mode of "cross-subnet" will only use IPIP tunneling when the destination node is on a different subnet to the originating node. The default value (if not specified) is "always". type: string type: object ipipMode: description: Contains configuration for IPIP tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). type: string nat-outgoing: description: 'Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is for internal use only.' type: boolean natOutgoing: description: When nat-outgoing is true, packets sent from Calico networked containers in this pool to destinations outside of this pool will be masqueraded. type: boolean nodeSelector: description: Allows IPPool to allocate for a specific node by label selector. type: string vxlanMode: description: Contains configuration for VXLAN tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. VXLAN tunneling is disabled). type: string required: - cidr type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPReservation listKind: IPReservationList plural: ipreservations singular: ipreservation scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPReservationSpec contains the specification for an IPReservation resource. properties: reservedCIDRs: description: ReservedCIDRs is a list of CIDRs and/or IP addresses that Calico IPAM will exclude from new allocations. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: kubecontrollersconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: KubeControllersConfiguration listKind: KubeControllersConfigurationList plural: kubecontrollersconfigurations singular: kubecontrollersconfiguration scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: KubeControllersConfigurationSpec contains the values of the Kubernetes controllers configuration. properties: controllers: description: Controllers enables and configures individual Kubernetes controllers properties: namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object node: description: Node enables and configures the node controller. Enabled by default, set to nil to disable. properties: hostEndpoint: description: HostEndpoint controls syncing nodes to host endpoints. Disabled by default, set to nil to disable. properties: autoCreate: description: 'AutoCreate enables automatic creation of host endpoints for every node. [Default: Disabled]' type: string type: object leakGracePeriod: description: 'LeakGracePeriod is the period used by the controller to determine if an IP address has been leaked. Set to 0 to disable IP garbage collection. [Default: 15m]' type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string syncLabels: description: 'SyncLabels controls whether to copy Kubernetes node labels to Calico nodes. [Default: Enabled]' type: string type: object policy: description: Policy enables and configures the policy controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object serviceAccount: description: ServiceAccount enables and configures the service account controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object workloadEndpoint: description: WorkloadEndpoint enables and configures the workload endpoint controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object type: object etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' type: string healthChecks: description: 'HealthChecks enables or disables support for health checks [Default: Enabled]' type: string logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string prometheusMetricsPort: description: 'PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. Set to 0 to disable. [Default: 9094]' type: integer required: - controllers type: object status: description: KubeControllersConfigurationStatus represents the status of the configuration. It's useful for admins to be able to see the actual config that was applied, which can be modified by environment variables on the kube-controllers process. properties: environmentVars: additionalProperties: type: string description: EnvironmentVars contains the environment variables on the kube-controllers that influenced the RunningConfig. type: object runningConfig: description: RunningConfig contains the effective config that is running in the kube-controllers pod, after merging the API resource with any environment variables. properties: controllers: description: Controllers enables and configures individual Kubernetes controllers properties: namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object node: description: Node enables and configures the node controller. Enabled by default, set to nil to disable. properties: hostEndpoint: description: HostEndpoint controls syncing nodes to host endpoints. Disabled by default, set to nil to disable. properties: autoCreate: description: 'AutoCreate enables automatic creation of host endpoints for every node. [Default: Disabled]' type: string type: object leakGracePeriod: description: 'LeakGracePeriod is the period used by the controller to determine if an IP address has been leaked. Set to 0 to disable IP garbage collection. [Default: 15m]' type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string syncLabels: description: 'SyncLabels controls whether to copy Kubernetes node labels to Calico nodes. [Default: Enabled]' type: string type: object policy: description: Policy enables and configures the policy controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object serviceAccount: description: ServiceAccount enables and configures the service account controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object workloadEndpoint: description: WorkloadEndpoint enables and configures the workload endpoint controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object type: object etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' type: string healthChecks: description: 'HealthChecks enables or disables support for health checks [Default: Enabled]' type: string logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string prometheusMetricsPort: description: 'PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. Set to 0 to disable. [Default: 9094]' type: integer required: - controllers type: object type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: networkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: NetworkPolicy listKind: NetworkPolicyList plural: networkpolicies singular: networkpolicy scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: properties: egress: description: The ordered set of egress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array ingress: description: The ordered set of ingress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array order: description: Order is an optional field that specifies the order in which the policy is applied. Policies with higher "order" are applied after those with lower order. If the order is omitted, it may be considered to be "infinite" - i.e. the policy will be applied last. Policies with identical order will be applied in alphanumerical order based on the Policy "Name". type: number selector: description: "The selector is an expression used to pick pick out the endpoints that the policy should be applied to. \n Selector expressions follow this syntax: \n \tlabel == \"string_literal\" \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" \ -> not equal; also matches if label is not present \tlabel in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name) -> True if that label is present \t! expr -> negation of expr \texpr && expr -> Short-circuit and \texpr || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() or the empty selector -> matches all endpoints. \n Label names are allowed to contain alphanumerics, -, _ and /. String literals are more permissive but they do not support escape characters. \n Examples (with made-up labels): \n \ttype == \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != \"dev\" \t! has(label_name)" type: string serviceAccountSelector: description: ServiceAccountSelector is an optional field for an expression used to select a pod based on service accounts. type: string types: description: "Types indicates whether this policy applies to ingress, or to egress, or to both. When not explicitly specified (and so the value on creation is empty or nil), Calico defaults Types according to what Ingress and Egress are present in the policy. The default is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are also no Ingress rules) \n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. \n When the policy is read back again, Types will always be one of these values, never empty or nil." items: description: PolicyType enumerates the possible values of the PolicySpec Types field. type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: networksets.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: NetworkSet listKind: NetworkSetList plural: networksets singular: networkset scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: NetworkSetSpec contains the specification for a NetworkSet resource. properties: nets: description: The list of IP networks that belong to this set. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- # Source: calico/templates/calico-kube-controllers-rbac.yaml # Include a clusterrole for the kube-controllers component, # and bind it to the calico-kube-controllers serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers rules: # Nodes are watched to monitor for deletions. - apiGroups: [""] resources: - nodes verbs: - watch - list - get # Pods are watched to check for existence as part of IPAM controller. - apiGroups: [""] resources: - pods verbs: - get - list - watch # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] resources: - ippools - ipreservations verbs: - list - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles verbs: - get - list - create - update - delete - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: - hostendpoints verbs: - get - list - create - update - delete # Needs access to update clusterinformations. - apiGroups: ["crd.projectcalico.org"] resources: - clusterinformations verbs: - get - create - update # KubeControllersConfiguration is where it gets its config - apiGroups: ["crd.projectcalico.org"] resources: - kubecontrollersconfigurations verbs: # read its own config - get # create a default if none exists - create # update status - update # watch for changes - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-kube-controllers subjects: - kind: ServiceAccount name: calico-kube-controllers namespace: kube-system --- --- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node rules: # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get # EndpointSlices are used for Service-based network policy rule # enforcement. - apiGroups: ["discovery.k8s.io"] resources: - endpointslices verbs: - watch - list - apiGroups: [""] resources: - endpoints - services verbs: # Used to discover service IPs for advertisement. - watch - list # Used to discover Typhas. - get # Pod CIDR auto-detection on kubeadm needs access to config maps. - apiGroups: [""] resources: - configmaps verbs: - get - apiGroups: [""] resources: - nodes/status verbs: # Needed for clearing NodeNetworkUnavailable flag. - patch # Calico stores some configuration information in node annotations. - update # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list # Used by Calico for policy information. - apiGroups: [""] resources: - pods - namespaces - serviceaccounts verbs: - list - watch # The CNI plugin patches pods/status. - apiGroups: [""] resources: - pods/status verbs: - patch # Calico monitors various CRDs for config. - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs - felixconfigurations - bgppeers - globalbgpconfigs - bgpconfigurations - ippools - ipreservations - ipamblocks - globalnetworkpolicies - globalnetworksets - networkpolicies - networksets - clusterinformations - hostendpoints - blockaffinities - caliconodestatuses verbs: - get - list - watch # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: - ippools - felixconfigurations - clusterinformations verbs: - create - update # Calico must update some CRDs. - apiGroups: [ "crd.projectcalico.org" ] resources: - caliconodestatuses verbs: - update # Calico stores some configuration information on the node. - apiGroups: [""] resources: - nodes verbs: - get - list - watch # These permissions are only required for upgrade from v2.6, and can # be removed after upgrade or on fresh installations. - apiGroups: ["crd.projectcalico.org"] resources: - bgpconfigurations - bgppeers verbs: - create - update # These permissions are required for Calico CNI to perform IPAM allocations. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles verbs: - get - list - create - update - delete - apiGroups: ["crd.projectcalico.org"] resources: - ipamconfigs verbs: - get # Block affinities must also be watchable by confd for route aggregation. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities verbs: - watch # The Calico IPAM migration needs to get daemonsets. These permissions can be # removed if not upgrading from an installation using host-local IPAM. - apiGroups: ["apps"] resources: - daemonsets verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-node subjects: - kind: ServiceAccount name: calico-node namespace: kube-system --- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well # as the CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: apps/v1 metadata: name: calico-node namespace: kube-system labels: k8s-app: calico-node spec: selector: matchLabels: k8s-app: calico-node updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 template: metadata: labels: k8s-app: calico-node spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true tolerations: # Make sure calico-node gets scheduled on all nodes. - effect: NoSchedule operator: Exists # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 priorityClassName: system-node-critical initContainers: # This container performs upgrade from host-local IPAM to calico-ipam. # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam image: docker.io/calico/cni:v3.23.4 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend volumeMounts: - mountPath: /var/lib/cni/networks name: host-local-net-dir - mountPath: /host/opt/cni/bin name: cni-bin-dir securityContext: privileged: true # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni image: docker.io/calico/cni:v3.23.4 command: ["/opt/cni/bin/install"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-calico.conflist" # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: calico-config key: cni_network_config # Set the hostname based on the k8s node name. - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName # CNI MTU Config variable - name: CNI_MTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Prevents the container from sleeping forever. - name: SLEEP value: "false" - name: CNI_NET_DIR value: "/var/snap/microk8s/current/args/cni-network" volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir securityContext: privileged: true # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver image: docker.io/calico/pod2daemon-flexvol:v3.23.4 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver securityContext: privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: docker.io/calico/node:v3.23.4 envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" # Set based on the k8s node name. - name: NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,bgp" # Auto-detect the BGP IP address. - name: IP value: "autodetect" - name: IP_AUTODETECTION_METHOD value: "first-found" # Enable IPIP #- name: CALICO_IPV4POOL_IPIP # value: "Always" # Enable or Disable VXLAN on the default IP pool. - name: CALICO_IPV4POOL_VXLAN value: "Always" # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Set MTU for the VXLAN tunnel device. - name: FELIX_VXLANMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Set MTU for the Wireguard tunnel device. - name: FELIX_WIREGUARDMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. - name: CALICO_IPV4POOL_CIDR value: "10.1.0.0/16" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" # Set Felix logging to "error" - name: FELIX_LOGSEVERITYSCREEN value: "error" - name: FELIX_HEALTHENABLED value: "true" securityContext: privileged: true resources: requests: cpu: 250m lifecycle: preStop: exec: command: - /bin/calico-node - -shutdown livenessProbe: exec: command: - /bin/calico-node - -felix-live # - -bird-live periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 timeoutSeconds: 10 readinessProbe: exec: command: - /bin/calico-node - -felix-ready # - -bird-ready periodSeconds: 10 timeoutSeconds: 10 volumeMounts: # For maintaining CNI plugin API credentials. - mountPath: /host/etc/cni/net.d name: cni-net-dir readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /run/xtables.lock name: xtables-lock readOnly: false - mountPath: /var/run/calico name: var-run-calico readOnly: false - mountPath: /var/lib/calico name: var-lib-calico readOnly: false - name: policysync mountPath: /var/run/nodeagent # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the # parent directory. # - name: sysfs # mountPath: /sys/fs/ # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. # If the host is known to mount that filesystem already then Bidirectional can be omitted. # mountPropagation: Bidirectional - name: cni-log-dir mountPath: /var/log/calico/cni readOnly: true volumes: # Used by calico-node. - name: lib-modules hostPath: path: /lib/modules - name: var-run-calico hostPath: path: /var/snap/microk8s/current/var/run/calico - name: var-lib-calico hostPath: path: /var/snap/microk8s/current/var/lib/calico - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate - name: sysfs hostPath: path: /sys/fs/ type: DirectoryOrCreate # Used to install CNI. - name: cni-bin-dir hostPath: path: /var/snap/microk8s/current/opt/cni/bin - name: cni-net-dir hostPath: path: /var/snap/microk8s/current/args/cni-network # Used to access CNI logs. - name: cni-log-dir hostPath: path: /var/snap/microk8s/common/var/log/calico/cni # Mount in the directory for host-local IPAM allocations. This is # used when upgrading from host-local to calico-ipam, and can be removed # if not using the upgrade-ipam init container. - name: host-local-net-dir hostPath: path: /var/snap/microk8s/current/var/lib/cni/networks # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: type: DirectoryOrCreate path: /var/snap/microk8s/current/var/run/nodeagent # Used to install Flex Volume Driver - name: flexvol-driver-host hostPath: type: DirectoryOrCreate path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system --- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers apiVersion: apps/v1 kind: Deployment metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: # The controllers can only have a single active instance. replicas: 1 selector: matchLabels: k8s-app: calico-kube-controllers strategy: type: Recreate template: metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: nodeSelector: kubernetes.io/os: linux tolerations: # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule serviceAccountName: calico-kube-controllers priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers image: docker.io/calico/kube-controllers:v3.23.4 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes livenessProbe: exec: command: - /usr/bin/check-status - -l periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 timeoutSeconds: 10 readinessProbe: exec: command: - /usr/bin/check-status - -r periodSeconds: 10 --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system --- # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: maxUnavailable: 1 selector: matchLabels: k8s-app: calico-kube-controllers --- # Source: calico/templates/calico-etcd-secrets.yaml --- # Source: calico/templates/calico-typha.yaml --- # Source: calico/templates/configure-canal.yaml ================================================ FILE: tests/unit/yamls/cni.yaml ================================================ --- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap apiVersion: v1 metadata: name: calico-config namespace: kube-system data: # Typha is disabled. typha_service_name: "none" # Configure the backend to use. calico_backend: "vxlan" # Configure the MTU to use for workload interfaces and tunnels. # By default, MTU is auto-detected, and explicitly setting this field should not be required. # You can override auto-detection by providing a non-zero value. veth_mtu: "1440" # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", "cniVersion": "0.3.1", "plugins": [ { "type": "calico", "log_level": "info", "datastore_type": "kubernetes", "nodename_file_optional": true, "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { "type": "calico-ipam" }, "policy": { "type": "k8s" }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { "type": "portmap", "snat": true, "capabilities": {"portMappings": true} }, { "type": "bandwidth", "capabilities": {"bandwidth": true} } ] } --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bgpconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPConfiguration listKind: BGPConfigurationList plural: bgpconfigurations singular: bgpconfiguration scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: BGPConfiguration contains the configuration for any BGP routing. properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: BGPConfigurationSpec contains the values of the BGP configuration. properties: asNumber: description: "ASNumber is the default AS number used by a node. [Default: 64512]" format: int32 type: integer communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. items: description: Community contains standard or large community value and its name. properties: name: description: Name given to community value. type: string value: description: Value must be of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where, `aa` is an AS Number, `nn` and `mm` are per-AS identifier. pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ type: string type: object type: array listenPort: description: ListenPort is the port where BGP protocol should listen. Defaults to 179 maximum: 65535 minimum: 1 type: integer logSeverityScreen: description: "LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]" type: string nodeToNodeMeshEnabled: description: "NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]" type: boolean prefixAdvertisements: description: PrefixAdvertisements contains per-prefix advertisement configuration. items: description: PrefixAdvertisement configures advertisement properties for the specified CIDR. properties: cidr: description: CIDR for which properties should be advertised. type: string communities: description: Communities can be list of either community names already defined in `Specs.Communities` or community value of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and `mm` are per-AS identifier. items: type: string type: array type: object type: array serviceClusterIPs: description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them. items: description: ServiceClusterIPBlock represents a single allowed ClusterIP CIDR block. properties: cidr: type: string type: object type: array serviceExternalIPs: description: ServiceExternalIPs are the CIDR blocks for Kubernetes Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. items: description: ServiceExternalIPBlock represents a single allowed External IP CIDR block. properties: cidr: type: string type: object type: array serviceLoadBalancerIPs: description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress IPs will only be advertised if they are within one of these blocks. items: description: ServiceLoadBalancerIPBlock represents a single allowed LoadBalancer IP CIDR block. properties: cidr: type: string type: object type: array type: object type: object served: true storage: true --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bgppeers.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPPeer listKind: BGPPeerList plural: bgppeers singular: bgppeer scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: BGPPeerSpec contains the specification for a BGPPeer resource. properties: asNumber: description: The AS Number of the peer. format: int32 type: integer keepOriginalNextHop: description: Option to keep the original nexthop field when routes are sent to a BGP Peer. Setting "true" configures the selected BGP Peers node to use the "next hop keep;" instead of "next hop self;"(default) in the specific branch of the Node on "bird.cfg". type: boolean node: description: The node name identifying the Calico node instance that is targeted by this peer. If this is not set, and no nodeSelector is specified, then this BGP peer selects all nodes in the cluster. type: string nodeSelector: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string password: description: Optional BGP password for the peerings generated by this BGPPeer resource. properties: secretKeyRef: description: Selects a key of a secret in the node pod's namespace. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?" type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object peerIP: description: The IP address of the peer followed by an optional port number to peer with. If port number is given, format should be `[]:port` or `:` for IPv4. If optional port number is not set, and this peer IP and ASNumber belongs to a calico/node with ListenPort set in BGPConfiguration, then we use that port to peer. type: string peerSelector: description: Selector for the remote nodes to peer with. When this is set, the PeerIP and ASNumber fields must be empty. For each peering between the local node and selected remote nodes, we configure an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The remote AS number comes from the remote node's NodeBGPSpec.ASNumber, or the global default if that is not set. type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: blockaffinities.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BlockAffinity listKind: BlockAffinityList plural: blockaffinities singular: blockaffinity scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: BlockAffinitySpec contains the specification for a BlockAffinity resource. properties: cidr: type: string deleted: description: Deleted indicates that this block affinity is being deleted. This field is a string for compatibility with older releases that mistakenly treat this field as a string. type: string node: type: string state: type: string required: - cidr - deleted - node - state type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clusterinformations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: ClusterInformation listKind: ClusterInformationList plural: clusterinformations singular: clusterinformation scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: ClusterInformation contains the cluster specific information. properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: ClusterInformationSpec contains the values of describing the cluster. properties: calicoVersion: description: CalicoVersion is the version of Calico that the cluster is running type: string clusterGUID: description: ClusterGUID is the GUID of the cluster type: string clusterType: description: ClusterType describes the type of the cluster type: string datastoreReady: description: DatastoreReady is used during significant datastore migrations to signal to components such as Felix that it should wait before accessing the datastore. type: boolean variant: description: Variant declares which variant of Calico should be active. type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: felixconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: FelixConfiguration listKind: FelixConfigurationList plural: felixconfigurations singular: felixconfiguration scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: Felix Configuration contains the configuration for Felix. properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: FelixConfigurationSpec contains the values of the Felix configuration. properties: allowIPIPPacketsFromWorkloads: description: "AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads [Default: false]" type: boolean allowVXLANPacketsFromWorkloads: description: "AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads [Default: false]" type: boolean awsSrcDstCheck: description: 'Set source-destination-check on AWS EC2 instances. Accepted value must be one of "DoNothing", "Enabled" or "Disabled". [Default: DoNothing]' enum: - DoNothing - Enable - Disable type: string bpfConnectTimeLoadBalancingEnabled: description: "BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes. [Default: true]" type: boolean bpfDataIfacePattern: description: BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually named cali...). type: string bpfDisableUnprivileged: description: "BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable unprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and cannot insert their own BPF programs to interfere with Calico's. [Default: true]" type: boolean bpfEnabled: description: "BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]" type: boolean bpfExternalServiceMode: description: 'BPFExternalServiceMode in BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) are forwarded to remote workloads. If set to "Tunnel" then both request and response traffic is tunneled to the remote node. If set to "DSR", the request traffic is tunneled but the response traffic is sent directly from the remote node. In "DSR" mode, the remote node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' type: string bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls whether Felix's embedded kube-proxy accepts EndpointSlices or not. type: boolean bpfKubeProxyIptablesCleanupEnabled: description: "BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF mode, Felix will proactively clean up the upstream Kubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running. [Default: true]" type: boolean bpfKubeProxyMinSyncPeriod: description: "BPFKubeProxyMinSyncPeriod, in BPF mode, controls the minimum time between updates to the dataplane for Felix's embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by batching up more work. [Default: 1s]" type: string bpfLogLevel: description: 'BPFLogLevel controls the log level of the BPF programs when in BPF dataplane mode. One of "Off", "Info", or "Debug". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string chainInsertMode: description: "ChainInsertMode controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents Calico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]" type: string dataplaneDriver: type: string debugDisableLogDropping: type: boolean debugMemoryProfilePath: type: string debugSimulateCalcGraphHangAfter: type: string debugSimulateDataplaneHangAfter: type: string defaultEndpointToHostAction: description: "DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an iptables DROP action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables INPUT chain; Calico will insert its rules at the top of that chain, then RETURN packets to the INPUT chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]" type: string deviceRouteProtocol: description: This defines the route protocol added to programmed device routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: description: This is the source address to use on programmed device routes. By default the source address is left blank, leaving the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean endpointReportingDelay: type: string endpointReportingEnabled: type: boolean externalNodesList: description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have the tunneled traffic be accepted at calico nodes. items: type: string type: array failsafeInboundHostPorts: description: "FailsafeInboundHostPorts is a comma-delimited list of UDP/TCP ports that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. Each port should be specified as tcp: or udp:. For back-compatibility, if the protocol is not specified, it defaults to tcp. To disable all inbound host ports, use the value none. The default value allows ssh access and DHCP. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]" items: description: ProtoPort is combination of protocol and port, both must be specified. properties: port: type: integer protocol: type: string required: - port - protocol type: object type: array failsafeOutboundHostPorts: description: "FailsafeOutboundHostPorts is a comma-delimited list of UDP/TCP ports that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. Each port should be specified as tcp: or udp:. For back-compatibility, if the protocol is not specified, it defaults to tcp. To disable all outbound host ports, use the value none. The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP and DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, udp:53, udp:67]" items: description: ProtoPort is combination of protocol and port, both must be specified. properties: port: type: integer protocol: type: string required: - port - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override the feature detection. Values are specified in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string genericXDPEnabled: description: "GenericXDPEnabled enables Generic XDP so network cards that don't support XDP offload or driver modes can use XDP. This is not recommended since it doesn't provide better performance than iptables. [Default: false]" type: boolean healthEnabled: type: boolean healthHost: type: string healthPort: type: integer interfaceExclude: description: "InterfaceExclude is a comma-separated list of interfaces that Felix should exclude when monitoring for host endpoints. The default value ensures that Felix ignores Kubernetes' IPVS dummy interface, which is used internally by kube-proxy. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with '/'. For example having values '/^kube/,veth1' will exclude all interfaces that begin with 'kube' and also the interface 'veth1'. [Default: kube-ipvs0]" type: string interfacePrefix: description: "InterfacePrefix is the interface name prefix that identifies workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value, and our OpenStack integration sets the 'tap' value. [Default: cali]" type: string interfaceRefreshInterval: description: InterfaceRefreshInterval is the period at which Felix rescans local interfaces to verify their state. The rescan can be disabled by setting the interval to 0. type: string ipipEnabled: type: boolean ipipMTU: description: "IPIPMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440]" type: integer ipsetsRefreshInterval: description: "IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable iptables refresh. [Default: 90s]" type: string iptablesBackend: description: IptablesBackend specifies which backend of iptables will be used. The default is legacy. type: string iptablesFilterAllowAction: type: string iptablesLockFilePath: description: "IptablesLockFilePath is the location of the iptables lock file. You may need to change this if the lock file is not in its standard location (for example if you have mapped it into Felix's container at a different path). [Default: /run/xtables.lock]" type: string iptablesLockProbeInterval: description: "IptablesLockProbeInterval is the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU. [Default: 50ms]" type: string iptablesLockTimeout: description: "IptablesLockTimeout is the time that Felix will wait for the iptables lock, or 0, to disable. To use this feature, Felix must share the iptables lock file with all other processes that also take the lock. When running Felix inside a container, this requires the /run directory of the host to be mounted into the calico/node or calico/felix container. [Default: 0s disabled]" type: string iptablesMangleAllowAction: type: string iptablesMarkMask: description: "IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xff000000]" format: int32 type: integer iptablesNATOutgoingInterfaceFilter: type: string iptablesPostWriteCheckInterval: description: "IptablesPostWriteCheckInterval is the period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. This should only occur if another application on the system doesn't respect the iptables lock. [Default: 1s]" type: string iptablesRefreshInterval: description: "IptablesRefreshInterval is the period at which Felix re-checks the IP sets in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. [Default: 10s]" type: string ipv6Support: type: boolean kubeNodePortRanges: description: "KubeNodePortRanges holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. Felix uses these ranges to separate host and workload traffic. [Default: 30000:32767]." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array logFilePath: description: "LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]" type: string logPrefix: description: "LogPrefix is the log prefix that Felix uses when rendering LOG rules. [Default: calico-packet]" type: string logSeverityFile: description: "LogSeverityFile is the log severity above which logs are sent to the log file. [Default: Info]" type: string logSeverityScreen: description: "LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]" type: string logSeveritySys: description: "LogSeveritySys is the log severity above which logs are sent to the syslog. Set to None for no logging to syslog. [Default: Info]" type: string maxIpsetSize: type: integer metadataAddr: description: "MetadataAddr is the IP address or domain name of the server that can answer VM queries for cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in Ubuntu, nova-api-metadata). A value of none (case insensitive) means that Felix should not set up any NAT rule for the metadata path. [Default: 127.0.0.1]" type: string metadataPort: description: "MetadataPort is the port of the metadata server. This, combined with global.MetadataAddr (if not 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed [Default: 8775]." type: integer mtuIfacePattern: description: MTUIfacePattern is a regular expression that controls which interfaces Felix should scan in order to calculate the host's MTU. This should not match workload interfaces (usually named cali...). type: string natOutgoingAddress: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (ie it uses the iptables MASQUERADE target) type: string natPortRange: anyOf: - type: integer - type: string description: NATPortRange specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the network stack is used. pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: type: string openstackRegion: description: "OpenstackRegion is the name of the region that a particular Felix belongs to. In a multi-region Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, or in felix.cfg or the environment on each compute node), and must match the [calico] openstack_region value configured in neutron.conf on each node. [Default: Empty]" type: string policySyncPathPrefix: description: "PolicySyncPathPrefix is used to by Felix to communicate policy changes to external services, like Application layer policy. [Default: Empty]" type: string prometheusGoMetricsEnabled: description: "PrometheusGoMetricsEnabled disables Go runtime metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]" type: boolean prometheusMetricsEnabled: description: "PrometheusMetricsEnabled enables the Prometheus metrics server in Felix if set to true. [Default: false]" type: boolean prometheusMetricsHost: description: "PrometheusMetricsHost is the host that the Prometheus metrics server should bind to. [Default: empty]" type: string prometheusMetricsPort: description: "PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. [Default: 9091]" type: integer prometheusProcessMetricsEnabled: description: "PrometheusProcessMetricsEnabled disables process metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]" type: boolean removeExternalRoutes: description: Whether or not to remove device routes that have not been programmed by Felix. Disabling this will allow external applications to also add device routes. This is enabled by default which means we will remove externally added routes. type: boolean reportingInterval: description: "ReportingInterval is the interval at which Felix reports its status into the datastore or 0 to disable. Must be non-zero in OpenStack deployments. [Default: 30s]" type: string reportingTTL: description: "ReportingTTL is the time-to-live setting for process-wide status reports. [Default: 90s]" type: string routeRefreshInterval: description: "RouteRefreshInterval is the period at which Felix re-checks the routes in the dataplane to ensure that no other process has accidentally broken Calico's rules. Set to 0 to disable route refresh. [Default: 90s]" type: string routeSource: description: "RouteSource configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes." type: string routeTableRange: description: Calico programs additional Linux route tables for various purposes. RouteTableRange specifies the indices of the route tables that Calico should use. properties: max: type: integer min: type: integer required: - max - min type: object serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", in which case such routing loops continue to be allowed. [Default: Drop]' type: string sidecarAccelerationEnabled: description: "SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false]" type: boolean usageReportingEnabled: description: "UsageReportingEnabled reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage server. For example, if a significant security vulnerability has been discovered in the version of Calico being used. [Default: true]" type: boolean usageReportingInitialDelay: description: "UsageReportingInitialDelay controls the minimum delay before Felix makes a report. [Default: 300s]" type: string usageReportingInterval: description: "UsageReportingInterval controls the interval at which Felix makes reports. [Default: 86400s]" type: string useInternalDataplaneDriver: type: boolean vxlanEnabled: type: boolean vxlanMTU: description: "VXLANMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440]" type: integer vxlanPort: type: integer vxlanVNI: type: integer wireguardEnabled: description: "WireguardEnabled controls whether Wireguard is enabled. [Default: false]" type: boolean wireguardInterfaceName: description: "WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]" type: string wireguardListeningPort: description: "WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]" type: integer wireguardMTU: description: "WireguardMTU controls the MTU on the Wireguard interface. See Configuring MTU [Default: 1420]" type: integer wireguardRoutingRulePriority: description: "WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]" type: integer xdpEnabled: description: "XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]" type: boolean xdpRefreshInterval: description: "XDPRefreshInterval is the period at which Felix re-checks all XDP state to ensure that no other process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to disable XDP refresh. [Default: 90s]" type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: globalnetworkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: GlobalNetworkPolicy listKind: GlobalNetworkPolicyList plural: globalnetworkpolicies singular: globalnetworkpolicy scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: properties: applyOnForward: description: ApplyOnForward indicates to apply the rules in this policy on forward traffic. type: boolean doNotTrack: description: DoNotTrack indicates whether packets matched by the rules in this policy should go through the data plane's connection tracking, such as Linux conntrack. If True, the rules in this policy are applied before any data plane connection tracking, and packets allowed by this policy are marked as not to be tracked. type: boolean egress: description: The ordered set of egress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with Not. All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: "Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR'd together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it." items: description: "HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix" properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object required: - action type: object type: array ingress: description: The ordered set of ingress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with Not. All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: "Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR'd together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it." items: description: "HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix" properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object required: - action type: object type: array namespaceSelector: description: NamespaceSelector is an optional field for an expression used to select a pod based on namespaces. type: string order: description: Order is an optional field that specifies the order in which the policy is applied. Policies with higher "order" are applied after those with lower order. If the order is omitted, it may be considered to be "infinite" - i.e. the policy will be applied last. Policies with identical order will be applied in alphanumerical order based on the Policy "Name". type: number preDNAT: description: PreDNAT indicates to apply the rules in this policy before any DNAT. type: boolean selector: description: "The selector is an expression used to pick pick out the endpoints that the policy should be applied to. \n Selector expressions follow this syntax: \n \tlabel == \"string_literal\" \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" \ -> not equal; also matches if label is not present \tlabel in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name) -> True if that label is present \t! expr -> negation of expr \texpr && expr -> Short-circuit and \texpr || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() or the empty selector -> matches all endpoints. \n Label names are allowed to contain alphanumerics, -, _ and /. String literals are more permissive but they do not support escape characters. \n Examples (with made-up labels): \n \ttype == \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != \"dev\" \t! has(label_name)" type: string serviceAccountSelector: description: ServiceAccountSelector is an optional field for an expression used to select a pod based on service accounts. type: string types: description: "Types indicates whether this policy applies to ingress, or to egress, or to both. When not explicitly specified (and so the value on creation is empty or nil), Calico defaults Types according to what Ingress and Egress rules are present in the policy. The default is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are also no Ingress rules) \n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. \n When the policy is read back again, Types will always be one of these values, never empty or nil." items: description: PolicyType enumerates the possible values of the PolicySpec Types field. type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: globalnetworksets.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: GlobalNetworkSet listKind: GlobalNetworkSetList plural: globalnetworksets singular: globalnetworkset scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs that share labels to allow rules to refer to them via selectors. The labels of GlobalNetworkSet are not namespaced. properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: GlobalNetworkSetSpec contains the specification for a NetworkSet resource. properties: nets: description: The list of IP networks that belong to this set. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: hostendpoints.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: HostEndpoint listKind: HostEndpointList plural: hostendpoints singular: hostendpoint scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: HostEndpointSpec contains the specification for a HostEndpoint resource. properties: expectedIPs: description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. If \"InterfaceName\" is not present, Calico will look for an interface matching any of the IPs in the list and apply policy to that. Note: \tWhen using the selector match criteria in an ingress or egress security Policy \tor Profile, Calico converts the selector into a set of IP addresses. For host \tendpoints, the ExpectedIPs field is used for that purpose. (If only the interface \tname is specified, Calico does not learn the IPs of the interface for use in match \tcriteria.)" items: type: string type: array interfaceName: description: "Either \"*\", or the name of a specific Linux interface to apply policy to; or empty. \"*\" indicates that this HostEndpoint governs all traffic to, from or through the default network namespace of the host named by the \"Node\" field; entering and leaving that namespace via any interface, including those from/to non-host-networked local workloads. \n If InterfaceName is not \"*\", this HostEndpoint only governs traffic that enters or leaves the host through the specific interface named by InterfaceName, or - when InterfaceName is empty - through the specific interface that has one of the IPs in ExpectedIPs. Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only external interfaces (such as eth0) are supported here; it isn't possible for a HostEndpoint to protect traffic through a specific local workload interface. \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; initially just pre-DNAT policy. Please check Calico documentation for the latest position." type: string node: description: The node name identifying the Calico node instance. type: string ports: description: Ports contains the endpoint's named ports, which may be referenced in security policy rules. items: properties: name: type: string port: type: integer protocol: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true required: - name - port - protocol type: object type: array profiles: description: A list of identifiers of security Profile objects that apply to this endpoint. Each profile is applied in the order that they appear in this list. Profile rules are applied after the selector-based security policy. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamblocks.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMBlock listKind: IPAMBlockList plural: ipamblocks singular: ipamblock scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: IPAMBlockSpec contains the specification for an IPAMBlock resource. properties: affinity: type: string allocations: items: type: integer # TODO: This nullable is manually added in. We should update controller-gen # to handle []*int properly itself. nullable: true type: array attributes: items: properties: handle_id: type: string secondary: additionalProperties: type: string type: object type: object type: array cidr: type: string deleted: type: boolean strictAffinity: type: boolean unallocated: items: type: integer type: array required: - allocations - attributes - cidr - strictAffinity - unallocated type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamconfigs.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMConfig listKind: IPAMConfigList plural: ipamconfigs singular: ipamconfig scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: IPAMConfigSpec contains the specification for an IPAMConfig resource. properties: autoAllocateBlocks: type: boolean maxBlocksPerHost: description: MaxBlocksPerHost, if non-zero, is the max number of blocks that can be affine to each host. type: integer strictAffinity: type: boolean required: - autoAllocateBlocks - strictAffinity type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamhandles.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMHandle listKind: IPAMHandleList plural: ipamhandles singular: ipamhandle scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: IPAMHandleSpec contains the specification for an IPAMHandle resource. properties: block: additionalProperties: type: integer type: object deleted: type: boolean handleID: type: string required: - block - handleID type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ippools.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPPool listKind: IPPoolList plural: ippools singular: ippool scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: IPPoolSpec contains the specification for an IPPool resource. properties: blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 112 for IPv6. type: integer cidr: description: The pool CIDR. type: string disabled: description: When disabled is true, Calico IPAM will not assign addresses from this pool. type: boolean ipip: description: "Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is for internal use only." properties: enabled: description: When enabled is true, ipip tunneling will be used to deliver packets to destinations within this pool. type: boolean mode: description: The IPIP mode. This can be one of "always" or "cross-subnet". A mode of "always" will also use IPIP tunneling for routing to destination IP addresses within this pool. A mode of "cross-subnet" will only use IPIP tunneling when the destination node is on a different subnet to the originating node. The default value (if not specified) is "always". type: string type: object ipipMode: description: Contains configuration for IPIP tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). type: string nat-outgoing: description: "Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is for internal use only." type: boolean natOutgoing: description: When nat-outgoing is true, packets sent from Calico networked containers in this pool to destinations outside of this pool will be masqueraded. type: boolean nodeSelector: description: Allows IPPool to allocate for a specific node by label selector. type: string vxlanMode: description: Contains configuration for VXLAN tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. VXLAN tunneling is disabled). type: string required: - cidr type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: kubecontrollersconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: KubeControllersConfiguration listKind: KubeControllersConfigurationList plural: kubecontrollersconfigurations singular: kubecontrollersconfiguration scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: KubeControllersConfigurationSpec contains the values of the Kubernetes controllers configuration. properties: controllers: description: Controllers enables and configures individual Kubernetes controllers properties: namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object node: description: Node enables and configures the node controller. Enabled by default, set to nil to disable. properties: hostEndpoint: description: HostEndpoint controls syncing nodes to host endpoints. Disabled by default, set to nil to disable. properties: autoCreate: description: "AutoCreate enables automatic creation of host endpoints for every node. [Default: Disabled]" type: string type: object reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string syncLabels: description: "SyncLabels controls whether to copy Kubernetes node labels to Calico nodes. [Default: Enabled]" type: string type: object policy: description: Policy enables and configures the policy controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object serviceAccount: description: ServiceAccount enables and configures the service account controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object workloadEndpoint: description: WorkloadEndpoint enables and configures the workload endpoint controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object type: object etcdV3CompactionPeriod: description: "EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]" type: string healthChecks: description: "HealthChecks enables or disables support for health checks [Default: Enabled]" type: string logSeverityScreen: description: "LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]" type: string required: - controllers type: object status: description: KubeControllersConfigurationStatus represents the status of the configuration. It's useful for admins to be able to see the actual config that was applied, which can be modified by environment variables on the kube-controllers process. properties: environmentVars: additionalProperties: type: string description: EnvironmentVars contains the environment variables on the kube-controllers that influenced the RunningConfig. type: object runningConfig: description: RunningConfig contains the effective config that is running in the kube-controllers pod, after merging the API resource with any environment variables. properties: controllers: description: Controllers enables and configures individual Kubernetes controllers properties: namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object node: description: Node enables and configures the node controller. Enabled by default, set to nil to disable. properties: hostEndpoint: description: HostEndpoint controls syncing nodes to host endpoints. Disabled by default, set to nil to disable. properties: autoCreate: description: "AutoCreate enables automatic creation of host endpoints for every node. [Default: Disabled]" type: string type: object reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string syncLabels: description: "SyncLabels controls whether to copy Kubernetes node labels to Calico nodes. [Default: Enabled]" type: string type: object policy: description: Policy enables and configures the policy controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object serviceAccount: description: ServiceAccount enables and configures the service account controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object workloadEndpoint: description: WorkloadEndpoint enables and configures the workload endpoint controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: "ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]" type: string type: object type: object etcdV3CompactionPeriod: description: "EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]" type: string healthChecks: description: "HealthChecks enables or disables support for health checks [Default: Enabled]" type: string logSeverityScreen: description: "LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]" type: string required: - controllers type: object type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: networkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: NetworkPolicy listKind: NetworkPolicyList plural: networkpolicies singular: networkpolicy scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: properties: egress: description: The ordered set of egress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with Not. All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: "Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR'd together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it." items: description: "HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix" properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object required: - action type: object type: array ingress: description: The ordered set of ingress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with Not. All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: "Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR'd together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it." items: description: "HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix" properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and Selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label my_label. \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label my_label. \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object type: object required: - action type: object type: array order: description: Order is an optional field that specifies the order in which the policy is applied. Policies with higher "order" are applied after those with lower order. If the order is omitted, it may be considered to be "infinite" - i.e. the policy will be applied last. Policies with identical order will be applied in alphanumerical order based on the Policy "Name". type: number selector: description: "The selector is an expression used to pick pick out the endpoints that the policy should be applied to. \n Selector expressions follow this syntax: \n \tlabel == \"string_literal\" \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" \ -> not equal; also matches if label is not present \tlabel in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name) -> True if that label is present \t! expr -> negation of expr \texpr && expr -> Short-circuit and \texpr || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() or the empty selector -> matches all endpoints. \n Label names are allowed to contain alphanumerics, -, _ and /. String literals are more permissive but they do not support escape characters. \n Examples (with made-up labels): \n \ttype == \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != \"dev\" \t! has(label_name)" type: string serviceAccountSelector: description: ServiceAccountSelector is an optional field for an expression used to select a pod based on service accounts. type: string types: description: "Types indicates whether this policy applies to ingress, or to egress, or to both. When not explicitly specified (and so the value on creation is empty or nil), Calico defaults Types according to what Ingress and Egress are present in the policy. The default is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are also no Ingress rules) \n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. \n When the policy is read back again, Types will always be one of these values, never empty or nil." items: description: PolicyType enumerates the possible values of the PolicySpec Types field. type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: networksets.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: NetworkSet listKind: NetworkSetList plural: networksets singular: networkset scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. properties: apiVersion: description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object spec: description: NetworkSetSpec contains the specification for a NetworkSet resource. properties: nets: description: The list of IP networks that belong to this set. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- # Source: calico/templates/calico-kube-controllers-rbac.yaml # Include a clusterrole for the kube-controllers component, # and bind it to the calico-kube-controllers serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers rules: # Nodes are watched to monitor for deletions. - apiGroups: [""] resources: - nodes verbs: - watch - list - get # Pods are queried to check for existence. - apiGroups: [""] resources: - pods verbs: - get # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] resources: - ippools verbs: - list - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles verbs: - get - list - create - update - delete # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: - hostendpoints verbs: - get - list - create - update - delete # Needs access to update clusterinformations. - apiGroups: ["crd.projectcalico.org"] resources: - clusterinformations verbs: - get - create - update # KubeControllersConfiguration is where it gets its config - apiGroups: ["crd.projectcalico.org"] resources: - kubecontrollersconfigurations verbs: # read its own config - get # create a default if none exists - create # update status - update # watch for changes - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-kube-controllers subjects: - kind: ServiceAccount name: calico-kube-controllers namespace: kube-system --- --- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node rules: # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get - apiGroups: [""] resources: - endpoints - services verbs: # Used to discover service IPs for advertisement. - watch - list # Used to discover Typhas. - get # Pod CIDR auto-detection on kubeadm needs access to config maps. - apiGroups: [""] resources: - configmaps verbs: - get - apiGroups: [""] resources: - nodes/status verbs: # Needed for clearing NodeNetworkUnavailable flag. - patch # Calico stores some configuration information in node annotations. - update # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list # Used by Calico for policy information. - apiGroups: [""] resources: - pods - namespaces - serviceaccounts verbs: - list - watch # The CNI plugin patches pods/status. - apiGroups: [""] resources: - pods/status verbs: - patch # Calico monitors various CRDs for config. - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs - felixconfigurations - bgppeers - globalbgpconfigs - bgpconfigurations - ippools - ipamblocks - globalnetworkpolicies - globalnetworksets - networkpolicies - networksets - clusterinformations - hostendpoints - blockaffinities verbs: - get - list - watch # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: - ippools - felixconfigurations - clusterinformations verbs: - create - update # Calico stores some configuration information on the node. - apiGroups: [""] resources: - nodes verbs: - get - list - watch # These permissions are only required for upgrade from v2.6, and can # be removed after upgrade or on fresh installations. - apiGroups: ["crd.projectcalico.org"] resources: - bgpconfigurations - bgppeers verbs: - create - update # These permissions are required for Calico CNI to perform IPAM allocations. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles verbs: - get - list - create - update - delete - apiGroups: ["crd.projectcalico.org"] resources: - ipamconfigs verbs: - get # Block affinities must also be watchable by confd for route aggregation. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities verbs: - watch # The Calico IPAM migration needs to get daemonsets. These permissions can be # removed if not upgrading from an installation using host-local IPAM. - apiGroups: ["apps"] resources: - daemonsets verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-node subjects: - kind: ServiceAccount name: calico-node namespace: kube-system --- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well # as the CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: apps/v1 metadata: name: calico-node namespace: kube-system labels: k8s-app: calico-node spec: selector: matchLabels: k8s-app: calico-node updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 template: metadata: labels: k8s-app: calico-node annotations: # This, along with the CriticalAddonsOnly toleration below, # marks the pod as a critical add-on, ensuring it gets # priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: "" spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true tolerations: # Make sure calico-node gets scheduled on all nodes. - effect: NoSchedule operator: Exists # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists serviceAccountName: calico-node # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 priorityClassName: system-node-critical initContainers: # This container performs upgrade from host-local IPAM to calico-ipam. # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam image: docker.io/calico/cni:v3.21.1 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend volumeMounts: - mountPath: /var/lib/cni/networks name: host-local-net-dir - mountPath: /host/opt/cni/bin name: cni-bin-dir securityContext: privileged: true # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni image: docker.io/calico/cni:v3.21.1 command: ["/opt/cni/bin/install"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-calico.conflist" # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: calico-config key: cni_network_config # Set the hostname based on the k8s node name. - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName # CNI MTU Config variable - name: CNI_MTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Prevents the container from sleeping forever. - name: SLEEP value: "false" - name: CNI_NET_DIR value: "/var/snap/microk8s/current/args/cni-network" volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir securityContext: privileged: true # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver image: docker.io/calico/pod2daemon-flexvol:v3.21.1 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver securityContext: privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: docker.io/calico/node:v3.21.1 envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" # Set based on the k8s node name. - name: NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,bgp" # Auto-detect the BGP IP address. - name: IP value: "autodetect" - name: IP_AUTODETECTION_METHOD value: "can-reach=192.168.1.43" # Enable IPIP #- name: CALICO_IPV4POOL_IPIP - name: CALICO_IPV4POOL_VXLAN value: "Always" # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Set MTU for the VXLAN tunnel device. - name: FELIX_VXLANMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Set MTU for the Wireguard tunnel device. - name: FELIX_WIREGUARDMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. - name: CALICO_IPV4POOL_CIDR value: "10.1.0.0/16" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" # Set Felix logging to "error" - name: FELIX_LOGSEVERITYSCREEN value: "error" - name: FELIX_HEALTHENABLED value: "true" securityContext: privileged: true resources: requests: cpu: 250m livenessProbe: exec: command: - /bin/calico-node - -felix-live # - -bird-live periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: exec: command: - /bin/calico-node - -felix-ready # - -bird-ready periodSeconds: 10 volumeMounts: - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /run/xtables.lock name: xtables-lock readOnly: false - mountPath: /var/run/calico name: var-run-calico readOnly: false - mountPath: /var/lib/calico name: var-lib-calico readOnly: false - name: policysync mountPath: /var/run/nodeagent volumes: # Used by calico-node. - name: lib-modules hostPath: path: /lib/modules - name: var-run-calico hostPath: path: /var/snap/microk8s/current/var/run/calico - name: var-lib-calico hostPath: path: /var/snap/microk8s/current/var/lib/calico - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate # Used to install CNI. - name: cni-bin-dir hostPath: path: /var/snap/microk8s/current/opt/cni/bin - name: cni-net-dir hostPath: path: /var/snap/microk8s/current/args/cni-network # Mount in the directory for host-local IPAM allocations. This is # used when upgrading from host-local to calico-ipam, and can be removed # if not using the upgrade-ipam init container. - name: host-local-net-dir hostPath: path: /var/snap/microk8s/current/var/lib/cni/networks # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: type: DirectoryOrCreate path: /var/snap/microk8s/current/var/run/nodeagent # Used to install Flex Volume Driver - name: flexvol-driver-host hostPath: type: DirectoryOrCreate path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system --- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers apiVersion: apps/v1 kind: Deployment metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: # The controllers can only have a single active instance. replicas: 1 selector: matchLabels: k8s-app: calico-kube-controllers strategy: type: Recreate template: metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers annotations: scheduler.alpha.kubernetes.io/critical-pod: "" spec: nodeSelector: kubernetes.io/os: linux tolerations: # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule serviceAccountName: calico-kube-controllers priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers image: docker.io/calico/kube-controllers:v3.17.3 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes readinessProbe: exec: command: - /usr/bin/check-status - -r --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system --- # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: maxUnavailable: 1 selector: matchLabels: k8s-app: calico-kube-controllers --- # Source: calico/templates/calico-etcd-secrets.yaml --- # Source: calico/templates/calico-typha.yaml --- # Source: calico/templates/configure-canal.yaml ================================================ FILE: tests/unit/yamls/invalid.yaml ================================================ not a yaml ================================================ FILE: tests/utils.py ================================================ import os.path import datetime import time import yaml import platform import psutil from subprocess import check_output, CalledProcessError, check_call arch_translate = {"aarch64": "arm64", "x86_64": "amd64"} def get_arch(): """ Returns the architecture we are running on """ return arch_translate[platform.machine()] def run_until_success(cmd, timeout_insec=60, err_out=None): """ Run a command until it succeeds or times out. Args: cmd: Command to run timeout_insec: Time out in seconds err_out: If command fails and this is the output, return. Returns: The string output of the command """ deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) while True: try: output = check_output(cmd.split()).strip().decode("utf8") return output.replace("\\n", "\n") except CalledProcessError as err: output = err.output.strip().decode("utf8").replace("\\n", "\n") print(output) if output == err_out: return output if datetime.datetime.now() > deadline: raise print("Retrying {}".format(cmd)) time.sleep(3) def kubectl(cmd, timeout_insec=300, err_out=None): """ Do a kubectl Args: cmd: left part of kubectl command timeout_insec: timeout for this job err_out: If command fails and this is the output, return. Returns: the kubectl response in a string """ cmd = "/snap/bin/microk8s.kubectl " + cmd return run_until_success(cmd, timeout_insec, err_out) def docker(cmd): """ Do a docker Args: cmd: left part of docker command Returns: the docker response in a string """ docker_bin = "/usr/bin/docker" if os.path.isfile("/snap/bin/microk8s.docker"): docker_bin = "/snap/bin/microk8s.docker" cmd = docker_bin + " " + cmd return run_until_success(cmd) def kubectl_get(target, timeout_insec=300): """ Do a kubectl get and return the results in a yaml structure. Args: target: which resource we are getting timeout_insec: timeout for this job Returns: YAML structured response """ cmd = "get -o yaml " + target output = kubectl(cmd, timeout_insec) return yaml.safe_load(output) def wait_for_pod_state( pod, namespace, desired_state, desired_reason=None, label=None, timeout_insec=600 ): """ Wait for a a pod state. If you do not specify a pod name and you set instead a label only the first pod will be checked. """ deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) while True: if datetime.datetime.now() > deadline: all_res = kubectl("get all -A") err_msg = f"Pod {pod} not in {desired_state} after {timeout_insec} seconds, " err_msg += f"label: {label}, desired_reason: {desired_reason}.\n" print(err_msg) print(f"kubectl get all -A\n{all_res}\n") raise TimeoutError(err_msg) cmd = "po {} -n {}".format(pod, namespace) if label: cmd += " -l {}".format(label) data = kubectl_get(cmd, timeout_insec) if pod == "": if len(data["items"]) > 0: status = data["items"][0]["status"] else: status = [] else: status = data["status"] if "containerStatuses" in status: container_status = status["containerStatuses"][0] state, details = list(container_status["state"].items())[0] if desired_reason: reason = details.get("reason") if state == desired_state and reason == desired_reason: break elif state == desired_state: break time.sleep(3) def wait_for_installation(cluster_nodes=1, timeout_insec=360): """ Wait for kubernetes service to appear. """ while True: cmd = "svc kubernetes" data = kubectl_get(cmd, timeout_insec) service = data["metadata"]["name"] if "kubernetes" in service: break else: time.sleep(3) while True: cmd = "get no" nodes = kubectl(cmd, timeout_insec) if nodes.count(" Ready") == cluster_nodes: break else: time.sleep(3) # Allow rest of the services to come up time.sleep(30) def wait_for_namespace_termination(namespace, timeout_insec=360): """ Wait for the termination of the provided namespace. """ print("Waiting for namespace {} to be removed".format(namespace)) deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) while True: try: cmd = "/snap/bin/microk8s.kubectl get ns {}".format(namespace) check_output(cmd.split()).strip().decode("utf8") print("Waiting...") except CalledProcessError: if datetime.datetime.now() > deadline: raise else: return time.sleep(10) def microk8s_enable(addon, timeout_insec=300): """ Disable an addon Args: addon: name of the addon timeout_insec: seconds to keep retrying """ # NVidia pre-check so as to not wait for a timeout. if addon == "gpu": nv_out = run_until_success("lsmod", timeout_insec=10) if "nvidia" not in nv_out: print("Not a cuda capable system. Will not test gpu addon") raise CalledProcessError(1, "Nothing to do for gpu") cmd = "/snap/bin/microk8s.enable {}".format(addon) return run_until_success(cmd, timeout_insec) def microk8s_disable(addon): """ Enable an addon Args: addon: name of the addon """ cmd = "/snap/bin/microk8s.disable {}".format(addon) return run_until_success(cmd, timeout_insec=300) def microk8s_clustering_capable(): """ Are we in a clustering capable microk8s? """ return os.path.isfile("/snap/bin/microk8s.join") def microk8s_reset(cluster_nodes=1): """ Call microk8s reset """ cmd = "/snap/bin/microk8s.reset" run_until_success(cmd, timeout_insec=300) wait_for_installation(cluster_nodes) def update_yaml_with_arch(manifest_file): """ Updates any $ARCH entry with the architecture in the manifest """ arch = arch_translate[platform.machine()] with open(manifest_file) as f: s = f.read() with open(manifest_file, "w") as f: s = s.replace("$ARCH", arch) f.write(s) def is_container(): """ Returns: True if the deployment is in a VM/container. """ try: if os.path.isdir("/run/systemd/system"): container = check_output("sudo systemd-detect-virt --container".split()) print("Tests are running in {}".format(container)) return True except CalledProcessError: print("systemd-detect-virt did not detect a container") if os.path.exists("/run/container_type"): return True try: check_call("sudo grep -E (lxc|hypervisor) /proc/1/environ /proc/cpuinfo".split()) print("Tests are running in an undetectable container") return True except CalledProcessError: print("no indication of a container in /proc") return False def is_strict(): if "STRICT" in os.environ and os.environ["STRICT"] == "yes": return True return False def is_ipv6_configured(): try: output = check_output(["ip", "-6", "address"]) return b"inet6" in output except CalledProcessError: return False def _get_process(name): return [p for p in psutil.process_iter() if name == p.name()] ================================================ FILE: tests/validators.py ================================================ import time import os import re import requests import platform import yaml import subprocess from pathlib import Path from utils import ( get_arch, kubectl, wait_for_pod_state, docker, update_yaml_with_arch, ) TEMPLATES = Path(__file__).absolute().parent / "templates" def validate_dns_dashboard(): """ Validate the dashboard addon by trying to access the kubernetes dashboard. The dashboard will return an HTML indicating that it is up and running. """ service = "kubernetes-dashboard:" ns = "kube-system" app_names = ["k8s-app=kubernetes-dashboard", "k8s-app=dashboard-metrics-scraper"] output = kubectl("get ns") if "kubernetes-dashboard" in output: # we are running a newer version of the dashboard introduced in 1.33 service = "kubernetes-dashboard-kong-proxy:443" ns = "kubernetes-dashboard" components = ["api", "auth", "metrics-scraper", "web"] app_names = [f"app.kubernetes.io/name=kubernetes-dashboard-{app}" for app in components] app_names.append("app.kubernetes.io/name=kong") for app_name in app_names: wait_for_pod_state("", ns, "running", label=f"{app_name}") attempt = 30 while attempt > 0: try: output = kubectl(f"get --raw /api/v1/namespaces/{ns}/services/https:{service}/proxy/") if "Kubernetes Dashboard" in output: break except subprocess.CalledProcessError: pass time.sleep(10) attempt -= 1 assert attempt > 0 def validate_storage(): """ Validate storage by creating a PVC. """ output = kubectl("describe deployment hostpath-provisioner -n kube-system") if "hostpath-provisioner-{}:1.0.0".format(get_arch()) in output: # we are running with a hostpath-provisioner that is old and we need to patch it cmd = ( "set image deployment hostpath-provisioner" "-n kube-system" "hostpath-provisioner=cdkbot/hostpath-provisioner:1.1.0" ) kubectl(cmd) wait_for_pod_state("", "kube-system", "running", label="k8s-app=hostpath-provisioner") manifest = TEMPLATES / "pvc.yaml" kubectl("apply -f {}".format(manifest)) wait_for_pod_state("hostpath-test-pod", "default", "running") attempt = 50 while attempt >= 0: output = kubectl("get pvc") if "Bound" in output: break time.sleep(2) attempt -= 1 # Make sure the test pod writes data sto the storage found = False for root, dirs, files in os.walk("/var/snap/microk8s/common/default-storage"): for file in files: if file == "dates": found = True assert found assert "myclaim" in output assert "Bound" in output kubectl("delete -f {}".format(manifest)) def common_ingress(): """ Perform the Ingress validations that are common for all the Ingress controllers. """ attempt = 50 while attempt >= 0: output = kubectl("get ing") if "microbot.127.0.0.1.nip.io" in output: break time.sleep(5) attempt -= 1 assert "microbot.127.0.0.1.nip.io" in output service_ok = False attempt = 50 while attempt >= 0: try: resp = requests.get("http://microbot.127.0.0.1.nip.io/") if resp.status_code == 200 and "microbot.png" in resp.content.decode("utf-8"): service_ok = True break except requests.RequestException: time.sleep(5) attempt -= 1 assert service_ok def validate_ingress(): """ Validate ingress by creating a ingress rule. """ ds = kubectl("get ds -n ingress") if "nginx-ingress-microk8s-controller" in ds: wait_for_pod_state("", "ingress", "running", label="name=nginx-ingress-microk8s") else: # Support migration to Traefik ingress controller wait_for_pod_state("", "ingress", "running", label="app.kubernetes.io/name=traefik") manifest = TEMPLATES / "ingress.yaml" update_yaml_with_arch(manifest) kubectl("apply -f {}".format(manifest)) wait_for_pod_state("", "default", "running", label="app=microbot") common_ingress() kubectl("delete -f {}".format(manifest)) def validate_registry(): """ Validate the private registry. """ wait_for_pod_state("", "container-registry", "running", label="app=registry") pvc_stdout = kubectl("get pvc registry-claim -n container-registry -o yaml") pvc_yaml = yaml.safe_load(pvc_stdout) storage = pvc_yaml["spec"]["resources"]["requests"]["storage"] assert re.match("(^[2-9][0-9]{1,}|^[1-9][0-9]{2,})(Gi$)", storage) docker("pull busybox") docker("tag busybox localhost:32000/my-busybox") docker("push localhost:32000/my-busybox") manifest = TEMPLATES / "bbox-local.yaml" kubectl("apply -f {}".format(manifest)) wait_for_pod_state("busybox", "default", "running") output = kubectl("describe po busybox") assert "localhost:32000/my-busybox" in output kubectl("delete -f {}".format(manifest)) def validate_forward(): """ Validate ports are forwarded """ manifest = TEMPLATES / "nginx-pod.yaml" kubectl("apply -f {}".format(manifest)) wait_for_pod_state("", "default", "running", label="app=nginx") os.system("killall kubectl") os.system("/snap/bin/microk8s.kubectl port-forward pod/nginx 5123:80 &") attempt = 10 while attempt >= 0: try: resp = requests.get("http://localhost:5123") if resp.status_code == 200: break except requests.RequestException: pass attempt -= 1 time.sleep(2) assert resp.status_code == 200 os.system("killall kubectl") def validate_metrics_server(): """ Validate the metrics server works """ wait_for_pod_state("", "kube-system", "running", label="k8s-app=metrics-server") attempt = 30 while attempt > 0: try: output = kubectl("get --raw /apis/metrics.k8s.io/v1beta1/pods") if "PodMetricsList" in output: break except subprocess.CalledProcessError: pass time.sleep(10) attempt -= 1 assert attempt > 0 def validate_metallb_config(ip_ranges="192.168.0.105"): """ Validate Metallb """ if platform.machine() != "x86_64": print("Metallb tests are only relevant in x86 architectures") return out = kubectl( "get ipaddresspool -n metallb-system default-addresspool -o jsonpath='{.spec.addresses}" ) for ip_range in ip_ranges.split(","): assert ip_range in out def validate_dual_stack(): # Deploy the test deployment and service manifest = TEMPLATES / "dual-stack.yaml" kubectl("apply -f {}".format(manifest)) wait_for_pod_state("", "default", "running", label="run=nginxdualstack") ipv6_endpoint = kubectl( "get endpoints nginx6 " "-o jsonpath={.subsets[0].addresses[0].ip} " "--output=jsonpath=[{.subsets[0].addresses[0].ip}]" ) print("Pinging endpoint: http://{}/".format(ipv6_endpoint)) url = f"http://{ipv6_endpoint}/" attempt = 10 service_ok = False while attempt >= 0: try: resp = requests.get(url) if "Kubernetes IPv6 nginx" in str(resp.content): print(resp.content) service_ok = True break except requests.RequestException: time.sleep(5) attempt -= 1 assert service_ok kubectl("delete -f {}".format(manifest)) ================================================ FILE: tests/verify-branches.py ================================================ import requests from subprocess import check_output class TestMicrok8sBranches(object): def test_branches(self): """Ensures LP builders push to correct snap tracks. We need to make sure the LP builders pointing to the master github branch are only pushing to the latest and current k8s stable snap tracks. An indication that this is not enforced is that we do not have a branch for the k8s release for the previous stable release. Let me clarify with an example. Assuming upstream stable k8s release is v1.12.x, there has to be a 1.11 github branch used by the respective LP builders for building the v1.11.y. """ upstream_version = self._upstream_release() assert upstream_version version_parts = upstream_version.split(".") major_minor_upstream_version = "{}.{}".format(version_parts[0][1:], version_parts[1]) if version_parts[1] != "0": prev_major_minor_version = "{}.{}".format( version_parts[0][1:], int(version_parts[1]) - 1 ) else: major = int(version_parts[0][1:]) - 1 minor = self._get_max_minor(major) prev_major_minor_version = "{}.{}".format(major, minor) print( "Current stable is {}. Making sure we have a branch for {}".format( major_minor_upstream_version, prev_major_minor_version ) ) cmd = "git ls-remote --heads http://github.com/canonical/microk8s.git {}".format( prev_major_minor_version ) branch = check_output(cmd.split()).decode("utf-8") assert prev_major_minor_version in branch def _upstream_release(self): """Return the latest stable k8s in the release series""" release_url = "https://dl.k8s.io/release/stable.txt" r = requests.get(release_url) if r.status_code == 200: return r.content.decode().strip() else: None def _get_max_minor(self, major): """Get the latest minor release of the provided major. For example if you use 1 as major you will get back X where X gives you latest 1.X release. """ minor = 0 while self._upstream_release_exists(major, minor): minor += 1 return minor - 1 def _upstream_release_exists(self, major, minor): """Return true if the major.minor release exists""" release_url = "https://dl.k8s.io/release/stable-{}.{}.txt".format(major, minor) r = requests.get(release_url) if r.status_code == 200: return True else: return False ================================================ FILE: tox.ini ================================================ [tox] skipsdist=True skip_missing_interpreters = True envlist = lint, scripts, wrappers [testenv] basepython = python3 envdir = {toxinidir}/.tox_env passenv = MK8S_* deps = black ==21.4b2 click==7.1.2 flake8 flake8-colors pep8-naming codespell -r{toxinidir}/tests/requirements.txt [testenv:lint] commands = flake8 --max-line-length=120 --ignore=C901,N801,N802,N803,N806,N816,W503,E203 codespell --ignore-words-list="aks,ccompiler,NotIn" --quiet-level=2 --skip="*.patch,*.spec,.tox_env,.git,*.nsi" black --diff --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod)/" . [testenv:scripts] setenv = PYTHONPATH={toxinidir}/scripts commands = pytest -s tests/unit/test_upgrade_calico_cni.py {posargs} [testenv:wrappers] setenv = PYTHONPATH={toxinidir}/scripts/wrappers commands = pytest -s tests/unit/ \ --ignore=tests/unit/test_upgrade_calico_cni.py \ --ignore-glob=tests/unit/cluster/* {posargs} [testenv:cluster] setenv = PYTHONPATH={toxinidir}/scripts/wrappers commands = pytest -s tests/unit/cluster {posargs} [testenv:clustering] commands = pytest -s tests/test-cluster.py [testenv:simple] commands = pytest -rA -s tests/test-simple.py [flake8] exclude = .git, __pycache__, .tox, .tox_env, max-complexity = 10 import-order-style = google ================================================ FILE: upgrade-scripts/000-switch-to-calico/commit-master.sh ================================================ #!/bin/bash set -ex echo "Switching master to calico" source $SNAP/actions/common/utils.sh RESOURCES="$SNAP/upgrade-scripts/000-switch-to-calico/resources" BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" mkdir -p "$BACKUP_DIR" mkdir -p "$BACKUP_DIR/args/cni-network/" cp "$SNAP_DATA"/args/cni-network/* "$BACKUP_DIR/args/cni-network/" 2>/dev/null || true find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; ARCH="$($SNAP/bin/uname -m)" cp "$RESOURCES/calico.yaml" "$SNAP_DATA/args/cni-network/cni.yaml" echo "Restarting services" cp "$SNAP_DATA"/args/kube-apiserver "$BACKUP_DIR/args" refresh_opt_in_config "allow-privileged" "true" kube-apiserver # Reconfigure kubelet/containerd to pick up the new CNI config and binary. cp "$SNAP_DATA"/args/kubelet "$BACKUP_DIR/args" cp "$SNAP_DATA"/args/containerd-template.toml "$BACKUP_DIR/args" if grep -qE "bin_dir.*SNAP}\/" $SNAP_DATA/args/containerd-template.toml; then echo "Restarting containerd" "${SNAP}/bin/sed" -i 's;bin_dir = "${SNAP}/opt;bin_dir = "${SNAP_DATA}/opt;g' "$SNAP_DATA/args/containerd-template.toml" snapctl restart ${SNAP_NAME}.daemon-containerd fi cp "$SNAP_DATA"/args/kube-proxy "$BACKUP_DIR/args" refresh_opt_in_config "cluster-cidr" "10.1.0.0/16" kube-proxy echo "Restarting kubelite" snapctl restart ${SNAP_NAME}.daemon-kubelite set_service_not_expected_to_start flanneld snapctl stop ${SNAP_NAME}.daemon-flanneld remove_vxlan_interfaces # Allow for services to restart sleep 15 ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" $KUBECTL apply -f "$SNAP_DATA/args/cni-network/cni.yaml" echo "Calico is enabled" ================================================ FILE: upgrade-scripts/000-switch-to-calico/commit-node.sh ================================================ #!/bin/bash set -ex echo "Switching master to calico" source $SNAP/actions/common/utils.sh RESOURCES="$SNAP/upgrade-scripts/000-switch-to-calico/resources" BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" mkdir -p "$BACKUP_DIR" mkdir -p "$BACKUP_DIR/args/cni-network/" cp "$SNAP_DATA"/args/cni-network/* "$BACKUP_DIR/args/cni-network/" 2>/dev/null || true find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; ARCH="$($SNAP/bin/uname -m)" CALICO_MANIFEST="$RESOURCES/calico.yaml" run_with_sudo cp "$CALICO_MANIFEST" "$SNAP_DATA/args/cni-network/cni.yaml" cp "$SNAP_DATA"/args/kube-apiserver "$BACKUP_DIR/args" refresh_opt_in_config "allow-privileged" "true" kube-apiserver # Reconfigure kubelet/containerd to pick up the new CNI config and binary. cp "$SNAP_DATA"/args/kubelet "$BACKUP_DIR/args" cp "$SNAP_DATA"/args/containerd-template.toml "$BACKUP_DIR/args" if grep -qE "bin_dir.*SNAP}\/" $SNAP_DATA/args/containerd-template.toml; then echo "Restarting containerd" "${SNAP}/bin/sed" -i 's;bin_dir = "${SNAP}/opt;bin_dir = "${SNAP_DATA}/opt;g' "$SNAP_DATA/args/containerd-template.toml" snapctl restart ${SNAP_NAME}.daemon-containerd fi cp "$SNAP_DATA"/args/kube-proxy "$BACKUP_DIR/args" refresh_opt_in_config "cluster-cidr" "10.1.0.0/16" kube-proxy echo "Restarting kubelite" snapctl restart ${SNAP_NAME}.daemon-kubelite set_service_not_expected_to_start flanneld snapctl stop ${SNAP_NAME}.daemon-flanneld remove_vxlan_interfaces echo "Calico is enabled" ================================================ FILE: upgrade-scripts/000-switch-to-calico/description.txt ================================================ Removes the flannel cni and installs calico. ================================================ FILE: upgrade-scripts/000-switch-to-calico/prepare-master.sh ================================================ #!/usr/bin/env bash echo "Preparing master for calico" ================================================ FILE: upgrade-scripts/000-switch-to-calico/prepare-node.sh ================================================ #!/bin/bash set -e echo "Praparing node for calico" ================================================ FILE: upgrade-scripts/000-switch-to-calico/resources/calico.yaml ================================================ --- # Source: calico/templates/calico-kube-controllers.yaml # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: maxUnavailable: 1 selector: matchLabels: k8s-app: calico-kube-controllers --- # Source: calico/templates/calico-kube-controllers.yaml apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system --- # Source: calico/templates/calico-node.yaml apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system --- # Source: calico/templates/calico-node.yaml apiVersion: v1 kind: ServiceAccount metadata: name: calico-cni-plugin namespace: kube-system --- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap apiVersion: v1 metadata: name: calico-config namespace: kube-system data: # Typha is disabled. typha_service_name: "none" # Configure the backend to use. # calico_backend: "bird" calico_backend: "vxlan" # Configure the MTU to use for workload interfaces and tunnels. # By default, MTU is auto-detected, and explicitly setting this field should not be required. # You can override auto-detection by providing a non-zero value. veth_mtu: "0" # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- { "name": "k8s-pod-network", "cniVersion": "0.3.1", "plugins": [ { "type": "calico", "log_level": "info", "log_file_path": "/var/snap/microk8s/common/var/log/calico/cni/cni.log", "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "nodename_file": "/var/snap/microk8s/current/var/lib/calico/nodename", "mtu": __CNI_MTU__, "ipam": { "type": "calico-ipam" }, "policy": { "type": "k8s" }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { "type": "portmap", "snat": true, "capabilities": {"portMappings": true} }, { "type": "bandwidth", "capabilities": {"bandwidth": true} } ] } --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bgpconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPConfiguration listKind: BGPConfigurationList plural: bgpconfigurations singular: bgpconfiguration preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: BGPConfiguration contains the configuration for any BGP routing. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPConfigurationSpec contains the values of the BGP configuration. properties: asNumber: description: 'ASNumber is the default AS number used by a node. [Default: 64512]' format: int32 type: integer bindMode: description: BindMode indicates whether to listen for BGP connections on all addresses (None) or only on the node's canonical IP address Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen for BGP connections on all addresses. type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. items: description: Community contains standard or large community value and its name. properties: name: description: Name given to community value. type: string value: description: Value must be of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where, `aa` is an AS Number, `nn` and `mm` are per-AS identifier. pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ type: string type: object type: array ignoredInterfaces: description: IgnoredInterfaces indicates the network interfaces that needs to be excluded when reading device routes. items: type: string type: array listenPort: description: ListenPort is the port where BGP protocol should listen. Defaults to 179 maximum: 65535 minimum: 1 type: integer logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string nodeMeshMaxRestartTime: description: Time to allow for software restart for node-to-mesh peerings. When specified, this is configured as the graceful restart timeout. When not specified, the BIRD default of 120s is used. This field can only be set on the default BGPConfiguration instance and requires that NodeMesh is enabled type: string nodeMeshPassword: description: Optional BGP password for full node-to-mesh peerings. This field can only be set on the default BGPConfiguration instance and requires that NodeMesh is enabled properties: secretKeyRef: description: Selects a key of a secret in the node pod's namespace. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Drop `kubebuilder:default` when controller-gen doesn''t need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' type: boolean prefixAdvertisements: description: PrefixAdvertisements contains per-prefix advertisement configuration. items: description: PrefixAdvertisement configures advertisement properties for the specified CIDR. properties: cidr: description: CIDR for which properties should be advertised. type: string communities: description: Communities can be list of either community names already defined in `Specs.Communities` or community value of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and `mm` are per-AS identifier. items: type: string type: array type: object type: array serviceClusterIPs: description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them. items: description: ServiceClusterIPBlock represents a single allowed ClusterIP CIDR block. properties: cidr: type: string type: object type: array serviceExternalIPs: description: ServiceExternalIPs are the CIDR blocks for Kubernetes Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. items: description: ServiceExternalIPBlock represents a single allowed External IP CIDR block. properties: cidr: type: string type: object type: array serviceLoadBalancerIPs: description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress IPs will only be advertised if they are within one of these blocks. items: description: ServiceLoadBalancerIPBlock represents a single allowed LoadBalancer IP CIDR block. properties: cidr: type: string type: object type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (devel) creationTimestamp: null name: bgpfilters.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPFilter listKind: BGPFilterList plural: bgpfilters singular: bgpfilter scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of the BGP Filter. properties: exportV4: description: The ordered set of IPv4 BGPFilter rules acting on exporting routes to a peer. items: description: BGPFilterRuleV4 defines a BGP filter rule consisting a single IPv4 CIDR block and a filter action for this CIDR. properties: action: type: string cidr: type: string interface: type: string matchOperator: type: string prefixLength: properties: max: format: int32 maximum: 32 minimum: 0 type: integer min: format: int32 maximum: 32 minimum: 0 type: integer type: object source: type: string required: - action type: object type: array exportV6: description: The ordered set of IPv6 BGPFilter rules acting on exporting routes to a peer. items: description: BGPFilterRuleV6 defines a BGP filter rule consisting a single IPv6 CIDR block and a filter action for this CIDR. properties: action: type: string cidr: type: string interface: type: string matchOperator: type: string prefixLength: properties: max: format: int32 maximum: 128 minimum: 0 type: integer min: format: int32 maximum: 128 minimum: 0 type: integer type: object source: type: string required: - action type: object type: array importV4: description: The ordered set of IPv4 BGPFilter rules acting on importing routes from a peer. items: description: BGPFilterRuleV4 defines a BGP filter rule consisting a single IPv4 CIDR block and a filter action for this CIDR. properties: action: type: string cidr: type: string interface: type: string matchOperator: type: string prefixLength: properties: max: format: int32 maximum: 32 minimum: 0 type: integer min: format: int32 maximum: 32 minimum: 0 type: integer type: object source: type: string required: - action type: object type: array importV6: description: The ordered set of IPv6 BGPFilter rules acting on importing routes from a peer. items: description: BGPFilterRuleV6 defines a BGP filter rule consisting a single IPv6 CIDR block and a filter action for this CIDR. properties: action: type: string cidr: type: string interface: type: string matchOperator: type: string prefixLength: properties: max: format: int32 maximum: 128 minimum: 0 type: integer min: format: int32 maximum: 128 minimum: 0 type: integer type: object source: type: string required: - action type: object type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bgppeers.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BGPPeer listKind: BGPPeerList plural: bgppeers singular: bgppeer preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPPeerSpec contains the specification for a BGPPeer resource. properties: asNumber: description: The AS Number of the peer. format: int32 type: integer filters: description: The ordered set of BGPFilters applied on this BGP peer. items: type: string type: array keepOriginalNextHop: description: Option to keep the original nexthop field when routes are sent to a BGP Peer. Setting "true" configures the selected BGP Peers node to use the "next hop keep;" instead of "next hop self;"(default) in the specific branch of the Node on "bird.cfg". type: boolean maxRestartTime: description: Time to allow for software restart. When specified, this is configured as the graceful restart timeout. When not specified, the BIRD default of 120s is used. type: string node: description: The node name identifying the Calico node instance that is targeted by this peer. If this is not set, and no nodeSelector is specified, then this BGP peer selects all nodes in the cluster. type: string nodeSelector: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string numAllowedLocalASNumbers: description: Maximum number of local AS numbers that are allowed in the AS path for received routes. This removes BGP loop prevention and should only be used if absolutely necessary. format: int32 type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. properties: secretKeyRef: description: Selects a key of a secret in the node pod's namespace. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: default: "" description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Drop `kubebuilder:default` when controller-gen doesn''t need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object peerIP: description: The IP address of the peer followed by an optional port number to peer with. If port number is given, format should be `[]:port` or `:` for IPv4. If optional port number is not set, and this peer IP and ASNumber belongs to a calico/node with ListenPort set in BGPConfiguration, then we use that port to peer. type: string peerSelector: description: Selector for the remote nodes to peer with. When this is set, the PeerIP and ASNumber fields must be empty. For each peering between the local node and selected remote nodes, we configure an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The remote AS number comes from the remote node's NodeBGPSpec.ASNumber, or the global default if that is not set. type: string reachableBy: description: Add an exact, i.e. /32, static route toward peer IP in order to prevent route flapping. ReachableBy contains the address of the gateway which peer can be reached by. type: string sourceAddress: description: Specifies whether and how to configure a source address for the peerings generated by this BGPPeer resource. Default value "UseNodeIP" means to configure the node IP as the source address. "None" means not to configure a source address. type: string ttlSecurity: description: TTLSecurity enables the generalized TTL security mechanism (GTSM) which protects against spoofed packets by ignoring received packets with a smaller than expected TTL value. The provided value is the number of hops (edges) between the peers. type: integer type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: blockaffinities.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: BlockAffinity listKind: BlockAffinityList plural: blockaffinities singular: blockaffinity preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BlockAffinitySpec contains the specification for a BlockAffinity resource. properties: cidr: type: string deleted: description: Deleted indicates that this block affinity is being deleted. This field is a string for compatibility with older releases that mistakenly treat this field as a string. type: string node: type: string state: type: string required: - cidr - deleted - node - state type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (devel) creationTimestamp: null name: caliconodestatuses.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: CalicoNodeStatus listKind: CalicoNodeStatusList plural: caliconodestatuses singular: caliconodestatus preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus resource. properties: classes: description: Classes declares the types of information to monitor for this calico/node, and allows for selective status reporting about certain subsets of information. items: type: string type: array node: description: The node name identifies the Calico node instance for node status. type: string updatePeriodSeconds: description: UpdatePeriodSeconds is the period at which CalicoNodeStatus should be updated. Set to 0 to disable CalicoNodeStatus refresh. Maximum update period is one day. format: int32 type: integer type: object status: description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. No validation needed for status since it is updated by Calico. properties: agent: description: Agent holds agent status on the node. properties: birdV4: description: BIRDV4 represents the latest observed status of bird4. properties: lastBootTime: description: LastBootTime holds the value of lastBootTime from bird.ctl output. type: string lastReconfigurationTime: description: LastReconfigurationTime holds the value of lastReconfigTime from bird.ctl output. type: string routerID: description: Router ID used by bird. type: string state: description: The state of the BGP Daemon. type: string version: description: Version of the BGP daemon type: string type: object birdV6: description: BIRDV6 represents the latest observed status of bird6. properties: lastBootTime: description: LastBootTime holds the value of lastBootTime from bird.ctl output. type: string lastReconfigurationTime: description: LastReconfigurationTime holds the value of lastReconfigTime from bird.ctl output. type: string routerID: description: Router ID used by bird. type: string state: description: The state of the BGP Daemon. type: string version: description: Version of the BGP daemon type: string type: object type: object bgp: description: BGP holds node BGP status. properties: numberEstablishedV4: description: The total number of IPv4 established bgp sessions. type: integer numberEstablishedV6: description: The total number of IPv6 established bgp sessions. type: integer numberNotEstablishedV4: description: The total number of IPv4 non-established bgp sessions. type: integer numberNotEstablishedV6: description: The total number of IPv6 non-established bgp sessions. type: integer peersV4: description: PeersV4 represents IPv4 BGP peers status on the node. items: description: CalicoNodePeer contains the status of BGP peers on the node. properties: peerIP: description: IP address of the peer whose condition we are reporting. type: string since: description: Since the state or reason last changed. type: string state: description: State is the BGP session state. type: string type: description: Type indicates whether this peer is configured via the node-to-node mesh, or via en explicit global or per-node BGPPeer object. type: string type: object type: array peersV6: description: PeersV6 represents IPv6 BGP peers status on the node. items: description: CalicoNodePeer contains the status of BGP peers on the node. properties: peerIP: description: IP address of the peer whose condition we are reporting. type: string since: description: Since the state or reason last changed. type: string state: description: State is the BGP session state. type: string type: description: Type indicates whether this peer is configured via the node-to-node mesh, or via en explicit global or per-node BGPPeer object. type: string type: object type: array required: - numberEstablishedV4 - numberEstablishedV6 - numberNotEstablishedV4 - numberNotEstablishedV6 type: object lastUpdated: description: LastUpdated is a timestamp representing the server time when CalicoNodeStatus object last updated. It is represented in RFC3339 form and is in UTC. format: date-time nullable: true type: string routes: description: Routes reports routes known to the Calico BGP daemon on the node. properties: routesV4: description: RoutesV4 represents IPv4 routes on the node. items: description: CalicoNodeRoute contains the status of BGP routes on the node. properties: destination: description: Destination of the route. type: string gateway: description: Gateway for the destination. type: string interface: description: Interface for the destination type: string learnedFrom: description: LearnedFrom contains information regarding where this route originated. properties: peerIP: description: If sourceType is NodeMesh or BGPPeer, IP address of the router that sent us this route. type: string sourceType: description: Type of the source where a route is learned from. type: string type: object type: description: Type indicates if the route is being used for forwarding or not. type: string type: object type: array routesV6: description: RoutesV6 represents IPv6 routes on the node. items: description: CalicoNodeRoute contains the status of BGP routes on the node. properties: destination: description: Destination of the route. type: string gateway: description: Gateway for the destination. type: string interface: description: Interface for the destination type: string learnedFrom: description: LearnedFrom contains information regarding where this route originated. properties: peerIP: description: If sourceType is NodeMesh or BGPPeer, IP address of the router that sent us this route. type: string sourceType: description: Type of the source where a route is learned from. type: string type: object type: description: Type indicates if the route is being used for forwarding or not. type: string type: object type: array type: object type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clusterinformations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: ClusterInformation listKind: ClusterInformationList plural: clusterinformations singular: clusterinformation preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: ClusterInformation contains the cluster specific information. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: ClusterInformationSpec contains the values of describing the cluster. properties: calicoVersion: description: CalicoVersion is the version of Calico that the cluster is running type: string clusterGUID: description: ClusterGUID is the GUID of the cluster type: string clusterType: description: ClusterType describes the type of the cluster type: string datastoreReady: description: DatastoreReady is used during significant datastore migrations to signal to components such as Felix that it should wait before accessing the datastore. type: boolean variant: description: Variant declares which variant of Calico should be active. type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: felixconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: FelixConfiguration listKind: FelixConfigurationList plural: felixconfigurations singular: felixconfiguration preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: Felix Configuration contains the configuration for Felix. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: FelixConfigurationSpec contains the values of the Felix configuration. properties: allowIPIPPacketsFromWorkloads: description: 'AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic from workloads [Default: false]' type: boolean allowVXLANPacketsFromWorkloads: description: 'AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic from workloads [Default: false]' type: boolean awsSrcDstCheck: description: 'Set source-destination-check on AWS EC2 instances. Accepted value must be one of "DoNothing", "Enable" or "Disable". [Default: DoNothing]' enum: - DoNothing - Enable - Disable type: string bpfCTLBLogFilter: description: 'BPFCTLBLogFilter specifies, what is logged by connect time load balancer when BPFLogLevel is debug. Currently has to be specified as ''all'' when BPFLogFilters is set to see CTLB logs. [Default: unset - means logs are emitted when BPFLogLevel id debug and BPFLogFilters not set.]' type: string bpfConnectTimeLoadBalancing: description: 'BPFConnectTimeLoadBalancing when in BPF mode, controls whether Felix installs the connect-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing is available only for services with TCP ports. [Default: TCP]' enum: - TCP - Enabled - Disabled type: string bpfConnectTimeLoadBalancingEnabled: description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging purposes. This will be deprecated. Use BPFConnectTimeLoadBalancing [Default: true]' type: boolean bpfDSROptoutCIDRs: description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients in those CIDRs will accesses nodeports as if BPFExternalServiceMode was set to Tunnel. items: type: string type: array bpfDataIfacePattern: description: BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. It should not match the workload interfaces (usually named cali...). type: string bpfDisableGROForIfaces: description: BPFDisableGROForIfaces is a regular expression that controls which interfaces Felix should disable the Generic Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...). type: string bpfDisableUnprivileged: description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable unprivileged use of BPF. This ensures that unprivileged users cannot access Calico''s BPF maps and cannot insert their own BPF programs to interfere with Calico''s. [Default: true]' type: boolean bpfEnabled: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean bpfEnforceRPF: description: 'BPFEnforceRPF enforce strict RPF on all host interfaces with BPF programs regardless of what is the per-interfaces or global setting. Possible values are Disabled, Strict or Loose. [Default: Loose]' pattern: ^(?i)(Disabled|Strict|Loose)?$ type: string bpfExcludeCIDRsFromNAT: description: BPFExcludeCIDRsFromNAT is a list of CIDRs that are to be excluded from NAT resolution so that host can handle them. A typical usecase is node local DNS cache. items: type: string type: array bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local service. This mark allows us to control how packets of that connection are routed within the host and how is routing interpreted by RPF check. [Default: 0]' type: integer bpfExternalServiceMode: description: 'BPFExternalServiceMode in BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) are forwarded to remote workloads. If set to "Tunnel" then both request and response traffic is tunneled to the remote node. If set to "DSR", the request traffic is tunneled but the response traffic is sent directly from the remote node. In "DSR" mode, the remote node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' pattern: ^(?i)(Tunnel|DSR)?$ type: string bpfForceTrackPacketsFromIfaces: description: 'BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic from these interfaces to skip Calico''s iptables NOTRACK rule, allowing traffic from those interfaces to be tracked by Linux conntrack. Should only be used for interfaces that are not used for the Calico fabric. For example, a docker bridge device for non-Calico-networked containers. [Default: docker+]' items: type: string type: array bpfHostConntrackBypass: description: 'BPFHostConntrackBypass Controls whether to bypass Linux conntrack in BPF mode for workloads and services. [Default: true - bypass Linux conntrack]' type: boolean bpfHostNetworkedNATWithoutCTLB: description: 'BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing determines the CTLB behavior. [Default: Enabled]' enum: - Enabled - Disabled type: string bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled is deprecated and has no effect. BPF kube-proxy always accepts endpoint slices. This option will be removed in the next release. type: boolean bpfKubeProxyIptablesCleanupEnabled: description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s iptables chains. Should only be enabled if kube-proxy is not running. [Default: true]' type: boolean bpfKubeProxyMinSyncPeriod: description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the minimum time between updates to the dataplane for Felix''s embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by batching up more work. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string bpfL3IfacePattern: description: BPFL3IfacePattern is a regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices) in addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. type: string bpfLogFilters: additionalProperties: type: string description: "BPFLogFilters is a map of key=values where the value is a pcap filter expression and the key is an interface name with 'all' denoting all interfaces, 'weps' all workload endpoints and 'heps' all host endpoints. \n When specified as an env var, it accepts a comma-separated list of key=values. [Default: unset - means all debug logs are emitted]" type: object bpfLogLevel: description: 'BPFLogLevel controls the log level of the BPF programs when in BPF dataplane mode. One of "Off", "Info", or "Debug". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' pattern: ^(?i)(Off|Info|Debug)?$ type: string bpfMapSizeConntrack: description: 'BPFMapSizeConntrack sets the size for the conntrack map. This map must be large enough to hold an entry for each active connection. Warning: changing the size of the conntrack map can cause disruption.' type: integer bpfMapSizeIPSets: description: BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry for each endpoint matched by every selector in the source/destination matches in network policy. Selectors such as "all()" can result in large numbers of entries (one entry per endpoint in that case). type: integer bpfMapSizeIfState: description: BPFMapSizeIfState sets the size for ifstate map. The ifstate map must be large enough to hold an entry for each device (host + workloads) on a host. type: integer bpfMapSizeNATAffinity: type: integer bpfMapSizeNATBackend: description: BPFMapSizeNATBackend sets the size for nat back end map. This is the total number of endpoints. This is mostly more than the size of the number of services. type: integer bpfMapSizeNATFrontend: description: BPFMapSizeNATFrontend sets the size for nat front end map. FrontendMap should be large enough to hold an entry for each nodeport, external IP and each port in each service. type: integer bpfMapSizeRoute: description: BPFMapSizeRoute sets the size for the routes map. The routes map should be large enough to hold one entry per workload and a handful of entries per host (enough to cover its own IPs and tunnel IPs). type: integer bpfPSNATPorts: anyOf: - type: integer - type: string description: 'BPFPSNATPorts sets the range from which we randomly pick a port if there is a source port collision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and preferably outside the ephemeral ranges used by common operating systems. Linux uses 32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily a problem if this range overlaps with the operating systems. Both ends of the range are inclusive. [Default: 20000:29999]' pattern: ^.* x-kubernetes-int-or-string: true bpfPolicyDebugEnabled: description: BPFPolicyDebugEnabled when true, Felix records detailed information about the BPF policy programs, which can be examined with the calico-bpf command-line tool. type: boolean bpfRedirectToPeer: description: 'BPFRedirectToPeer controls which whether it is allowed to forward straight to the peer side of the workload devices. It is allowed for any host L2 devices by default (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses it on ingress. Value of Enabled also allows redirection from L3 host devices like IPIP tunnel or Wireguard directly to the peer side of the workload''s device. This makes redirection faster, however, it breaks tools like tcpdump on the peer side. Use Enabled with caution. [Default: L2Only]' type: string chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents Calico''s rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. [Default: insert]' pattern: ^(?i)(insert|append)?$ type: string dataplaneDriver: description: DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver is set to false. type: string dataplaneWatchdogTimeout: description: "DataplaneWatchdogTimeout is the readiness/liveness timeout used for Felix's (internal) dataplane driver. Increase this value if you experience spurious non-ready or non-live events when Felix is under heavy load. Decrease the value to get felix to report non-live or non-ready more quickly. [Default: 90s] \n Deprecated: replaced by the generic HealthTimeoutOverrides." type: string debugDisableLogDropping: type: boolean debugHost: description: DebugHost is the host IP or hostname to bind the debug port to. Only used if DebugPort is set. [Default:localhost] type: string debugMemoryProfilePath: type: string debugPort: description: DebugPort if set, enables Felix's debug HTTP port, which allows memory and CPU profiles to be retrieved. The debug port is not secure, it should not be exposed to the internet. type: integer debugSimulateCalcGraphHangAfter: pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneApplyDelay: pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string debugSimulateDataplaneHangAfter: pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string defaultEndpointToHostAction: description: 'DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host itself (after the traffic hits the endpoint egress policy). By default Calico blocks traffic from workload endpoints to the host itself with an iptables "DROP" action. If you want to allow some or all traffic from endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables "INPUT" chain; Calico will insert its rules at the top of that chain, then "RETURN" packets to the "INPUT" chain once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets from workloads after processing workload endpoint egress policy. [Default: Drop]' pattern: ^(?i)(Drop|Accept|Return)?$ type: string deviceRouteProtocol: description: This defines the route protocol added to programmed device routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: description: This is the IPv4 source address to use on programmed device routes. By default the source address is left blank, leaving the kernel to choose the source address used. type: string deviceRouteSourceAddressIPv6: description: This is the IPv6 source address to use on programmed device routes. By default the source address is left blank, leaving the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean endpointReportingDelay: pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string endpointReportingEnabled: type: boolean endpointStatusPathPrefix: description: "EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status file reporting is disabled if field is left empty. \n Chosen directory should match the directory used by the CNI for PodStartupDelay. [Default: \"\"]" type: string externalNodesList: description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes which may source tunnel traffic and have the tunneled traffic be accepted at calico nodes. items: type: string type: array failsafeInboundHostPorts: description: 'FailsafeInboundHostPorts is a list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, use the value "[]". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API. [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ]' items: description: ProtoPort is combination of protocol, port, and CIDR. Protocol and port must be specified. properties: net: type: string port: type: integer protocol: type: string required: - port - protocol type: object type: array failsafeOutboundHostPorts: description: 'FailsafeOutboundHostPorts is a list of List of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, use the value "[]". The default value opens etcd''s standard ports to ensure that Felix does not get cut off from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes API. [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ]' items: description: ProtoPort is combination of protocol, port, and CIDR. Protocol and port must be specified. properties: net: type: string port: type: integer protocol: type: string required: - port - protocol type: object type: array featureDetectOverride: description: FeatureDetectOverride is used to override feature detection based on auto-detected platform capabilities. Values are specified in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" or "false" will force the feature, empty or omitted values are auto-detected. pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ type: string featureGates: description: FeatureGates is used to enable or disable tech-preview Calico features. Values are specified in a comma separated list with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". This is used to enable features that are not fully production ready. pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$ type: string floatingIPs: description: FloatingIPs configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived floating IPs are always programmed, regardless of this setting.) enum: - Enabled - Disabled type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This is not recommended since it doesn''t provide better performance than iptables. [Default: false]' type: boolean goGCThreshold: description: "GoGCThreshold Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is allowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time spent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used to disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting. \n This setting is overridden by the GOGC environment variable. \n [Default: 40]" type: integer goMaxProcs: description: "GoMaxProcs sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means \"use the system default\"; typically the number of real CPUs on the system. \n this setting is overridden by the GOMAXPROCS environment variable. \n [Default: -1]" type: integer goMemoryLimitMB: description: "GoMemoryLimitMB sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory usage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to take more than 50% of the process's CPU time. A value of -1 disables the memory limit. \n Note that the memory limit, if used, must be considerably less than any hard resource limit set at the container or pod level. This is because felix is not the only process that must run in the container or pod. \n This setting is overridden by the GOMEMLIMIT environment variable. \n [Default: -1]" type: integer healthEnabled: type: boolean healthHost: type: string healthPort: type: integer healthTimeoutOverrides: description: HealthTimeoutOverrides allows the internal watchdog timeouts of individual subcomponents to be overridden. This is useful for working around "false positive" liveness timeouts that can occur in particularly stressful workloads or if CPU is constrained. For a list of active subcomponents, see Felix's logs. items: properties: name: type: string timeout: type: string required: - name - timeout type: object type: array interfaceExclude: description: 'InterfaceExclude is a comma-separated list of interfaces that Felix should exclude when monitoring for host endpoints. The default value ensures that Felix ignores Kubernetes'' IPVS dummy interface, which is used internally by kube-proxy. If you want to exclude multiple interface names using a single value, the list supports regular expressions. For regular expressions you must wrap the value with ''/''. For example having values ''/^kube/,veth1'' will exclude all interfaces that begin with ''kube'' and also the interface ''veth1''. [Default: kube-ipvs0]' type: string interfacePrefix: description: 'InterfacePrefix is the interface name prefix that identifies workload endpoints and so distinguishes them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators configure this appropriately. For example our Kubernetes and Docker integrations set the ''cali'' value, and our OpenStack integration sets the ''tap'' value. [Default: cali]' type: string interfaceRefreshInterval: description: InterfaceRefreshInterval is the period at which Felix rescans local interfaces to verify their state. The rescan can be disabled by setting the interval to 0. pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string ipForwarding: description: 'IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required when using Calico for workload networking. This should only be disabled on hosts where Calico is used for host protection. [Default: Enabled]' enum: - Enabled - Disabled type: string ipipEnabled: description: 'IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix determines this based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See Configuring MTU [Default: 1440]' type: integer ipsetsRefreshInterval: description: 'IpsetsRefreshInterval is the period at which Felix re-checks all iptables state to ensure that no other process has accidentally broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesBackend: description: IptablesBackend specifies which backend of iptables will be used. The default is Auto. pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$ type: string iptablesFilterAllowAction: pattern: ^(?i)(Accept|Return)?$ type: string iptablesFilterDenyAction: description: IptablesFilterDenyAction controls what happens to traffic that is denied by network policy. By default Calico blocks traffic with an iptables "DROP" action. If you want to use "REJECT" action instead you can configure it in here. pattern: ^(?i)(Drop|Reject)?$ type: string iptablesLockFilePath: description: 'IptablesLockFilePath is the location of the iptables lock file. You may need to change this if the lock file is not in its standard location (for example if you have mapped it into Felix''s container at a different path). [Default: /run/xtables.lock]' type: string iptablesLockProbeInterval: description: 'IptablesLockProbeInterval is the time that Felix will wait between attempts to acquire the iptables lock if it is not available. Lower values make Felix more responsive when the lock is contended, but use more CPU. [Default: 50ms]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesLockTimeout: description: 'IptablesLockTimeout is the time that Felix will wait for the iptables lock, or 0, to disable. To use this feature, Felix must share the iptables lock file with all other processes that also take the lock. When running Felix inside a container, this requires the /run directory of the host to be mounted into the calico/node or calico/felix container. [Default: 0s disabled]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesMangleAllowAction: pattern: ^(?i)(Accept|Return)?$ type: string iptablesMarkMask: description: 'IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xff000000]' format: int32 type: integer iptablesNATOutgoingInterfaceFilter: type: string iptablesPostWriteCheckInterval: description: 'IptablesPostWriteCheckInterval is the period after Felix has done a write to the dataplane that it schedules an extra read back in order to check the write was not clobbered by another process. This should only occur if another application on the system doesn''t respect the iptables lock. [Default: 1s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string iptablesRefreshInterval: description: 'IptablesRefreshInterval is the period at which Felix re-checks the IP sets in the dataplane to ensure that no other process has accidentally broken Calico''s rules. Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value to reduce Felix CPU usage. [Default: 10s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string ipv6Support: description: IPv6Support controls whether Felix enables support for IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. Felix uses these ranges to separate host and workload traffic. [Default: 30000:32767].' items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array logDebugFilenameRegex: description: LogDebugFilenameRegex controls which source code files have their Debug log output included in the logs. Only logs from files with names that match the given regular expression are included. The filter only applies to Debug level logs. type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' type: string logPrefix: description: 'LogPrefix is the log prefix that Felix uses when rendering LOG rules. [Default: calico-packet]' type: string logSeverityFile: description: 'LogSeverityFile is the log severity above which logs are sent to the log file. [Default: Info]' pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ type: string logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ type: string logSeveritySys: description: 'LogSeveritySys is the log severity above which logs are sent to the syslog. Set to None for no logging to syslog. [Default: Info]' pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$ type: string maxIpsetSize: description: MaxIpsetSize is the maximum number of IP addresses that can be stored in an IP set. Not applicable if using the nftables backend. type: integer metadataAddr: description: 'MetadataAddr is the IP address or domain name of the server that can answer VM queries for cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in Ubuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not set up any NAT rule for the metadata path. [Default: 127.0.0.1]' type: string metadataPort: description: 'MetadataPort is the port of the metadata server. This, combined with global.MetadataAddr (if not ''None''), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. In most cases this should not need to be changed [Default: 8775].' type: integer mtuIfacePattern: description: MTUIfacePattern is a regular expression that controls which interfaces Felix should scan in order to calculate the host's MTU. This should not match workload interfaces (usually named cali...). type: string natOutgoingAddress: description: NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that is leaving the network. By default the address used is an address on the interface the traffic is leaving on (ie it uses the iptables MASQUERADE target) type: string natPortRange: anyOf: - type: integer - type: string description: NATPortRange specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the network stack is used. pattern: ^.* x-kubernetes-int-or-string: true netlinkTimeout: pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string nftablesFilterAllowAction: pattern: ^(?i)(Accept|Return)?$ type: string nftablesFilterDenyAction: description: FilterDenyAction controls what happens to traffic that is denied by network policy. By default Calico blocks traffic with a "drop" action. If you want to use a "reject" action instead you can configure it here. pattern: ^(?i)(Drop|Reject)?$ type: string nftablesMangleAllowAction: pattern: ^(?i)(Accept|Return)?$ type: string nftablesMarkMask: description: 'MarkMask is the mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal number with at least 8 bits set, none of which clash with any other mark bits in use on the system. [Default: 0xffff0000]' format: int32 type: integer nftablesMode: description: 'NFTablesMode configures nftables support in Felix. [Default: Disabled]' type: string nftablesRefreshInterval: description: 'NftablesRefreshInterval controls the interval at which Felix periodically refreshes the nftables rules. [Default: 90s]' type: string openstackRegion: description: 'OpenstackRegion is the name of the region that a particular Felix belongs to. In a multi-region Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, or in felix.cfg or the environment on each compute node), and must match the [calico] openstack_region value configured in neutron.conf on each node. [Default: Empty]' type: string policySyncPathPrefix: description: 'PolicySyncPathPrefix is used to by Felix to communicate policy changes to external services, like Application layer policy. [Default: Empty]' type: string prometheusGoMetricsEnabled: description: 'PrometheusGoMetricsEnabled disables Go runtime metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]' type: boolean prometheusMetricsEnabled: description: 'PrometheusMetricsEnabled enables the Prometheus metrics server in Felix if set to true. [Default: false]' type: boolean prometheusMetricsHost: description: 'PrometheusMetricsHost is the host that the Prometheus metrics server should bind to. [Default: empty]' type: string prometheusMetricsPort: description: 'PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. [Default: 9091]' type: integer prometheusProcessMetricsEnabled: description: 'PrometheusProcessMetricsEnabled disables process metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]' type: boolean prometheusWireGuardMetricsEnabled: description: 'PrometheusWireGuardMetricsEnabled disables wireguard metrics collection, which the Prometheus client does by default, when set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true]' type: boolean removeExternalRoutes: description: Whether or not to remove device routes that have not been programmed by Felix. Disabling this will allow external applications to also add device routes. This is enabled by default which means we will remove externally added routes. type: boolean reportingInterval: description: 'ReportingInterval is the interval at which Felix reports its status into the datastore or 0 to disable. Must be non-zero in OpenStack deployments. [Default: 30s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string reportingTTL: description: 'ReportingTTL is the time-to-live setting for process-wide status reports. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string routeRefreshInterval: description: 'RouteRefreshInterval is the period at which Felix re-checks the routes in the dataplane to ensure that no other process has accidentally broken Calico''s rules. Set to 0 to disable route refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string routeSource: description: 'RouteSource configures where Felix gets its routing information. - WorkloadIPs: use workload endpoints to construct routes. - CalicoIPAM: the default - use IPAM data to construct routes.' pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$ type: string routeSyncDisabled: description: RouteSyncDisabled will disable all operations performed on the route table. Set to true to run in network-policy mode only. type: boolean routeTableRange: description: Deprecated in favor of RouteTableRanges. Calico programs additional Linux route tables for various purposes. RouteTableRange specifies the indices of the route tables that Calico should use. properties: max: type: integer min: type: integer required: - max - min type: object routeTableRanges: description: Calico programs additional Linux route tables for various purposes. RouteTableRanges specifies a set of table index ranges that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. items: properties: max: type: integer min: type: integer required: - max - min type: object type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", in which case such routing loops continue to be allowed. [Default: Drop]' pattern: ^(?i)(Drop|Reject|Disabled)?$ type: string sidecarAccelerationEnabled: description: 'SidecarAccelerationEnabled enables experimental sidecar acceleration [Default: false]' type: boolean usageReportingEnabled: description: 'UsageReportingEnabled reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage server. For example, if a significant security vulnerability has been discovered in the version of Calico being used. [Default: true]' type: boolean usageReportingInitialDelay: description: 'UsageReportingInitialDelay controls the minimum delay before Felix makes a report. [Default: 300s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string usageReportingInterval: description: 'UsageReportingInterval controls the interval at which Felix makes reports. [Default: 86400s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string useInternalDataplaneDriver: description: UseInternalDataplaneDriver, if true, Felix will use its internal dataplane programming logic. If false, it will launch an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: description: 'VXLANEnabled overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix determines this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel device. See Configuring MTU [Default: 1410]' type: integer vxlanMTUV6: description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer vxlanVNI: type: integer windowsManageFirewallRules: description: 'WindowsManageFirewallRules configures whether or not Felix will program Windows Firewall rules. (to allow inbound access to its own metrics ports) [Default: Disabled]' enum: - Enabled - Disabled type: string wireguardEnabled: description: 'WireguardEnabled controls whether Wireguard is enabled for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). [Default: false]' type: boolean wireguardEnabledV6: description: 'WireguardEnabledV6 controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). [Default: false]' type: boolean wireguardHostEncryptionEnabled: description: 'WireguardHostEncryptionEnabled controls whether Wireguard host-to-host encryption is enabled. [Default: false]' type: boolean wireguardInterfaceName: description: 'WireguardInterfaceName specifies the name to use for the IPv4 Wireguard interface. [Default: wireguard.cali]' type: string wireguardInterfaceNameV6: description: 'WireguardInterfaceNameV6 specifies the name to use for the IPv6 Wireguard interface. [Default: wg-v6.cali]' type: string wireguardKeepAlive: description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive option. Set 0 to disable. [Default: 0]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by IPv4 Wireguard. [Default: 51820]' type: integer wireguardListeningPortV6: description: 'WireguardListeningPortV6 controls the listening port used by IPv6 Wireguard. [Default: 51821]' type: integer wireguardMTU: description: 'WireguardMTU controls the MTU on the IPv4 Wireguard interface. See Configuring MTU [Default: 1440]' type: integer wireguardMTUV6: description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard interface. See Configuring MTU [Default: 1420]' type: integer wireguardRoutingRulePriority: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer wireguardThreadingEnabled: description: 'WireguardThreadingEnabled controls whether Wireguard has NAPI threading enabled. [Default: false]' type: boolean workloadSourceSpoofing: description: WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP address that is not theirs. This is disabled by default. When set to "Any", pods can request any prefix. pattern: ^(?i)(Disabled|Any)?$ type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' type: boolean xdpRefreshInterval: description: 'XDPRefreshInterval is the period at which Felix re-checks all XDP state to ensure that no other process has accidentally broken Calico''s BPF maps or attached programs. Set to 0 to disable XDP refresh. [Default: 90s]' pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: globalnetworkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: GlobalNetworkPolicy listKind: GlobalNetworkPolicyList plural: globalnetworkpolicies singular: globalnetworkpolicy preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: properties: applyOnForward: description: ApplyOnForward indicates to apply the rules in this policy on forward traffic. type: boolean doNotTrack: description: DoNotTrack indicates whether packets matched by the rules in this policy should go through the data plane's connection tracking, such as Linux conntrack. If True, the rules in this policy are applied before any data plane connection tracking, and packets allowed by this policy are marked as not to be tracked. type: boolean egress: description: The ordered set of egress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array ingress: description: The ordered set of ingress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array namespaceSelector: description: NamespaceSelector is an optional field for an expression used to select a pod based on namespaces. type: string order: description: Order is an optional field that specifies the order in which the policy is applied. Policies with higher "order" are applied after those with lower order within the same tier. If the order is omitted, it may be considered to be "infinite" - i.e. the policy will be applied last. Policies with identical order will be applied in alphanumerical order based on the Policy "Name" within the tier. type: number performanceHints: description: "PerformanceHints contains a list of hints to Calico's policy engine to help process the policy more efficiently. Hints never change the enforcement behaviour of the policy. \n Currently, the only available hint is \"AssumeNeededOnEveryNode\". When that hint is set on a policy, Felix will act as if the policy matches a local endpoint even if it does not. This is useful for \"preloading\" any large static policies that are known to be used on every node. If the policy is _not_ used on a particular node then the work done to preload the policy (and to maintain it) is wasted." items: type: string type: array preDNAT: description: PreDNAT indicates to apply the rules in this policy before any DNAT. type: boolean selector: description: "The selector is an expression used to pick out the endpoints that the policy should be applied to. \n Selector expressions follow this syntax: \n \tlabel == \"string_literal\" -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" -> not equal; also matches if label is not present \tlabel in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... } \ -> true if the value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name) -> True if that label is present \t! expr -> negation of expr \texpr && expr -> Short-circuit and \texpr || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() or the empty selector -> matches all endpoints. \n Label names are allowed to contain alphanumerics, -, _ and /. String literals are more permissive but they do not support escape characters. \n Examples (with made-up labels): \n \ttype == \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != \"dev\" \t! has(label_name)" type: string serviceAccountSelector: description: ServiceAccountSelector is an optional field for an expression used to select a pod based on service accounts. type: string tier: description: The name of the tier that this policy belongs to. If this is omitted, the default tier (name is "default") is assumed. The specified tier must exist in order to create security policies within the tier, the "default" tier is created automatically if it does not exist, this means for deployments requiring only a single Tier, the tier name may be omitted on all policy management requests. type: string types: description: "Types indicates whether this policy applies to ingress, or to egress, or to both. When not explicitly specified (and so the value on creation is empty or nil), Calico defaults Types according to what Ingress and Egress rules are present in the policy. The default is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are also no Ingress rules) \n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. \n When the policy is read back again, Types will always be one of these values, never empty or nil." items: description: PolicyType enumerates the possible values of the PolicySpec Types field. type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: globalnetworksets.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: GlobalNetworkSet listKind: GlobalNetworkSetList plural: globalnetworksets singular: globalnetworkset preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs that share labels to allow rules to refer to them via selectors. The labels of GlobalNetworkSet are not namespaced. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: GlobalNetworkSetSpec contains the specification for a NetworkSet resource. properties: nets: description: The list of IP networks that belong to this set. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: hostendpoints.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: HostEndpoint listKind: HostEndpointList plural: hostendpoints singular: hostendpoint preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: HostEndpointSpec contains the specification for a HostEndpoint resource. properties: expectedIPs: description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. If \"InterfaceName\" is not present, Calico will look for an interface matching any of the IPs in the list and apply policy to that. Note: \tWhen using the selector match criteria in an ingress or egress security Policy \tor Profile, Calico converts the selector into a set of IP addresses. For host \tendpoints, the ExpectedIPs field is used for that purpose. (If only the interface \tname is specified, Calico does not learn the IPs of the interface for use in match \tcriteria.)" items: type: string type: array interfaceName: description: "Either \"*\", or the name of a specific Linux interface to apply policy to; or empty. \"*\" indicates that this HostEndpoint governs all traffic to, from or through the default network namespace of the host named by the \"Node\" field; entering and leaving that namespace via any interface, including those from/to non-host-networked local workloads. \n If InterfaceName is not \"*\", this HostEndpoint only governs traffic that enters or leaves the host through the specific interface named by InterfaceName, or - when InterfaceName is empty - through the specific interface that has one of the IPs in ExpectedIPs. Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only external interfaces (such as \"eth0\") are supported here; it isn't possible for a HostEndpoint to protect traffic through a specific local workload interface. \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; initially just pre-DNAT policy. Please check Calico documentation for the latest position." type: string node: description: The node name identifying the Calico node instance. type: string ports: description: Ports contains the endpoint's named ports, which may be referenced in security policy rules. items: properties: name: type: string port: type: integer protocol: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true required: - name - port - protocol type: object type: array profiles: description: A list of identifiers of security Profile objects that apply to this endpoint. Each profile is applied in the order that they appear in this list. Profile rules are applied after the selector-based security policy. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamblocks.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMBlock listKind: IPAMBlockList plural: ipamblocks singular: ipamblock preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAMBlockSpec contains the specification for an IPAMBlock resource. properties: affinity: description: Affinity of the block, if this block has one. If set, it will be of the form "host:". If not set, this block is not affine to a host. type: string allocations: description: Array of allocations in-use within this block. nil entries mean the allocation is free. For non-nil entries at index i, the index is the ordinal of the allocation within this block and the value is the index of the associated attributes in the Attributes array. items: type: integer # TODO: This nullable is manually added in. We should update controller-gen # to handle []*int properly itself. nullable: true type: array attributes: description: Attributes is an array of arbitrary metadata associated with allocations in the block. To find attributes for a given allocation, use the value of the allocation's entry in the Allocations array as the index of the element in this array. items: properties: handle_id: type: string secondary: additionalProperties: type: string type: object type: object type: array cidr: description: The block's CIDR. type: string deleted: description: Deleted is an internal boolean used to workaround a limitation in the Kubernetes API whereby deletion will not return a conflict error if the block has been updated. It should not be set manually. type: boolean sequenceNumber: default: 0 description: We store a sequence number that is updated each time the block is written. Each allocation will also store the sequence number of the block at the time of its creation. When releasing an IP, passing the sequence number associated with the allocation allows us to protect against a race condition and ensure the IP hasn't been released and re-allocated since the release request. format: int64 type: integer sequenceNumberForAllocation: additionalProperties: format: int64 type: integer description: Map of allocated ordinal within the block to sequence number of the block at the time of allocation. Kubernetes does not allow numerical keys for maps, so the key is cast to a string. type: object strictAffinity: description: StrictAffinity on the IPAMBlock is deprecated and no longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: description: Unallocated is an ordered list of allocations which are free in the block. items: type: integer type: array required: - allocations - attributes - cidr - strictAffinity - unallocated type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamconfigs.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMConfig listKind: IPAMConfigList plural: ipamconfigs singular: ipamconfig preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAMConfigSpec contains the specification for an IPAMConfig resource. properties: autoAllocateBlocks: type: boolean maxBlocksPerHost: description: MaxBlocksPerHost, if non-zero, is the max number of blocks that can be affine to each host. maximum: 2147483647 minimum: 0 type: integer strictAffinity: type: boolean required: - autoAllocateBlocks - strictAffinity type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ipamhandles.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPAMHandle listKind: IPAMHandleList plural: ipamhandles singular: ipamhandle preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAMHandleSpec contains the specification for an IPAMHandle resource. properties: block: additionalProperties: type: integer type: object deleted: type: boolean handleID: type: string required: - block - handleID type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ippools.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPPool listKind: IPPoolList plural: ippools singular: ippool preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPPoolSpec contains the specification for an IPPool resource. properties: allowedUses: description: AllowedUse controls what the IP pool will be used for. If not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility items: type: string type: array blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. type: string disableBGPExport: description: 'Disable exporting routes from this IP Pool''s CIDR over BGP. [Default: false]' type: boolean disabled: description: When disabled is true, Calico IPAM will not assign addresses from this pool. type: boolean ipip: description: 'Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is for internal use only.' properties: enabled: description: When enabled is true, ipip tunneling will be used to deliver packets to destinations within this pool. type: boolean mode: description: The IPIP mode. This can be one of "always" or "cross-subnet". A mode of "always" will also use IPIP tunneling for routing to destination IP addresses within this pool. A mode of "cross-subnet" will only use IPIP tunneling when the destination node is on a different subnet to the originating node. The default value (if not specified) is "always". type: string type: object ipipMode: description: Contains configuration for IPIP tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). type: string nat-outgoing: description: 'Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is for internal use only.' type: boolean natOutgoing: description: When natOutgoing is true, packets sent from Calico networked containers in this pool to destinations outside of this pool will be masqueraded. type: boolean nodeSelector: description: Allows IPPool to allocate for a specific node by label selector. type: string vxlanMode: description: Contains configuration for VXLAN tunneling for this pool. If not specified, then this is defaulted to "Never" (i.e. VXLAN tunneling is disabled). type: string required: - cidr type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (devel) creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: IPReservation listKind: IPReservationList plural: ipreservations singular: ipreservation preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPReservationSpec contains the specification for an IPReservation resource. properties: reservedCIDRs: description: ReservedCIDRs is a list of CIDRs and/or IP addresses that Calico IPAM will exclude from new allocations. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: kubecontrollersconfigurations.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: KubeControllersConfiguration listKind: KubeControllersConfigurationList plural: kubecontrollersconfigurations singular: kubecontrollersconfiguration preserveUnknownFields: false scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: KubeControllersConfigurationSpec contains the values of the Kubernetes controllers configuration. properties: controllers: description: Controllers enables and configures individual Kubernetes controllers properties: namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object node: description: Node enables and configures the node controller. Enabled by default, set to nil to disable. properties: hostEndpoint: description: HostEndpoint controls syncing nodes to host endpoints. Disabled by default, set to nil to disable. properties: autoCreate: description: 'AutoCreate enables automatic creation of host endpoints for every node. [Default: Disabled]' type: string type: object leakGracePeriod: description: 'LeakGracePeriod is the period used by the controller to determine if an IP address has been leaked. Set to 0 to disable IP garbage collection. [Default: 15m]' type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string syncLabels: description: 'SyncLabels controls whether to copy Kubernetes node labels to Calico nodes. [Default: Enabled]' type: string type: object policy: description: Policy enables and configures the policy controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object serviceAccount: description: ServiceAccount enables and configures the service account controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object workloadEndpoint: description: WorkloadEndpoint enables and configures the workload endpoint controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object type: object debugProfilePort: description: DebugProfilePort configures the port to serve memory and cpu profiles on. If not specified, profiling is disabled. format: int32 type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' type: string healthChecks: description: 'HealthChecks enables or disables support for health checks [Default: Enabled]' type: string logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string prometheusMetricsPort: description: 'PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. Set to 0 to disable. [Default: 9094]' type: integer required: - controllers type: object status: description: KubeControllersConfigurationStatus represents the status of the configuration. It's useful for admins to be able to see the actual config that was applied, which can be modified by environment variables on the kube-controllers process. properties: environmentVars: additionalProperties: type: string description: EnvironmentVars contains the environment variables on the kube-controllers that influenced the RunningConfig. type: object runningConfig: description: RunningConfig contains the effective config that is running in the kube-controllers pod, after merging the API resource with any environment variables. properties: controllers: description: Controllers enables and configures individual Kubernetes controllers properties: namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object node: description: Node enables and configures the node controller. Enabled by default, set to nil to disable. properties: hostEndpoint: description: HostEndpoint controls syncing nodes to host endpoints. Disabled by default, set to nil to disable. properties: autoCreate: description: 'AutoCreate enables automatic creation of host endpoints for every node. [Default: Disabled]' type: string type: object leakGracePeriod: description: 'LeakGracePeriod is the period used by the controller to determine if an IP address has been leaked. Set to 0 to disable IP garbage collection. [Default: 15m]' type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string syncLabels: description: 'SyncLabels controls whether to copy Kubernetes node labels to Calico nodes. [Default: Enabled]' type: string type: object policy: description: Policy enables and configures the policy controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object serviceAccount: description: ServiceAccount enables and configures the service account controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object workloadEndpoint: description: WorkloadEndpoint enables and configures the workload endpoint controller. Enabled by default, set to nil to disable. properties: reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' type: string type: object type: object debugProfilePort: description: DebugProfilePort configures the port to serve memory and cpu profiles on. If not specified, profiling is disabled. format: int32 type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' type: string healthChecks: description: 'HealthChecks enables or disables support for health checks [Default: Enabled]' type: string logSeverityScreen: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: Info]' type: string prometheusMetricsPort: description: 'PrometheusMetricsPort is the TCP port that the Prometheus metrics server should bind to. Set to 0 to disable. [Default: 9094]' type: integer required: - controllers type: object type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: networkpolicies.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: NetworkPolicy listKind: NetworkPolicyList plural: networkpolicies singular: networkpolicy preserveUnknownFields: false scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: properties: egress: description: The ordered set of egress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array ingress: description: The ordered set of ingress rules. Each rule contains a set of packet match criteria and a corresponding action to apply. items: description: "A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy and security Profiles reference rules - separated out as a list of rules for both ingress and egress packet matching. \n Each positive match criteria has a negated version, prefixed with \"Not\". All the match criteria within a rule must be satisfied for a packet to match. A single rule can contain the positive and negative version of a match and both must be satisfied for the rule to match." properties: action: type: string destination: description: Destination contains the match criteria that apply to destination entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object http: description: HTTP contains match criteria that apply to HTTP requests. properties: methods: description: Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods are OR'd together. items: type: string type: array paths: description: 'Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed HTTP Paths. Multiple paths are OR''d together. e.g: - exact: /foo - prefix: /bar NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it.' items: description: 'HTTPPath specifies an HTTP path to match. It may be either of the form: exact: : which matches the path exactly or prefix: : which matches the path prefix' properties: exact: type: string prefix: type: string type: object type: array type: object icmp: description: ICMP is an optional field that restricts the rule to apply to a specific type and code of ICMP traffic. This should only be specified if the Protocol field is set to "ICMP" or "ICMPv6". properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object ipVersion: description: IPVersion is an optional field that restricts the rule to only match a specific IP version. type: integer metadata: description: Metadata contains additional information for this rule properties: annotations: additionalProperties: type: string description: Annotations is a set of key value pairs that give extra information about the rule type: object type: object notICMP: description: NotICMP is the negated version of the ICMP field. properties: code: description: Match on a specific ICMP code. If specified, the Type value must also be specified. This is a technical limitation imposed by the kernel's iptables firewall, which Calico uses to enforce the rule. type: integer type: description: Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request (i.e. pings). type: integer type: object notProtocol: anyOf: - type: integer - type: string description: NotProtocol is the negated version of the Protocol field. pattern: ^.* x-kubernetes-int-or-string: true protocol: anyOf: - type: integer - type: string description: "Protocol is an optional field that restricts the rule to only apply to traffic of a specific IP protocol. Required if any of the EntityRules contain Ports (because ports only apply to certain protocols). \n Must be one of these string values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\" or an integer in the range 1-255." pattern: ^.* x-kubernetes-int-or-string: true source: description: Source contains the match criteria that apply to source entity. properties: namespaceSelector: description: "NamespaceSelector is an optional field that contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector and another selector are defined on the same rule, then only workload endpoints that are matched by both selectors will be selected by the rule. \n For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting only workload endpoints in the same namespace as the NetworkPolicy. \n For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting only GlobalNetworkSet or HostEndpoint. \n For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload endpoints across all namespaces." type: string nets: description: Nets is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) IP addresses in any of the given subnets. items: type: string type: array notNets: description: NotNets is the negated version of the Nets field. items: type: string type: array notPorts: description: NotPorts is the negated version of the Ports field. Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to "TCP" or "UDP". items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array notSelector: description: NotSelector is the negated version of the Selector field. See Selector field for subtleties with negated selectors. type: string ports: description: "Ports is an optional field that restricts the rule to only apply to traffic that has a source (destination) port that matches one of these ranges/values. This value is a list of integers or strings that represent ranges of ports. \n Since only some protocols have ports, if any ports are specified it requires the Protocol match in the Rule to be set to \"TCP\" or \"UDP\"." items: anyOf: - type: integer - type: string pattern: ^.* x-kubernetes-int-or-string: true type: array selector: description: "Selector is an optional field that contains a selector expression (see Policy for sample syntax). \ Only traffic that originates from (terminates at) endpoints matching the selector will be matched. \n Note that: in addition to the negated version of the Selector (see NotSelector below), the selector expression syntax itself supports negation. The two types of negation are subtly different. One negates the set of matched endpoints, the other negates the whole match: \n \tSelector = \"!has(my_label)\" matches packets that are from other Calico-controlled \tendpoints that do not have the label \"my_label\". \n \tNotSelector = \"has(my_label)\" matches packets that are not from Calico-controlled \tendpoints that do have the label \"my_label\". \n The effect is that the latter will accept packets from non-Calico sources whereas the former is limited to packets from Calico-controlled endpoints." type: string serviceAccounts: description: ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a matching service account. properties: names: description: Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account whose name is in the list. items: type: string type: array selector: description: Selector is an optional field that restricts the rule to only apply to traffic that originates from (or terminates at) a pod running as a service account that matches the given label selector. If both Names and Selector are specified then they are AND'ed. type: string type: object services: description: "Services is an optional field that contains options for matching Kubernetes Services. If specified, only traffic that originates from or terminates at endpoints within the selected service(s) will be matched, and only to/from each endpoint's port. \n Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, NotNets or ServiceAccounts. \n Ports and NotPorts can only be specified with Services on ingress rules." properties: name: description: Name specifies the name of a Kubernetes Service to match. type: string namespace: description: Namespace specifies the namespace of the given Service. If left empty, the rule will match within this policy's namespace. type: string type: object type: object required: - action type: object type: array order: description: Order is an optional field that specifies the order in which the policy is applied. Policies with higher "order" are applied after those with lower order within the same tier. If the order is omitted, it may be considered to be "infinite" - i.e. the policy will be applied last. Policies with identical order will be applied in alphanumerical order based on the Policy "Name" within the tier. type: number performanceHints: description: "PerformanceHints contains a list of hints to Calico's policy engine to help process the policy more efficiently. Hints never change the enforcement behaviour of the policy. \n Currently, the only available hint is \"AssumeNeededOnEveryNode\". When that hint is set on a policy, Felix will act as if the policy matches a local endpoint even if it does not. This is useful for \"preloading\" any large static policies that are known to be used on every node. If the policy is _not_ used on a particular node then the work done to preload the policy (and to maintain it) is wasted." items: type: string type: array selector: description: "The selector is an expression used to pick out the endpoints that the policy should be applied to. \n Selector expressions follow this syntax: \n \tlabel == \"string_literal\" -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" -> not equal; also matches if label is not present \tlabel in { \"a\", \"b\", \"c\", ... } -> true if the value of label X is one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... } \ -> true if the value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name) -> True if that label is present \t! expr -> negation of expr \texpr && expr -> Short-circuit and \texpr || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() or the empty selector -> matches all endpoints. \n Label names are allowed to contain alphanumerics, -, _ and /. String literals are more permissive but they do not support escape characters. \n Examples (with made-up labels): \n \ttype == \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != \"dev\" \t! has(label_name)" type: string serviceAccountSelector: description: ServiceAccountSelector is an optional field for an expression used to select a pod based on service accounts. type: string tier: description: The name of the tier that this policy belongs to. If this is omitted, the default tier (name is "default") is assumed. The specified tier must exist in order to create security policies within the tier, the "default" tier is created automatically if it does not exist, this means for deployments requiring only a single Tier, the tier name may be omitted on all policy management requests. type: string types: description: "Types indicates whether this policy applies to ingress, or to egress, or to both. When not explicitly specified (and so the value on creation is empty or nil), Calico defaults Types according to what Ingress and Egress are present in the policy. The default is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are also no Ingress rules) \n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. \n When the policy is read back again, Types will always be one of these values, never empty or nil." items: description: PolicyType enumerates the possible values of the PolicySpec Types field. type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: networksets.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: NetworkSet listKind: NetworkSetList plural: networksets singular: networkset preserveUnknownFields: false scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: NetworkSetSpec contains the specification for a NetworkSet resource. properties: nets: description: The list of IP networks that belong to this set. items: type: string type: array type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: (devel) creationTimestamp: null name: tiers.crd.projectcalico.org spec: group: crd.projectcalico.org names: kind: Tier listKind: TierList plural: tiers singular: tier scope: Cluster versions: - name: v1 schema: openAPIV3Schema: properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: TierSpec contains the specification for a security policy tier resource. properties: defaultAction: description: 'DefaultAction specifies the action applied to workloads selected by a policy in the tier, but not rule matched the workload''s traffic. [Default: Deny]' enum: - Pass - Deny type: string order: description: Order is an optional field that specifies the order in which the tier is applied. Tiers with higher "order" are applied after those with lower order. If the order is omitted, it may be considered to be "infinite" - i.e. the tier will be applied last. Tiers with identical order will be applied in alphanumerical order based on the Tier "Name". type: number type: object type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 policy.networking.k8s.io/bundle-version: v0.1.1 policy.networking.k8s.io/channel: experimental creationTimestamp: null name: adminnetworkpolicies.policy.networking.k8s.io spec: group: policy.networking.k8s.io names: kind: AdminNetworkPolicy listKind: AdminNetworkPolicyList plural: adminnetworkpolicies shortNames: - anp singular: adminnetworkpolicy scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .spec.priority name: Priority type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1alpha1 schema: openAPIV3Schema: description: |- AdminNetworkPolicy is a cluster level resource that is part of the AdminNetworkPolicy API. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: Specification of the desired behavior of AdminNetworkPolicy. properties: egress: description: |- Egress is the list of Egress rules to be applied to the selected pods. A total of 100 rules will be allowed in each ANP instance. The relative precedence of egress rules within a single ANP object (all of which share the priority) will be determined by the order in which the rule is written. Thus, a rule that appears at the top of the egress rules would take the highest precedence. ANPs with no egress rules do not affect egress traffic. Support: Core items: description: |- AdminNetworkPolicyEgressRule describes an action to take on a particular set of traffic originating from pods selected by a AdminNetworkPolicy's Subject field. properties: action: description: |- Action specifies the effect this rule will have on matching traffic. Currently the following actions are supported: Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) Deny: denies the selected traffic Pass: instructs the selected traffic to skip any remaining ANP rules, and then pass execution to any NetworkPolicies that select the pod. If the pod is not selected by any NetworkPolicies then execution is passed to any BaselineAdminNetworkPolicies that select the pod. Support: Core enum: - Allow - Deny - Pass type: string name: description: |- Name is an identifier for this rule, that may be no more than 100 characters in length. This field should be used by the implementation to help improve observability, readability and error-reporting for any applied AdminNetworkPolicies. Support: Core maxLength: 100 type: string ports: description: |- Ports allows for matching traffic based on port and protocols. This field is a list of destination ports for the outgoing egress traffic. If Ports is not set then the rule does not filter traffic via port. Support: Core items: description: |- AdminNetworkPolicyPort describes how to select network ports on pod(s). Exactly one field must be set. maxProperties: 1 minProperties: 1 properties: namedPort: description: |- NamedPort selects a port on a pod(s) based on name. Support: Extended type: string portNumber: description: |- Port selects a port on a pod(s) based on number. Support: Core properties: port: description: |- Number defines a network port value. Support: Core format: int32 maximum: 65535 minimum: 1 type: integer protocol: default: TCP description: |- Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP. Support: Core type: string required: - port - protocol type: object portRange: description: |- PortRange selects a port range on a pod(s) based on provided start and end values. Support: Core properties: end: description: |- End defines a network port that is the end of a port range, the End value must be greater than Start. Support: Core format: int32 maximum: 65535 minimum: 1 type: integer protocol: default: TCP description: |- Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP. Support: Core type: string start: description: |- Start defines a network port that is the start of a port range, the Start value must be less than End. Support: Core format: int32 maximum: 65535 minimum: 1 type: integer required: - end - start type: object type: object maxItems: 100 type: array to: description: |- To is the List of destinations whose traffic this rule applies to. If any AdminNetworkPolicyEgressPeer matches the destination of outgoing traffic then the specified action is applied. This field must be defined and contain at least one item. Support: Core items: description: |- AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. Exactly one of the selector pointers must be set for a given peer. If a consumer observes none of its fields are set, they must assume an unknown option has been specified and fail closed. maxProperties: 1 minProperties: 1 properties: namespaces: description: |- Namespaces defines a way to select all pods within a set of Namespaces. Note that host-networked pods are not included in this type of peer. Support: Core properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic networks: description: |- Networks defines a way to select peers via CIDR blocks. This is intended for representing entities that live outside the cluster, which can't be selected by pods, namespaces and nodes peers, but note that cluster-internal traffic will be checked against the rule as well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow or deny all IPv4 pod-to-pod traffic as well. If you don't want that, add a rule that Passes all pod traffic before the Networks rule. Each item in Networks should be provided in the CIDR format and should be IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". Networks can have upto 25 CIDRs specified. Support: Extended items: description: |- CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). This string must be validated by implementations using net.ParseCIDR TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. maxLength: 43 type: string x-kubernetes-validations: - message: CIDR must be either an IPv4 or IPv6 address. IPv4 address embedded in IPv6 addresses are not supported rule: self.contains(':') != self.contains('.') maxItems: 25 minItems: 1 type: array x-kubernetes-list-type: set nodes: description: |- Nodes defines a way to select a set of nodes in the cluster. This field follows standard label selector semantics; if present but empty, it selects all Nodes. Support: Extended properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in a set of namespaces. Note that host-networked pods are not included in this type of peer. Support: Core properties: namespaceSelector: description: |- NamespaceSelector follows standard label selector semantics; if empty, it selects all Namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic podSelector: description: |- PodSelector is used to explicitly select pods within a namespace; if empty, it selects all Pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic required: - namespaceSelector - podSelector type: object type: object maxItems: 100 minItems: 1 type: array required: - action - to type: object x-kubernetes-validations: - message: networks/nodes peer cannot be set with namedPorts since there are no namedPorts for networks/nodes rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' maxItems: 100 type: array ingress: description: |- Ingress is the list of Ingress rules to be applied to the selected pods. A total of 100 rules will be allowed in each ANP instance. The relative precedence of ingress rules within a single ANP object (all of which share the priority) will be determined by the order in which the rule is written. Thus, a rule that appears at the top of the ingress rules would take the highest precedence. ANPs with no ingress rules do not affect ingress traffic. Support: Core items: description: |- AdminNetworkPolicyIngressRule describes an action to take on a particular set of traffic destined for pods selected by an AdminNetworkPolicy's Subject field. properties: action: description: |- Action specifies the effect this rule will have on matching traffic. Currently the following actions are supported: Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) Deny: denies the selected traffic Pass: instructs the selected traffic to skip any remaining ANP rules, and then pass execution to any NetworkPolicies that select the pod. If the pod is not selected by any NetworkPolicies then execution is passed to any BaselineAdminNetworkPolicies that select the pod. Support: Core enum: - Allow - Deny - Pass type: string from: description: |- From is the list of sources whose traffic this rule applies to. If any AdminNetworkPolicyIngressPeer matches the source of incoming traffic then the specified action is applied. This field must be defined and contain at least one item. Support: Core items: description: |- AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. Exactly one of the selector pointers must be set for a given peer. If a consumer observes none of its fields are set, they must assume an unknown option has been specified and fail closed. maxProperties: 1 minProperties: 1 properties: namespaces: description: |- Namespaces defines a way to select all pods within a set of Namespaces. Note that host-networked pods are not included in this type of peer. Support: Core properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic pods: description: |- Pods defines a way to select a set of pods in a set of namespaces. Note that host-networked pods are not included in this type of peer. Support: Core properties: namespaceSelector: description: |- NamespaceSelector follows standard label selector semantics; if empty, it selects all Namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic podSelector: description: |- PodSelector is used to explicitly select pods within a namespace; if empty, it selects all Pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic required: - namespaceSelector - podSelector type: object type: object maxItems: 100 minItems: 1 type: array name: description: |- Name is an identifier for this rule, that may be no more than 100 characters in length. This field should be used by the implementation to help improve observability, readability and error-reporting for any applied AdminNetworkPolicies. Support: Core maxLength: 100 type: string ports: description: |- Ports allows for matching traffic based on port and protocols. This field is a list of ports which should be matched on the pods selected for this policy i.e the subject of the policy. So it matches on the destination port for the ingress traffic. If Ports is not set then the rule does not filter traffic via port. Support: Core items: description: |- AdminNetworkPolicyPort describes how to select network ports on pod(s). Exactly one field must be set. maxProperties: 1 minProperties: 1 properties: namedPort: description: |- NamedPort selects a port on a pod(s) based on name. Support: Extended type: string portNumber: description: |- Port selects a port on a pod(s) based on number. Support: Core properties: port: description: |- Number defines a network port value. Support: Core format: int32 maximum: 65535 minimum: 1 type: integer protocol: default: TCP description: |- Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP. Support: Core type: string required: - port - protocol type: object portRange: description: |- PortRange selects a port range on a pod(s) based on provided start and end values. Support: Core properties: end: description: |- End defines a network port that is the end of a port range, the End value must be greater than Start. Support: Core format: int32 maximum: 65535 minimum: 1 type: integer protocol: default: TCP description: |- Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP. Support: Core type: string start: description: |- Start defines a network port that is the start of a port range, the Start value must be less than End. Support: Core format: int32 maximum: 65535 minimum: 1 type: integer required: - end - start type: object type: object maxItems: 100 type: array required: - action - from type: object maxItems: 100 type: array priority: description: |- Priority is a value from 0 to 1000. Rules with lower priority values have higher precedence, and are checked before rules with higher priority values. All AdminNetworkPolicy rules have higher precedence than NetworkPolicy or BaselineAdminNetworkPolicy rules The behavior is undefined if two ANP objects have same priority. Support: Core format: int32 maximum: 1000 minimum: 0 type: integer subject: description: |- Subject defines the pods to which this AdminNetworkPolicy applies. Note that host-networked pods are not included in subject selection. Support: Core maxProperties: 1 minProperties: 1 properties: namespaces: description: Namespaces is used to select pods via namespace selectors. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic pods: description: Pods is used to select pods via namespace AND pod selectors. properties: namespaceSelector: description: |- NamespaceSelector follows standard label selector semantics; if empty, it selects all Namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic podSelector: description: |- PodSelector is used to explicitly select pods within a namespace; if empty, it selects all Pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic required: - namespaceSelector - podSelector type: object type: object required: - priority - subject type: object status: description: Status is the status to be reported by the implementation. properties: conditions: items: description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t \ // other fields\n\t}" properties: lastTransitionTime: description: |- lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: description: |- message is a human readable message indicating details about the transition. This may be an empty string. maxLength: 32768 type: string observedGeneration: description: |- observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: description: |- reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ type: string status: description: status of the condition, one of True, False, Unknown. enum: - "True" - "False" - Unknown type: string type: description: |- type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: - lastTransitionTime - message - reason - status - type type: object type: array x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map required: - conditions type: object required: - metadata - spec type: object served: true storage: true subresources: status: {} status: acceptedNames: kind: "" plural: "" conditions: null storedVersions: null --- # Source: calico/templates/calico-kube-controllers-rbac.yaml # Include a clusterrole for the kube-controllers component, # and bind it to the calico-kube-controllers serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers rules: # Nodes are watched to monitor for deletions. - apiGroups: [""] resources: - nodes verbs: - watch - list - get # Pods are watched to check for existence as part of IPAM controller. - apiGroups: [""] resources: - pods verbs: - get - list - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: - ipreservations verbs: - list - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles - tiers verbs: - get - list - create - update - delete - watch # Pools are watched to maintain a mapping of blocks to IP pools. - apiGroups: ["crd.projectcalico.org"] resources: - ippools verbs: - list - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: - hostendpoints verbs: - get - list - create - update - delete # Needs access to update clusterinformations. - apiGroups: ["crd.projectcalico.org"] resources: - clusterinformations verbs: - get - list - create - update - watch # KubeControllersConfiguration is where it gets its config - apiGroups: ["crd.projectcalico.org"] resources: - kubecontrollersconfigurations verbs: # read its own config - get - list # create a default if none exists - create # update status - update # watch for changes - watch --- # Source: calico/templates/calico-node-rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node rules: # Used for creating service account tokens to be used by the CNI plugin - apiGroups: [""] resources: - serviceaccounts/token resourceNames: - calico-cni-plugin verbs: - create # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get # EndpointSlices are used for Service-based network policy rule # enforcement. - apiGroups: ["discovery.k8s.io"] resources: - endpointslices verbs: - watch - list - apiGroups: [""] resources: - endpoints - services verbs: # Used to discover service IPs for advertisement. - watch - list # Used to discover Typhas. - get # Pod CIDR auto-detection on kubeadm needs access to config maps. - apiGroups: [""] resources: - configmaps verbs: - get - apiGroups: [""] resources: - nodes/status verbs: # Needed for clearing NodeNetworkUnavailable flag. - patch # Calico stores some configuration information in node annotations. - update # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list # Watch for changes to Kubernetes AdminNetworkPolicies. - apiGroups: ["policy.networking.k8s.io"] resources: - adminnetworkpolicies verbs: - watch - list # Used by Calico for policy information. - apiGroups: [""] resources: - pods - namespaces - serviceaccounts verbs: - list - watch # The CNI plugin patches pods/status. - apiGroups: [""] resources: - pods/status verbs: - patch # Calico monitors various CRDs for config. - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs - felixconfigurations - bgppeers - bgpfilters - globalbgpconfigs - bgpconfigurations - ippools - ipreservations - ipamblocks - globalnetworkpolicies - globalnetworksets - networkpolicies - networksets - clusterinformations - hostendpoints - blockaffinities - caliconodestatuses - tiers verbs: - get - list - watch # Calico creates some tiers on startup. - apiGroups: ["crd.projectcalico.org"] resources: - tiers verbs: - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: - ippools - felixconfigurations - clusterinformations verbs: - create - update # Calico must update some CRDs. - apiGroups: ["crd.projectcalico.org"] resources: - caliconodestatuses verbs: - update # Calico stores some configuration information on the node. - apiGroups: [""] resources: - nodes verbs: - get - list - watch # These permissions are only required for upgrade from v2.6, and can # be removed after upgrade or on fresh installations. - apiGroups: ["crd.projectcalico.org"] resources: - bgpconfigurations - bgppeers verbs: - create - update # These permissions are required for Calico CNI to perform IPAM allocations. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles verbs: - get - list - create - update - delete # The CNI plugin and calico/node need to be able to create a default # IPAMConfiguration - apiGroups: ["crd.projectcalico.org"] resources: - ipamconfigs verbs: - get - create # Block affinities must also be watchable by confd for route aggregation. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities verbs: - watch # The Calico IPAM migration needs to get daemonsets. These permissions can be # removed if not upgrading from an installation using host-local IPAM. - apiGroups: ["apps"] resources: - daemonsets verbs: - get --- # Source: calico/templates/calico-node-rbac.yaml # CNI cluster role kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-cni-plugin rules: - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get - apiGroups: [""] resources: - pods/status verbs: - patch - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles - clusterinformations - ippools - ipreservations - ipamconfigs verbs: - get - list - create - update - delete --- # Source: calico/templates/calico-kube-controllers-rbac.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-kube-controllers subjects: - kind: ServiceAccount name: calico-kube-controllers namespace: kube-system --- # Source: calico/templates/calico-node-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-node subjects: - kind: ServiceAccount name: calico-node namespace: kube-system --- # Source: calico/templates/calico-node-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-cni-plugin roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-cni-plugin subjects: - kind: ServiceAccount name: calico-cni-plugin namespace: kube-system --- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well # as the CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: apps/v1 metadata: name: calico-node namespace: kube-system labels: k8s-app: calico-node spec: selector: matchLabels: k8s-app: calico-node updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 template: metadata: labels: k8s-app: calico-node spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true tolerations: # Make sure calico-node gets scheduled on all nodes. - effect: NoSchedule operator: Exists # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists serviceAccountName: calico-node securityContext: seccompProfile: type: RuntimeDefault # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 priorityClassName: system-node-critical initContainers: # This container performs upgrade from host-local IPAM to calico-ipam. # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam image: docker.io/calico/cni:v3.29.3 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend volumeMounts: - mountPath: /var/lib/cni/networks name: host-local-net-dir - mountPath: /host/opt/cni/bin name: cni-bin-dir securityContext: privileged: true # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni image: docker.io/calico/cni:v3.29.3 imagePullPolicy: IfNotPresent command: ["/opt/cni/bin/install"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-calico.conflist" # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: calico-config key: cni_network_config # Set the hostname based on the k8s node name. - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName # CNI MTU Config variable - name: CNI_MTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Prevents the container from sleeping forever. - name: SLEEP value: "false" - name: CNI_NET_DIR value: "/var/snap/microk8s/current/args/cni-network" volumeMounts: - mountPath: /host/opt/cni/bin name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir securityContext: privileged: true # # This init container mounts the necessary filesystems needed by the BPF data plane # # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. # - name: "mount-bpffs" # image: docker.io/calico/node:v3.29.3 # imagePullPolicy: IfNotPresent # command: ["calico-node", "-init", "-best-effort"] # volumeMounts: # - mountPath: /sys/fs # name: sys-fs # # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host # # so that it outlives the init container. # mountPropagation: Bidirectional # - mountPath: /var/run/calico # name: var-run-calico # # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host # # so that it outlives the init container. # mountPropagation: Bidirectional # # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, # # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. # - mountPath: /nodeproc # name: nodeproc # readOnly: true # securityContext: # privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: docker.io/calico/node:v3.29.3 imagePullPolicy: IfNotPresent envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" # Set based on the k8s node name. - name: NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,bgp" # Auto-detect the BGP IP address. - name: IP value: "autodetect" - name: IP_AUTODETECTION_METHOD value: "first-found" # Enable IPIP # - name: CALICO_IPV4POOL_IPIP # value: "Always" # Enable or Disable VXLAN on the default IP pool. - name: CALICO_IPV4POOL_VXLAN value: "Always" # Enable or Disable VXLAN on the default IPv6 IP pool. - name: CALICO_IPV6POOL_VXLAN value: "Never" # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Set MTU for the VXLAN tunnel device. - name: FELIX_VXLANMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # Set MTU for the Wireguard tunnel device. - name: FELIX_WIREGUARDMTU valueFrom: configMapKeyRef: name: calico-config key: veth_mtu # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. - name: CALICO_IPV4POOL_CIDR value: "10.1.0.0/16" # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "ACCEPT" # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - name: FELIX_HEALTHENABLED value: "true" - name: FELIX_FEATUREDETECTOVERRIDE value: ChecksumOffloadBroken=true securityContext: privileged: true resources: requests: cpu: 250m lifecycle: preStop: exec: command: - /bin/calico-node - -shutdown livenessProbe: exec: command: - /bin/calico-node - -felix-live # - -bird-live periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 timeoutSeconds: 10 readinessProbe: exec: command: - /bin/calico-node - -felix-ready # - -bird-ready periodSeconds: 10 timeoutSeconds: 10 volumeMounts: # For maintaining CNI plugin API credentials. - mountPath: /host/etc/cni/net.d name: cni-net-dir readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /run/xtables.lock name: xtables-lock readOnly: false - mountPath: /var/run/calico name: var-run-calico readOnly: false - mountPath: /var/lib/calico name: var-lib-calico readOnly: false - name: policysync mountPath: /var/run/nodeagent # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the # parent directory. # - name: bpffs # mountPath: /sys/fs/bpf - name: cni-log-dir mountPath: /var/log/calico/cni readOnly: true volumes: # Used by calico-node. - name: lib-modules hostPath: path: /lib/modules - name: var-run-calico hostPath: path: /var/snap/microk8s/current/var/run/calico - name: var-lib-calico hostPath: path: /var/snap/microk8s/current/var/lib/calico - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate - name: sys-fs hostPath: path: /sys/fs/ type: DirectoryOrCreate # - name: bpffs # hostPath: # path: /sys/fs/bpf # type: Directory # # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. # - name: nodeproc # hostPath: # path: /proc # Used to install CNI. - name: cni-bin-dir hostPath: path: /var/snap/microk8s/current/opt/cni/bin type: DirectoryOrCreate - name: cni-net-dir hostPath: path: /var/snap/microk8s/current/args/cni-network # Used to access CNI logs. - name: cni-log-dir hostPath: path: /var/snap/microk8s/common/var/log/calico/cni # Mount in the directory for host-local IPAM allocations. This is # used when upgrading from host-local to calico-ipam, and can be removed # if not using the upgrade-ipam init container. - name: host-local-net-dir hostPath: path: /var/snap/microk8s/current/var/lib/cni/networks # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: type: DirectoryOrCreate path: /var/snap/microk8s/current/var/run/nodeagent --- # Source: calico/templates/calico-kube-controllers.yaml # See https://github.com/projectcalico/kube-controllers apiVersion: apps/v1 kind: Deployment metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: # The controllers can only have a single active instance. replicas: 1 selector: matchLabels: k8s-app: calico-kube-controllers strategy: type: Recreate template: metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: nodeSelector: kubernetes.io/os: linux tolerations: # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - key: node-role.kubernetes.io/master effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule serviceAccountName: calico-kube-controllers securityContext: seccompProfile: type: RuntimeDefault priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers image: docker.io/calico/kube-controllers:v3.29.3 imagePullPolicy: IfNotPresent env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes livenessProbe: exec: command: - /usr/bin/check-status - -l periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 timeoutSeconds: 10 readinessProbe: exec: command: - /usr/bin/check-status - -r periodSeconds: 10 securityContext: runAsNonRoot: true ================================================ FILE: upgrade-scripts/000-switch-to-calico/rollback-master.sh ================================================ #!/bin/bash set -ex echo "Rolling back calico upgrade on master" source $SNAP/actions/common/utils.sh if [ -e "$SNAP_DATA/args/cni-network/cni.yaml" ]; then KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" $KUBECTL delete -f "$SNAP_DATA/args/cni-network/cni.yaml" fi BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" if [ -e "$BACKUP_DIR/args/cni-network/flannel.conflist" ]; then find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; cp -rf "$BACKUP_DIR"/args/cni-network/* "$SNAP_DATA/args/cni-network/" fi echo "Restarting kubelet" if [ -e "$BACKUP_DIR/args/kubelet" ]; then cp "$BACKUP_DIR"/args/kubelet "$SNAP_DATA/args/" fi echo "Restarting kube-proxy" if [ -e "$BACKUP_DIR/args/kube-proxy" ]; then cp "$BACKUP_DIR"/args/kube-proxy "$SNAP_DATA/args/" fi echo "Restarting kube-apiserver" if [ -e "$BACKUP_DIR/args/kube-apiserver" ]; then cp "$BACKUP_DIR"/args/kube-apiserver "$SNAP_DATA/args/" fi snapctl restart ${SNAP_NAME}.daemon-kubelite ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 echo "Restarting flannel" set_service_expected_to_start flanneld remove_vxlan_interfaces snapctl start ${SNAP_NAME}.daemon-flanneld echo "Restarting kubelet" if grep -qE "bin_dir.*SNAP_DATA}\/" $SNAP_DATA/args/containerd-template.toml; then echo "Restarting containerd" "${SNAP}/bin/sed" -i 's;bin_dir = "${SNAP_DATA}/opt;bin_dir = "${SNAP}/opt;g' "$SNAP_DATA/args/containerd-template.toml" snapctl restart ${SNAP_NAME}.daemon-containerd fi echo "Calico rolledback" ================================================ FILE: upgrade-scripts/000-switch-to-calico/rollback-node.sh ================================================ #!/bin/bash set -ex echo "Rolling back calico upgrade on a node" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" if [ -e "$BACKUP_DIR/args/cni-network/flannel.conflist" ]; then find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; cp -rf "$BACKUP_DIR"/args/cni-network/* "$SNAP_DATA/args/cni-network/" fi echo "Restarting kubelet" if [ -e "$BACKUP_DIR/args/kubelet" ]; then cp "$BACKUP_DIR"/args/kubelet "$SNAP_DATA/args/" fi echo "Restarting kube-proxy" if [ -e "$BACKUP_DIR/args/kube-proxy" ]; then cp "$BACKUP_DIR"/args/kube-proxy "$SNAP_DATA/args/" fi echo "Restarting kube-apiserver" if [ -e "$BACKUP_DIR/args/kube-apiserver" ]; then cp "$BACKUP_DIR"/args/kube-apiserver "$SNAP_DATA/args/" fi snapctl restart ${SNAP_NAME}.daemon-kubelite echo "Restarting flannel" set_service_expected_to_start flanneld remove_vxlan_interfaces snapctl start ${SNAP_NAME}.daemon-flanneld echo "Restarting kubelet" if grep -qE "bin_dir.*SNAP_DATA}\/" $SNAP_DATA/args/containerd-template.toml; then echo "Restarting containerd" "${SNAP}/bin/sed" -i 's;bin_dir = "${SNAP_DATA}/opt;bin_dir = "${SNAP}/opt;g' "$SNAP_DATA/args/containerd-template.toml" snapctl restart ${SNAP_NAME}.daemon-containerd fi echo "Calico rolledback" ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/commit-master.sh ================================================ #!/bin/bash set -ex echo "Switching master to dqlite" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" DB_DIR="$BACKUP_DIR/db" mkdir -p "$BACKUP_DIR/args/" echo "Configuring services" ${SNAP}/microk8s-stop.wrapper cp "$SNAP_DATA"/args/kube-apiserver "$BACKUP_DIR/args" # Configure the API sever to talk to the external dqlite if [ -e ${SNAP}/default-args/k8s-dqlite ] && ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] then echo "Reconfiguring the API server for dqlite" cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite fi skip_opt_in_config storage-backend kube-apiserver skip_opt_in_config storage-dir kube-apiserver skip_opt_in_config "etcd-servers" kube-apiserver skip_opt_in_config "etcd-cafile" kube-apiserver skip_opt_in_config "etcd-certfile" kube-apiserver skip_opt_in_config "etcd-keyfile" kube-apiserver refresh_opt_in_config etcd-servers unix://\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver cp "$SNAP_DATA"/args/etcd "$BACKUP_DIR/args" cat < "$SNAP_DATA"/args/etcd --data-dir=\${SNAP_COMMON}/var/run/etcd --advertise-client-urls=http://127.0.0.1:12379 --listen-client-urls=http://0.0.0.0:12379 --enable-v2=true EOT if ! [ -e "${SNAP_DATA}/var/kubernetes/backend/cluster.key" ] then init_cluster fi snapctl restart ${SNAP_NAME}.daemon-k8s-dqlite snapctl restart ${SNAP_NAME}.daemon-kubelite run_etcd="$(is_service_expected_to_start etcd)" if [ "${run_etcd}" == "1" ] then snapctl start microk8s.daemon-etcd # TODO do some proper wait here sleep 15 rm -rf "$DB_DIR" $SNAP/bin/k8s-dqlite migrator --mode backup --endpoint "http://127.0.0.1:12379" --db-dir "$DB_DIR" --debug chmod 600 "$DB_DIR" # Wait up to two minutes for the apiserver to come up. # TODO: this polling is not good enough. We should find a new way to ensure the apiserver is up. timeout="120" start_timer="$(date +%s)" while ! (is_apiserver_ready) do sleep 5 now="$(date +%s)" if [[ "$now" > "$(($start_timer + $timeout))" ]] ; then break fi done # if the API server came up try to load the CNI manifest now="$(date +%s)" if [[ "$now" < "$(($start_timer + $timeout))" ]] ; then if (is_apiserver_ready) then $SNAP/bin/k8s-dqlite migrator --mode restore --endpoint "unix://${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379" --db-dir "$DB_DIR" --debug fi fi sleep 10 set_service_not_expected_to_start etcd snapctl stop microk8s.daemon-etcd set_service_expected_to_start k8s-dqlite fi rm -rf ${SNAP_DATA}/var/lock/cni-loaded ${SNAP}/microk8s-start.wrapper ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 120 echo "Dqlite is enabled" ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/commit-node.sh ================================================ #!/bin/bash set -ex echo "Switching node to dqlite" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" mkdir -p "$BACKUP_DIR/args/" echo "Configuring services" cp "$SNAP_DATA"/args/kube-apiserver "$BACKUP_DIR/args" if [ -e ${SNAP}/default-args/k8s-dqlite ] then echo "Configuring the API server for dqlite" cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite refresh_opt_in_config etcd-servers unix://\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver skip_opt_in_config storage-backend kube-apiserver skip_opt_in_config storage-dir kube-apiserver else refresh_opt_in_config "storage-backend" "dqlite" kube-apiserver refresh_opt_in_config "storage-dir" "\${SNAP_DATA}/var/kubernetes/backend/" kube-apiserver skip_opt_in_config "etcd-servers" kube-apiserver skip_opt_in_config "etcd-cafile" kube-apiserver skip_opt_in_config "etcd-certfile" kube-apiserver skip_opt_in_config "etcd-keyfile" kube-apiserver fi if ! [ -e "${SNAP_DATA}/var/kubernetes/backend/cluster.key" ] then init_cluster fi set_service_not_expected_to_start etcd set_service_expected_to_start k8s-dqlite ${SNAP}/microk8s-stop.wrapper sleep 5 ${SNAP}/microk8s-start.wrapper echo "Dqlite is enabled on the node" ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/description.txt ================================================ Migrates from etcd to dqlite ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/prepare-master.sh ================================================ #!/usr/bin/env bash echo "Master ready for dqlite" ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/prepare-node.sh ================================================ #!/bin/bash echo "Nothing to do to praparing node for dqlite" ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/rollback-master.sh ================================================ #!/bin/bash set -ex echo "Rolling back dqlite upgrade on master" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" echo "Restarting etcd" set_service_expected_to_start etcd if [ -e "$BACKUP_DIR/args/etcd" ]; then cp "$BACKUP_DIR"/args/etcd "$SNAP_DATA/args/" snapctl restart ${SNAP_NAME}.daemon-etcd fi echo "Restarting kube-apiserver" if [ -e "$BACKUP_DIR/args/kube-apiserver" ]; then cp "$BACKUP_DIR"/args/kube-apiserver "$SNAP_DATA/args/" fi snapctl restart ${SNAP_NAME}.daemon-kubelite ${SNAP}/microk8s-start.wrapper ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 echo "Dqlite rolled back" ================================================ FILE: upgrade-scripts/001-switch-to-dqlite/rollback-node.sh ================================================ #!/bin/bash set -ex echo "Rolling back dqlite upgrade on master" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" echo "Restarting etcd" set_service_expected_to_start etcd if [ -e "$BACKUP_DIR/args/etcd" ]; then cp "$BACKUP_DIR"/args/etcd "$SNAP_DATA/args/" snapctl restart ${SNAP_NAME}.daemon-etcd fi echo "Restarting kube-apiserver" if [ -e "$BACKUP_DIR/args/kube-apiserver" ]; then cp "$BACKUP_DIR"/args/kube-apiserver "$SNAP_DATA/args/" fi snapctl restart ${SNAP_NAME}.daemon-kubelite ${SNAP}/microk8s-start.wrapper echo "Dqlite rolled back" ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/commit-master.sh ================================================ #!/bin/bash set -ex echo "Switching master to flannel-etcd" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/002-switch-to-flannel-etcd" DB_DIR="$BACKUP_DIR/db" mkdir -p "$BACKUP_DIR/args/" if [ -e "$SNAP_DATA/args/cni-network/cni.yaml" ]; then KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" $KUBECTL delete -f "$SNAP_DATA/args/cni-network/cni.yaml" || true sleep 10 fi echo "Configuring services" ${SNAP}/microk8s-stop.wrapper cp "$SNAP_DATA"/args/kube-apiserver "$BACKUP_DIR/args" "${SNAP}/bin/sed" -i '/--storage-backend/d' "$SNAP_DATA/args/kube-apiserver" "${SNAP}/bin/sed" -i '/--storage-dir/d' "$SNAP_DATA/args/kube-apiserver" "${SNAP}/bin/sed" -i '/--etcd-servers/d' "$SNAP_DATA/args/kube-apiserver" echo "--etcd-servers=https://127.0.0.1:12379" >> "$SNAP_DATA/args/kube-apiserver" echo "--etcd-cafile=\${SNAP_DATA}/certs/ca.crt" >> "$SNAP_DATA/args/kube-apiserver" echo "--etcd-certfile=\${SNAP_DATA}/certs/server.crt" >> "$SNAP_DATA/args/kube-apiserver" echo "--etcd-keyfile=\${SNAP_DATA}/certs/server.key" >> "$SNAP_DATA/args/kube-apiserver" cp "$SNAP_DATA"/args/etcd "$BACKUP_DIR/args" rm -rf ${SNAP_COMMON}/var/run/etcd/* cp "$SNAP"/default-args/etcd "$SNAP_DATA"/args/ chmod 660 "$SNAP_DATA"/args/etcd cp -r "$SNAP_DATA"/args/cni-network "$BACKUP_DIR/args/" find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; cp --no-preserve=mode,ownership "$SNAP"/default-args/cni-network/* "$SNAP_DATA"/args/cni-network/ chmod -R 770 "$SNAP_DATA"/args/cni-network group=$(get_microk8s_group) if getent group ${group} >/dev/null 2>&1 then chgrp ${group} -R ${SNAP_DATA}/args/ || true fi set_service_expected_to_start etcd set_service_expected_to_start flanneld set_service_not_expected_to_start k8s-dqlite ${SNAP}/microk8s-start.wrapper ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 echo "Switch to etcd and flanneld completed" ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/commit-node.sh ================================================ #!/bin/bash set -ex echo "This will transition only the master node" exit 1 ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/description.txt ================================================ Migrates to a flannel and etcd setup ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/prepare-master.sh ================================================ #!/usr/bin/env bash echo "Master ready for flannel-etcd transition" ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/prepare-node.sh ================================================ #!/bin/bash echo "This will transition only the master node" exit 1 ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/rollback-master.sh ================================================ #!/bin/bash set -ex echo "Rolling back flannel-etcd upgrade on master" source $SNAP/actions/common/utils.sh BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/002-switch-to-flannel-etcd" ${SNAP}/microk8s-stop.wrapper echo "Restarting etcd" set_service_not_expected_to_start etcd if [ -e "$BACKUP_DIR/args/etcd" ]; then cp "$BACKUP_DIR"/args/etcd "$SNAP_DATA/args/" fi echo "Restarting kube-apiserver" if [ -e "$BACKUP_DIR/args/kube-apiserver" ]; then cp "$BACKUP_DIR"/args/kube-apiserver "$SNAP_DATA/args/" fi set_service_not_expected_to_start flanneld if [ -e "$BACKUP_DIR/args/cni-network" ]; then find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; cp -rf "$BACKUP_DIR"/args/cni-network/* "$SNAP_DATA/args/cni-network/" fi group=$(get_microk8s_group) chmod -R ug+rwX "${SNAP_DATA}/args/" chmod -R o-rwX "${SNAP_DATA}/args/" if getent group ${group} >/dev/null 2>&1 then chgrp ${group} -R ${SNAP_DATA}/args/ || true fi ${SNAP}/microk8s-start.wrapper || true ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 if [ -e "$SNAP_DATA/args/cni-network/cni.yaml" ]; then KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" $KUBECTL apply -f "$SNAP_DATA/args/cni-network/cni.yaml" fi echo "Flannel-etcd rolled back" ================================================ FILE: upgrade-scripts/002-switch-to-flannel-etcd/rollback-node.sh ================================================ #!/bin/bash set -ex echo "This will transition only the master node" exit 1