[ { "path": ".ansible-lint", "content": "---\nskip_list:\n # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules\n\n # DO NOT add any other rules to this skip_list, instead use local `# noqa` with a comment explaining WHY it is necessary\n\n # These rules are intentionally skipped:\n #\n # [role-name] \"meta/main.yml\" Role name role-name does not match ``^+$`` pattern\n # Meta roles in Kubespray don't need proper names\n # (Disabled in June 2021)\n - 'role-name'\n\n # [var-naming]\n # In Kubespray we use variables that use camelCase to match their k8s counterparts\n # (Disabled in June 2021)\n - 'var-naming[pattern]'\n # Variables names from within roles in kubespray don't need role name as a prefix\n - 'var-naming[no-role-prefix]'\n\n # [fqcn-builtins]\n # Roles in kubespray don't need fully qualified collection names\n # (Disabled in Feb 2023)\n - 'fqcn-builtins'\n\n # We use template in names\n - 'name[template]'\n\n # No changed-when on commands\n # (Disabled in June 2023 after ansible upgrade; FIXME)\n - 'no-changed-when'\n\n # Disable run-once check with free strategy\n # (Disabled in June 2023 after ansible upgrade; FIXME)\n - 'run-once[task]'\n\n - 'jinja[spacing]'\nexclude_paths:\n # Generated files\n - tests/files/custom_cni/cilium.yaml\n - venv\n - .github\n - .ansible\n - .cache\n - .gitlab-ci.yml\n - .gitlab-ci\nmock_modules:\n - gluster.gluster.gluster_volume\n" }, { "path": ".ansible-lint-ignore", "content": "# This file contains ignores rule violations for ansible-lint\ninventory/sample/group_vars/k8s_cluster/k8s-cluster.yml jinja[spacing]\nroles/kubernetes/control-plane/defaults/main/kube-proxy.yml jinja[spacing]\nroles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing]\nroles/kubernetes/kubeadm/defaults/main.yml jinja[spacing]\nroles/kubernetes/node/defaults/main.yml jinja[spacing]\nroles/kubernetes/preinstall/defaults/main.yml jinja[spacing]\nroles/kubespray-defaults/defaults/main/main.yml jinja[spacing]\n" }, { "path": ".editorconfig", "content": "root = true\n\n[*.{yaml,yml,yml.j2,yaml.j2}]\nindent_style = space\nindent_size = 2\ntrim_trailing_whitespace = true\ninsert_final_newline = true\ncharset = utf-8\n\n[{Dockerfile}]\nindent_style = space\nindent_size = 2\ntrim_trailing_whitespace = true\ninsert_final_newline = true\ncharset = utf-8\n" }, { "path": ".gitattributes", "content": "docs/_sidebar.md linguist-generated=true\n" }, { "path": ".github/ISSUE_TEMPLATE/bug-report.yaml", "content": "---\nname: Bug Report\ndescription: Report a bug encountered while using Kubespray\nlabels: kind/bug\nbody:\n - type: markdown\n attributes:\n value: |\n Please, be ready for followup questions, and please respond in a timely\n manner. If we can't reproduce a bug or think a feature already exists, we\n might close your issue. If we're wrong, PLEASE feel free to reopen it and\n explain why.\n - type: textarea\n id: problem\n attributes:\n label: What happened?\n description: |\n Please provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner.\n validations:\n required: true\n - type: textarea\n id: expected\n attributes:\n label: What did you expect to happen?\n validations:\n required: true\n\n - type: textarea\n id: repro\n attributes:\n label: How can we reproduce it (as minimally and precisely as possible)?\n validations:\n required: true\n\n - type: markdown\n attributes:\n value: '### Environment'\n\n - type: dropdown\n id: os\n attributes:\n label: OS\n options:\n - 'RHEL 9'\n - 'RHEL 8'\n - 'Fedora 40'\n - 'Ubuntu 24'\n - 'Ubuntu 22'\n - 'Ubuntu 20'\n - 'Debian 12'\n - 'Debian 11'\n - 'Flatcar Container Linux'\n - 'openSUSE Leap'\n - 'openSUSE Tumbleweed'\n - 'Oracle Linux 9'\n - 'Oracle Linux 8'\n - 'AlmaLinux 9'\n - 'AlmaLinux 8'\n - 'Rocky Linux 9'\n - 'Rocky Linux 8'\n - 'Amazon Linux 2'\n - 'Kylin Linux Advanced Server V10'\n - 'UOS Linux 20'\n - 'openEuler 24'\n - 'openEuler 22'\n - 'openEuler 20'\n - 'Other|Unsupported'\n validations:\n required: true\n\n - type: textarea\n id: ansible_version\n attributes:\n label: Version of Ansible\n placeholder: 'ansible --version'\n validations:\n required: true\n\n - type: input\n id: python_version\n attributes:\n label: Version of Python\n placeholder: 'python --version'\n validations:\n required: true\n\n - type: input\n id: kubespray_version\n attributes:\n label: Version of Kubespray (commit)\n placeholder: 'git rev-parse --short HEAD'\n validations:\n required: true\n\n - type: dropdown\n id: network_plugin\n attributes:\n label: Network plugin used\n options:\n - calico\n - cilium\n - cni\n - custom_cni\n - flannel\n - kube-ovn\n - kube-router\n - macvlan\n - meta\n - multus\n - ovn4nfv\n validations:\n required: true\n\n - type: textarea\n id: inventory\n attributes:\n label: Full inventory with variables\n placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a \"var=hostvars[inventory_hostname]\"'\n description: We recommend using snippets services like https://gist.github.com/ etc.\n validations:\n required: true\n\n - type: input\n id: ansible_command\n attributes:\n label: Command used to invoke ansible\n validations:\n required: true\n\n - type: textarea\n id: ansible_output\n attributes:\n label: Output of ansible run\n description: We recommend using snippets services like https://gist.github.com/ etc.\n validations:\n required: true\n\n - type: textarea\n id: anything_else\n attributes:\n label: Anything else we need to know\n description: |\n By running scripts/collect-info.yaml you can get a lot of useful informations.\n Script can be started by:\n ansible-playbook -i -u -e ansible_ssh_user= -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml\n (If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python').\n After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "---\nblank_issues_enabled: false\ncontact_links:\n - name: Support Request\n url: https://kubernetes.slack.com/channels/kubespray\n about: Support request or question relating to Kubernetes\n" }, { "path": ".github/ISSUE_TEMPLATE/enhancement.yaml", "content": "---\nname: Enhancement Request\ndescription: Suggest an enhancement to the Kubespray project\nlabels: kind/feature\nbody:\n - type: markdown\n attributes:\n value: Please only use this template for submitting enhancement requests\n - type: textarea\n id: what\n attributes:\n label: What would you like to be added\n validations:\n required: true\n - type: textarea\n id: why\n attributes:\n label: Why is this needed\n validations:\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/failing-test.yaml", "content": "---\nname: Failing Test\ndescription: Report test failures in Kubespray CI jobs\nlabels: kind/failing-test\nbody:\n - type: markdown\n attributes:\n value: Please only use this template for submitting reports about failing tests in Kubespray CI jobs\n - type: textarea\n id: failing_jobs\n attributes:\n label: Which jobs are failing ?\n validations:\n required: true\n\n - type: textarea\n id: failing_tests\n attributes:\n label: Which tests are failing ?\n validations:\n required: true\n\n - type: input\n id: since_when\n attributes:\n label: Since when has it been failing ?\n validations:\n required: true\n\n - type: textarea\n id: failure_reason\n attributes:\n label: Reason for failure\n description: If you don't know and have no guess, just put \"Unknown\"\n validations:\n required: true\n\n - type: textarea\n id: anything_else\n attributes:\n label: Anything else we need to know\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "\n\n**What type of PR is this?**\n> Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:\n>\n> /kind api-change\n> /kind bug\n> /kind cleanup\n> /kind design\n> /kind documentation\n> /kind failing-test\n> /kind feature\n> /kind flake\n\n**What this PR does / why we need it**:\n\n**Which issue(s) this PR fixes**:\n\nFixes #\n\n**Special notes for your reviewer**:\n\n**Does this PR introduce a user-facing change?**:\n\n```release-note\n\n```\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"pip\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n labels:\n - dependencies\n - release-note-none\n groups:\n molecule:\n patterns:\n - molecule\n - molecule-plugins*\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n labels:\n - release-note-none\n - ci-short\n schedule:\n interval: \"weekly\"\n" }, { "path": ".github/workflows/auto-label-os.yml", "content": "name: Issue labeler\non:\n issues:\n types: [opened]\n\npermissions:\n contents: read\n\njobs:\n label-component:\n runs-on: ubuntu-latest\n permissions:\n issues: write\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd\n\n - name: Parse issue form\n uses: stefanbuck/github-issue-parser@10dcc54158ba4c137713d9d69d70a2da63b6bda3\n id: issue-parser\n with:\n template-path: .github/ISSUE_TEMPLATE/bug-report.yaml\n\n - name: Set labels based on OS field\n uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e\n with:\n issue-form: ${{ steps.issue-parser.outputs.jsonString }}\n section: os\n block-list: |\n None\n Other\n token: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/upgrade-patch-versions-schedule.yml", "content": "name: Upgrade Kubespray components with new patches versions - all branches\n\non:\n schedule:\n - cron: '22 2 * * *' # every day, 02:22 UTC\n workflow_dispatch:\n\npermissions: {}\njobs:\n get-releases-branches:\n if: github.repository == 'kubernetes-sigs/kubespray'\n runs-on: ubuntu-latest\n outputs:\n branches: ${{ steps.get-branches.outputs.data }}\n steps:\n - uses: octokit/graphql-action@ddde8ebb2493e79f390e6449c725c21663a67505\n id: get-branches\n with:\n query: |\n query get_release_branches($owner:String!, $name:String!) {\n repository(owner:$owner, name:$name) {\n refs(refPrefix: \"refs/heads/\",\n first: 3,\n query: \"release-\",\n orderBy: {\n field: ALPHABETICAL,\n direction: DESC\n }) {\n nodes {\n name\n }\n }\n }\n }\n variables: |\n owner: ${{ github.repository_owner }}\n name: ${{ github.event.repository.name }}\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n\n update-versions:\n needs: get-releases-branches\n strategy:\n fail-fast: false\n matrix:\n branch:\n - name: ${{ github.event.repository.default_branch }}\n - ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }}\n uses: ./.github/workflows/upgrade-patch-versions.yml\n permissions:\n contents: write\n pull-requests: write\n name: Update patch updates on ${{ matrix.branch.name }}\n with:\n branch: ${{ matrix.branch.name }}\n" }, { "path": ".github/workflows/upgrade-patch-versions.yml", "content": "on:\n workflow_call:\n inputs:\n branch:\n description: Which branch to update with new patch versions\n default: master\n required: true\n type: string\n\njobs:\n update-patch-versions:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd\n with:\n ref: ${{ inputs.branch }}\n - uses: actions/setup-python@v6\n with:\n python-version: '3.13'\n cache: 'pip'\n - run: pip install scripts/component_hash_update pre-commit\n - run: update-hashes\n env:\n API_KEY: ${{ secrets.GITHUB_TOKEN }}\n - uses: actions/cache@v5\n with:\n key: pre-commit-hook-propagate\n path: |\n ~/.cache/pre-commit\n - run: pre-commit run --all-files propagate-ansible-variables\n continue-on-error: true\n - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0\n with:\n commit-message: Patch versions updates\n title: Patch versions updates - ${{ inputs.branch }}\n labels: bot\n branch: component_hash_update/${{ inputs.branch }}\n sign-commits: true\n body: |\n /kind feature\n\n ```release-note\n NONE\n ```\n" }, { "path": ".gitignore", "content": ".vagrant\n*.retry\n**/vagrant_ansible_inventory\n*.iml\ntemp\ncontrib/offline/container-images\ncontrib/offline/container-images.tar.gz\ncontrib/offline/offline-files\ncontrib/offline/offline-files.tar.gz\n.idea\n.vscode\n.tox\n.cache\n*.bak\n*.tfstate\n*.tfstate*backup\n*.lock.hcl\n.terraform/\ncontrib/terraform/aws/credentials.tfvars\n.terraform.lock.hcl\n/ssh-bastion.conf\n**/*.sw[pon]\n*~\nvagrant/\nplugins/mitogen\n\n# Ansible inventory\ninventory/*\n!inventory/local\n!inventory/sample\ninventory/*/artifacts/\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ncredentials/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\n.hypothesis/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# IPython Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# dotenv\n.env\n\n# virtualenv\nvenv/\nENV/\n\n# molecule\nroles/**/molecule/**/__pycache__/\n\n# macOS\n.DS_Store\n\n# Temp location used by our scripts\nscripts/tmp/\ntmp.md\n\n# Ansible collection files\nkubernetes_sigs-kubespray*tar.gz\nansible_collections\n" }, { "path": ".gitlab-ci/build.yml", "content": "---\npipeline-image:\n cache:\n key: $CI_COMMIT_REF_SLUG\n paths:\n - image-cache\n tags:\n - ffci\n stage: build\n image: moby/buildkit:rootless\n variables:\n BUILDKITD_FLAGS: --oci-worker-no-process-sandbox\n CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache\n # TODO: remove the override\n # currently rebase.sh depends on bash (not available in the kaniko image)\n # once we have a simpler rebase (which should be easy if the target branch ref is available as variable\n # we'll be able to rebase here as well hopefully\n before_script:\n - mkdir -p ~/.docker\n - echo \"{\\\"auths\\\":{\\\"$CI_REGISTRY\\\":{\\\"auth\\\":\\\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\\\"}}}\" > ~/.docker/config.json\n script:\n - |\n buildctl-daemonless.sh build \\\n --frontend dockerfile.v0 \\\n --local context=$CI_PROJECT_DIR \\\n --local dockerfile=$CI_PROJECT_DIR \\\n --opt filename=pipeline.Dockerfile \\\n --export-cache type=registry,ref=$CACHE_IMAGE \\\n --import-cache type=registry,ref=$CACHE_IMAGE \\\n --output type=image,name=$PIPELINE_IMAGE,push=true\n" }, { "path": ".gitlab-ci/kubevirt.yml", "content": "---\n.kubevirt:\n extends: .job-moderated\n interruptible: true\n script:\n - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml\n -e @\"tests/files/${TESTCASE}.yml\"\n - ./tests/scripts/testcases_run.sh\n variables:\n ANSIBLE_TIMEOUT: \"120\"\n tags:\n - ffci\n needs:\n - pipeline-image\n\n# TODO: generate testcases matrixes from the files in tests/files/\n# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch\n# (currently, a removed job in the target branch breaks the tests, because the\n# pipeline definition is parsed by gitlab before the rebase.sh script)\n# CI template for PRs\npr:\n stage: deploy-part1\n rules:\n - if: $PR_LABELS =~ /.*ci-short.*/\n when: manual\n allow_failure: true\n - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/\n when: on_success\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n - when: manual\n allow_failure: true\n extends: .kubevirt\n parallel:\n matrix:\n - TESTCASE:\n - almalinux9-crio\n - almalinux9-kube-ovn\n - debian11-calico-collection\n - debian11-macvlan\n - debian12-cilium\n - debian13-cilium\n - fedora39-kube-router\n - fedora41-kube-router\n - fedora42-calico\n - rockylinux9-cilium\n - rockylinux10-cilium\n - ubuntu22-calico-all-in-one\n - ubuntu22-calico-all-in-one-upgrade\n - ubuntu24-calico-etcd-datastore\n - ubuntu24-calico-all-in-one-hardening\n - ubuntu24-cilium-sep\n - ubuntu24-crio-scale\n - ubuntu24-crio-upgrade\n - ubuntu24-flannel-collection\n - ubuntu24-kube-router-sep\n - ubuntu24-kube-router-svc-proxy\n - ubuntu24-ha-separate-etcd\n - fedora40-flannel-crio-collection-scale\n - openeuler24-calico\n\n# This is for flakey test so they don't disrupt the PR worklflow too much.\n# Jobs here MUST have a open issue so we don't lose sight of them\npr-flakey:\n extends: pr\n retry: 1\n parallel:\n matrix:\n - TESTCASE:\n - flatcar4081-calico # https://github.com/kubernetes-sigs/kubespray/issues/12309\n\n# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken\nubuntu24-calico-all-in-one:\n stage: deploy-part1\n extends: .kubevirt\n variables:\n TESTCASE: ubuntu24-calico-all-in-one\n rules:\n - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/\n when: on_success\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n - when: manual\n allow_failure: true\n\npr_full:\n extends: .kubevirt\n stage: deploy-extended\n rules:\n - if: $PR_LABELS =~ /.*ci-full.*/\n when: on_success\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n # Else run as manual\n - when: manual\n allow_failure: true\n parallel:\n matrix:\n - TESTCASE:\n - almalinux9-calico-ha-ebpf\n - almalinux9-calico-nodelocaldns-secondary\n - debian11-custom-cni\n - debian11-kubelet-csr-approver\n - debian12-custom-cni-helm\n - fedora39-calico-swap-selinux\n - fedora39-crio\n - fedora41-calico-swap-selinux\n - fedora41-crio\n - ubuntu24-calico-ha-wireguard\n - ubuntu24-flannel-ha\n - ubuntu24-flannel-ha-once\n\n# Need an update of the container image to use schema v2\n# update: quay.io/kubespray/vm-amazon-linux-2:latest\nmanual:\n extends: pr_full\n parallel:\n matrix:\n - TESTCASE:\n - amazon-linux-2-all-in-one\n rules:\n - when: manual\n allow_failure: true\n\npr_extended:\n extends: .kubevirt\n stage: deploy-extended\n rules:\n - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/\n when: on_success\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n - when: manual\n allow_failure: true\n parallel:\n matrix:\n - TESTCASE:\n - almalinux9-calico\n - almalinux9-calico-remove-node\n - almalinux9-docker\n - debian11-docker\n - debian12-calico\n - debian12-docker\n - debian13-calico\n - rockylinux9-calico\n - rockylinux10-calico\n - ubuntu22-all-in-one-docker\n - ubuntu24-all-in-one-docker\n - ubuntu24-calico-all-in-one\n - ubuntu24-calico-etcd-kubeadm\n - ubuntu24-flannel\n\n# TODO: migrate to pr-full, fix the broken ones\nperiodic:\n allow_failure: true\n extends: .kubevirt\n rules:\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n parallel:\n matrix:\n - TESTCASE:\n - debian11-calico-upgrade\n - debian11-calico-upgrade-once\n - debian12-cilium-svc-proxy\n - fedora39-calico-selinux\n - fedora40-docker-calico\n - fedora41-calico-selinux\n - ubuntu24-calico-etcd-kubeadm-upgrade-ha\n - ubuntu24-calico-ha-recover\n - ubuntu24-calico-ha-recover-noquorum\n" }, { "path": ".gitlab-ci/lint.yml", "content": "---\npre-commit:\n stage: test\n tags:\n - ffci\n image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'\n variables:\n PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit\n ANSIBLE_STDOUT_CALLBACK: default\n script:\n - pre-commit run --all-files --show-diff-on-failure\n cache:\n key: pre-commit-2\n paths:\n - ${PRE_COMMIT_HOME}\n when: 'always'\n needs: []\n\nvagrant-validate:\n extends: .job\n stage: test\n tags: [ffci]\n variables:\n VAGRANT_VERSION: 2.3.7\n script:\n - ./tests/scripts/vagrant-validate.sh\n" }, { "path": ".gitlab-ci/molecule.yml", "content": "---\n.molecule:\n tags: [ffci]\n rules: # run on ci-short as well\n - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/\n when: on_success\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n - when: manual\n allow_failure: true\n stage: deploy-part1\n image: $PIPELINE_IMAGE\n needs:\n - pipeline-image\n script:\n - ./tests/scripts/molecule_run.sh\n after_script:\n - rm -fr molecule_logs\n - mkdir -p molecule_logs\n - find ~/.cache/molecule/ \\( -name '*.out' -o -name '*.err' \\) -type f | xargs tar -uf molecule_logs/molecule.tar\n - gzip molecule_logs/molecule.tar\n artifacts:\n when: always\n paths:\n - molecule_logs/\n\nmolecule:\n extends: .molecule\n script:\n - ./tests/scripts/molecule_run.sh -i $ROLE\n parallel:\n matrix:\n - ROLE:\n - container-engine/cri-dockerd\n - container-engine/containerd\n - container-engine/cri-o\n - container-engine/gvisor\n - container-engine/youki\n - adduser\n - bastion-ssh-config\n - bootstrap_os\n\nmolecule_full:\n allow_failure: true\n rules:\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n - when: manual\n allow_failure: true\n extends: molecule\n parallel:\n matrix:\n - ROLE:\n # FIXME : tests below are perma-failing\n - container-engine/kata-containers\n" }, { "path": ".gitlab-ci/terraform.yml", "content": "---\n# Tests for contrib/terraform/\n.terraform_install:\n extends: .job\n needs:\n - pipeline-image\n variables:\n TF_VAR_public_key_path: \"${ANSIBLE_PRIVATE_KEY_FILE}.pub\"\n TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE\n CLUSTER: $CI_COMMIT_REF_NAME\n TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR\n stage: deploy-part1\n before_script:\n - ./tests/scripts/rebase.sh\n - mkdir -p cluster-dump $ANSIBLE_INVENTORY\n - ./tests/scripts/opentofu_install.sh\n - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .\n - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts\n - tofu -chdir=\"contrib/terraform/$PROVIDER\" init\n\nterraform_validate:\n extends: .terraform_install\n tags: [ffci]\n only: ['master', /^pr-.*$/]\n script:\n - tofu -chdir=\"contrib/terraform/$PROVIDER\" validate\n - tofu -chdir=\"contrib/terraform/$PROVIDER\" fmt -check -diff\n stage: test\n needs:\n - pipeline-image\n parallel:\n matrix:\n - PROVIDER:\n - openstack\n - aws\n - exoscale\n - hetzner\n - vsphere\n - upcloud\n\n.terraform_apply:\n extends: .terraform_install\n tags: [ffci]\n stage: deploy-extended\n when: manual\n only: [/^pr-.*$/]\n variables:\n ANSIBLE_INVENTORY_UNPARSED_FAILED: \"true\"\n ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly\n ANSIBLE_SSH_RETRIES: 15\n TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER\n TF_VAR_cluster_name: $CI_JOB_ID\n script:\n # Set Ansible config\n - cp ansible.cfg ~/.ansible.cfg\n - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa\n - mkdir -p contrib/terraform/$PROVIDER/group_vars\n # Random subnet to avoid routing conflicts\n - export TF_VAR_subnet_cidr=\"10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24\"\n - tofu -chdir=\"contrib/terraform/$PROVIDER\" apply -auto-approve -parallelism=1\n - tests/scripts/testcases_run.sh\n after_script:\n # Cleanup regardless of exit code\n - tofu -chdir=\"contrib/terraform/$PROVIDER\" destroy -auto-approve\n\n# Elastx is generously donating resources for Kubespray on Openstack CI\n# Contacts: @gix @bl0m1\n.elastx_variables: &elastx_variables\n OS_AUTH_URL: https://ops.elastx.cloud:5000\n OS_PROJECT_ID: 564c6b461c6b44b1bb19cdb9c2d928e4\n OS_PROJECT_NAME: kubespray_ci\n OS_USER_DOMAIN_NAME: Default\n OS_PROJECT_DOMAIN_ID: default\n OS_USERNAME: kubespray@root314.com\n OS_REGION_NAME: se-sto\n OS_INTERFACE: public\n OS_IDENTITY_API_VERSION: \"3\"\n TF_VAR_router_id: \"ab95917c-41fb-4881-b507-3a6dfe9403df\"\n\ntf-elastx_cleanup:\n tags: [ffci]\n image: python\n variables:\n <<: *elastx_variables\n before_script:\n - pip install -r scripts/openstack-cleanup/requirements.txt\n script:\n - ./scripts/openstack-cleanup/main.py\n allow_failure: true\n\ntf-elastx_ubuntu24-calico:\n extends: .terraform_apply\n stage: deploy-part1\n when: on_success\n variables:\n <<: *elastx_variables\n PROVIDER: openstack\n ANSIBLE_TIMEOUT: \"60\"\n TF_VAR_number_of_k8s_masters: \"1\"\n TF_VAR_number_of_k8s_masters_no_floating_ip: \"0\"\n TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: \"0\"\n TF_VAR_number_of_etcd: \"0\"\n TF_VAR_number_of_k8s_nodes: \"1\"\n TF_VAR_number_of_k8s_nodes_no_floating_ip: \"0\"\n TF_VAR_number_of_gfs_nodes_no_floating_ip: \"0\"\n TF_VAR_number_of_bastions: \"0\"\n TF_VAR_number_of_k8s_masters_no_etcd: \"0\"\n TF_VAR_floatingip_pool: \"elx-public1\"\n TF_VAR_dns_nameservers: '[\"1.1.1.1\", \"8.8.8.8\", \"8.8.4.4\"]'\n TF_VAR_use_access_ip: \"0\"\n TF_VAR_external_net: \"600b8501-78cb-4155-9c9f-23dfcba88828\"\n TF_VAR_network_name: \"ci-$CI_JOB_ID\"\n TF_VAR_az_list: '[\"sto1\"]'\n TF_VAR_az_list_node: '[\"sto1\"]'\n TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b # v1-standard-2\n TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b # v1-standard-2\n TF_VAR_image: ubuntu-24.04-server-latest\n TF_VAR_k8s_allowed_remote_ips: '[\"0.0.0.0/0\"]'\n TESTCASE: $CI_JOB_NAME\n" }, { "path": ".gitlab-ci/vagrant.yml", "content": "---\nvagrant:\n extends: .job-moderated\n variables:\n CI_PLATFORM: \"vagrant\"\n SSH_USER: \"vagrant\"\n VAGRANT_DEFAULT_PROVIDER: \"libvirt\"\n KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb\n DOCKER_NAME: vagrant\n VAGRANT_ANSIBLE_TAGS: facts\n VAGRANT_HOME: \"$CI_PROJECT_DIR/.vagrant.d\"\n PIP_CACHE_DIR: \"$CI_PROJECT_DIR/.cache/pip\"\n tags: [ffci-vm-large]\n image: quay.io/kubespray/vm-kubespray-ci:v13\n services: []\n before_script:\n - echo $USER\n - python3 -m venv citest\n - source citest/bin/activate\n - vagrant plugin expunge --reinstall --force --no-tty\n - vagrant plugin install vagrant-libvirt\n - pip install --no-compile --no-cache-dir pip -U\n - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt\n - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt\n - ./tests/scripts/vagrant_clean.sh\n script:\n - vagrant up\n - ./tests/scripts/testcases_run.sh\n after_script:\n - vagrant destroy -f\n cache:\n key: $CI_JOB_NAME_SLUG\n paths:\n - .vagrant.d/boxes\n - .cache/pip\n policy: pull-push # TODO: change to \"pull\" when not on main\n stage: deploy-extended\n rules:\n - if: $PR_LABELS =~ /.*ci-full.*/\n when: on_success\n - if: $CI_PIPELINE_SOURCE == \"schedule\" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == \"daily-ci\"\n when: on_success\n - when: manual\n allow_failure: true\n parallel:\n matrix:\n - TESTCASE:\n - ubuntu24-calico-dual-stack\n - ubuntu24-calico-ipv6only-stack\n" }, { "path": ".gitlab-ci.yml", "content": "---\nstages:\n - build # build docker image used in most other jobs\n - test # unit tests\n - deploy-part1 # kubespray runs - common setup\n - deploy-extended # kubespray runs - rarer or costlier (to test) setups\n\nvariables:\n FAILFASTCI_NAMESPACE: 'kargo-ci'\n GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'\n GIT_CONFIG_COUNT: 2\n GIT_CONFIG_KEY_0: user.email\n GIT_CONFIG_VALUE_0: \"ci@kubespray.io\"\n GIT_CONFIG_KEY_1: user.name\n GIT_CONFIG_VALUE_1: \"Kubespray CI\"\n ANSIBLE_FORCE_COLOR: \"true\"\n MAGIC: \"ci check this\"\n GS_ACCESS_KEY_ID: $GS_KEY\n GS_SECRET_ACCESS_KEY: $GS_SECRET\n CONTAINER_ENGINE: docker\n GCE_PREEMPTIBLE: \"false\"\n ANSIBLE_KEEP_REMOTE_FILES: \"1\"\n ANSIBLE_CONFIG: ./tests/ansible.cfg\n ANSIBLE_REMOTE_USER: kubespray\n ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa\n ANSIBLE_INVENTORY: /tmp/inventory\n ANSIBLE_STDOUT_CALLBACK: \"default\"\n RESET_CHECK: \"false\"\n REMOVE_NODE_CHECK: \"false\"\n UPGRADE_TEST: \"false\"\n MITOGEN_ENABLE: \"false\"\n ANSIBLE_VERBOSITY: 2\n RECOVER_CONTROL_PLANE_TEST: \"false\"\n RECOVER_CONTROL_PLANE_TEST_GROUPS: \"etcd[2:]:kube_control_plane[1:]\"\n OPENTOFU_VERSION: v1.9.1\n PIPELINE_IMAGE: \"$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}\"\n\nbefore_script:\n - ./tests/scripts/rebase.sh\n - mkdir -p cluster-dump $ANSIBLE_INVENTORY\n\n.job: &job\n tags:\n - ffci\n image: $PIPELINE_IMAGE\n artifacts:\n when: always\n paths:\n - cluster-dump/\n needs:\n - pipeline-image\n\n.job-moderated:\n extends: .job\n needs:\n - pipeline-image\n - pre-commit # lint\n - vagrant-validate # lint\n\ninclude:\n - .gitlab-ci/build.yml\n - .gitlab-ci/lint.yml\n - .gitlab-ci/terraform.yml\n - .gitlab-ci/kubevirt.yml\n - .gitlab-ci/vagrant.yml\n - .gitlab-ci/molecule.yml\n" }, { "path": ".gitmodules", "content": "" }, { "path": ".md_style.rb", "content": "all\nexclude_rule 'MD013'\nexclude_rule 'MD029'\nrule 'MD007', :indent => 2\n" }, { "path": ".mdlrc", "content": "style \"#{File.dirname(__FILE__)}/.md_style.rb\"\n" }, { "path": ".nojekyll", "content": "" }, { "path": ".pre-commit-config.yaml", "content": "---\nrepos:\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v6.0.0\n hooks:\n - id: check-added-large-files\n - id: check-case-conflict\n - id: check-executables-have-shebangs\n - id: check-xml\n - id: check-merge-conflict\n - id: detect-private-key\n - id: end-of-file-fixer\n - id: forbid-new-submodules\n - id: requirements-txt-fixer\n - id: trailing-whitespace\n\n - repo: https://github.com/adrienverge/yamllint.git\n rev: v1.37.1\n hooks:\n - id: yamllint\n args: [--strict]\n\n - repo: https://github.com/shellcheck-py/shellcheck-py\n rev: v0.11.0.1\n hooks:\n - id: shellcheck\n args: [\"--severity=error\"]\n exclude: \"^.git\"\n files: \"\\\\.sh$\"\n\n - repo: https://github.com/ansible/ansible-lint\n rev: v25.11.0\n hooks:\n - id: ansible-lint\n additional_dependencies:\n - jmespath==1.0.1\n - netaddr==1.3.0\n - distlib\n\n - repo: https://github.com/golangci/misspell\n rev: v0.7.0\n hooks:\n - id: misspell\n exclude: \"OWNERS_ALIASES$\"\n\n - repo: local\n hooks:\n - id: collection-build-install\n name: Build and install kubernetes-sigs.kubespray Ansible collection\n language: python\n additional_dependencies:\n - ansible-core>=2.16.4\n - distlib\n entry: tests/scripts/collection-build-install.sh\n pass_filenames: false\n\n - id: generate-docs-sidebar\n name: generate-docs-sidebar\n entry: scripts/gen_docs_sidebar.sh\n language: script\n pass_filenames: false\n\n - id: ci-matrix\n name: ci-matrix\n entry: tests/scripts/md-table/main.py\n language: python\n pass_filenames: false\n additional_dependencies:\n - jinja2\n - pathlib\n - pyaml\n\n - id: check-galaxy-version\n name: Verify correct version for galaxy.yml\n entry: scripts/galaxy_version.py\n language: python\n pass_filenames: false\n additional_dependencies:\n - ruamel.yaml\n\n - id: jinja-syntax-check\n name: jinja-syntax-check\n entry: tests/scripts/check-templates.py\n language: python\n types:\n - jinja\n additional_dependencies:\n - jinja2\n\n - id: propagate-ansible-variables\n name: Update static files referencing default kubespray values\n language: python\n additional_dependencies:\n - ansible-core>=2.16.4\n entry: scripts/propagate_ansible_variables.yml\n pass_filenames: false\n\n - id: check-checksums-sorted\n name: Check that our checksums are correctly sorted by version\n entry: scripts/assert-sorted-checksums.yml\n language: python\n pass_filenames: false\n additional_dependencies:\n - ansible\n\n - repo: https://github.com/markdownlint/markdownlint\n rev: v0.12.0\n hooks:\n - id: markdownlint\n exclude: \"^.github|(^docs/_sidebar\\\\.md$)\"\n" }, { "path": ".yamllint", "content": "---\nextends: default\n\nignore: |\n .git/\n .github/\n # Generated file\n tests/files/custom_cni/cilium.yaml\n# https://ansible.readthedocs.io/projects/lint/rules/yaml/\nrules:\n braces:\n min-spaces-inside: 0\n max-spaces-inside: 1\n brackets:\n min-spaces-inside: 0\n max-spaces-inside: 1\n comments:\n min-spaces-from-content: 1\n # https://github.com/adrienverge/yamllint/issues/384\n comments-indentation: false\n indentation:\n spaces: 2\n indent-sequences: consistent\n line-length: disable\n new-line-at-end-of-file: disable\n octal-values:\n forbid-implicit-octal: true # yamllint defaults to false\n forbid-explicit-octal: true # yamllint defaults to false\n" }, { "path": "CHANGELOG.md", "content": "# See our release notes on [GitHub](https://github.com/kubernetes-sigs/kubespray/releases)\n" }, { "path": "CNAME", "content": "kubespray.io\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing guidelines\n\n## How to become a contributor and submit your own code\n\n### Environment setup\n\nIt is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications)\n\nTo install development dependencies you can set up a python virtual env with the necessary dependencies:\n\n```ShellSession\nvirtualenv venv\nsource venv/bin/activate\npip install -r tests/requirements.txt\n```\n\n#### Linting\n\nKubespray uses [pre-commit](https://pre-commit.com) hook configuration to run several linters, please install this tool and use it to run validation tests before submitting a PR.\n\n```ShellSession\npre-commit install\npre-commit run -a # To run pre-commit hook on all files in the repository, even if they were not modified\n```\n\n#### Molecule\n\n[molecule](https://github.com/ansible-community/molecule) is designed to help the development and testing of Ansible roles. In Kubespray you can run it all for all roles with `./tests/scripts/molecule_run.sh` or for a specific role (that you are working with) with `molecule test` from the role directory (`cd roles/my-role`).\n\nWhen developing or debugging a role it can be useful to run `molecule create` and `molecule converge` separately. Then you can use `molecule login` to SSH into the test environment.\n\n#### Vagrant\n\nVagrant with VirtualBox or libvirt driver helps you to quickly spin test clusters to test things end to end. See [README.md#vagrant](README.md)\n\n### Contributing A Patch\n\n1. Submit an issue describing your proposed change to the repo in question.\n2. The [repo owners](OWNERS) will respond to your issue promptly.\n3. Fork the desired repo, develop and test your code changes.\n4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.\n5. Address any pre-commit validation failures.\n6. Sign the CNCF CLA ()\n7. Submit a pull request.\n8. Work with the reviewers on their suggestions.\n9. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits () before final merger of your contribution.\n" }, { "path": "Dockerfile", "content": "# syntax=docker/dockerfile:1\n\n# Use immutable image tags rather than mutable tags (like ubuntu:24.04)\nFROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b\n\n# Some tools like yamllint need this\n# Pip needs this as well at the moment to install ansible\n# (and potentially other packages)\n# See: https://github.com/pypa/pip/issues/10219\nENV LANG=C.UTF-8 \\\n DEBIAN_FRONTEND=noninteractive \\\n PYTHONDONTWRITEBYTECODE=1\n\nWORKDIR /kubespray\n\n# hadolint ignore=DL3008\nRUN --mount=type=cache,target=/var/cache/apt,sharing=locked \\\n apt-get update -q \\\n && apt-get install -yq --no-install-recommends \\\n curl \\\n python3 \\\n python3-pip \\\n sshpass \\\n vim \\\n rsync \\\n openssh-client \\\n && apt-get clean \\\n && rm -rf /var/lib/apt/lists/* /var/log/*\n\nRUN --mount=type=bind,source=requirements.txt,target=requirements.txt \\\n --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \\\n pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt \\\n && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \\;\n\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nRUN OS_ARCHITECTURE=$(dpkg --print-architecture) \\\n && curl -L \"https://dl.k8s.io/release/v1.35.1/bin/linux/${OS_ARCHITECTURE}/kubectl\" -o /usr/local/bin/kubectl \\\n && echo \"$(curl -L \"https://dl.k8s.io/release/v1.35.1/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256\")\" /usr/local/bin/kubectl | sha256sum --check \\\n && chmod a+x /usr/local/bin/kubectl\n\nCOPY *.yml ./\nCOPY *.cfg ./\nCOPY roles ./roles\nCOPY contrib ./contrib\nCOPY inventory ./inventory\nCOPY library ./library\nCOPY extra_playbooks ./extra_playbooks\nCOPY playbooks ./playbooks\nCOPY plugins ./plugins\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2016 Kubespray\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "OWNERS", "content": "# See the OWNERS docs at https://go.k8s.io/owners\n\napprovers:\n - kubespray-approvers\nreviewers:\n - kubespray-reviewers\nemeritus_approvers:\n - kubespray-emeritus_approvers\n" }, { "path": "OWNERS_ALIASES", "content": "aliases:\n kubespray-approvers:\n - ant31\n - mzaian\n - tico88612\n - vannten\n - yankay\n kubespray-reviewers:\n - cyclinder\n - erikjiang\n - mzaian\n - tico88612\n - vannten\n - yankay\n kubespray-emeritus_approvers:\n - atoms\n - chadswen\n - cristicalin\n - floryut\n - liupeng0518\n - luckysb\n - mattymo\n - miouge1\n - oomichi\n - riverzhang\n - woopstar\n" }, { "path": "README.md", "content": "# Deploy a Production Ready Kubernetes Cluster\n\n![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png)\n\nIf you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\\#kubespray**.\nYou can get your invite [here](http://slack.k8s.io/)\n\n- Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_controllers/openstack.md), [vSphere](docs/cloud_controllers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**\n- **Highly available** cluster\n- **Composable** (Choice of the network plugin for instance)\n- Supports most popular **Linux distributions**\n- **Continuous integration tests**\n\n## Quick Start\n\nBelow are several ways to use Kubespray to deploy a Kubernetes cluster.\n\n### Docker\n\nEnsure you have installed Docker then\n\n```ShellSession\ndocker run --rm -it --mount type=bind,source=\"$(pwd)\"/inventory/sample,dst=/inventory \\\n --mount type=bind,source=\"${HOME}\"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \\\n quay.io/kubespray/kubespray:v2.30.0 bash\n# Inside the container you may now run the kubespray playbooks:\nansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml\n```\n\n### Ansible\n\n#### Usage\n\nSee [Getting started](/docs/getting_started/getting-started.md)\n\n#### Collection\n\nSee [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection\n\n### Vagrant\n\nFor Vagrant we need to install Python dependencies for provisioning tasks.\nCheck that ``Python`` and ``pip`` are installed:\n\n```ShellSession\npython -V && pip -V\n```\n\nIf this returns the version of the software, you're good to go. If not, download and install Python from here \n\nInstall Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)\nthen run the following step:\n\n```ShellSession\nvagrant up\n```\n\n## Documents\n\n- [Requirements](#requirements)\n- [Kubespray vs ...](docs/getting_started/comparisons.md)\n- [Getting started](docs/getting_started/getting-started.md)\n- [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md)\n- [Ansible inventory and tags](docs/ansible/ansible.md)\n- [Integration with existing ansible repo](docs/operations/integration.md)\n- [Deployment data variables](docs/ansible/vars.md)\n- [DNS stack](docs/advanced/dns-stack.md)\n- [HA mode](docs/operations/ha-mode.md)\n- [Network plugins](#network-plugins)\n- [Vagrant install](docs/developers/vagrant.md)\n- [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md)\n- [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md)\n- [openSUSE setup](docs/operating_systems/opensuse.md)\n- [Downloaded artifacts](docs/advanced/downloads.md)\n- [Equinix Metal](docs/cloud_providers/equinix-metal.md)\n- [OpenStack](docs/cloud_controllers/openstack.md)\n- [vSphere](docs/cloud_controllers/vsphere.md)\n- [Large deployments](docs/operations/large-deployments.md)\n- [Adding/replacing a node](docs/operations/nodes.md)\n- [Upgrades basics](docs/operations/upgrades.md)\n- [Air-Gap installation](docs/operations/offline-environment.md)\n- [NTP](docs/advanced/ntp.md)\n- [Hardening](docs/operations/hardening.md)\n- [Mirror](docs/operations/mirror.md)\n- [Roadmap](docs/roadmap/roadmap.md)\n\n## Supported Linux Distributions\n\n- **Flatcar Container Linux by Kinvolk**\n- **Debian** Bookworm, Bullseye, Trixie\n- **Ubuntu** 22.04, 24.04\n- **CentOS Stream / RHEL** 9, 10\n- **Fedora** 39, 40, 41, 42\n- **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))\n- **openSUSE** Leap 15.x/Tumbleweed\n- **Oracle Linux** 9, 10\n- **Alma Linux** 9, 10\n- **Rocky Linux** 9, 10 (experimental in 10: see [Rocky Linux 10 notes](docs/operating_systems/rhel.md#rocky-linux-10))\n- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))\n- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))\n- **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))\n- **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md))\n\nNote:\n\n- Upstart/SysV init based OS types are not supported.\n- [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19).\n\n## Supported Components\n\n\n\n- Core\n - [kubernetes](https://github.com/kubernetes/kubernetes) 1.35.1\n - [etcd](https://github.com/etcd-io/etcd) 3.6.8\n - [docker](https://www.docker.com/) 28.3\n - [containerd](https://containerd.io/) 2.2.1\n - [cri-o](http://cri-o.io/) 1.35.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)\n- Network Plugin\n - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0\n - [calico](https://github.com/projectcalico/calico) 3.30.6\n - [cilium](https://github.com/cilium/cilium) 1.19.1\n - [flannel](https://github.com/flannel-io/flannel) 0.27.3\n - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21\n - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1\n - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2\n - [kube-vip](https://github.com/kube-vip/kube-vip) 1.0.3\n- Application\n - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3\n - [coredns](https://github.com/coredns/coredns) 1.12.4\n - [argocd](https://argoproj.github.io/) 2.14.5\n - [helm](https://helm.sh/) 3.18.4\n - [metallb](https://metallb.universe.tf/) 0.13.9\n - [registry](https://github.com/distribution/distribution) 2.8.1\n- Storage Plugin\n - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0\n - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0\n - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0\n - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2\n - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32\n - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0\n - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4\n\n\n\n## Container Runtime Notes\n\n- The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)\n\n## Requirements\n\n- **Minimum required version of Kubernetes is v1.30**\n- **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**\n- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md))\n- The target servers are configured to allow **IPv4 forwarding**.\n- If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**.\n- The **firewalls are not managed**, you'll need to implement your own rules the way you used to.\n in order to avoid any issue during deployment you should disable your firewall.\n- If kubespray is run from non-root user account, correct privilege escalation method\n should be configured in the target servers. Then the `ansible_become` flag\n or command parameters `--become or -b` should be specified.\n\nHardware:\nThese limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide.\n\n- Control Plane\n - Memory: 2 GB\n- Worker Node\n - Memory: 1 GB\n\n## Network Plugins\n\nYou can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`)\n\n- [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking.\n\n- [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options\n designed to give you the most efficient networking across a range of situations, including non-overlay\n and overlay networks, with or without BGP. Calico uses the same engine to enforce network policy for hosts,\n pods, and (if using Istio and Envoy) applications at the service mesh layer.\n\n- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.\n\n- [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.\n\n- [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational\n simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy),\n iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).\n It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.\n\n- [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.\n\n- [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.\n\n- [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray.\n See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart.\n\nThe network plugin to use is defined by the variable `kube_network_plugin`. There is also an\noption to leverage built-in cloud provider networking instead.\nSee also [Network checker](docs/advanced/netcheck.md).\n\n## Ingress Plugins\n\n- [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider.\n\n## Community docs and resources\n\n- [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/)\n- [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr\n- [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty\n- [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=CJ5G4GpqDy0)\n\n## Tools and projects on top of Kubespray\n\n- [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/v4/doc/integrations/ansible.rst)\n- [Terraform Contrib](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/terraform)\n- [Kubean](https://github.com/kubean-io/kubean)\n\n## CI Tests\n\n[![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/pipelines)\n\nCI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).\n\nSee the [test matrix](docs/developers/test_cases.md) for details.\n" }, { "path": "RELEASE.md", "content": "# Release Process\n\nThe Kubespray Project is released on an as-needed basis. The process is as follows:\n\n1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325)\n1. At least one of the [approvers](OWNERS_ALIASES) must approve this release\n1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1`\n1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables.\n1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.\n1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes\n1. (Only for major releases) An approver creates a release branch in the form `release-X.Y`\n1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request.\n1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request.\n1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.\n1. The release issue is closed\n1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`\n1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`\n1. Create/Update Issue for upgrading kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)\n\n## Major/minor releases and milestones\n\n* For major releases (vX.Y) Kubespray maintains one branch (`release-X.Y`). Minor releases (vX.Y.Z) are available only as tags.\n\n* Security patches and bugs might be backported.\n\n* Fixes for major releases (vX.Y) and minor releases (vX.Y.Z) are delivered\n via maintenance releases (vX.Y.Z) and assigned to the corresponding open\n [GitHub milestone](https://github.com/kubernetes-sigs/kubespray/milestones).\n That milestone remains open for the major/minor releases support lifetime,\n which ends once the milestone is closed. Then only a next major or minor release\n can be done.\n\n* Kubespray major and minor releases are bound to the given `kube_version` major/minor\n version numbers and other components' arbitrary versions, like etcd or network plugins.\n Older or newer component versions are not supported and not tested for the given\n release (even if included in the checksum variables, like `kubeadm_checksums`).\n\n* There is no unstable releases and no APIs, thus Kubespray doesn't follow\n [semver](https://semver.org/). Every version describes only a stable release.\n Breaking changes, if any introduced by changed defaults or non-contrib ansible roles'\n playbooks, shall be described in the release notes. Other breaking changes, if any in\n the contributed addons or bound versions of Kubernetes and other components, are\n considered out of Kubespray scope and are up to the components' teams to deal with and\n document.\n\n* Minor releases can change components' versions, but not the major `kube_version`.\n Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0\n is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`,\n then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1\n and *any* changes to other components, like etcd v4, or calico 1.2.3.\n And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively.\n\n## Release note creation\n\nYou can create a release note with:\n\n```shell\nexport GITHUB_TOKEN=\nexport ORG=kubernetes-sigs\nexport REPO=kubespray\nrelease-notes --start-sha --end-sha --dependencies=false --output=/tmp/kubespray-release-note --required-author=\"\"\n```\n\nIf the release note file(/tmp/kubespray-release-note) contains \"### Uncategorized\" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.).\nIt is necessary to put a valid label on each pull request and run the above release-notes command again to get a better release note\n\n## Container image creation\n\nThe container image `quay.io/kubespray/kubespray:vX.Y.Z` can be created from Dockerfile of the kubespray root directory:\n\n```shell\ncd kubespray/\nnerdctl build -t quay.io/kubespray/kubespray:vX.Y.Z .\nnerdctl push quay.io/kubespray/kubespray:vX.Y.Z\n```\n\nThe container image `quay.io/kubespray/vagrant:vX.Y.Z` can be created from build.sh of test-infra/vagrant-docker/:\n\n```shell\ncd kubespray/test-infra/vagrant-docker/\n./build vX.Y.Z\n```\n\nPlease note that the above operation requires the permission to push container images into quay.io/kubespray/.\nIf you don't have the permission, please ask it on the #kubespray-dev channel.\n" }, { "path": "SECURITY_CONTACTS", "content": "# Defined below are the security contacts for this repo.\n#\n# They are the contact point for the Product Security Committee to reach out\n# to for triaging and handling of incoming issues.\n#\n# The below names agree to abide by the\n# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)\n# and will be removed and replaced if they violate that agreement.\n#\n# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE\n# INSTRUCTIONS AT https://kubernetes.io/security/\nfloryut\nant31\nVannTen\nyankay\n" }, { "path": "Vagrantfile", "content": "# -*- mode: ruby -*-\n# # vi: set ft=ruby :\n\n# For help on using kubespray with vagrant, check out docs/developers/vagrant.md\n\nrequire 'fileutils'\nrequire 'ipaddr'\nrequire 'socket'\n\nVagrant.require_version \">= 2.0.0\"\n\nCONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb')\n\nFLATCAR_URL_TEMPLATE = \"https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json\"\n\n# Uniq disk UUID for libvirt\nDISK_UUID = Time.now.utc.to_i\n\nSUPPORTED_OS = {\n \"flatcar-stable\" => {box: \"flatcar-stable\", user: \"core\", box_url: FLATCAR_URL_TEMPLATE % [\"stable\"]},\n \"flatcar-beta\" => {box: \"flatcar-beta\", user: \"core\", box_url: FLATCAR_URL_TEMPLATE % [\"beta\"]},\n \"flatcar-alpha\" => {box: \"flatcar-alpha\", user: \"core\", box_url: FLATCAR_URL_TEMPLATE % [\"alpha\"]},\n \"flatcar-edge\" => {box: \"flatcar-edge\", user: \"core\", box_url: FLATCAR_URL_TEMPLATE % [\"edge\"]},\n \"ubuntu2004\" => {box: \"generic/ubuntu2004\", user: \"vagrant\"},\n \"ubuntu2204\" => {box: \"generic/ubuntu2204\", user: \"vagrant\"},\n \"ubuntu2404\" => {box: \"bento/ubuntu-24.04\", user: \"vagrant\"},\n \"centos8\" => {box: \"centos/8\", user: \"vagrant\"},\n \"centos8-bento\" => {box: \"bento/centos-8\", user: \"vagrant\"},\n \"almalinux8\" => {box: \"almalinux/8\", user: \"vagrant\"},\n \"almalinux8-bento\" => {box: \"bento/almalinux-8\", user: \"vagrant\"},\n \"almalinux9\" => {box: \"almalinux/9\", user: \"vagrant\"},\n \"rockylinux8\" => {box: \"rockylinux/8\", user: \"vagrant\"},\n \"rockylinux9\" => {box: \"rockylinux/9\", user: \"vagrant\"},\n \"fedora39\" => {box: \"fedora/39-cloud-base\", user: \"vagrant\"},\n \"fedora40\" => {box: \"fedora/40-cloud-base\", user: \"vagrant\"},\n \"fedora39-arm64\" => {box: \"bento/fedora-39-arm64\", user: \"vagrant\"},\n \"fedora40-arm64\" => {box: \"bento/fedora-40\", user: \"vagrant\"},\n \"fedora41\" => {box: \"fedora/41-cloud-base\", user: \"vagrant\"},\n \"fedora42\" => {box: \"fedora/42-cloud-base\", user: \"vagrant\"},\n \"fedora41-bento\" => {box: \"bento/fedora-41\", user: \"vagrant\"},\n \"opensuse\" => {box: \"opensuse/Leap-15.6.x86_64\", user: \"vagrant\"},\n \"opensuse-tumbleweed\" => {box: \"opensuse/Tumbleweed.x86_64\", user: \"vagrant\"},\n \"oraclelinux\" => {box: \"generic/oracle7\", user: \"vagrant\"},\n \"oraclelinux8\" => {box: \"generic/oracle8\", user: \"vagrant\"},\n \"rhel8\" => {box: \"generic/rhel8\", user: \"vagrant\"},\n \"debian11\" => {box: \"debian/bullseye64\", user: \"vagrant\"},\n \"debian12\" => {box: \"debian/bookworm64\", user: \"vagrant\"},\n}\n\nif File.exist?(CONFIG)\n require CONFIG\nend\n\n# Defaults for config options defined in CONFIG\n$num_instances ||= 3\n$instance_name_prefix ||= \"k8s\"\n$vm_gui ||= false\n$vm_memory ||= 2048\n$vm_cpus ||= 2\n$shared_folders ||= {}\n$forwarded_ports ||= {}\n$subnet ||= \"172.18.8\"\n$subnet_ipv6 ||= \"fd3c:b398:0698:0756\"\n$os ||= \"ubuntu2004\"\n$network_plugin ||= \"flannel\"\n$inventories ||= []\n# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni\n$multi_networking ||= \"False\"\n$download_run_once ||= \"True\"\n$download_force_cache ||= \"False\"\n# Modify those to have separate groups (for instance, to test separate etcd:)\n# first_control_plane = 1\n# first_etcd = 4\n# control_plane_instances = 3\n# etcd_instances = 3\n$first_node ||= 1\n$first_control_plane ||= 1\n$first_etcd ||= 1\n\n# The first three nodes are etcd servers\n$etcd_instances ||= [$num_instances, 3].min\n# The first two nodes are kube masters\n$control_plane_instances ||= [$num_instances, 2].min\n# All nodes are kube nodes\n$kube_node_instances ||= $num_instances - $first_node + 1\n\n# The following only works when using the libvirt provider\n$kube_node_instances_with_disks ||= false\n$kube_node_instances_with_disks_size ||= \"20G\"\n$kube_node_instances_with_disks_number ||= 2\n$override_disk_size ||= false\n$disk_size ||= \"20GB\"\n$local_path_provisioner_enabled ||= \"False\"\n$local_path_provisioner_claim_root ||= \"/opt/local-path-provisioner/\"\n$libvirt_nested ||= false\n# boolean or string (e.g. \"-vvv\")\n$ansible_verbosity ||= false\n$ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || \"\"\n\n$vagrant_dir ||= File.join(File.dirname(__FILE__), \".vagrant\")\n\n$playbook ||= \"cluster.yml\"\n$extra_vars ||= {}\n\nhost_vars = {}\n\ndef collect_networks(subnet, subnet_ipv6)\n Socket.getifaddrs.filter_map do |iface|\n next unless iface&.netmask&.ip_address && iface.addr\n\n is_ipv6 = iface.addr.ipv6?\n ip = IPAddr.new(iface.addr.ip_address.split('%').first)\n ip_test = is_ipv6 ? IPAddr.new(\"#{subnet_ipv6}::0\") : IPAddr.new(\"#{subnet}.0\")\n\n prefix = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')\n network = ip.mask(prefix)\n\n [IPAddr.new(\"#{network}/#{prefix}\"), ip_test]\n end\nend\n\ndef subnet_in_use?(network_ips)\n network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }\nend\n\nnetwork_ips = collect_networks($subnet, $subnet_ipv6)\n\nif subnet_in_use?(network_ips)\n puts \"Invalid subnet provided, subnet is already in use: #{$subnet}.0\"\n puts \"Subnets in use: #{network_ips.inspect}\"\n exit 1\nend\n\n# throw error if os is not supported\nif ! SUPPORTED_OS.key?($os)\n puts \"Unsupported OS: #{$os}\"\n puts \"Supported OS are: #{SUPPORTED_OS.keys.join(', ')}\"\n exit 1\nend\n\n$box = SUPPORTED_OS[$os][:box]\n\nif Vagrant.has_plugin?(\"vagrant-proxyconf\")\n $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || \"127.0.0.1,localhost\"\n (1..$num_instances).each do |i|\n $no_proxy += \",#{$subnet}.#{i+100}\"\n end\nend\n\nVagrant.configure(\"2\") do |config|\n\n config.vm.box = $box\n if SUPPORTED_OS[$os].has_key? :box_url\n config.vm.box_url = SUPPORTED_OS[$os][:box_url]\n end\n config.ssh.username = SUPPORTED_OS[$os][:user]\n\n # plugin conflict\n if Vagrant.has_plugin?(\"vagrant-vbguest\") then\n config.vbguest.auto_update = false\n end\n\n # always use Vagrants insecure key\n config.ssh.insert_key = false\n\n if ($override_disk_size)\n unless Vagrant.has_plugin?(\"vagrant-disksize\")\n system \"vagrant plugin install vagrant-disksize\"\n end\n config.disksize.size = $disk_size\n end\n\n (1..$num_instances).each do |i|\n config.vm.define vm_name = \"%s-%01d\" % [$instance_name_prefix, i] do |node|\n\n node.vm.hostname = vm_name\n\n if Vagrant.has_plugin?(\"vagrant-proxyconf\")\n node.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || \"\"\n node.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || \"\"\n node.proxy.no_proxy = $no_proxy\n end\n\n [\"vmware_fusion\", \"vmware_workstation\"].each do |vmware|\n node.vm.provider vmware do |v|\n v.vmx['memsize'] = $vm_memory\n v.vmx['numvcpus'] = $vm_cpus\n end\n end\n\n node.vm.provider :virtualbox do |vb|\n vb.memory = $vm_memory\n vb.cpus = $vm_cpus\n vb.gui = $vm_gui\n vb.linked_clone = true\n vb.customize [\"modifyvm\", :id, \"--vram\", \"8\"] # ubuntu defaults to 256 MB which is a waste of precious RAM\n vb.customize [\"modifyvm\", :id, \"--audio\", \"none\"]\n end\n\n node.vm.provider :libvirt do |lv|\n lv.nested = $libvirt_nested\n lv.cpu_mode = \"host-model\"\n lv.memory = $vm_memory\n lv.cpus = $vm_cpus\n lv.default_prefix = 'kubespray'\n # Fix kernel panic on fedora 28\n if $os == \"fedora\"\n lv.cpu_mode = \"host-passthrough\"\n end\n end\n\n if $kube_node_instances_with_disks\n # Libvirt\n driverletters = ('a'..'z').to_a\n node.vm.provider :libvirt do |lv|\n # always make /dev/sd{a/b/c} so that CI can ensure that\n # virtualbox and libvirt will have the same devices to use for OSDs\n (1..$kube_node_instances_with_disks_number).each do |d|\n lv.storage :file, :device => \"hd#{driverletters[d]}\", :path => \"disk-#{i}-#{d}-#{DISK_UUID}.disk\", :size => $kube_node_instances_with_disks_size, :bus => \"scsi\"\n end\n end\n node.vm.provider :virtualbox do |vb|\n # always make /dev/sd{a/b/c} so that CI can ensure that\n # virtualbox and libvirt will have the same devices to use for OSDs\n (1..$kube_node_instances_with_disks_number).each do |d|\n vb.customize ['createhd', '--filename', \"disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk\", '--size', $kube_node_instances_with_disks_size] # 10GB disk\n vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', \"disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk\", '--nonrotational', 'on', '--mtype', 'normal']\n end\n end\n end\n\n if $expose_docker_tcp\n node.vm.network \"forwarded_port\", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true\n end\n\n $forwarded_ports.each do |guest, host|\n node.vm.network \"forwarded_port\", guest: guest, host: host, auto_correct: true\n end\n\n if [\"rhel8\"].include? $os\n # Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot\n # be installed until the host is registered with a valid Red Hat support subscription\n node.vm.synced_folder \".\", \"/vagrant\", disabled: false\n $shared_folders.each do |src, dst|\n node.vm.synced_folder src, dst\n end\n else\n node.vm.synced_folder \".\", \"/vagrant\", disabled: false, type: \"rsync\", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']\n $shared_folders.each do |src, dst|\n node.vm.synced_folder src, dst, type: \"rsync\", rsync__args: ['--verbose', '--archive', '--delete', '-z']\n end\n end\n\n ip = \"#{$subnet}.#{i+100}\"\n ip6 = \"#{$subnet_ipv6}::#{i+100}\"\n node.vm.network :private_network,\n :ip => ip,\n :libvirt__guest_ipv6 => 'yes',\n :libvirt__ipv6_address => ip6,\n :libvirt__ipv6_prefix => \"64\",\n :libvirt__forward_mode => \"none\",\n :libvirt__dhcp_enabled => false\n\n # libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64)\n # add default route for detect ansible_default_ipv6\n # TODO: fix libvirt__ipv6 or use $subnet in shell\n config.vm.provision \"shell\", inline: \"ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true\"\n\n # Disable swap for each vm\n node.vm.provision \"shell\", inline: \"swapoff -a\"\n\n # ubuntu2004 and ubuntu2204 have IPv6 explicitly disabled. This undoes that.\n if [\"ubuntu2004\", \"ubuntu2204\"].include? $os\n node.vm.provision \"shell\", inline: \"rm -f /etc/modprobe.d/local.conf\"\n node.vm.provision \"shell\", inline: \"sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf\"\n end\n # Hack for fedora39/40 to get the IP address of the second interface\n if [\"fedora39\", \"fedora40\", \"fedora39-arm64\", \"fedora40-arm64\"].include? $os\n config.vm.provision \"shell\", inline: <<-SHELL\n nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d \"=\" -f2)/24\n nmcli conn modify 'Wired connection 2' ipv4.method manual\n service NetworkManager restart\n SHELL\n end\n\n\n # Rockylinux boxes needs UEFI\n if [\"rockylinux8\", \"rockylinux9\"].include? $os\n config.vm.provider \"libvirt\" do |domain|\n domain.loader = \"/usr/share/OVMF/x64/OVMF_CODE.fd\"\n end\n end\n\n # Disable firewalld on oraclelinux/redhat vms\n if [\"oraclelinux\",\"oraclelinux8\", \"rhel8\",\"rockylinux8\"].include? $os\n node.vm.provision \"shell\", inline: \"systemctl stop firewalld; systemctl disable firewalld\"\n end\n\n host_vars[vm_name] = {\n \"ip\": ip,\n \"flannel_interface\": \"eth1\",\n \"kube_network_plugin\": $network_plugin,\n \"kube_network_plugin_multus\": $multi_networking,\n \"download_run_once\": $download_run_once,\n \"download_localhost\": \"False\",\n \"download_cache_dir\": ENV['HOME'] + \"/kubespray_cache\",\n # Make kubespray cache even when download_run_once is false\n \"download_force_cache\": $download_force_cache,\n # Keeping the cache on the nodes can improve provisioning speed while debugging kubespray\n \"download_keep_remote_cache\": \"False\",\n \"docker_rpm_keepcache\": \"1\",\n # These two settings will put kubectl and admin.config in $inventory/artifacts\n \"kubeconfig_localhost\": \"True\",\n \"kubectl_localhost\": \"True\",\n \"local_path_provisioner_enabled\": \"#{$local_path_provisioner_enabled}\",\n \"local_path_provisioner_claim_root\": \"#{$local_path_provisioner_claim_root}\",\n \"ansible_ssh_user\": SUPPORTED_OS[$os][:user],\n \"ansible_ssh_private_key_file\": File.join(Dir.home, \".vagrant.d\", \"insecure_private_key\"),\n \"unsafe_show_logs\": \"True\"\n }\n\n # Only execute the Ansible provisioner once, when all the machines are up and ready.\n # And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh\n if i == $num_instances\n node.vm.provision \"ansible\" do |ansible|\n ansible.playbook = $playbook\n ansible.compatibility_mode = \"2.0\"\n ansible.verbose = $ansible_verbosity\n ansible.become = true\n ansible.limit = \"all,localhost\"\n ansible.host_key_checking = false\n ansible.raw_arguments = [\"--forks=#{$num_instances}\",\n \"--flush-cache\",\n \"-e ansible_become_pass=vagrant\"] +\n $inventories.map {|inv| [\"-i\", inv]}.flatten\n ansible.host_vars = host_vars\n ansible.extra_vars = $extra_vars\n if $ansible_tags != \"\"\n ansible.tags = [$ansible_tags]\n end\n ansible.groups = {\n \"etcd\" => [\"#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]\"],\n \"kube_control_plane\" => [\"#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]\"],\n \"kube_node\" => [\"#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]\"],\n \"k8s_cluster:children\" => [\"kube_control_plane\", \"kube_node\"],\n }\n end\n end\n\n end\n end\nend\n" }, { "path": "_config.yml", "content": "---\ntheme: jekyll-theme-slate\n" }, { "path": "ansible.cfg", "content": "[ssh_connection]\npipelining=True\nssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null\n#control_path = ~/.ssh/ansible-%%r@%%h:%%p\n[defaults]\n# https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)\nforce_valid_group_names = ignore\n\nhost_key_checking=False\ngathering = smart\nfact_caching = jsonfile\nfact_caching_connection = /tmp\nfact_caching_timeout = 86400\ntimeout = 300\nstdout_callback = default\ndisplay_skipped_hosts = no\nlibrary = ./library\ncallbacks_enabled = profile_tasks\nroles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles\ndeprecation_warnings=False\ninventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg\n[inventory]\nignore_patterns = artifacts, credentials\n" }, { "path": "cluster.yml", "content": "---\n- name: Install Kubernetes\n ansible.builtin.import_playbook: playbooks/cluster.yml\n" }, { "path": "code-of-conduct.md", "content": "# Kubernetes Community Code of Conduct\n\nPlease refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)\n" }, { "path": "contrib/aws_iam/kubernetes-master-policy.json", "content": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\"ec2:*\"],\n \"Resource\": [\"*\"]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\"elasticloadbalancing:*\"],\n \"Resource\": [\"*\"]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\"route53:*\"],\n \"Resource\": [\"*\"]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:*\",\n \"Resource\": [\n \"arn:aws:s3:::kubernetes-*\"\n ]\n }\n ]\n}\n" }, { "path": "contrib/aws_iam/kubernetes-master-role.json", "content": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": { \"Service\": \"ec2.amazonaws.com\"},\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" }, { "path": "contrib/aws_iam/kubernetes-minion-policy.json", "content": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:*\",\n \"Resource\": [\n \"arn:aws:s3:::kubernetes-*\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:Describe*\",\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:AttachVolume\",\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:DetachVolume\",\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\"route53:*\"],\n \"Resource\": [\"*\"]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ecr:GetAuthorizationToken\",\n \"ecr:BatchCheckLayerAvailability\",\n \"ecr:GetDownloadUrlForLayer\",\n \"ecr:GetRepositoryPolicy\",\n \"ecr:DescribeRepositories\",\n \"ecr:ListImages\",\n \"ecr:BatchGetImage\"\n ],\n \"Resource\": \"*\"\n }\n ]\n}\n" }, { "path": "contrib/aws_iam/kubernetes-minion-role.json", "content": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": { \"Service\": \"ec2.amazonaws.com\"},\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" }, { "path": "contrib/aws_inventory/kubespray-aws-inventory.py", "content": "#!/usr/bin/env python\n\nfrom __future__ import print_function\nimport boto3\nimport os\nimport argparse\nimport json\n\nclass SearchEC2Tags(object):\n\n def __init__(self):\n self.parse_args()\n if self.args.list:\n self.search_tags()\n if self.args.host:\n data = {}\n print(json.dumps(data, indent=2))\n\n def parse_args(self):\n\n ##Check if VPC_VISIBILITY is set, if not default to private\n if \"VPC_VISIBILITY\" in os.environ:\n self.vpc_visibility = os.environ['VPC_VISIBILITY']\n else:\n self.vpc_visibility = \"private\"\n\n ##Support --list and --host flags. We largely ignore the host one.\n parser = argparse.ArgumentParser()\n parser.add_argument('--list', action='store_true', default=False, help='List instances')\n parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance')\n self.args = parser.parse_args()\n\n def search_tags(self):\n hosts = {}\n hosts['_meta'] = { 'hostvars': {} }\n\n ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.\n for group in [\"kube_control_plane\", \"kube_node\", \"etcd\"]:\n hosts[group] = []\n tag_key = \"kubespray-role\"\n tag_value = [\"*\"+group+\"*\"]\n region = os.environ['AWS_REGION']\n\n ec2 = boto3.resource('ec2', region)\n filters = [{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}]\n cluster_name = os.getenv('CLUSTER_NAME')\n if cluster_name:\n filters.append({'Name': 'tag-key', 'Values': ['kubernetes.io/cluster/'+cluster_name]})\n instances = ec2.instances.filter(Filters=filters)\n for instance in instances:\n\n ##Suppose default vpc_visibility is private\n dns_name = instance.private_dns_name\n ansible_host = {\n 'ansible_ssh_host': instance.private_ip_address\n }\n\n ##Override when vpc_visibility actually is public\n if self.vpc_visibility == \"public\":\n dns_name = instance.public_dns_name\n ansible_host = {\n 'ansible_ssh_host': instance.public_ip_address\n }\n\n ##Set when instance actually has node_labels\n node_labels_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-labels', instance.tags))\n if node_labels_tag:\n ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ])\n\n ##Set when instance actually has node_taints\n node_taints_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-taints', instance.tags))\n if node_taints_tag:\n ansible_host['node_taints'] = list([ taint.strip() for taint in node_taints_tag[0]['Value'].split(',') ])\n\n hosts[group].append(dns_name)\n hosts['_meta']['hostvars'][dns_name] = ansible_host\n\n hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}\n print(json.dumps(hosts, sort_keys=True, indent=2))\n\nSearchEC2Tags()\n" }, { "path": "contrib/aws_inventory/requirements.txt", "content": "boto3 # Apache-2.0\n" }, { "path": "contrib/azurerm/.gitignore", "content": ".generated\n/inventory\n" }, { "path": "contrib/azurerm/README.md", "content": "# Kubernetes on Azure with Azure Resource Group Templates\n\nProvision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates)\n\n## Status\n\nThis will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified\nResource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course).\n\n## Requirements\n\n- [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest)\n- [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest)\n- Dedicated Resource Group created in the Azure Portal or through azure-cli\n\n## Configuration through group_vars/all\n\nYou have to modify at least two variables in group_vars/all. The one is the **cluster_name** variable, it must be globally\nunique due to some restrictions in Azure. The other one is the **ssh_public_keys** variable, it must be your ssh public\nkey to access your azure virtual machines. Most other variables should be self explanatory if you have some basic Kubernetes\nexperience.\n\n## Bastion host\n\nYou can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated\ntemplates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option\nalso removes all public IPs from all other VMs.\n\n## Generating and applying\n\nTo generate and apply the templates, call:\n\n```shell\n./apply-rg.sh \n```\n\nIf you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will\ntake care about creating/modifying whatever is needed.\n\n## Clearing a resource group\n\nIf you need to delete all resources from a resource group, simply call:\n\n```shell\n./clear-rg.sh \n```\n\n**WARNING** this really deletes everything from your resource group, including everything that was later created by you!\n\n## Installing Ansible and the dependencies\n\nInstall Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)\n\n## Generating an inventory for kubespray\n\nAfter you have applied the templates, you can generate an inventory with this call:\n\n```shell\n./generate-inventory.sh \n```\n\nIt will create the file ./inventory which can then be used with kubespray, e.g.:\n\n```shell\ncd kubespray-root-dir\nansible-playbook -i contrib/azurerm/inventory -u devops --become -e \"@inventory/sample/group_vars/all/all.yml\" cluster.yml\n```\n" }, { "path": "contrib/azurerm/apply-rg.sh", "content": "#!/usr/bin/env bash\n\nset -e\n\nAZURE_RESOURCE_GROUP=\"$1\"\n\nif [ \"$AZURE_RESOURCE_GROUP\" == \"\" ]; then\n echo \"AZURE_RESOURCE_GROUP is missing\"\n exit 1\nfi\n\nansible-playbook generate-templates.yml\n\naz deployment group create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP\naz deployment group create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP\naz deployment group create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP\naz deployment group create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP\naz deployment group create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP\naz deployment group create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP\n" }, { "path": "contrib/azurerm/clear-rg.sh", "content": "#!/usr/bin/env bash\n\nset -e\n\nAZURE_RESOURCE_GROUP=\"$1\"\n\nif [ \"$AZURE_RESOURCE_GROUP\" == \"\" ]; then\n echo \"AZURE_RESOURCE_GROUP is missing\"\n exit 1\nfi\n\nansible-playbook generate-templates.yml\n\naz group deployment create -g \"$AZURE_RESOURCE_GROUP\" --template-file ./.generated/clear-rg.json --mode Complete\n" }, { "path": "contrib/azurerm/generate-inventory.sh", "content": "#!/usr/bin/env bash\n\nset -e\n\nAZURE_RESOURCE_GROUP=\"$1\"\n\nif [ \"$AZURE_RESOURCE_GROUP\" == \"\" ]; then\n echo \"AZURE_RESOURCE_GROUP is missing\"\n exit 1\nfi\n# check if azure cli 2.0 exists else use azure cli 1.0\nif az &>/dev/null; then\n ansible-playbook generate-inventory_2.yml -e azure_resource_group=\"$AZURE_RESOURCE_GROUP\"\nelif azure &>/dev/null; then\n ansible-playbook generate-inventory.yml -e azure_resource_group=\"$AZURE_RESOURCE_GROUP\"\nelse\n echo \"Azure cli not found\"\nfi\n" }, { "path": "contrib/azurerm/generate-inventory.yml", "content": "---\n- name: Generate Azure inventory\n hosts: localhost\n gather_facts: false\n roles:\n - generate-inventory\n" }, { "path": "contrib/azurerm/generate-inventory_2.yml", "content": "---\n- name: Generate Azure inventory\n hosts: localhost\n gather_facts: false\n roles:\n - generate-inventory_2\n" }, { "path": "contrib/azurerm/generate-templates.yml", "content": "---\n- name: Generate Azure templates\n hosts: localhost\n gather_facts: false\n roles:\n - generate-templates\n" }, { "path": "contrib/azurerm/group_vars/all", "content": "\n# Due to some Azure limitations (ex:- Storage Account's name must be unique),\n# this name must be globally unique - it will be used as a prefix for azure components\ncluster_name: example\n\n# Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion\n# node that can be used to access the masters and minions\nuse_bastion: false\n\n# Set this to a preferred name that will be used as the first part of the dns name for your bastotion host. For example: k8s-bastion..cloudapp.azure.com.\n# This is convenient when exceptions have to be configured on a firewall to allow ssh to the given bastion host.\n# bastion_domain_prefix: k8s-bastion\n\nnumber_of_k8s_masters: 3\nnumber_of_k8s_nodes: 3\n\nmasters_vm_size: Standard_A2\nmasters_os_disk_size: 1000\n\nminions_vm_size: Standard_A2\nminions_os_disk_size: 1000\n\nadmin_username: devops\nadmin_password: changeme\n\n# MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines\nssh_public_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy\"\n\n# Disable using ssh using password. Change it to false to allow to connect to ssh by password\ndisablePasswordAuthentication: true\n\n# Azure CIDRs\nazure_vnet_cidr: 10.0.0.0/8\nazure_admin_cidr: 10.241.2.0/24\nazure_masters_cidr: 10.0.4.0/24\nazure_minions_cidr: 10.240.0.0/16\n\n# Azure loadbalancer port to use to access your cluster\nkube_apiserver_port: 6443\n\n# Azure Netwoking and storage naming to use with inventory/all.yml\n#azure_virtual_network_name: KubeVNET\n#azure_subnet_admin_name: ad-subnet\n#azure_subnet_masters_name: master-subnet\n#azure_subnet_minions_name: minion-subnet\n#azure_route_table_name: routetable\n#azure_security_group_name: secgroup\n\n# Storage types available are: \"Standard_LRS\",\"Premium_LRS\"\n#azure_storage_account_type: Standard_LRS\n" }, { "path": "contrib/azurerm/roles/generate-inventory/tasks/main.yml", "content": "---\n\n- name: Query Azure VMs\n command: azure vm list-ip-address --json {{ azure_resource_group }}\n register: vm_list_cmd\n\n- name: Set vm_list\n set_fact:\n vm_list: \"{{ vm_list_cmd.stdout }}\"\n\n- name: Generate inventory\n template:\n src: inventory.j2\n dest: \"{{ playbook_dir }}/inventory\"\n mode: \"0644\"\n" }, { "path": "contrib/azurerm/roles/generate-inventory/templates/inventory.j2", "content": "\n{% for vm in vm_list %}\n{% if not use_bastion or vm.name == 'bastion' %}\n{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}\n{% else %}\n{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}\n{% endif %}\n{% endfor %}\n\n[kube_control_plane]\n{% for vm in vm_list %}\n{% if 'kube_control_plane' in vm.tags.roles %}\n{{ vm.name }}\n{% endif %}\n{% endfor %}\n\n[etcd]\n{% for vm in vm_list %}\n{% if 'etcd' in vm.tags.roles %}\n{{ vm.name }}\n{% endif %}\n{% endfor %}\n\n[kube_node]\n{% for vm in vm_list %}\n{% if 'kube_node' in vm.tags.roles %}\n{{ vm.name }}\n{% endif %}\n{% endfor %}\n\n[k8s_cluster:children]\nkube_node\nkube_control_plane\n" }, { "path": "contrib/azurerm/roles/generate-inventory_2/tasks/main.yml", "content": "---\n\n- name: Query Azure VMs IPs\n command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }}\n register: vm_ip_list_cmd\n\n- name: Query Azure VMs Roles\n command: az vm list -o json --resource-group {{ azure_resource_group }}\n register: vm_list_cmd\n\n- name: Query Azure Load Balancer Public IP\n command: az network public-ip show -o json -g {{ azure_resource_group }} -n kubernetes-api-pubip\n register: lb_pubip_cmd\n\n- name: Set VM IP, roles lists and load balancer public IP\n set_fact:\n vm_ip_list: \"{{ vm_ip_list_cmd.stdout }}\"\n vm_roles_list: \"{{ vm_list_cmd.stdout }}\"\n lb_pubip: \"{{ lb_pubip_cmd.stdout }}\"\n\n- name: Generate inventory\n template:\n src: inventory.j2\n dest: \"{{ playbook_dir }}/inventory\"\n mode: \"0644\"\n\n- name: Generate Load Balancer variables\n template:\n src: loadbalancer_vars.j2\n dest: \"{{ playbook_dir }}/loadbalancer_vars.yml\"\n mode: \"0644\"\n" }, { "path": "contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2", "content": "\n{% for vm in vm_ip_list %}\n{% if not use_bastion or vm.virtualMachine.name == 'bastion' %}\n{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }}\n{% else %}\n{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.privateIpAddresses[0] }}\n{% endif %}\n{% endfor %}\n\n[kube_control_plane]\n{% for vm in vm_roles_list %}\n{% if 'kube_control_plane' in vm.tags.roles %}\n{{ vm.name }}\n{% endif %}\n{% endfor %}\n\n[etcd]\n{% for vm in vm_roles_list %}\n{% if 'etcd' in vm.tags.roles %}\n{{ vm.name }}\n{% endif %}\n{% endfor %}\n\n[kube_node]\n{% for vm in vm_roles_list %}\n{% if 'kube_node' in vm.tags.roles %}\n{{ vm.name }}\n{% endif %}\n{% endfor %}\n\n[k8s_cluster:children]\nkube_node\nkube_control_plane\n" }, { "path": "contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2", "content": "## External LB example config\napiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }}\nloadbalancer_apiserver:\n address: {{ lb_pubip.ipAddress }}\n port: 6443\n\n## Internal loadbalancers for apiservers\nloadbalancer_apiserver_localhost: false\n" }, { "path": "contrib/azurerm/roles/generate-templates/defaults/main.yml", "content": "---\napiVersion: \"2015-06-15\"\n\nvirtualNetworkName: \"{{ azure_virtual_network_name | default('KubeVNET') }}\"\n\nsubnetAdminName: \"{{ azure_subnet_admin_name | default('ad-subnet') }}\"\nsubnetMastersName: \"{{ azure_subnet_masters_name | default('master-subnet') }}\"\nsubnetMinionsName: \"{{ azure_subnet_minions_name | default('minion-subnet') }}\"\n\nrouteTableName: \"{{ azure_route_table_name | default('routetable') }}\"\nsecurityGroupName: \"{{ azure_security_group_name | default('secgroup') }}\"\n\nnameSuffix: \"{{ cluster_name }}\"\n\navailabilitySetMasters: \"master-avs\"\navailabilitySetMinions: \"minion-avs\"\n\nfaultDomainCount: 3\nupdateDomainCount: 10\n\nbastionVmSize: Standard_A0\nbastionVMName: bastion\nbastionIPAddressName: bastion-pubip\n\ndisablePasswordAuthentication: true\n\nsshKeyPath: \"/home/{{ admin_username }}/.ssh/authorized_keys\"\n\nimageReference:\n publisher: \"OpenLogic\"\n offer: \"CentOS\"\n sku: \"7.5\"\n version: \"latest\"\nimageReferenceJson: \"{{ imageReference | to_json }}\"\n\nstorageAccountName: \"sa{{ nameSuffix | replace('-', '') }}\"\nstorageAccountType: \"{{ azure_storage_account_type | default('Standard_LRS') }}\"\n" }, { "path": "contrib/azurerm/roles/generate-templates/tasks/main.yml", "content": "---\n- name: Set base_dir\n set_fact:\n base_dir: \"{{ playbook_dir }}/.generated/\"\n\n- name: Create base_dir\n file:\n path: \"{{ base_dir }}\"\n state: directory\n recurse: true\n mode: \"0755\"\n\n- name: Store json files in base_dir\n template:\n src: \"{{ item }}\"\n dest: \"{{ base_dir }}/{{ item }}\"\n mode: \"0644\"\n with_items:\n - network.json\n - storage.json\n - availability-sets.json\n - bastion.json\n - masters.json\n - minions.json\n - clear-rg.json\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/availability-sets.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {\n },\n \"variables\": {\n },\n \"resources\": [\n {\n \"type\": \"Microsoft.Compute/availabilitySets\",\n \"name\": \"{{availabilitySetMasters}}\",\n \"apiVersion\": \"{{apiVersion}}\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"PlatformFaultDomainCount\": \"{{faultDomainCount}}\",\n \"PlatformUpdateDomainCount\": \"{{updateDomainCount}}\"\n }\n },\n {\n \"type\": \"Microsoft.Compute/availabilitySets\",\n \"name\": \"{{availabilitySetMinions}}\",\n \"apiVersion\": \"{{apiVersion}}\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"PlatformFaultDomainCount\": \"{{faultDomainCount}}\",\n \"PlatformUpdateDomainCount\": \"{{updateDomainCount}}\"\n }\n }\n ]\n}\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/bastion.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {\n },\n \"variables\": {\n \"vnetID\": \"[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]\",\n \"subnetAdminRef\": \"[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]\"\n },\n \"resources\": [\n {% if use_bastion %}\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/publicIPAddresses\",\n \"name\": \"{{bastionIPAddressName}}\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"publicIPAllocationMethod\": \"Static\",\n \"dnsSettings\": {\n {% if bastion_domain_prefix %}\n \"domainNameLabel\": \"{{ bastion_domain_prefix }}\"\n {% endif %}\n }\n }\n },\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/networkInterfaces\",\n \"name\": \"{{bastionVMName}}-nic\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n \"[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]\"\n ],\n \"properties\": {\n \"ipConfigurations\": [\n {\n \"name\": \"BastionIpConfig\",\n \"properties\": {\n \"privateIPAllocationMethod\": \"Dynamic\",\n \"publicIPAddress\": {\n \"id\": \"[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]\"\n },\n \"subnet\": {\n \"id\": \"[variables('subnetAdminRef')]\"\n }\n }\n }\n ]\n }\n },\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Compute/virtualMachines\",\n \"name\": \"{{bastionVMName}}\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n \"[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]\"\n ],\n \"tags\": {\n \"roles\": \"bastion\"\n },\n \"properties\": {\n \"hardwareProfile\": {\n \"vmSize\": \"{{bastionVmSize}}\"\n },\n \"osProfile\": {\n \"computerName\": \"{{bastionVMName}}\",\n \"adminUsername\": \"{{admin_username}}\",\n \"adminPassword\": \"{{admin_password}}\",\n \"linuxConfiguration\": {\n \"disablePasswordAuthentication\": \"true\",\n \"ssh\": {\n \"publicKeys\": [\n {% for key in ssh_public_keys %}\n {\n \"path\": \"{{sshKeyPath}}\",\n \"keyData\": \"{{key}}\"\n }{% if loop.index < ssh_public_keys | length %},{% endif %}\n {% endfor %}\n ]\n }\n }\n },\n \"storageProfile\": {\n \"imageReference\": {{imageReferenceJson}},\n \"osDisk\": {\n \"name\": \"osdisk\",\n \"vhd\": {\n \"uri\": \"[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]\"\n },\n \"caching\": \"ReadWrite\",\n \"createOption\": \"FromImage\"\n }\n },\n \"networkProfile\": {\n \"networkInterfaces\": [\n {\n \"id\": \"[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]\"\n }\n ]\n }\n }\n }\n {% endif %}\n ]\n}\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/clear-rg.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {},\n \"variables\": {},\n \"resources\": [],\n \"outputs\": {}\n}\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/masters.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {\n },\n \"variables\": {\n \"lbDomainName\": \"{{nameSuffix}}-api\",\n \"lbPublicIPAddressName\": \"kubernetes-api-pubip\",\n \"lbPublicIPAddressType\": \"Static\",\n \"lbPublicIPAddressID\": \"[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]\",\n \"lbName\": \"kubernetes-api\",\n \"lbID\": \"[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]\",\n\n \"vnetID\": \"[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]\",\n \"kubeMastersSubnetRef\": \"[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]\"\n },\n \"resources\": [\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/publicIPAddresses\",\n \"name\": \"[variables('lbPublicIPAddressName')]\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"publicIPAllocationMethod\": \"[variables('lbPublicIPAddressType')]\",\n \"dnsSettings\": {\n \"domainNameLabel\": \"[variables('lbDomainName')]\"\n }\n }\n },\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"name\": \"[variables('lbName')]\",\n \"type\": \"Microsoft.Network/loadBalancers\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n \"[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]\"\n ],\n \"properties\": {\n \"frontendIPConfigurations\": [\n {\n \"name\": \"kube-api-frontend\",\n \"properties\": {\n \"publicIPAddress\": {\n \"id\": \"[variables('lbPublicIPAddressID')]\"\n }\n }\n }\n ],\n \"backendAddressPools\": [\n {\n \"name\": \"kube-api-backend\"\n }\n ],\n \"loadBalancingRules\": [\n {\n \"name\": \"kube-api\",\n \"properties\": {\n \"frontendIPConfiguration\": {\n \"id\": \"[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]\"\n },\n \"backendAddressPool\": {\n \"id\": \"[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]\"\n },\n \"protocol\": \"tcp\",\n \"frontendPort\": \"{{kube_apiserver_port}}\",\n \"backendPort\": \"{{kube_apiserver_port}}\",\n \"enableFloatingIP\": false,\n \"idleTimeoutInMinutes\": 5,\n \"probe\": {\n \"id\": \"[concat(variables('lbID'), '/probes/kube-api')]\"\n }\n }\n }\n ],\n \"probes\": [\n {\n \"name\": \"kube-api\",\n \"properties\": {\n \"protocol\": \"tcp\",\n \"port\": \"{{kube_apiserver_port}}\",\n \"intervalInSeconds\": 5,\n \"numberOfProbes\": 2\n }\n }\n ]\n }\n },\n {% for i in range(number_of_k8s_masters) %}\n {% if not use_bastion %}\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/publicIPAddresses\",\n \"name\": \"master-{{i}}-pubip\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"publicIPAllocationMethod\": \"Static\"\n }\n },\n {% endif %}\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/networkInterfaces\",\n \"name\": \"master-{{i}}-nic\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n {% if not use_bastion %}\n \"[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]\",\n {% endif %}\n \"[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]\"\n ],\n \"properties\": {\n \"ipConfigurations\": [\n {\n \"name\": \"MastersIpConfig\",\n \"properties\": {\n \"privateIPAllocationMethod\": \"Dynamic\",\n {% if not use_bastion %}\n \"publicIPAddress\": {\n \"id\": \"[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]\"\n },\n {% endif %}\n \"subnet\": {\n \"id\": \"[variables('kubeMastersSubnetRef')]\"\n },\n\t \"loadBalancerBackendAddressPools\": [\n {\n \"id\": \"[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]\"\n }\n ]\n }\n }\n ],\n \"networkSecurityGroup\": {\n \"id\": \"[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]\"\n },\n \"enableIPForwarding\": true\n }\n },\n {\n \"type\": \"Microsoft.Compute/virtualMachines\",\n \"name\": \"master-{{i}}\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n \"[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]\"\n ],\n \"tags\": {\n \"roles\": \"kube_control_plane,etcd\"\n },\n \"apiVersion\": \"{{apiVersion}}\",\n \"properties\": {\n \"availabilitySet\": {\n \"id\": \"[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]\"\n },\n \"hardwareProfile\": {\n \"vmSize\": \"{{masters_vm_size}}\"\n },\n \"osProfile\": {\n \"computerName\": \"master-{{i}}\",\n \"adminUsername\": \"{{admin_username}}\",\n \"adminPassword\": \"{{admin_password}}\",\n \"linuxConfiguration\": {\n \"disablePasswordAuthentication\": \"{{disablePasswordAuthentication}}\",\n \"ssh\": {\n \"publicKeys\": [\n {% for key in ssh_public_keys %}\n {\n \"path\": \"{{sshKeyPath}}\",\n \"keyData\": \"{{key}}\"\n }{% if loop.index < ssh_public_keys | length %},{% endif %}\n {% endfor %}\n ]\n }\n }\n },\n \"storageProfile\": {\n \"imageReference\": {{imageReferenceJson}},\n \"osDisk\": {\n \"name\": \"ma{{nameSuffix}}{{i}}\",\n \"vhd\": {\n \"uri\": \"[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]\"\n },\n \"caching\": \"ReadWrite\",\n \"createOption\": \"FromImage\",\n \"diskSizeGB\": \"{{masters_os_disk_size}}\"\n }\n },\n \"networkProfile\": {\n \"networkInterfaces\": [\n {\n \"id\": \"[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]\"\n }\n ]\n }\n }\n } {% if not loop.last %},{% endif %}\n {% endfor %}\n ]\n}\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/minions.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {\n },\n \"variables\": {\n \"vnetID\": \"[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]\",\n \"kubeMinionsSubnetRef\": \"[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]\"\n },\n \"resources\": [\n {% for i in range(number_of_k8s_nodes) %}\n {% if not use_bastion %}\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/publicIPAddresses\",\n \"name\": \"minion-{{i}}-pubip\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"publicIPAllocationMethod\": \"Static\"\n }\n },\n {% endif %}\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/networkInterfaces\",\n \"name\": \"minion-{{i}}-nic\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n {% if not use_bastion %}\n \"[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]\"\n {% endif %}\n ],\n \"properties\": {\n \"ipConfigurations\": [\n {\n \"name\": \"MinionsIpConfig\",\n \"properties\": {\n \"privateIPAllocationMethod\": \"Dynamic\",\n {% if not use_bastion %}\n \"publicIPAddress\": {\n \"id\": \"[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]\"\n },\n {% endif %}\n \"subnet\": {\n \"id\": \"[variables('kubeMinionsSubnetRef')]\"\n }\n }\n }\n ],\n \"networkSecurityGroup\": {\n \"id\": \"[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]\"\n },\n \"enableIPForwarding\": true\n }\n },\n {\n \"type\": \"Microsoft.Compute/virtualMachines\",\n \"name\": \"minion-{{i}}\",\n \"location\": \"[resourceGroup().location]\",\n \"dependsOn\": [\n \"[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]\"\n ],\n \"tags\": {\n \"roles\": \"kube_node\"\n },\n \"apiVersion\": \"{{apiVersion}}\",\n \"properties\": {\n \"availabilitySet\": {\n \"id\": \"[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]\"\n },\n \"hardwareProfile\": {\n \"vmSize\": \"{{minions_vm_size}}\"\n },\n \"osProfile\": {\n \"computerName\": \"minion-{{i}}\",\n \"adminUsername\": \"{{admin_username}}\",\n \"adminPassword\": \"{{admin_password}}\",\n \"linuxConfiguration\": {\n \"disablePasswordAuthentication\": \"{{disablePasswordAuthentication}}\",\n \"ssh\": {\n \"publicKeys\": [\n {% for key in ssh_public_keys %}\n {\n \"path\": \"{{sshKeyPath}}\",\n \"keyData\": \"{{key}}\"\n }{% if loop.index < ssh_public_keys | length %},{% endif %}\n {% endfor %}\n ]\n }\n }\n },\n \"storageProfile\": {\n \"imageReference\": {{imageReferenceJson}},\n \"osDisk\": {\n \"name\": \"mi{{nameSuffix}}{{i}}\",\n \"vhd\": {\n \"uri\": \"[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]\"\n },\n \"caching\": \"ReadWrite\",\n \"createOption\": \"FromImage\",\n \"diskSizeGB\": \"{{minions_os_disk_size}}\"\n }\n },\n \"networkProfile\": {\n \"networkInterfaces\": [\n {\n \"id\": \"[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]\"\n }\n ]\n }\n }\n } {% if not loop.last %},{% endif %}\n {% endfor %}\n ]\n}\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/network.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {\n },\n \"variables\": {\n },\n \"resources\": [\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/routeTables\",\n \"name\": \"{{routeTableName}}\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"routes\": [\n ]\n }\n },\n {\n \"type\": \"Microsoft.Network/virtualNetworks\",\n \"name\": \"{{virtualNetworkName}}\",\n \"location\": \"[resourceGroup().location]\",\n \"apiVersion\": \"{{apiVersion}}\",\n \"dependsOn\": [\n \"[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]\"\n ],\n \"properties\": {\n \"addressSpace\": {\n \"addressPrefixes\": [\n \"{{azure_vnet_cidr}}\"\n ]\n },\n \"subnets\": [\n {\n \"name\": \"{{subnetMastersName}}\",\n \"properties\": {\n \"addressPrefix\": \"{{azure_masters_cidr}}\",\n \"routeTable\": {\n \"id\": \"[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]\"\n }\n }\n },\n {\n \"name\": \"{{subnetMinionsName}}\",\n \"properties\": {\n \"addressPrefix\": \"{{azure_minions_cidr}}\",\n \"routeTable\": {\n \"id\": \"[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]\"\n }\n }\n }\n {% if use_bastion %}\n ,{\n \"name\": \"{{subnetAdminName}}\",\n \"properties\": {\n \"addressPrefix\": \"{{azure_admin_cidr}}\",\n \"routeTable\": {\n \"id\": \"[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]\"\n }\n }\n }\n {% endif %}\n ]\n }\n },\n {\n \"apiVersion\": \"{{apiVersion}}\",\n \"type\": \"Microsoft.Network/networkSecurityGroups\",\n \"name\": \"{{securityGroupName}}\",\n \"location\": \"[resourceGroup().location]\",\n \"properties\": {\n \"securityRules\": [\n {% if not use_bastion %}\n {\n \"name\": \"ssh\",\n \"properties\": {\n \"description\": \"Allow SSH\",\n \"protocol\": \"Tcp\",\n \"sourcePortRange\": \"*\",\n \"destinationPortRange\": \"22\",\n \"sourceAddressPrefix\": \"Internet\",\n \"destinationAddressPrefix\": \"*\",\n \"access\": \"Allow\",\n \"priority\": 100,\n \"direction\": \"Inbound\"\n }\n },\n {% endif %}\n {\n \"name\": \"kube-api\",\n \"properties\": {\n \"description\": \"Allow secure kube-api\",\n \"protocol\": \"Tcp\",\n \"sourcePortRange\": \"*\",\n \"destinationPortRange\": \"{{kube_apiserver_port}}\",\n \"sourceAddressPrefix\": \"Internet\",\n \"destinationAddressPrefix\": \"*\",\n \"access\": \"Allow\",\n \"priority\": 101,\n \"direction\": \"Inbound\"\n }\n }\n ]\n },\n \"resources\": [],\n \"dependsOn\": []\n }\n ]\n}\n" }, { "path": "contrib/azurerm/roles/generate-templates/templates/storage.json", "content": "{\n \"$schema\": \"https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#\",\n \"contentVersion\": \"1.0.0.0\",\n \"parameters\": {\n },\n \"variables\": {\n },\n \"resources\": [\n {\n \"type\": \"Microsoft.Storage/storageAccounts\",\n \"name\": \"{{storageAccountName}}\",\n \"location\": \"[resourceGroup().location]\",\n \"apiVersion\": \"{{apiVersion}}\",\n \"properties\": {\n \"accountType\": \"{{storageAccountType}}\"\n }\n }\n ]\n}\n" }, { "path": "contrib/collection.sh", "content": "#!/bin/bash -eux\n# Install collection from source assuming dependencies are present.\n# Run in SemaphoreUI this bash script can install Kubespray from the repo\nNAMESPACE=kubernetes_sigs\nCOLLECTION=kubespray\nMY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //')\n\nansible-galaxy collection build --force --output-path .\nansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz\n" }, { "path": "contrib/offline/README.md", "content": "# Offline deployment\n\n## manage-offline-container-images.sh\n\nContainer image collecting script for offline deployment\n\nThis script has two features:\n(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE\n environment variable to get images from a file (e.g. temp/images.list after running the\n ./generate_list.sh script).\n(2) Deploy local container registry and register the container images to the registry.\n\nStep(1) should be done online site as a preparation, then we bring the gotten images\nto the target offline environment. if images are from a private registry,\nyou need to set `PRIVATE_REGISTRY` environment variable.\nThen we will run step(2) for registering the images to local registry, or to an existing\nregistry set by the `DESTINATION_REGISTRY` environment variable. By default, the local registry\nwill run on port 5000. This can be changed with the `REGISTRY_PORT` environment variable\n\nStep(1) can be operated with:\n\n```shell\nmanage-offline-container-images.sh create\n```\n\nStep(2) can be operated with:\n\n```shell\nmanage-offline-container-images.sh register\n```\n\n## generate_list.sh\n\nThis script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.\n\nRun this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,\nall downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.\n\n```shell\n./generate_list.sh\ntree temp\ntemp\n├── files.list\n├── files.list.template\n├── images.list\n└── images.list.template\n0 directories, 5 files\n```\n\nIn some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars,\nthen run `./generate_list.sh -i [inventory_file]` to update file.list and images.list.\n\n## manage-offline-files.sh\n\nThis script will download all files according to `temp/files.list` and run nginx container to provide offline file download.\n\nStep(1) generate `files.list`\n\n```shell\n./generate_list.sh\n```\n\nStep(2) download files and run nginx container\n\n```shell\n./manage-offline-files.sh\n```\n\nwhen nginx container is running, it can be accessed through .\n\n## upload2artifactory.py\n\nAfter the steps above, this script can recursively upload each file under a directory to a generic repository in Artifactory.\n\nEnvironment Variables:\n\n- USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.\n- TOKEN -- Generate this with 'Set Me Up' in your user.\n- BASE_URL -- The URL including the repository name.\n\nStep(3) (optional) upload files to Artifactory\n\n```shell\ncd kubespray/contrib/offline/offline-files\nexport USERNAME=admin\nexport TOKEN=...\nexport BASE_URL=https://artifactory.example.com/artifactory/a-generic-repo/\n./upload2artifactory.py\n```\n" }, { "path": "contrib/offline/docker-daemon.json", "content": "{ \"insecure-registries\":[\"HOSTNAME:5000\"] }\n" }, { "path": "contrib/offline/generate_list.sh", "content": "#!/bin/bash\nset -eo pipefail\n\nCURRENT_DIR=$(cd $(dirname $0); pwd)\nTEMP_DIR=\"${CURRENT_DIR}/temp\"\nREPO_ROOT_DIR=\"${CURRENT_DIR%/contrib/offline}\"\n\n: ${DOWNLOAD_YML:=\"roles/kubespray_defaults/defaults/main/download.yml\"}\n\nmkdir -p ${TEMP_DIR}\n\n# generate all download files url template\ngrep 'download_url:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \\\n | sed 's/^.*_url: //g;s/\\\"//g' > ${TEMP_DIR}/files.list.template\n\n# generate all images list template\nsed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \\\n | sed -n \"s/repo: //p;s/tag: //p\" | tr -d ' ' \\\n | sed 'N;s#\\n# #g' | tr ' ' ':' | sed 's/\\\"//g' > ${TEMP_DIR}/images.list.template\n\n# add kube-* images to images list template\n# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml\n# doesn't contain those images. That is reason why here needs to put those images into the\n# list separately.\nKUBE_IMAGES=\"kube-apiserver kube-controller-manager kube-scheduler kube-proxy\"\nfor i in $KUBE_IMAGES; do\n echo \"{{ kube_image_repo }}/$i:v{{ kube_version }}\" >> ${TEMP_DIR}/images.list.template\ndone\n\n# run ansible to expand templates\n/bin/cp ${CURRENT_DIR}/generate_list.yml ${REPO_ROOT_DIR}\n\n(cd ${REPO_ROOT_DIR} && ansible-playbook $* generate_list.yml && /bin/rm generate_list.yml) || exit 1\n" }, { "path": "contrib/offline/generate_list.yml", "content": "---\n- name: Collect container images for offline deployment\n hosts: localhost\n become: false\n\n roles:\n # Just load default variables from roles.\n - role: kubespray_defaults\n when: false\n - role: download\n when: false\n\n tasks:\n # Generate files.list and images.list files from templates.\n - name: Collect container images for offline deployment\n template:\n src: ./contrib/offline/temp/{{ item }}.list.template\n dest: ./contrib/offline/temp/{{ item }}.list\n mode: \"0644\"\n with_items:\n - files\n - images\n" }, { "path": "contrib/offline/manage-offline-container-images.sh", "content": "#!/usr/bin/env bash\n\nOPTION=$1\nCURRENT_DIR=$(cd $(dirname $0); pwd)\nTEMP_DIR=\"${CURRENT_DIR}/temp\"\n\nIMAGE_TAR_FILE=\"${CURRENT_DIR}/container-images.tar.gz\"\nIMAGE_DIR=\"${CURRENT_DIR}/container-images\"\nIMAGE_LIST=\"${IMAGE_DIR}/container-images.txt\"\nRETRY_COUNT=5\n\nfunction create_container_image_tar() {\n\tset -e\n\n\tif [ -z \"${IMAGES_FROM_FILE}\" ]; then\n\t\techo \"Getting images from current \\\"$(kubectl config current-context)\\\"\"\n\n\t\tIMAGES=$(mktemp --suffix=-images)\n\t\ttrap 'rm -f \"${IMAGES}\"' EXIT\n\n\t\tkubectl describe cronjobs,jobs,pods --all-namespaces | grep \" Image:\" | awk '{print $2}' | sort | uniq > \"${IMAGES}\"\n\t\t# NOTE: etcd and pause cannot be seen as pods.\n\t\tkubectl cluster-info dump | grep -E \"quay.io/coreos/etcd:|registry.k8s.io/pause:\" | sed s@\\\"@@g >> \"${IMAGES}\"\n\telse\n\t\techo \"Getting images from file \\\"${IMAGES_FROM_FILE}\\\"\"\n\t\tif [ ! -f \"${IMAGES_FROM_FILE}\" ]; then\n\t\t\techo \"${IMAGES_FROM_FILE} is not a file\"\n\t\t\texit 1\n\t\tfi\n\t\tIMAGES=$(realpath $IMAGES_FROM_FILE)\n\tfi\n\n\trm -f ${IMAGE_TAR_FILE}\n\trm -rf ${IMAGE_DIR}\n\tmkdir ${IMAGE_DIR}\n\tcd ${IMAGE_DIR}\n\n\tsudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest\n\tsudo ${runtime} save -o registry-latest.tar registry:latest\n\n\twhile read -r image\n\tdo\n\t\tFILE_NAME=\"$(echo ${image} | sed s@\"/\"@\"-\"@g | sed s/\":\"/\"-\"/g | sed -E 's/\\@.*//g')\".tar\n\t\tset +e\n\t\tfor step in $(seq 1 ${RETRY_COUNT})\n\t\tdo\n\t\t\tsudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}\n\t\t\tif [ $? -eq 0 ]; then\n\t\t\t\tbreak\n\t\t\tfi\n\t\t\techo \"Failed to pull ${image} at step ${step}\"\n\t\t\tif [ ${step} -eq ${RETRY_COUNT} ]; then\n\t\t\t\texit 1\n\t\t\tfi\n\t\tdone\n\t\tset -e\n\t\tsudo ${runtime} save -o ${FILE_NAME} ${image}\n\n\t\t# NOTE: Here removes the following repo parts from each image\n\t\t# so that these parts will be replaced with Kubespray.\n\t\t# - kube_image_repo: \"registry.k8s.io\"\n\t\t# - gcr_image_repo: \"gcr.io\"\n\t\t# - ghcr_image_repo: \"ghcr.io\"\n\t\t# - docker_image_repo: \"docker.io\"\n\t\t# - quay_image_repo: \"quay.io\"\n\t\tFIRST_PART=$(echo ${image} | awk -F\"/\" '{print $1}')\n\t\tif [ \"${FIRST_PART}\" = \"registry.k8s.io\" ] ||\n\t\t [ \"${FIRST_PART}\" = \"gcr.io\" ] ||\n\t\t [ \"${FIRST_PART}\" = \"ghcr.io\" ] ||\n\t\t [ \"${FIRST_PART}\" = \"docker.io\" ] ||\n\t\t [ \"${FIRST_PART}\" = \"quay.io\" ] ||\n\t\t [ \"${FIRST_PART}\" = \"${PRIVATE_REGISTRY}\" ]; then\n\t\t\timage=$(echo ${image} | sed s@\"${FIRST_PART}/\"@@ | sed -E 's/\\@.*/\\n/g')\n\t\tfi\n\t\techo \"${FILE_NAME} ${image}\" >> ${IMAGE_LIST}\n\tdone < \"${IMAGES}\"\n\n\tcd ..\n\tsudo chown ${USER} ${IMAGE_DIR}/*\n\ttar -zcvf ${IMAGE_TAR_FILE} ./container-images\n\trm -rf ${IMAGE_DIR}\n\n\techo \"\"\n\techo \"${IMAGE_TAR_FILE} is created to contain your container images.\"\n\techo \"Please keep this file and bring it to your offline environment.\"\n}\n\nfunction register_container_images() {\n\tcreate_registry=false\n\tREGISTRY_PORT=${REGISTRY_PORT:-\"5000\"}\n\n\tif [ -z \"${DESTINATION_REGISTRY}\" ]; then\n\t\techo \"DESTINATION_REGISTRY not set, will create local registry\"\n\t\tcreate_registry=true\n\t\tDESTINATION_REGISTRY=\"$(hostname):${REGISTRY_PORT}\"\n\tfi\n\techo \"Images will be pushed to ${DESTINATION_REGISTRY}\"\n\n\tif [ ! -f ${IMAGE_TAR_FILE} ]; then\n\t\techo \"${IMAGE_TAR_FILE} should exist.\"\n\t\texit 1\n\tfi\n\tif [ ! -d ${TEMP_DIR} ]; then\n\t\tmkdir ${TEMP_DIR}\n\tfi\n\n\t# To avoid \"http: server gave http response to https client\" error.\n\tif [ -d /etc/docker/ ]; then\n\t\tset -e\n\t\t# Ubuntu18.04, RHEL7/CentOS7\n\t\tcp ${CURRENT_DIR}/docker-daemon.json ${TEMP_DIR}/docker-daemon.json\n\t\tsed -i s@\"HOSTNAME\"@\"$(hostname)\"@ ${TEMP_DIR}/docker-daemon.json\n\t\tsudo cp ${TEMP_DIR}/docker-daemon.json /etc/docker/daemon.json\n\telif [ -d /etc/containers/ ]; then\n\t\tset -e\n\t\t# RHEL8/CentOS8\n\t\tcp ${CURRENT_DIR}/registries.conf ${TEMP_DIR}/registries.conf\n\t\tsed -i s@\"HOSTNAME\"@\"$(hostname)\"@ ${TEMP_DIR}/registries.conf\n\t\tsudo cp ${TEMP_DIR}/registries.conf /etc/containers/registries.conf\n elif [ \"$(uname)\" == \"Darwin\" ]; then\n echo \"This is a Mac, no configuration changes are required\"\n\telse\n\t\techo \"runtime package(docker-ce, podman, nerctl, etc.) should be installed\"\n\t\texit 1\n\tfi\n\n\ttar -zxvf ${IMAGE_TAR_FILE}\n\n\tif ${create_registry}; then\n\t\tsudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar\n\t\tset +e\n\n\t\tsudo ${runtime} container inspect registry >/dev/null 2>&1\n\t\tif [ $? -ne 0 ]; then\n\t\t\tsudo ${runtime} run --restart=always -d -p \"${REGISTRY_PORT}\":\"${REGISTRY_PORT}\" --name registry registry:latest\n\t\tfi\n\t\tset -e\n\tfi\n\n\twhile read -r line; do\n\t\tfile_name=$(echo ${line} | awk '{print $1}')\n\t\traw_image=$(echo ${line} | awk '{print $2}')\n\t\tnew_image=\"${DESTINATION_REGISTRY}/${raw_image}\"\n\t\tload_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1)\n\t\torg_image=$(echo \"${load_image}\" | awk '{print $3}')\n\t\t# special case for tags containing the digest when using docker or podman as the container runtime\n\t\tif [ \"${org_image}\" == \"ID:\" ]; then\n\t\t org_image=$(echo \"${load_image}\" | awk '{print $4}')\n\t\tfi\n\t\timage_id=$(sudo ${runtime} image inspect --format \"{{.Id}}\" \"${org_image}\")\n\t\tif [ -z \"${file_name}\" ]; then\n\t\t\techo \"Failed to get file_name for line ${line}\"\n\t\t\texit 1\n\t\tfi\n\t\tif [ -z \"${raw_image}\" ]; then\n\t\t\techo \"Failed to get raw_image for line ${line}\"\n\t\t\texit 1\n\t\tfi\n\t\tif [ -z \"${org_image}\" ]; then\n\t\t\techo \"Failed to get org_image for line ${line}\"\n\t\t\texit 1\n\t\tfi\n\t\tif [ -z \"${image_id}\" ]; then\n\t\t\techo \"Failed to get image_id for file ${file_name}\"\n\t\t\texit 1\n\t\tfi\n\t\tsudo ${runtime} load -i ${IMAGE_DIR}/${file_name}\n\t\tsudo ${runtime} tag ${image_id} ${new_image}\n\t\tsudo ${runtime} push ${new_image}\n\tdone <<< \"$(cat ${IMAGE_LIST})\"\n\n\techo \"Succeeded to register container images to local registry.\"\n\techo \"Please specify \\\"${DESTINATION_REGISTRY}\\\" for the following options in your inventry:\"\n\techo \"- kube_image_repo\"\n\techo \"- gcr_image_repo\"\n\techo \"- docker_image_repo\"\n\techo \"- quay_image_repo\"\n}\n\n# get runtime command\nif command -v nerdctl 1>/dev/null 2>&1; then\n runtime=\"nerdctl\"\nelif command -v podman 1>/dev/null 2>&1; then\n runtime=\"podman\"\nelif command -v docker 1>/dev/null 2>&1; then\n runtime=\"docker\"\nelse\n echo \"No supported container runtime found\"\n exit 1\nfi\n\nif [ \"${OPTION}\" == \"create\" ]; then\n\tcreate_container_image_tar\nelif [ \"${OPTION}\" == \"register\" ]; then\n\tregister_container_images\nelse\n\techo \"This script has two features:\"\n\techo \"(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE\"\n\techo \" environment variable to get images from a file (e.g. temp/images.list after running the\"\n\techo \" ./generate_list.sh script).\"\n\techo \"(2) Deploy local container registry and register the container images to the registry.\"\n\techo \"\"\n\techo \"Step(1) should be done online site as a preparation, then we bring\"\n\techo \"the gotten images to the target offline environment. if images are from\"\n\techo \"a private registry, you need to set PRIVATE_REGISTRY environment variable.\"\n\techo \"Then we will run step(2) for registering the images to local registry, or to an existing\"\n\techo \"registry set by the DESTINATION_REGISTRY environment variable. By default, the local registry\"\n\techo \"will run on port 5000. This can be changed with the REGISTRY_PORT environment variable\"\n\techo \"\"\n\techo \"${IMAGE_TAR_FILE} is created to contain your container images.\"\n\techo \"Please keep this file and bring it to your offline environment.\"\n\techo \"\"\n\techo \"Step(1) can be operated with:\"\n\techo \" $ ./manage-offline-container-images.sh create\"\n\techo \"\"\n\techo \"Step(2) can be operated with:\"\n\techo \" $ ./manage-offline-container-images.sh register\"\n\techo \"\"\n\techo \"Please specify 'create' or 'register'.\"\n\techo \"\"\n\texit 1\nfi\n" }, { "path": "contrib/offline/manage-offline-files.sh", "content": "#!/bin/bash\n\nCURRENT_DIR=$( dirname \"$(readlink -f \"$0\")\" )\nOFFLINE_FILES_DIR_NAME=\"offline-files\"\nOFFLINE_FILES_DIR=\"${CURRENT_DIR}/${OFFLINE_FILES_DIR_NAME}\"\nOFFLINE_FILES_ARCHIVE=\"${CURRENT_DIR}/offline-files.tar.gz\"\nFILES_LIST=${FILES_LIST:-\"${CURRENT_DIR}/temp/files.list\"}\nNGINX_PORT=8080\n\n# download files\nif [ ! -f \"${FILES_LIST}\" ]; then\n echo \"${FILES_LIST} should exist, run ./generate_list.sh first.\"\n exit 1\nfi\n\nrm -rf \"${OFFLINE_FILES_DIR}\"\nrm \"${OFFLINE_FILES_ARCHIVE}\"\nmkdir \"${OFFLINE_FILES_DIR}\"\n\nwhile read -r url; do\n if ! wget -x -P \"${OFFLINE_FILES_DIR}\" \"${url}\"; then\n exit 1\n fi\ndone < \"${FILES_LIST}\"\n\ntar -czvf \"${OFFLINE_FILES_ARCHIVE}\" \"${OFFLINE_FILES_DIR_NAME}\"\n\n[ -n \"$NO_HTTP_SERVER\" ] && echo \"skip to run nginx\" && exit 0\n\n# run nginx container server\nif command -v nerdctl 1>/dev/null 2>&1; then\n runtime=\"nerdctl\"\nelif command -v podman 1>/dev/null 2>&1; then\n runtime=\"podman\"\nelif command -v docker 1>/dev/null 2>&1; then\n runtime=\"docker\"\nelse\n echo \"No supported container runtime found\"\n exit 1\nfi\n\nsudo \"${runtime}\" container inspect nginx >/dev/null 2>&1\nif [ $? -ne 0 ]; then\n sudo --preserve-env=http_proxy,https_proxy,no_proxy \"${runtime}\" run \\\n --restart=always -d -p ${NGINX_PORT}:80 \\\n --volume \"${OFFLINE_FILES_DIR}\":/usr/share/nginx/html/download \\\n --volume \"${CURRENT_DIR}\"/nginx.conf:/etc/nginx/nginx.conf \\\n --name nginx nginx:alpine\nfi\n" }, { "path": "contrib/offline/nginx.conf", "content": "user nginx;\nworker_processes auto;\nerror_log /var/log/nginx/error.log;\npid /run/nginx.pid;\ninclude /usr/share/nginx/modules/*.conf;\nevents {\n worker_connections 1024;\n}\nhttp {\n log_format main '$remote_addr - $remote_user [$time_local] \"$request\" '\n '$status $body_bytes_sent \"$http_referer\" '\n '\"$http_user_agent\" \"$http_x_forwarded_for\"';\n access_log /var/log/nginx/access.log main;\n sendfile on;\n tcp_nopush on;\n tcp_nodelay on;\n keepalive_timeout 65;\n types_hash_max_size 2048;\n default_type application/octet-stream;\n include /etc/nginx/conf.d/*.conf;\n server {\n listen 80 default_server;\n listen [::]:80 default_server;\n server_name _;\n include /etc/nginx/default.d/*.conf;\n location / {\n root /usr/share/nginx/html/download;\n autoindex on;\n autoindex_exact_size off;\n autoindex_localtime on;\n }\n error_page 404 /404.html;\n location = /40x.html {\n }\n error_page 500 502 503 504 /50x.html;\n location = /50x.html {\n }\n }\n}\n" }, { "path": "contrib/offline/registries.conf", "content": "[registries.search]\nregistries = ['registry.access.redhat.com', 'registry.redhat.io', 'docker.io']\n\n[registries.insecure]\nregistries = ['HOSTNAME:5000']\n\n[registries.block]\nregistries = []\n" }, { "path": "contrib/offline/upload2artifactory.py", "content": "#!/usr/bin/env python3\n\"\"\"This is a helper script to manage-offline-files.sh.\n\nAfter running manage-offline-files.sh, you can run upload2artifactory.py\nto recursively upload each file to a generic repository in Artifactory.\n\nThis script recurses the current working directory and is intended to\nbe started from 'kubespray/contrib/offline/offline-files'\n\nEnvironment Variables:\n USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'.\n TOKEN -- Generate this with 'Set Me Up' in your user.\n BASE_URL -- The URL including the repository name.\n\n\"\"\"\nimport os\nimport urllib.request\nimport base64\n\n\ndef upload_file(file_path, destination_url, username, token):\n \"\"\"Helper function to upload a single file\"\"\"\n try:\n with open(file_path, 'rb') as f:\n file_data = f.read()\n\n request = urllib.request.Request(destination_url, data=file_data, method='PUT') # NOQA\n auth_header = base64.b64encode(f\"{username}:{token}\".encode()).decode()\n request.add_header(\"Authorization\", f\"Basic {auth_header}\")\n\n with urllib.request.urlopen(request) as response:\n if response.status in [200, 201]:\n print(f\"Success: Uploaded {file_path}\")\n else:\n print(f\"Failed: {response.status} {response.read().decode('utf-8')}\") # NOQA\n except urllib.error.HTTPError as e:\n print(f\"HTTPError: {e.code} {e.reason} for {file_path}\")\n except urllib.error.URLError as e:\n print(f\"URLError: {e.reason} for {file_path}\")\n except OSError as e:\n print(f\"OSError: {e.strerror} for {file_path}\")\n\n\ndef upload_files(base_url, username, token):\n \"\"\" Recurse current dir and upload each file using urllib.request \"\"\"\n for root, _, files in os.walk(os.getcwd()):\n for file in files:\n file_path = os.path.join(root, file)\n relative_path = os.path.relpath(file_path, os.getcwd())\n destination_url = f\"{base_url}/{relative_path}\"\n\n print(f\"Uploading {file_path} to {destination_url}\")\n upload_file(file_path, destination_url, username, token)\n\n\nif __name__ == \"__main__\":\n a_user = os.getenv(\"USERNAME\")\n a_token = os.getenv(\"TOKEN\")\n a_url = os.getenv(\"BASE_URL\")\n if not a_user or not a_token or not a_url:\n print(\n \"Error: Environment variables USERNAME, TOKEN, and BASE_URL must be set.\" # NOQA\n )\n exit()\n upload_files(a_url, a_user, a_token)\n" }, { "path": "contrib/os-services/os-services.yml", "content": "---\n- name: Disable firewalld/ufw\n hosts: all\n roles:\n - { role: prepare }\n" }, { "path": "contrib/os-services/roles/prepare/defaults/main.yml", "content": "---\ndisable_service_firewall: false\n" }, { "path": "contrib/os-services/roles/prepare/tasks/main.yml", "content": "---\n- name: Disable firewalld and ufw\n when:\n - disable_service_firewall is defined and disable_service_firewall\n block:\n - name: List services\n service_facts:\n\n - name: Disable service firewalld\n systemd_service:\n name: firewalld\n state: stopped\n enabled: false\n when:\n \"'firewalld.service' in services and services['firewalld.service'].status != 'not-found'\"\n\n - name: Disable service ufw\n systemd_service:\n name: ufw\n state: stopped\n enabled: false\n when:\n \"'ufw.service' in services and services['ufw.service'].status != 'not-found'\"\n" }, { "path": "contrib/terraform/aws/.gitignore", "content": "*.tfstate*\n.terraform.lock.hcl\n.terraform\n" }, { "path": "contrib/terraform/aws/README.md", "content": "# Kubernetes on AWS with Terraform\n\n## Overview\n\nThis project will create:\n\n- VPC with Public and Private Subnets in # Availability Zones\n- Bastion Hosts and NAT Gateways in the Public Subnet\n- A dynamic number of masters, etcd, and worker nodes in the Private Subnet\n - even distributed over the # of Availability Zones\n- AWS ELB in the Public Subnet for accessing the Kubernetes API from the internet\n\n## Requirements\n\n- Terraform 0.12.0 or newer\n\n## How to Use\n\n- Export the variables for your AWS credentials or edit `credentials.tfvars`:\n\n```commandline\nexport TF_VAR_AWS_ACCESS_KEY_ID=\"www\"\nexport TF_VAR_AWS_SECRET_ACCESS_KEY =\"xxx\"\nexport TF_VAR_AWS_SSH_KEY_NAME=\"yyy\"\nexport TF_VAR_AWS_DEFAULT_REGION=\"zzz\"\n```\n\n- Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use Ubuntu 18.04 LTS (Bionic) as base image. If you want to change this behaviour, see note \"Using other distrib than Ubuntu\" below.\n- Create an AWS EC2 SSH Key\n- Run with `terraform apply --var-file=\"credentials.tfvars\"` or `terraform apply` depending if you exported your AWS credentials\n\nExample:\n\n```commandline\nterraform apply -var-file=credentials.tfvars\n```\n\n- Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory`\n- Ansible will automatically generate an ssh config file for your bastion hosts. To connect to hosts with ssh using bastion host use generated `ssh-bastion.conf`. Ansible automatically detects bastion and changes `ssh_args`\n\n```commandline\nssh -F ./ssh-bastion.conf user@$ip\n```\n\n- Once the infrastructure is created, you can run the kubespray playbooks and supply inventory/hosts with the `-i` flag.\n\nExample (this one assumes you are using Ubuntu)\n\n```commandline\nansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_user=ubuntu -b --become-user=root --flush-cache\n```\n\n## Using other distrib than Ubuntu***\n\nTo leverage a Linux distribution other than Ubuntu 18.04 (Bionic) LTS for your Terraform configurations, you can adjust the AMI search filters within the 'data \"aws_ami\" \"distro\"' block by utilizing variables in your `terraform.tfvars` file. This approach ensures a flexible configuration that adapts to various Linux distributions without directly modifying the core Terraform files.\n\n### Example Usages\n\n- **Debian Jessie**: To configure the usage of Debian Jessie, insert the subsequent lines into your `terraform.tfvars`:\n\n ```hcl\n ami_name_pattern = \"debian-jessie-amd64-hvm-*\"\n ami_owners = [\"379101102735\"]\n ```\n\n- **Ubuntu 16.04**: To utilize Ubuntu 16.04 instead, apply the following configuration in your `terraform.tfvars`:\n\n ```hcl\n ami_name_pattern = \"ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-*\"\n ami_owners = [\"099720109477\"]\n ```\n\n- **Centos 7**: For employing Centos 7, incorporate these lines into your `terraform.tfvars`:\n\n ```hcl\n ami_name_pattern = \"dcos-centos7-*\"\n ami_owners = [\"688023202711\"]\n ```\n\n## Connecting to Kubernetes\n\nYou can use the following set of commands to get the kubeconfig file from your newly created cluster. Before running the commands, make sure you are in the project's root folder.\n\n```commandline\n# Get the controller's IP address.\nCONTROLLER_HOST_NAME=$(cat ./inventory/hosts | grep \"\\[kube_control_plane\\]\" -A 1 | tail -n 1)\nCONTROLLER_IP=$(cat ./inventory/hosts | grep $CONTROLLER_HOST_NAME | grep ansible_host | cut -d'=' -f2)\n\n# Get the hostname of the load balancer.\nLB_HOST=$(cat inventory/hosts | grep apiserver_loadbalancer_domain_name | cut -d'\"' -f2)\n\n# Get the controller's SSH fingerprint.\nssh-keygen -R $CONTROLLER_IP > /dev/null 2>&1\nssh-keyscan -H $CONTROLLER_IP >> ~/.ssh/known_hosts 2>/dev/null\n\n# Get the kubeconfig from the controller.\nmkdir -p ~/.kube\nssh -F ssh-bastion.conf centos@$CONTROLLER_IP \"sudo chmod 644 /etc/kubernetes/admin.conf\"\nscp -F ssh-bastion.conf centos@$CONTROLLER_IP:/etc/kubernetes/admin.conf ~/.kube/config\nsed -i \"s^server:.*^server: https://$LB_HOST:6443^\" ~/.kube/config\nkubectl get nodes\n```\n\n## Troubleshooting\n\n### Remaining AWS IAM Instance Profile\n\nIf the cluster was destroyed without using Terraform it is possible that\nthe AWS IAM Instance Profiles still remain. To delete them you can use\nthe `AWS CLI` with the following command:\n\n```commandline\naws iam delete-instance-profile --region --instance-profile-name \n```\n\n### Ansible Inventory doesn't get created\n\nIt could happen that Terraform doesn't create an Ansible Inventory file automatically. If this is the case copy the output after `inventory=` and create a file named `hosts`in the directory `inventory` and paste the inventory into the file.\n\n## Architecture\n\nPictured is an AWS Infrastructure created with this Terraform project distributed over two Availability Zones.\n\n![AWS Infrastructure with Terraform ](docs/aws_kubespray.png)\n" }, { "path": "contrib/terraform/aws/create-infrastructure.tf", "content": "terraform {\n required_version = \">= 0.12.0\"\n required_providers {\n aws = {\n source = \"hashicorp/aws\"\n version = \"~> 5.0\"\n }\n }\n}\n\nprovider \"aws\" {\n access_key = var.AWS_ACCESS_KEY_ID\n secret_key = var.AWS_SECRET_ACCESS_KEY\n region = var.AWS_DEFAULT_REGION\n}\n\ndata \"aws_availability_zones\" \"available\" {}\n\n/*\n* Calling modules who create the initial AWS VPC / AWS ELB\n* and AWS IAM Roles for Kubernetes Deployment\n*/\n\nmodule \"aws-vpc\" {\n source = \"./modules/vpc\"\n\n aws_cluster_name = var.aws_cluster_name\n aws_vpc_cidr_block = var.aws_vpc_cidr_block\n aws_avail_zones = data.aws_availability_zones.available.names\n aws_cidr_subnets_private = var.aws_cidr_subnets_private\n aws_cidr_subnets_public = var.aws_cidr_subnets_public\n default_tags = var.default_tags\n}\n\nmodule \"aws-nlb\" {\n source = \"./modules/nlb\"\n\n aws_cluster_name = var.aws_cluster_name\n aws_vpc_id = module.aws-vpc.aws_vpc_id\n aws_avail_zones = data.aws_availability_zones.available.names\n aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public\n aws_nlb_api_port = var.aws_nlb_api_port\n k8s_secure_api_port = var.k8s_secure_api_port\n default_tags = var.default_tags\n}\n\nmodule \"aws-iam\" {\n source = \"./modules/iam\"\n\n aws_cluster_name = var.aws_cluster_name\n}\n\n/*\n* Create Bastion Instances in AWS\n*\n*/\n\nresource \"aws_instance\" \"bastion-server\" {\n ami = data.aws_ami.distro.id\n instance_type = var.aws_bastion_size\n count = var.aws_bastion_num\n associate_public_ip_address = true\n subnet_id = element(module.aws-vpc.aws_subnet_ids_public, count.index)\n\n vpc_security_group_ids = module.aws-vpc.aws_security_group\n\n key_name = var.AWS_SSH_KEY_NAME\n\n tags = merge(var.default_tags, tomap({\n Name = \"kubernetes-${var.aws_cluster_name}-bastion-${count.index}\"\n Cluster = var.aws_cluster_name\n Role = \"bastion-${var.aws_cluster_name}-${count.index}\"\n }))\n}\n\n/*\n* Create K8s Master and worker nodes and etcd instances\n*\n*/\n\nresource \"aws_instance\" \"k8s-master\" {\n ami = data.aws_ami.distro.id\n instance_type = var.aws_kube_master_size\n\n count = var.aws_kube_master_num\n\n subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)\n\n vpc_security_group_ids = module.aws-vpc.aws_security_group\n\n root_block_device {\n volume_size = var.aws_kube_master_disk_size\n }\n\n iam_instance_profile = module.aws-iam.kube_control_plane-profile\n key_name = var.AWS_SSH_KEY_NAME\n\n tags = merge(var.default_tags, tomap({\n Name = \"kubernetes-${var.aws_cluster_name}-master${count.index}\"\n \"kubernetes.io/cluster/${var.aws_cluster_name}\" = \"member\"\n Role = \"master\"\n }))\n}\n\nresource \"aws_lb_target_group_attachment\" \"tg-attach_master_nodes\" {\n count = var.aws_kube_master_num\n target_group_arn = module.aws-nlb.aws_nlb_api_tg_arn\n target_id = element(aws_instance.k8s-master.*.private_ip, count.index)\n}\n\nresource \"aws_instance\" \"k8s-etcd\" {\n ami = data.aws_ami.distro.id\n instance_type = var.aws_etcd_size\n\n count = var.aws_etcd_num\n\n subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)\n\n vpc_security_group_ids = module.aws-vpc.aws_security_group\n\n root_block_device {\n volume_size = var.aws_etcd_disk_size\n }\n\n key_name = var.AWS_SSH_KEY_NAME\n\n tags = merge(var.default_tags, tomap({\n Name = \"kubernetes-${var.aws_cluster_name}-etcd${count.index}\"\n \"kubernetes.io/cluster/${var.aws_cluster_name}\" = \"member\"\n Role = \"etcd\"\n }))\n}\n\nresource \"aws_instance\" \"k8s-worker\" {\n ami = data.aws_ami.distro.id\n instance_type = var.aws_kube_worker_size\n\n count = var.aws_kube_worker_num\n\n subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)\n\n vpc_security_group_ids = module.aws-vpc.aws_security_group\n\n root_block_device {\n volume_size = var.aws_kube_worker_disk_size\n }\n\n iam_instance_profile = module.aws-iam.kube-worker-profile\n key_name = var.AWS_SSH_KEY_NAME\n\n tags = merge(var.default_tags, tomap({\n Name = \"kubernetes-${var.aws_cluster_name}-worker${count.index}\"\n \"kubernetes.io/cluster/${var.aws_cluster_name}\" = \"member\"\n Role = \"worker\"\n }))\n}\n\n/*\n* Create Kubespray Inventory File\n*\n*/\ndata \"template_file\" \"inventory\" {\n template = file(\"${path.module}/templates/inventory.tpl\")\n\n vars = {\n public_ip_address_bastion = join(\"\\n\", formatlist(\"bastion ansible_host=%s\", aws_instance.bastion-server.*.public_ip))\n connection_strings_master = join(\"\\n\", formatlist(\"%s ansible_host=%s\", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))\n connection_strings_node = join(\"\\n\", formatlist(\"%s ansible_host=%s\", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))\n list_master = join(\"\\n\", aws_instance.k8s-master.*.private_dns)\n list_node = join(\"\\n\", aws_instance.k8s-worker.*.private_dns)\n connection_strings_etcd = join(\"\\n\", formatlist(\"%s ansible_host=%s\", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))\n list_etcd = join(\"\\n\", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))\n nlb_api_fqdn = \"apiserver_loadbalancer_domain_name=\\\"${module.aws-nlb.aws_nlb_api_fqdn}\\\"\"\n }\n}\n\nresource \"null_resource\" \"inventories\" {\n provisioner \"local-exec\" {\n command = \"echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}\"\n }\n\n triggers = {\n template = data.template_file.inventory.rendered\n }\n}\n" }, { "path": "contrib/terraform/aws/credentials.tfvars.example", "content": "#AWS Access Key\nAWS_ACCESS_KEY_ID = \"\"\n#AWS Secret Key\nAWS_SECRET_ACCESS_KEY = \"\"\n#EC2 SSH Key Name\nAWS_SSH_KEY_NAME = \"\"\n#AWS Region\nAWS_DEFAULT_REGION = \"eu-central-1\"\n" }, { "path": "contrib/terraform/aws/modules/iam/main.tf", "content": "#Add AWS Roles for Kubernetes\n\nresource \"aws_iam_role\" \"kube_control_plane\" {\n name = \"kubernetes-${var.aws_cluster_name}-master\"\n\n assume_role_policy = < 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))\n}\n\noutput \"aws_nlb_api_fqdn\" {\n value = \"${module.aws-nlb.aws_nlb_api_fqdn}:${var.aws_nlb_api_port}\"\n}\n\noutput \"inventory\" {\n value = data.template_file.inventory.rendered\n}\n\noutput \"default_tags\" {\n value = var.default_tags\n}\n" }, { "path": "contrib/terraform/aws/sample-inventory/cluster.tfvars", "content": "#Global Vars\naws_cluster_name = \"devtest\"\n\n#VPC Vars\naws_vpc_cidr_block = \"10.250.192.0/18\"\n\naws_cidr_subnets_private = [\"10.250.192.0/20\", \"10.250.208.0/20\"]\n\naws_cidr_subnets_public = [\"10.250.224.0/20\", \"10.250.240.0/20\"]\n\n#Bastion Host\naws_bastion_num = 1\n\naws_bastion_size = \"t2.medium\"\n\n#Kubernetes Cluster\n\naws_kube_master_num = 3\n\naws_kube_master_size = \"t2.medium\"\n\naws_kube_master_disk_size = 50\n\naws_etcd_num = 3\n\naws_etcd_size = \"t2.medium\"\n\naws_etcd_disk_size = 50\n\naws_kube_worker_num = 4\n\naws_kube_worker_size = \"t2.medium\"\n\naws_kube_worker_disk_size = 50\n\n#Settings AWS NLB\n\naws_nlb_api_port = 6443\n\nk8s_secure_api_port = 6443\n\ndefault_tags = {\n # Env = \"devtest\" # Product = \"kubernetes\"\n}\n\ninventory_file = \"../../../inventory/hosts\"\n\n## Credentials\n#AWS Access Key\nAWS_ACCESS_KEY_ID = \"\"\n\n#AWS Secret Key\nAWS_SECRET_ACCESS_KEY = \"\"\n\n#EC2 SSH Key Name\nAWS_SSH_KEY_NAME = \"\"\n\n#AWS Region\nAWS_DEFAULT_REGION = \"eu-central-1\"\n" }, { "path": "contrib/terraform/aws/templates/inventory.tpl", "content": "[all]\n${connection_strings_master}\n${connection_strings_node}\n${connection_strings_etcd}\n${public_ip_address_bastion}\n\n[bastion]\n${public_ip_address_bastion}\n\n[kube_control_plane]\n${list_master}\n\n[kube_node]\n${list_node}\n\n[etcd]\n${list_etcd}\n\n[calico_rr]\n\n[k8s_cluster:children]\nkube_node\nkube_control_plane\ncalico_rr\n\n[k8s_cluster:vars]\n${nlb_api_fqdn}\n" }, { "path": "contrib/terraform/aws/terraform.tfvars", "content": "#Global Vars\naws_cluster_name = \"devtest\"\n\n#VPC Vars\naws_vpc_cidr_block = \"10.250.192.0/18\"\naws_cidr_subnets_private = [\"10.250.192.0/20\", \"10.250.208.0/20\"]\naws_cidr_subnets_public = [\"10.250.224.0/20\", \"10.250.240.0/20\"]\n\n# single AZ deployment\n#aws_cidr_subnets_private = [\"10.250.192.0/20\"]\n#aws_cidr_subnets_public = [\"10.250.224.0/20\"]\n\n# 3+ AZ deployment\n#aws_cidr_subnets_private = [\"10.250.192.0/24\",\"10.250.193.0/24\",\"10.250.194.0/24\",\"10.250.195.0/24\"]\n#aws_cidr_subnets_public = [\"10.250.224.0/24\",\"10.250.225.0/24\",\"10.250.226.0/24\",\"10.250.227.0/24\"]\n\n#Bastion Host\naws_bastion_num = 1\naws_bastion_size = \"t3.small\"\n\n#Kubernetes Cluster\naws_kube_master_num = 3\naws_kube_master_size = \"t3.medium\"\naws_kube_master_disk_size = 50\n\naws_etcd_num = 0\naws_etcd_size = \"t3.medium\"\naws_etcd_disk_size = 50\n\naws_kube_worker_num = 4\naws_kube_worker_size = \"t3.medium\"\naws_kube_worker_disk_size = 50\n\n#Settings AWS ELB\naws_nlb_api_port = 6443\nk8s_secure_api_port = 6443\n\ndefault_tags = {\n # Env = \"devtest\"\n # Product = \"kubernetes\"\n}\n\ninventory_file = \"../../../inventory/hosts\"\n" }, { "path": "contrib/terraform/aws/terraform.tfvars.example", "content": "#Global Vars\naws_cluster_name = \"devtest\"\n\n#VPC Vars\naws_vpc_cidr_block = \"10.250.192.0/18\"\naws_cidr_subnets_private = [\"10.250.192.0/20\",\"10.250.208.0/20\"]\naws_cidr_subnets_public = [\"10.250.224.0/20\",\"10.250.240.0/20\"]\naws_avail_zones = [\"eu-central-1a\",\"eu-central-1b\"]\n\n#Bastion Host\naws_bastion_num = 1\naws_bastion_size = \"t3.small\"\n\n#Kubernetes Cluster\naws_kube_master_num = 3\naws_kube_master_size = \"t3.medium\"\naws_kube_master_disk_size = 50\n\naws_etcd_num = 3\naws_etcd_size = \"t3.medium\"\naws_etcd_disk_size = 50\n\naws_kube_worker_num = 4\naws_kube_worker_size = \"t3.medium\"\naws_kube_worker_disk_size = 50\n\n#Settings AWS ELB\naws_nlb_api_port = 6443\nk8s_secure_api_port = 6443\n\ndefault_tags = { }\n\ninventory_file = \"../../../inventory/hosts\"\n" }, { "path": "contrib/terraform/aws/variables.tf", "content": "variable \"AWS_ACCESS_KEY_ID\" {\n description = \"AWS Access Key\"\n}\n\nvariable \"AWS_SECRET_ACCESS_KEY\" {\n description = \"AWS Secret Key\"\n}\n\nvariable \"AWS_SSH_KEY_NAME\" {\n description = \"Name of the SSH keypair to use in AWS.\"\n}\n\nvariable \"AWS_DEFAULT_REGION\" {\n description = \"AWS Region\"\n}\n\n//General Cluster Settings\n\nvariable \"aws_cluster_name\" {\n description = \"Name of AWS Cluster\"\n}\n\nvariable \"ami_name_pattern\" {\n description = \"The name pattern to use for AMI lookup\"\n type = string\n default = \"debian-10-amd64-*\"\n}\n\nvariable \"ami_virtualization_type\" {\n description = \"The virtualization type to use for AMI lookup\"\n type = string\n default = \"hvm\"\n}\n\nvariable \"ami_owners\" {\n description = \"The owners to use for AMI lookup\"\n type = list(string)\n default = [\"136693071363\"]\n}\n\ndata \"aws_ami\" \"distro\" {\n most_recent = true\n\n filter {\n name = \"name\"\n values = [var.ami_name_pattern]\n }\n\n filter {\n name = \"virtualization-type\"\n values = [var.ami_virtualization_type]\n }\n\n owners = var.ami_owners\n}\n\n//AWS VPC Variables\n\nvariable \"aws_vpc_cidr_block\" {\n description = \"CIDR Block for VPC\"\n}\n\nvariable \"aws_cidr_subnets_private\" {\n description = \"CIDR Blocks for private subnets in Availability Zones\"\n type = list(string)\n}\n\nvariable \"aws_cidr_subnets_public\" {\n description = \"CIDR Blocks for public subnets in Availability Zones\"\n type = list(string)\n}\n\n//AWS EC2 Settings\n\nvariable \"aws_bastion_size\" {\n description = \"EC2 Instance Size of Bastion Host\"\n}\n\n/*\n* AWS EC2 Settings\n* The number should be divisable by the number of used\n* AWS Availability Zones without an remainder.\n*/\nvariable \"aws_bastion_num\" {\n description = \"Number of Bastion Nodes\"\n}\n\nvariable \"aws_kube_master_num\" {\n description = \"Number of Kubernetes Master Nodes\"\n}\n\nvariable \"aws_kube_master_disk_size\" {\n description = \"Disk size for Kubernetes Master Nodes (in GiB)\"\n}\n\nvariable \"aws_kube_master_size\" {\n description = \"Instance size of Kube Master Nodes\"\n}\n\nvariable \"aws_etcd_num\" {\n description = \"Number of etcd Nodes\"\n}\n\nvariable \"aws_etcd_disk_size\" {\n description = \"Disk size for etcd Nodes (in GiB)\"\n}\n\nvariable \"aws_etcd_size\" {\n description = \"Instance size of etcd Nodes\"\n}\n\nvariable \"aws_kube_worker_num\" {\n description = \"Number of Kubernetes Worker Nodes\"\n}\n\nvariable \"aws_kube_worker_disk_size\" {\n description = \"Disk size for Kubernetes Worker Nodes (in GiB)\"\n}\n\nvariable \"aws_kube_worker_size\" {\n description = \"Instance size of Kubernetes Worker Nodes\"\n}\n\n/*\n* AWS NLB Settings\n*\n*/\nvariable \"aws_nlb_api_port\" {\n description = \"Port for AWS NLB\"\n}\n\nvariable \"k8s_secure_api_port\" {\n description = \"Secure Port of K8S API Server\"\n}\n\nvariable \"default_tags\" {\n description = \"Default tags for all resources\"\n type = map(string)\n}\n\nvariable \"inventory_file\" {\n description = \"Where to store the generated inventory file\"\n}\n" }, { "path": "contrib/terraform/exoscale/README.md", "content": "# Kubernetes on Exoscale with Terraform\n\nProvision a Kubernetes cluster on [Exoscale](https://www.exoscale.com/) using Terraform and Kubespray\n\n## Overview\n\nThe setup looks like following\n\n```text\n Kubernetes cluster\n +-----------------------+\n+---------------+ | +--------------+ |\n| | | | +--------------+ |\n| API server LB +---------> | | | |\n| | | | | Master/etcd | |\n+---------------+ | | | node(s) | |\n | +-+ | |\n | +--------------+ |\n | ^ |\n | | |\n | v |\n+---------------+ | +--------------+ |\n| | | | +--------------+ |\n| Ingress LB +---------> | | | |\n| | | | | Worker | |\n+---------------+ | | | node(s) | |\n | +-+ | |\n | +--------------+ |\n +-----------------------+\n```\n\n## Requirements\n\n* Terraform 0.13.0 or newer (0.12 also works if you modify the provider block to include version and remove all `versions.tf` files)\n\n## Quickstart\n\nNOTE: *Assumes you are at the root of the kubespray repo*\n\nCopy the sample inventory for your cluster and copy the default terraform variables.\n\n```bash\nCLUSTER=my-exoscale-cluster\ncp -r inventory/sample inventory/$CLUSTER\ncp contrib/terraform/exoscale/default.tfvars inventory/$CLUSTER/\ncd inventory/$CLUSTER\n```\n\nEdit `default.tfvars` to match your setup. You MUST, at the very least, change `ssh_public_keys`.\n\n```bash\n# Ensure $EDITOR points to your favorite editor, e.g., vim, emacs, VS Code, etc.\n$EDITOR default.tfvars\n```\n\nFor authentication you can use the credentials file `~/.cloudstack.ini` or `./cloudstack.ini`.\nThe file should look like something like this:\n\n```ini\n[cloudstack]\nkey = \nsecret = \n```\n\nFollow the [Exoscale IAM Quick-start](https://community.exoscale.com/documentation/iam/quick-start/) to learn how to generate API keys.\n\n### Encrypted credentials\n\nTo have the credentials encrypted at rest, you can use [sops](https://github.com/mozilla/sops) and only decrypt the credentials at runtime.\n\n```bash\ncat << EOF > cloudstack.ini\n[cloudstack]\nkey =\nsecret =\nEOF\nsops --encrypt --in-place --pgp cloudstack.ini\nsops cloudstack.ini\n```\n\nRun terraform to create the infrastructure\n\n```bash\nterraform init ../../contrib/terraform/exoscale\nterraform apply -var-file default.tfvars ../../contrib/terraform/exoscale\n```\n\nIf your cloudstack credentials file is encrypted using sops, run the following:\n\n```bash\nterraform init ../../contrib/terraform/exoscale\nsops exec-file -no-fifo cloudstack.ini 'CLOUDSTACK_CONFIG={} terraform apply -var-file default.tfvars ../../contrib/terraform/exoscale'\n```\n\nYou should now have a inventory file named `inventory.ini` that you can use with kubespray.\nYou can now copy your inventory file and use it with kubespray to set up a cluster.\nYou can type `terraform output` to find out the IP addresses of the nodes, as well as control-plane and data-plane load-balancer.\n\nIt is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:\n\n```bash\nansible -i inventory.ini -m ping all\n```\n\nExample to use this with the default sample inventory:\n\n```bash\nansible-playbook -i inventory.ini ../../cluster.yml -b -v\n```\n\n## Teardown\n\nThe Kubernetes cluster cannot create any load-balancers or disks, hence, teardown is as simple as Terraform destroy:\n\n```bash\nterraform destroy -var-file default.tfvars ../../contrib/terraform/exoscale\n```\n\n## Variables\n\n### Required\n\n* `ssh_public_keys`: List of public SSH keys to install on all machines\n* `zone`: The zone where to run the cluster\n* `machines`: Machines to provision. Key of this object will be used as the name of the machine\n * `node_type`: The role of this node *(master|worker)*\n * `size`: The size to use\n * `boot_disk`: The boot disk to use\n * `image_name`: Name of the image\n * `root_partition_size`: Size *(in GB)* for the root partition\n * `ceph_partition_size`: Size *(in GB)* for the partition for rook to use as ceph storage. *(Set to 0 to disable)*\n * `node_local_partition_size`: Size *(in GB)* for the partition for node-local-storage. *(Set to 0 to disable)*\n* `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes\n* `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server\n* `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports)\n\n### Optional\n\n* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)*\n\nAn example variables file can be found `default.tfvars`\n\n## Known limitations\n\n### Only single disk\n\nSince Exoscale doesn't support additional disks to be mounted onto an instance, this script has the ability to create partitions for [Rook](https://rook.io/) and [node-local-storage](https://kubernetes.io/docs/concepts/storage/volumes/#local).\n\n### No Kubernetes API\n\nThe current solution doesn't use the [Exoscale Kubernetes cloud controller](https://github.com/exoscale/exoscale-cloud-controller-manager).\nThis means that we need to set up a HTTP(S) loadbalancer in front of all workers and set the Ingress controller to DaemonSet mode.\n" }, { "path": "contrib/terraform/exoscale/default.tfvars", "content": "prefix = \"default\"\nzone = \"ch-gva-2\"\n\ninventory_file = \"inventory.ini\"\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\nmachines = {\n \"master-0\" : {\n \"node_type\" : \"master\",\n \"size\" : \"standard.medium\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"standard.large\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"standard.large\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n },\n \"worker-2\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"standard.large\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n }\n}\n\nnodeport_whitelist = [\n \"0.0.0.0/0\"\n]\n\nssh_whitelist = [\n \"0.0.0.0/0\"\n]\n\napi_server_whitelist = [\n \"0.0.0.0/0\"\n]\n" }, { "path": "contrib/terraform/exoscale/main.tf", "content": "provider \"exoscale\" {}\n\nmodule \"kubernetes\" {\n source = \"./modules/kubernetes-cluster\"\n\n prefix = var.prefix\n zone = var.zone\n machines = var.machines\n\n ssh_public_keys = var.ssh_public_keys\n\n ssh_whitelist = var.ssh_whitelist\n api_server_whitelist = var.api_server_whitelist\n nodeport_whitelist = var.nodeport_whitelist\n}\n\n#\n# Generate ansible inventory\n#\n\ndata \"template_file\" \"inventory\" {\n template = file(\"${path.module}/templates/inventory.tpl\")\n\n vars = {\n connection_strings_master = join(\"\\n\", formatlist(\"%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d\",\n keys(module.kubernetes.master_ip_addresses),\n values(module.kubernetes.master_ip_addresses).*.public_ip,\n values(module.kubernetes.master_ip_addresses).*.private_ip,\n range(1, length(module.kubernetes.master_ip_addresses) + 1)))\n connection_strings_worker = join(\"\\n\", formatlist(\"%s ansible_user=ubuntu ansible_host=%s ip=%s\",\n keys(module.kubernetes.worker_ip_addresses),\n values(module.kubernetes.worker_ip_addresses).*.public_ip,\n values(module.kubernetes.worker_ip_addresses).*.private_ip))\n\n list_master = join(\"\\n\", keys(module.kubernetes.master_ip_addresses))\n list_worker = join(\"\\n\", keys(module.kubernetes.worker_ip_addresses))\n api_lb_ip_address = module.kubernetes.control_plane_lb_ip_address\n }\n}\n\nresource \"null_resource\" \"inventories\" {\n provisioner \"local-exec\" {\n command = \"echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}\"\n }\n\n triggers = {\n template = data.template_file.inventory.rendered\n }\n}\n" }, { "path": "contrib/terraform/exoscale/modules/kubernetes-cluster/main.tf", "content": "data \"exoscale_template\" \"os_image\" {\n for_each = var.machines\n\n zone = var.zone\n name = each.value.boot_disk.image_name\n}\n\ndata \"exoscale_compute_instance\" \"master_nodes\" {\n for_each = exoscale_compute_instance.master\n\n id = each.value.id\n zone = var.zone\n}\n\ndata \"exoscale_compute_instance\" \"worker_nodes\" {\n for_each = exoscale_compute_instance.worker\n\n id = each.value.id\n zone = var.zone\n}\n\nresource \"exoscale_private_network\" \"private_network\" {\n zone = var.zone\n name = \"${var.prefix}-network\"\n\n start_ip = cidrhost(var.private_network_cidr, 1)\n # cidr -1 = Broadcast address\n # cidr -2 = DHCP server address (exoscale specific)\n end_ip = cidrhost(var.private_network_cidr, -3)\n netmask = cidrnetmask(var.private_network_cidr)\n}\n\nresource \"exoscale_compute_instance\" \"master\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"master\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n template_id = data.exoscale_template.os_image[each.key].id\n type = each.value.size\n disk_size = each.value.boot_disk.root_partition_size + each.value.boot_disk.node_local_partition_size + each.value.boot_disk.ceph_partition_size\n state = \"Running\"\n zone = var.zone\n security_group_ids = [exoscale_security_group.master_sg.id]\n network_interface {\n network_id = exoscale_private_network.private_network.id\n }\n elastic_ip_ids = [exoscale_elastic_ip.control_plane_lb.id]\n\n user_data = templatefile(\n \"${path.module}/templates/cloud-init.tmpl\",\n {\n eip_ip_address = exoscale_elastic_ip.ingress_controller_lb.ip_address\n node_local_partition_size = each.value.boot_disk.node_local_partition_size\n ceph_partition_size = each.value.boot_disk.ceph_partition_size\n root_partition_size = each.value.boot_disk.root_partition_size\n node_type = \"master\"\n ssh_public_keys = var.ssh_public_keys\n }\n )\n}\n\nresource \"exoscale_compute_instance\" \"worker\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"worker\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n template_id = data.exoscale_template.os_image[each.key].id\n type = each.value.size\n disk_size = each.value.boot_disk.root_partition_size + each.value.boot_disk.node_local_partition_size + each.value.boot_disk.ceph_partition_size\n state = \"Running\"\n zone = var.zone\n security_group_ids = [exoscale_security_group.worker_sg.id]\n network_interface {\n network_id = exoscale_private_network.private_network.id\n }\n elastic_ip_ids = [exoscale_elastic_ip.ingress_controller_lb.id]\n\n user_data = templatefile(\n \"${path.module}/templates/cloud-init.tmpl\",\n {\n eip_ip_address = exoscale_elastic_ip.ingress_controller_lb.ip_address\n node_local_partition_size = each.value.boot_disk.node_local_partition_size\n ceph_partition_size = each.value.boot_disk.ceph_partition_size\n root_partition_size = each.value.boot_disk.root_partition_size\n node_type = \"worker\"\n ssh_public_keys = var.ssh_public_keys\n }\n )\n}\n\nresource \"exoscale_security_group\" \"master_sg\" {\n name = \"${var.prefix}-master-sg\"\n description = \"Security group for Kubernetes masters\"\n}\n\nresource \"exoscale_security_group_rule\" \"master_sg_rule_ssh\" {\n security_group_id = exoscale_security_group.master_sg.id\n\n for_each = toset(var.ssh_whitelist)\n # SSH\n type = \"INGRESS\"\n start_port = 22\n end_port = 22\n protocol = \"TCP\"\n cidr = each.value\n}\n\nresource \"exoscale_security_group_rule\" \"master_sg_rule_k8s_api\" {\n security_group_id = exoscale_security_group.master_sg.id\n\n for_each = toset(var.api_server_whitelist)\n # Kubernetes API\n type = \"INGRESS\"\n start_port = 6443\n end_port = 6443\n protocol = \"TCP\"\n cidr = each.value\n}\n\nresource \"exoscale_security_group\" \"worker_sg\" {\n name = \"${var.prefix}-worker-sg\"\n description = \"security group for kubernetes worker nodes\"\n}\n\nresource \"exoscale_security_group_rule\" \"worker_sg_rule_ssh\" {\n security_group_id = exoscale_security_group.worker_sg.id\n\n # SSH\n for_each = toset(var.ssh_whitelist)\n type = \"INGRESS\"\n start_port = 22\n end_port = 22\n protocol = \"TCP\"\n cidr = each.value\n}\n\nresource \"exoscale_security_group_rule\" \"worker_sg_rule_http\" {\n security_group_id = exoscale_security_group.worker_sg.id\n\n # HTTP(S)\n for_each = toset([\"80\", \"443\"])\n type = \"INGRESS\"\n start_port = each.value\n end_port = each.value\n protocol = \"TCP\"\n cidr = \"0.0.0.0/0\"\n}\n\n\nresource \"exoscale_security_group_rule\" \"worker_sg_rule_nodeport\" {\n security_group_id = exoscale_security_group.worker_sg.id\n\n # HTTP(S)\n for_each = toset(var.nodeport_whitelist)\n type = \"INGRESS\"\n start_port = 30000\n end_port = 32767\n protocol = \"TCP\"\n cidr = each.value\n}\n\nresource \"exoscale_elastic_ip\" \"ingress_controller_lb\" {\n zone = var.zone\n healthcheck {\n mode = \"http\"\n port = 80\n uri = \"/healthz\"\n interval = 10\n timeout = 2\n strikes_ok = 2\n strikes_fail = 3\n }\n}\n\nresource \"exoscale_elastic_ip\" \"control_plane_lb\" {\n zone = var.zone\n healthcheck {\n mode = \"tcp\"\n port = 6443\n interval = 10\n timeout = 2\n strikes_ok = 2\n strikes_fail = 3\n }\n}\n" }, { "path": "contrib/terraform/exoscale/modules/kubernetes-cluster/output.tf", "content": "output \"master_ip_addresses\" {\n value = {\n for key, instance in exoscale_compute_instance.master :\n instance.name => {\n \"private_ip\" = contains(keys(data.exoscale_compute_instance.master_nodes), key) ? data.exoscale_compute_instance.master_nodes[key].private_network_ip_addresses[0] : \"\"\n \"public_ip\" = exoscale_compute_instance.master[key].ip_address\n }\n }\n}\n\noutput \"worker_ip_addresses\" {\n value = {\n for key, instance in exoscale_compute_instance.worker :\n instance.name => {\n \"private_ip\" = contains(keys(data.exoscale_compute_instance.worker_nodes), key) ? data.exoscale_compute_instance.worker_nodes[key].private_network_ip_addresses[0] : \"\"\n \"public_ip\" = exoscale_compute_instance.worker[key].ip_address\n }\n }\n}\n\noutput \"cluster_private_network_cidr\" {\n value = var.private_network_cidr\n}\n\noutput \"ingress_controller_lb_ip_address\" {\n value = exoscale_elastic_ip.ingress_controller_lb.ip_address\n}\n\noutput \"control_plane_lb_ip_address\" {\n value = exoscale_elastic_ip.control_plane_lb.ip_address\n}\n" }, { "path": "contrib/terraform/exoscale/modules/kubernetes-cluster/templates/cloud-init.tmpl", "content": "#cloud-config\n%{ if ceph_partition_size > 0 || node_local_partition_size > 0}\nbootcmd:\n- [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, /dev/vda ]\n%{ if node_local_partition_size > 0 }\n # Create partition for node local storage\n- [ cloud-init-per, once, create-node-local-part, parted, --script, /dev/vda, 'mkpart extended ext4 ${root_partition_size}GB %{ if ceph_partition_size == 0 }-1%{ else }${root_partition_size + node_local_partition_size}GB%{ endif }' ]\n- [ cloud-init-per, once, create-fs-node-local-part, mkfs.ext4, /dev/vda2 ]\n%{ endif }\n%{ if ceph_partition_size > 0 }\n # Create partition for rook to use for ceph\n- [ cloud-init-per, once, create-ceph-part, parted, --script, /dev/vda, 'mkpart extended ${root_partition_size + node_local_partition_size}GB -1' ]\n%{ endif }\n%{ endif }\n\nssh_authorized_keys:\n%{ for ssh_public_key in ssh_public_keys ~}\n - ${ssh_public_key}\n%{ endfor ~}\n\nwrite_files:\n - path: /etc/netplan/eth1.yaml\n content: |\n network:\n version: 2\n ethernets:\n eth1:\n dhcp4: true\n%{ if node_type == \"worker\" }\n # TODO: When a VM is seen as healthy and is added to the EIP loadbalancer\n # pool it no longer can send traffic back to itself via the EIP IP\n # address.\n # Remove this if it ever gets solved.\n - path: /etc/netplan/20-eip-fix.yaml\n content: |\n network:\n version: 2\n ethernets:\n \"lo:0\":\n match:\n name: lo\n dhcp4: false\n addresses:\n - ${eip_ip_address}/32\n%{ endif }\nruncmd:\n - netplan apply\n%{ if node_local_partition_size > 0 }\n - mkdir -p /mnt/disks/node-local-storage\n - chown nobody:nogroup /mnt/disks/node-local-storage\n - mount /dev/vda2 /mnt/disks/node-local-storage\n%{ endif }\n" }, { "path": "contrib/terraform/exoscale/modules/kubernetes-cluster/variables.tf", "content": "variable \"zone\" {\n type = string\n # This is currently the only zone that is supposed to be supporting\n # so called \"managed private networks\".\n # See: https://www.exoscale.com/syslog/introducing-managed-private-networks\n default = \"ch-gva-2\"\n}\n\nvariable \"prefix\" {}\n\nvariable \"machines\" {\n type = map(object({\n node_type = string\n size = string\n boot_disk = object({\n image_name = string\n root_partition_size = number\n ceph_partition_size = number\n node_local_partition_size = number\n })\n }))\n}\n\nvariable \"ssh_public_keys\" {\n type = list(string)\n}\n\nvariable \"ssh_whitelist\" {\n type = list(string)\n}\n\nvariable \"api_server_whitelist\" {\n type = list(string)\n}\n\nvariable \"nodeport_whitelist\" {\n type = list(string)\n}\n\nvariable \"private_network_cidr\" {\n default = \"172.0.10.0/24\"\n}\n" }, { "path": "contrib/terraform/exoscale/modules/kubernetes-cluster/versions.tf", "content": "terraform {\n required_providers {\n exoscale = {\n source = \"exoscale/exoscale\"\n version = \">= 0.21\"\n }\n }\n required_version = \">= 0.13\"\n}\n" }, { "path": "contrib/terraform/exoscale/output.tf", "content": "output \"master_ips\" {\n value = module.kubernetes.master_ip_addresses\n}\n\noutput \"worker_ips\" {\n value = module.kubernetes.worker_ip_addresses\n}\n\noutput \"ingress_controller_lb_ip_address\" {\n value = module.kubernetes.ingress_controller_lb_ip_address\n}\n\noutput \"control_plane_lb_ip_address\" {\n value = module.kubernetes.control_plane_lb_ip_address\n}\n" }, { "path": "contrib/terraform/exoscale/sample-inventory/cluster.tfvars", "content": "prefix = \"default\"\nzone = \"ch-gva-2\"\n\ninventory_file = \"inventory.ini\"\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\nmachines = {\n \"master-0\" : {\n \"node_type\" : \"master\",\n \"size\" : \"Small\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"Large\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"Large\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n },\n \"worker-2\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"Large\",\n \"boot_disk\" : {\n \"image_name\" : \"Linux Ubuntu 20.04 LTS 64-bit\",\n \"root_partition_size\" : 50,\n \"node_local_partition_size\" : 0,\n \"ceph_partition_size\" : 0\n }\n }\n}\n\nnodeport_whitelist = [\n \"0.0.0.0/0\"\n]\n\nssh_whitelist = [\n \"0.0.0.0/0\"\n]\n\napi_server_whitelist = [\n \"0.0.0.0/0\"\n]\n" }, { "path": "contrib/terraform/exoscale/templates/inventory.tpl", "content": "[all]\n${connection_strings_master}\n${connection_strings_worker}\n\n[kube_control_plane]\n${list_master}\n\n[kube_control_plane:vars]\nsupplementary_addresses_in_ssl_keys = [ \"${api_lb_ip_address}\" ]\n\n[etcd]\n${list_master}\n\n[kube_node]\n${list_worker}\n\n[k8s_cluster:children]\nkube_control_plane\nkube_node\n" }, { "path": "contrib/terraform/exoscale/variables.tf", "content": "variable \"zone\" {\n description = \"The zone where to run the cluster\"\n}\n\nvariable \"prefix\" {\n description = \"Prefix for resource names\"\n default = \"default\"\n}\n\nvariable \"machines\" {\n description = \"Cluster machines\"\n type = map(object({\n node_type = string\n size = string\n boot_disk = object({\n image_name = string\n root_partition_size = number\n ceph_partition_size = number\n node_local_partition_size = number\n })\n }))\n}\n\nvariable \"ssh_public_keys\" {\n description = \"List of public SSH keys which are injected into the VMs.\"\n type = list(string)\n}\n\nvariable \"ssh_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for ssh\"\n type = list(string)\n}\n\nvariable \"api_server_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for kubernetes api server\"\n type = list(string)\n}\n\nvariable \"nodeport_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for kubernetes nodeports\"\n type = list(string)\n}\n\nvariable \"inventory_file\" {\n description = \"Where to store the generated inventory file\"\n}\n" }, { "path": "contrib/terraform/exoscale/versions.tf", "content": "terraform {\n required_providers {\n exoscale = {\n source = \"exoscale/exoscale\"\n version = \">= 0.21\"\n }\n null = {\n source = \"hashicorp/null\"\n }\n template = {\n source = \"hashicorp/template\"\n }\n }\n required_version = \">= 0.13\"\n}\n" }, { "path": "contrib/terraform/gcp/README.md", "content": "# Kubernetes on GCP with Terraform\n\nProvision a Kubernetes cluster on GCP using Terraform and Kubespray\n\n## Overview\n\nThe setup looks like following\n\n```text\n Kubernetes cluster\n +-----------------------+\n+---------------+ | +--------------+ |\n| | | | +--------------+ |\n| API server LB +---------> | | | |\n| | | | | Master/etcd | |\n+---------------+ | | | node(s) | |\n | +-+ | |\n | +--------------+ |\n | ^ |\n | | |\n | v |\n+---------------+ | +--------------+ |\n| | | | +--------------+ |\n| Ingress LB +---------> | | | |\n| | | | | Worker | |\n+---------------+ | | | node(s) | |\n | +-+ | |\n | +--------------+ |\n +-----------------------+\n```\n\n## Requirements\n\n* Terraform 0.12.0 or newer\n\n## Quickstart\n\nTo get a cluster up and running you'll need a JSON keyfile.\nSet the path to the file in the `tfvars.json` file and run the following:\n\n```bash\nterraform apply -var-file tfvars.json -state dev-cluster.tfstate -var gcp_project_id= -var keyfile_location=\n```\n\nTo generate kubespray inventory based on the terraform state file you can run the following:\n\n```bash\n./generate-inventory.sh dev-cluster.tfstate > inventory.ini\n```\n\nYou should now have a inventory file named `inventory.ini` that you can use with kubespray, e.g.\n\n```bash\nansible-playbook -i contrib/terraform/gcp/inventory.ini cluster.yml -b -v\n```\n\n## Variables\n\n### Required\n\n* `keyfile_location`: Location to the keyfile to use as credentials for the google terraform provider\n* `gcp_project_id`: ID of the GCP project to deploy the cluster in\n* `ssh_pub_key`: Path to public ssh key to use for all machines\n* `region`: The region where to run the cluster\n* `machines`: Machines to provision. Key of this object will be used as the name of the machine\n * `node_type`: The role of this node *(master|worker)*\n * `size`: The size to use\n * `zone`: The zone the machine should run in\n * `additional_disks`: Extra disks to add to the machine. Key of this object will be used as the disk name\n * `size`: Size of the disk (in GB)\n * `boot_disk`: The boot disk to use\n * `image_name`: Name of the image\n * `size`: Size of the boot disk (in GB)\n* `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes\n* `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server\n* `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports)\n* `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to ingress on ports 80 and 443\n* `extra_ingress_firewalls`: Additional ingress firewall rules. Key will be used as the name of the rule\n * `source_ranges`: List of IP ranges (CIDR). Example: `[\"8.8.8.8\"]`\n * `protocol`: Protocol. Example `\"tcp\"`\n * `ports`: List of ports, as string. Example `[\"53\"]`\n * `target_tags`: List of target tag (either the machine name or `control-plane` or `worker`). Example: `[\"control-plane\", \"worker-0\"]`\n\n### Optional\n\n* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)*\n* `master_sa_email`: Service account email to use for the control plane nodes *(Defaults to `\"\"`, auto generate one)*\n* `master_sa_scopes`: Service account email to use for the control plane nodes *(Defaults to `[\"https://www.googleapis.com/auth/cloud-platform\"]`)*\n* `master_preemptible`: Enable [preemptible](https://cloud.google.com/compute/docs/instances/preemptible)\n for the control plane nodes *(Defaults to `false`)*\n* `master_additional_disk_type`: [Disk type](https://cloud.google.com/compute/docs/disks/#disk-types)\n for extra disks added on the control plane nodes *(Defaults to `\"pd-ssd\"`)*\n* `worker_sa_email`: Service account email to use for the worker nodes *(Defaults to `\"\"`, auto generate one)*\n* `worker_sa_scopes`: Service account email to use for the worker nodes *(Defaults to `[\"https://www.googleapis.com/auth/cloud-platform\"]`)*\n* `worker_preemptible`: Enable [preemptible](https://cloud.google.com/compute/docs/instances/preemptible)\n for the worker nodes *(Defaults to `false`)*\n* `worker_additional_disk_type`: [Disk type](https://cloud.google.com/compute/docs/disks/#disk-types)\n for extra disks added on the worker nodes *(Defaults to `\"pd-ssd\"`)*\n\nAn example variables file can be found `tfvars.json`\n\n## Known limitations\n\nThis solution does not provide a solution to use a bastion host. Thus all the nodes must expose a public IP for kubespray to work.\n" }, { "path": "contrib/terraform/gcp/generate-inventory.sh", "content": "#!/bin/bash\n\n#\n# Generates a inventory file based on the terraform output.\n# After provisioning a cluster, simply run this command and supply the terraform state file\n# Default state file is terraform.tfstate\n#\n\nset -e\n\nusage () {\n echo \"Usage: $0 \" >&2\n exit 1\n}\n\nif [[ $# -ne 1 ]]; then\n usage\nfi\n\nTF_STATE_FILE=${1}\n\nif [[ ! -f \"${TF_STATE_FILE}\" ]]; then\n echo \"ERROR: state file ${TF_STATE_FILE} doesn't exist\" >&2\n usage\nfi\n\nTF_OUT=$(terraform output -state \"${TF_STATE_FILE}\" -json)\n\nMASTERS=$(jq -r '.master_ips.value | to_entries[]' <(echo \"${TF_OUT}\"))\nWORKERS=$(jq -r '.worker_ips.value | to_entries[]' <(echo \"${TF_OUT}\"))\nmapfile -t MASTER_NAMES < <(jq -r '.key' <(echo \"${MASTERS}\"))\nmapfile -t WORKER_NAMES < <(jq -r '.key' <(echo \"${WORKERS}\"))\n\nAPI_LB=$(jq -r '.control_plane_lb_ip_address.value' <(echo \"${TF_OUT}\"))\n\n# Generate master hosts\ni=1\nfor name in \"${MASTER_NAMES[@]}\"; do\n private_ip=$(jq -r '. | select( .key=='\"\\\"${name}\\\"\"' ) | .value.private_ip' <(echo \"${MASTERS}\"))\n public_ip=$(jq -r '. | select( .key=='\"\\\"${name}\\\"\"' ) | .value.public_ip' <(echo \"${MASTERS}\"))\n echo \"${name} ansible_user=ubuntu ansible_host=${public_ip} ip=${private_ip} etcd_member_name=etcd${i}\"\n i=$(( i + 1 ))\ndone\n\n# Generate worker hosts\nfor name in \"${WORKER_NAMES[@]}\"; do\n private_ip=$(jq -r '. | select( .key=='\"\\\"${name}\\\"\"' ) | .value.private_ip' <(echo \"${WORKERS}\"))\n public_ip=$(jq -r '. | select( .key=='\"\\\"${name}\\\"\"' ) | .value.public_ip' <(echo \"${WORKERS}\"))\n echo \"${name} ansible_user=ubuntu ansible_host=${public_ip} ip=${private_ip}\"\ndone\n\necho \"\"\necho \"[kube_control_plane]\"\nfor name in \"${MASTER_NAMES[@]}\"; do\n echo \"${name}\"\ndone\n\necho \"\"\necho \"[kube_control_plane:vars]\"\necho \"supplementary_addresses_in_ssl_keys = [ '${API_LB}' ]\" # Add LB address to API server certificate\necho \"\"\necho \"[etcd]\"\nfor name in \"${MASTER_NAMES[@]}\"; do\n echo \"${name}\"\ndone\n\necho \"\"\necho \"[kube_node]\"\nfor name in \"${WORKER_NAMES[@]}\"; do\n echo \"${name}\"\ndone\n\necho \"\"\necho \"[k8s_cluster:children]\"\necho \"kube_control_plane\"\necho \"kube_node\"\n" }, { "path": "contrib/terraform/gcp/main.tf", "content": "terraform {\n required_providers {\n google = {\n source = \"hashicorp/google\"\n version = \"~> 4.0\"\n }\n }\n}\n\nprovider \"google\" {\n credentials = file(var.keyfile_location)\n region = var.region\n project = var.gcp_project_id\n}\n\nmodule \"kubernetes\" {\n source = \"./modules/kubernetes-cluster\"\n region = var.region\n prefix = var.prefix\n\n machines = var.machines\n ssh_pub_key = var.ssh_pub_key\n\n master_sa_email = var.master_sa_email\n master_sa_scopes = var.master_sa_scopes\n master_preemptible = var.master_preemptible\n master_additional_disk_type = var.master_additional_disk_type\n worker_sa_email = var.worker_sa_email\n worker_sa_scopes = var.worker_sa_scopes\n worker_preemptible = var.worker_preemptible\n worker_additional_disk_type = var.worker_additional_disk_type\n\n ssh_whitelist = var.ssh_whitelist\n api_server_whitelist = var.api_server_whitelist\n nodeport_whitelist = var.nodeport_whitelist\n ingress_whitelist = var.ingress_whitelist\n\n extra_ingress_firewalls = var.extra_ingress_firewalls\n}\n" }, { "path": "contrib/terraform/gcp/modules/kubernetes-cluster/main.tf", "content": "#################################################\n##\n## General\n##\n\nresource \"google_compute_network\" \"main\" {\n name = \"${var.prefix}-network\"\n\n auto_create_subnetworks = false\n}\n\nresource \"google_compute_subnetwork\" \"main\" {\n name = \"${var.prefix}-subnet\"\n network = google_compute_network.main.name\n ip_cidr_range = var.private_network_cidr\n region = var.region\n}\n\nresource \"google_compute_firewall\" \"deny_all\" {\n name = \"${var.prefix}-default-firewall\"\n network = google_compute_network.main.name\n\n priority = 1000\n\n source_ranges = [\"0.0.0.0/0\"]\n\n deny {\n protocol = \"all\"\n }\n}\n\nresource \"google_compute_firewall\" \"allow_internal\" {\n name = \"${var.prefix}-internal-firewall\"\n network = google_compute_network.main.name\n\n priority = 500\n\n source_ranges = [var.private_network_cidr]\n\n allow {\n protocol = \"all\"\n }\n}\n\nresource \"google_compute_firewall\" \"ssh\" {\n count = length(var.ssh_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-ssh-firewall\"\n network = google_compute_network.main.name\n\n priority = 100\n\n source_ranges = var.ssh_whitelist\n\n allow {\n protocol = \"tcp\"\n ports = [\"22\"]\n }\n}\n\nresource \"google_compute_firewall\" \"api_server\" {\n count = length(var.api_server_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-api-server-firewall\"\n network = google_compute_network.main.name\n\n priority = 100\n\n source_ranges = var.api_server_whitelist\n\n allow {\n protocol = \"tcp\"\n ports = [\"6443\"]\n }\n}\n\nresource \"google_compute_firewall\" \"nodeport\" {\n count = length(var.nodeport_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-nodeport-firewall\"\n network = google_compute_network.main.name\n\n priority = 100\n\n source_ranges = var.nodeport_whitelist\n\n allow {\n protocol = \"tcp\"\n ports = [\"30000-32767\"]\n }\n}\n\nresource \"google_compute_firewall\" \"ingress_http\" {\n count = length(var.ingress_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-http-ingress-firewall\"\n network = google_compute_network.main.name\n\n priority = 100\n\n source_ranges = var.ingress_whitelist\n\n allow {\n protocol = \"tcp\"\n ports = [\"80\"]\n }\n}\n\nresource \"google_compute_firewall\" \"ingress_https\" {\n count = length(var.ingress_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-https-ingress-firewall\"\n network = google_compute_network.main.name\n\n priority = 100\n\n source_ranges = var.ingress_whitelist\n\n allow {\n protocol = \"tcp\"\n ports = [\"443\"]\n }\n}\n\n#################################################\n##\n## Local variables\n##\n\nlocals {\n master_target_list = [\n for name, machine in google_compute_instance.master :\n \"${machine.zone}/${machine.name}\"\n ]\n\n worker_target_list = [\n for name, machine in google_compute_instance.worker :\n \"${machine.zone}/${machine.name}\"\n ]\n\n master_disks = flatten([\n for machine_name, machine in var.machines : [\n for disk_name, disk in machine.additional_disks : {\n \"${machine_name}-${disk_name}\" = {\n \"machine_name\": machine_name,\n \"machine\": machine,\n \"disk_size\": disk.size,\n \"disk_name\": disk_name\n }\n }\n ]\n if machine.node_type == \"master\"\n ])\n\n worker_disks = flatten([\n for machine_name, machine in var.machines : [\n for disk_name, disk in machine.additional_disks : {\n \"${machine_name}-${disk_name}\" = {\n \"machine_name\": machine_name,\n \"machine\": machine,\n \"disk_size\": disk.size,\n \"disk_name\": disk_name\n }\n }\n ]\n if machine.node_type == \"worker\"\n ])\n}\n\n#################################################\n##\n## Master\n##\n\nresource \"google_compute_address\" \"master\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"master\"\n }\n\n name = \"${var.prefix}-${each.key}-pip\"\n address_type = \"EXTERNAL\"\n region = var.region\n}\n\nresource \"google_compute_disk\" \"master\" {\n for_each = {\n for item in local.master_disks :\n keys(item)[0] => values(item)[0]\n }\n\n name = \"${var.prefix}-${each.key}\"\n type = var.master_additional_disk_type\n zone = each.value.machine.zone\n size = each.value.disk_size\n\n physical_block_size_bytes = 4096\n}\n\nresource \"google_compute_attached_disk\" \"master\" {\n for_each = {\n for item in local.master_disks :\n keys(item)[0] => values(item)[0]\n }\n\n disk = google_compute_disk.master[each.key].id\n instance = google_compute_instance.master[each.value.machine_name].id\n}\n\nresource \"google_compute_instance\" \"master\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"master\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n machine_type = each.value.size\n zone = each.value.zone\n\n tags = [\"control-plane\", \"master\", each.key]\n\n boot_disk {\n initialize_params {\n image = each.value.boot_disk.image_name\n size = each.value.boot_disk.size\n }\n }\n\n network_interface {\n subnetwork = google_compute_subnetwork.main.name\n\n access_config {\n nat_ip = google_compute_address.master[each.key].address\n }\n }\n\n metadata = {\n ssh-keys = \"ubuntu:${trimspace(file(pathexpand(var.ssh_pub_key)))}\"\n }\n\n service_account {\n email = var.master_sa_email\n scopes = var.master_sa_scopes\n }\n\n # Since we use google_compute_attached_disk we need to ignore this\n lifecycle {\n ignore_changes = [attached_disk]\n }\n\n scheduling {\n preemptible = var.master_preemptible\n automatic_restart = !var.master_preemptible\n }\n}\n\nresource \"google_compute_forwarding_rule\" \"master_lb\" {\n count = length(var.api_server_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-master-lb-forward-rule\"\n\n port_range = \"6443\"\n\n target = google_compute_target_pool.master_lb[count.index].id\n}\n\nresource \"google_compute_target_pool\" \"master_lb\" {\n count = length(var.api_server_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-master-lb-pool\"\n instances = local.master_target_list\n}\n\n#################################################\n##\n## Worker\n##\n\nresource \"google_compute_disk\" \"worker\" {\n for_each = {\n for item in local.worker_disks :\n keys(item)[0] => values(item)[0]\n }\n\n name = \"${var.prefix}-${each.key}\"\n type = var.worker_additional_disk_type\n zone = each.value.machine.zone\n size = each.value.disk_size\n\n physical_block_size_bytes = 4096\n}\n\nresource \"google_compute_attached_disk\" \"worker\" {\n for_each = {\n for item in local.worker_disks :\n keys(item)[0] => values(item)[0]\n }\n\n disk = google_compute_disk.worker[each.key].id\n instance = google_compute_instance.worker[each.value.machine_name].id\n}\n\nresource \"google_compute_address\" \"worker\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"worker\"\n }\n\n name = \"${var.prefix}-${each.key}-pip\"\n address_type = \"EXTERNAL\"\n region = var.region\n}\n\nresource \"google_compute_instance\" \"worker\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"worker\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n machine_type = each.value.size\n zone = each.value.zone\n\n tags = [\"worker\", each.key]\n\n boot_disk {\n initialize_params {\n image = each.value.boot_disk.image_name\n size = each.value.boot_disk.size\n }\n }\n\n network_interface {\n subnetwork = google_compute_subnetwork.main.name\n\n access_config {\n nat_ip = google_compute_address.worker[each.key].address\n }\n }\n\n metadata = {\n ssh-keys = \"ubuntu:${trimspace(file(pathexpand(var.ssh_pub_key)))}\"\n }\n\n service_account {\n email = var.worker_sa_email\n scopes = var.worker_sa_scopes\n }\n\n # Since we use google_compute_attached_disk we need to ignore this\n lifecycle {\n ignore_changes = [attached_disk]\n }\n\n scheduling {\n preemptible = var.worker_preemptible\n automatic_restart = !var.worker_preemptible\n }\n}\n\nresource \"google_compute_address\" \"worker_lb\" {\n count = length(var.ingress_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-worker-lb-address\"\n address_type = \"EXTERNAL\"\n region = var.region\n}\n\nresource \"google_compute_forwarding_rule\" \"worker_http_lb\" {\n count = length(var.ingress_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-worker-http-lb-forward-rule\"\n\n ip_address = google_compute_address.worker_lb[count.index].address\n port_range = \"80\"\n\n target = google_compute_target_pool.worker_lb[count.index].id\n}\n\nresource \"google_compute_forwarding_rule\" \"worker_https_lb\" {\n count = length(var.ingress_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-worker-https-lb-forward-rule\"\n\n ip_address = google_compute_address.worker_lb[count.index].address\n port_range = \"443\"\n\n target = google_compute_target_pool.worker_lb[count.index].id\n}\n\nresource \"google_compute_target_pool\" \"worker_lb\" {\n count = length(var.ingress_whitelist) > 0 ? 1 : 0\n\n name = \"${var.prefix}-worker-lb-pool\"\n instances = local.worker_target_list\n}\n\nresource \"google_compute_firewall\" \"extra_ingress_firewall\" {\n for_each = {\n for name, firewall in var.extra_ingress_firewalls :\n name => firewall\n }\n\n name = \"${var.prefix}-${each.key}-ingress\"\n network = google_compute_network.main.name\n\n priority = 100\n\n source_ranges = each.value.source_ranges\n\n target_tags = each.value.target_tags\n\n allow {\n protocol = each.value.protocol\n ports = each.value.ports\n }\n}\n" }, { "path": "contrib/terraform/gcp/modules/kubernetes-cluster/output.tf", "content": "output \"master_ip_addresses\" {\n value = {\n for key, instance in google_compute_instance.master :\n instance.name => {\n \"private_ip\" = instance.network_interface.0.network_ip\n \"public_ip\" = instance.network_interface.0.access_config.0.nat_ip\n }\n }\n}\n\noutput \"worker_ip_addresses\" {\n value = {\n for key, instance in google_compute_instance.worker :\n instance.name => {\n \"private_ip\" = instance.network_interface.0.network_ip\n \"public_ip\" = instance.network_interface.0.access_config.0.nat_ip\n }\n }\n}\n\noutput \"ingress_controller_lb_ip_address\" {\n value = length(var.ingress_whitelist) > 0 ? google_compute_address.worker_lb.0.address : \"\"\n}\n\noutput \"control_plane_lb_ip_address\" {\n value = length(var.api_server_whitelist) > 0 ? google_compute_forwarding_rule.master_lb.0.ip_address : \"\"\n}\n" }, { "path": "contrib/terraform/gcp/modules/kubernetes-cluster/variables.tf", "content": "variable \"region\" {\n type = string\n}\n\nvariable \"prefix\" {}\n\nvariable \"machines\" {\n type = map(object({\n node_type = string\n size = string\n zone = string\n additional_disks = map(object({\n size = number\n }))\n boot_disk = object({\n image_name = string\n size = number\n })\n }))\n}\n\nvariable \"master_sa_email\" {\n type = string\n}\n\nvariable \"master_sa_scopes\" {\n type = list(string)\n}\n\nvariable \"master_preemptible\" {\n type = bool\n}\n\nvariable \"master_additional_disk_type\" {\n type = string\n}\n\nvariable \"worker_sa_email\" {\n type = string\n}\n\nvariable \"worker_sa_scopes\" {\n type = list(string)\n}\n\nvariable \"worker_preemptible\" {\n type = bool\n}\n\nvariable \"worker_additional_disk_type\" {\n type = string\n}\n\nvariable \"ssh_pub_key\" {}\n\nvariable \"ssh_whitelist\" {\n type = list(string)\n}\n\nvariable \"api_server_whitelist\" {\n type = list(string)\n}\n\nvariable \"nodeport_whitelist\" {\n type = list(string)\n}\n\nvariable \"ingress_whitelist\" {\n type = list(string)\n default = [\"0.0.0.0/0\"]\n}\n\nvariable \"private_network_cidr\" {\n default = \"10.0.10.0/24\"\n}\n\nvariable \"extra_ingress_firewalls\" {\n type = map(object({\n source_ranges = set(string)\n protocol = string\n ports = list(string)\n target_tags = set(string)\n }))\n\n default = {}\n}\n" }, { "path": "contrib/terraform/gcp/output.tf", "content": "output \"master_ips\" {\n value = module.kubernetes.master_ip_addresses\n}\n\noutput \"worker_ips\" {\n value = module.kubernetes.worker_ip_addresses\n}\n\noutput \"ingress_controller_lb_ip_address\" {\n value = module.kubernetes.ingress_controller_lb_ip_address\n}\n\noutput \"control_plane_lb_ip_address\" {\n value = module.kubernetes.control_plane_lb_ip_address\n}\n" }, { "path": "contrib/terraform/gcp/tfvars.json", "content": "{\n \"gcp_project_id\": \"GCP_PROJECT_ID\",\n \"region\": \"us-central1\",\n \"ssh_pub_key\": \"~/.ssh/id_rsa.pub\",\n\n \"keyfile_location\": \"service-account.json\",\n\n \"prefix\": \"development\",\n\n \"ssh_whitelist\": [\n \"1.2.3.4/32\"\n ],\n \"api_server_whitelist\": [\n \"1.2.3.4/32\"\n ],\n \"nodeport_whitelist\": [\n \"1.2.3.4/32\"\n ],\n \"ingress_whitelist\": [\n \"0.0.0.0/0\"\n ],\n\n \"machines\": {\n \"master-0\": {\n \"node_type\": \"master\",\n \"size\": \"n1-standard-2\",\n \"zone\": \"us-central1-a\",\n \"additional_disks\": {},\n \"boot_disk\": {\n \"image_name\": \"ubuntu-os-cloud/ubuntu-2004-focal-v20220118\",\n \"size\": 50\n }\n },\n \"worker-0\": {\n \"node_type\": \"worker\",\n \"size\": \"n1-standard-8\",\n \"zone\": \"us-central1-a\",\n \"additional_disks\": {\n \"extra-disk-1\": {\n \"size\": 100\n }\n },\n \"boot_disk\": {\n \"image_name\": \"ubuntu-os-cloud/ubuntu-2004-focal-v20220118\",\n \"size\": 50\n }\n },\n \"worker-1\": {\n \"node_type\": \"worker\",\n \"size\": \"n1-standard-8\",\n \"zone\": \"us-central1-a\",\n \"additional_disks\": {\n \"extra-disk-1\": {\n \"size\": 100\n }\n },\n \"boot_disk\": {\n \"image_name\": \"ubuntu-os-cloud/ubuntu-2004-focal-v20220118\",\n \"size\": 50\n }\n }\n }\n}\n" }, { "path": "contrib/terraform/gcp/variables.tf", "content": "variable keyfile_location {\n description = \"Location of the json keyfile to use with the google provider\"\n type = string\n}\n\nvariable region {\n description = \"Region of all resources\"\n type = string\n}\n\nvariable gcp_project_id {\n description = \"ID of the project\"\n type = string\n}\n\nvariable prefix {\n description = \"Prefix for resource names\"\n default = \"default\"\n}\n\nvariable machines {\n description = \"Cluster machines\"\n type = map(object({\n node_type = string\n size = string\n zone = string\n additional_disks = map(object({\n size = number\n }))\n boot_disk = object({\n image_name = string\n size = number\n })\n }))\n}\n\nvariable \"master_sa_email\" {\n type = string\n default = \"\"\n}\n\nvariable \"master_sa_scopes\" {\n type = list(string)\n default = [\"https://www.googleapis.com/auth/cloud-platform\"]\n}\n\nvariable \"master_preemptible\" {\n type = bool\n default = false\n}\n\nvariable \"master_additional_disk_type\" {\n type = string\n default = \"pd-ssd\"\n}\n\nvariable \"worker_sa_email\" {\n type = string\n default = \"\"\n}\n\nvariable \"worker_sa_scopes\" {\n type = list(string)\n default = [\"https://www.googleapis.com/auth/cloud-platform\"]\n}\n\nvariable \"worker_preemptible\" {\n type = bool\n default = false\n}\n\nvariable \"worker_additional_disk_type\" {\n type = string\n default = \"pd-ssd\"\n}\n\nvariable ssh_pub_key {\n description = \"Path to public SSH key file which is injected into the VMs.\"\n type = string\n}\n\nvariable ssh_whitelist {\n type = list(string)\n}\n\nvariable api_server_whitelist {\n type = list(string)\n}\n\nvariable nodeport_whitelist {\n type = list(string)\n}\n\nvariable \"ingress_whitelist\" {\n type = list(string)\n default = [\"0.0.0.0/0\"]\n}\n\nvariable \"extra_ingress_firewalls\" {\n type = map(object({\n source_ranges = set(string)\n protocol = string\n ports = list(string)\n target_tags = set(string)\n }))\n\n default = {}\n}\n" }, { "path": "contrib/terraform/hetzner/README.md", "content": "# Kubernetes on Hetzner with Terraform\n\nProvision a Kubernetes cluster on [Hetzner](https://www.hetzner.com/cloud) using Terraform and Kubespray\n\n## Overview\n\nThe setup looks like following\n\n```text\n Kubernetes cluster\n+--------------------------+\n| +--------------+ |\n| | +--------------+ |\n| --> | | | |\n| | | Master/etcd | |\n| | | node(s) | |\n| +-+ | |\n| +--------------+ |\n| ^ |\n| | |\n| v |\n| +--------------+ |\n| | +--------------+ |\n| --> | | | |\n| | | Worker | |\n| | | node(s) | |\n| +-+ | |\n| +--------------+ |\n+--------------------------+\n```\n\nThe nodes uses a private network for node to node communication and a public interface for all external communication.\n\n## Requirements\n\n* Terraform 0.14.0 or newer\n\n## Quickstart\n\nNOTE: Assumes you are at the root of the kubespray repo.\n\nFor authentication in your cluster you can use the environment variables.\n\n```bash\nexport HCLOUD_TOKEN=api-token\n```\n\nCopy the cluster configuration file.\n\n```bash\nCLUSTER=my-hetzner-cluster\ncp -r inventory/sample inventory/$CLUSTER\ncp contrib/terraform/hetzner/default.tfvars inventory/$CLUSTER/\ncd inventory/$CLUSTER\n```\n\nEdit `default.tfvars` to match your requirement.\n\nFlatcar Container Linux instead of the basic Hetzner Images.\n\n```bash\ncd ../../contrib/terraform/hetzner\n```\n\nEdit `main.tf` and reactivate the module `source = \"./modules/kubernetes-cluster-flatcar\"`and\ncomment out the `#source = \"./modules/kubernetes-cluster\"`.\n\nactivate `ssh_private_key_path = var.ssh_private_key_path`. The VM boots into\nRescue-Mode with the selected image of the `var.machines` but installs Flatcar instead.\n\nRun Terraform to create the infrastructure.\n\n```bash\ncd ./kubespray\nterraform -chdir=./contrib/terraform/hetzner/ init\nterraform -chdir=./contrib/terraform/hetzner/ apply --var-file=../../../inventory/$CLUSTER/default.tfvars\n```\n\nYou should now have a inventory file named `inventory.ini` that you can use with kubespray.\nYou can use the inventory file with kubespray to set up a cluster.\n\nIt is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:\n\n```bash\nansible -i inventory.ini -m ping all\n```\n\nYou can setup Kubernetes with kubespray using the generated inventory:\n\n```bash\nansible-playbook -i inventory.ini ../../cluster.yml -b -v\n```\n\n## Cloud controller\n\nFor better support with the cloud you can install the [hcloud cloud controller](https://github.com/hetznercloud/hcloud-cloud-controller-manager) and [CSI driver](https://github.com/hetznercloud/csi-driver).\n\nPlease read the instructions in both repos on how to install it.\n\n## Teardown\n\nYou can teardown your infrastructure using the following Terraform command:\n\n```bash\ncd ./kubespray\nterraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars\n```\n\n## Variables\n\n* `prefix`: Prefix to add to all resources, if set to \"\" don't set any prefix\n* `ssh_public_keys`: List of public SSH keys to install on all machines\n* `zone`: The zone where to run the cluster\n* `network_zone`: the network zone where the cluster is running\n* `machines`: Machines to provision. Key of this object will be used as the name of the machine\n * `node_type`: The role of this node *(master|worker)*\n * `size`: Size of the VM\n * `image`: The image to use for the VM\n* `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes\n* `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server\n* `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports)\n* `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to kubernetes workers on port 80 and 443\n" }, { "path": "contrib/terraform/hetzner/default.tfvars", "content": "prefix = \"default\"\nzone = \"hel1\"\nnetwork_zone = \"eu-central\"\ninventory_file = \"inventory.ini\"\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\nssh_private_key_path = \"~/.ssh/id_rsa\"\n\nmachines = {\n \"master-0\" : {\n \"node_type\" : \"master\",\n \"size\" : \"cx21\",\n \"image\" : \"ubuntu-22.04\",\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"cx21\",\n \"image\" : \"ubuntu-22.04\",\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"cx21\",\n \"image\" : \"ubuntu-22.04\",\n }\n}\n\nnodeport_whitelist = [\n \"0.0.0.0/0\"\n]\n\ningress_whitelist = [\n \"0.0.0.0/0\"\n]\n\nssh_whitelist = [\n \"0.0.0.0/0\"\n]\n\napi_server_whitelist = [\n \"0.0.0.0/0\"\n]\n" }, { "path": "contrib/terraform/hetzner/main.tf", "content": "provider \"hcloud\" {}\n\nmodule \"kubernetes\" {\n source = \"./modules/kubernetes-cluster\"\n # source = \"./modules/kubernetes-cluster-flatcar\"\n\n prefix = var.prefix\n\n zone = var.zone\n\n machines = var.machines\n\n #only for flatcar\n #ssh_private_key_path = var.ssh_private_key_path\n\n ssh_public_keys = var.ssh_public_keys\n network_zone = var.network_zone\n\n ssh_whitelist = var.ssh_whitelist\n api_server_whitelist = var.api_server_whitelist\n nodeport_whitelist = var.nodeport_whitelist\n ingress_whitelist = var.ingress_whitelist\n}\n\n#\n# Generate ansible inventory\n#\n\nlocals {\n inventory = templatefile(\n \"${path.module}/templates/inventory.tpl\",\n {\n connection_strings_master = join(\"\\n\", formatlist(\"%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d\",\n keys(module.kubernetes.master_ip_addresses),\n values(module.kubernetes.master_ip_addresses).*.public_ip,\n values(module.kubernetes.master_ip_addresses).*.private_ip,\n range(1, length(module.kubernetes.master_ip_addresses) + 1)))\n connection_strings_worker = join(\"\\n\", formatlist(\"%s ansible_user=ubuntu ansible_host=%s ip=%s\",\n keys(module.kubernetes.worker_ip_addresses),\n values(module.kubernetes.worker_ip_addresses).*.public_ip,\n values(module.kubernetes.worker_ip_addresses).*.private_ip))\n list_master = join(\"\\n\", keys(module.kubernetes.master_ip_addresses))\n list_worker = join(\"\\n\", keys(module.kubernetes.worker_ip_addresses))\n network_id = module.kubernetes.network_id\n }\n )\n}\n\nresource \"null_resource\" \"inventories\" {\n provisioner \"local-exec\" {\n command = \"echo '${local.inventory}' > ${var.inventory_file}\"\n }\n\n triggers = {\n template = local.inventory\n }\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf", "content": "resource \"hcloud_network\" \"kubernetes\" {\n name = \"${var.prefix}-network\"\n ip_range = var.private_network_cidr\n}\n\nresource \"hcloud_network_subnet\" \"kubernetes\" {\n type = \"cloud\"\n network_id = hcloud_network.kubernetes.id\n network_zone = var.network_zone\n ip_range = var.private_subnet_cidr\n}\n\nresource \"hcloud_server\" \"master\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"master\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n image = each.value.image\n server_type = each.value.size\n location = var.zone\n\n user_data = templatefile(\n \"${path.module}/templates/cloud-init.tmpl\",\n {\n ssh_public_keys = var.ssh_public_keys\n }\n )\n\n firewall_ids = [hcloud_firewall.master.id]\n}\n\nresource \"hcloud_server_network\" \"master\" {\n for_each = hcloud_server.master\n\n server_id = each.value.id\n\n subnet_id = hcloud_network_subnet.kubernetes.id\n}\n\nresource \"hcloud_server\" \"worker\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"worker\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n image = each.value.image\n server_type = each.value.size\n location = var.zone\n\n user_data = templatefile(\n \"${path.module}/templates/cloud-init.tmpl\",\n {\n ssh_public_keys = var.ssh_public_keys\n }\n )\n\n firewall_ids = [hcloud_firewall.worker.id]\n\n}\n\nresource \"hcloud_server_network\" \"worker\" {\n for_each = hcloud_server.worker\n\n server_id = each.value.id\n\n subnet_id = hcloud_network_subnet.kubernetes.id\n}\n\nresource \"hcloud_firewall\" \"master\" {\n name = \"${var.prefix}-master-firewall\"\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"22\"\n source_ips = var.ssh_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"6443\"\n source_ips = var.api_server_whitelist\n }\n}\n\nresource \"hcloud_firewall\" \"worker\" {\n name = \"${var.prefix}-worker-firewall\"\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"22\"\n source_ips = var.ssh_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"80\"\n source_ips = var.ingress_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"443\"\n source_ips = var.ingress_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"30000-32767\"\n source_ips = var.nodeport_whitelist\n }\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf", "content": "output \"master_ip_addresses\" {\n value = {\n for key, instance in hcloud_server.master :\n instance.name => {\n \"private_ip\" = hcloud_server_network.master[key].ip\n \"public_ip\" = hcloud_server.master[key].ipv4_address\n }\n }\n}\n\noutput \"worker_ip_addresses\" {\n value = {\n for key, instance in hcloud_server.worker :\n instance.name => {\n \"private_ip\" = hcloud_server_network.worker[key].ip\n \"public_ip\" = hcloud_server.worker[key].ipv4_address\n }\n }\n}\n\noutput \"cluster_private_network_cidr\" {\n value = var.private_subnet_cidr\n}\n\noutput \"network_id\" {\n value = hcloud_network.kubernetes.id\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl", "content": "#cloud-config\n\nusers:\n - default\n - name: ubuntu\n shell: /bin/bash\n sudo: \"ALL=(ALL) NOPASSWD:ALL\"\n ssh_authorized_keys:\n %{ for ssh_public_key in ssh_public_keys ~}\n - ${ssh_public_key}\n %{ endfor ~}\n\nssh_authorized_keys:\n%{ for ssh_public_key in ssh_public_keys ~}\n - ${ssh_public_key}\n%{ endfor ~}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf", "content": "variable \"zone\" {\n type = string\n}\n\nvariable \"prefix\" {}\n\nvariable \"machines\" {\n type = map(object({\n node_type = string\n size = string\n image = string\n }))\n}\n\nvariable \"ssh_public_keys\" {\n type = list(string)\n}\n\nvariable \"ssh_whitelist\" {\n type = list(string)\n}\n\nvariable \"api_server_whitelist\" {\n type = list(string)\n}\n\nvariable \"nodeport_whitelist\" {\n type = list(string)\n}\n\nvariable \"ingress_whitelist\" {\n type = list(string)\n}\n\nvariable \"private_network_cidr\" {\n default = \"10.0.0.0/16\"\n}\n\nvariable \"private_subnet_cidr\" {\n default = \"10.0.10.0/24\"\n}\nvariable \"network_zone\" {\n default = \"eu-central\"\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf", "content": "terraform {\n required_providers {\n hcloud = {\n source = \"hetznercloud/hcloud\"\n version = \"1.38.2\"\n }\n }\n required_version = \">= 0.14\"\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf", "content": "resource \"hcloud_network\" \"kubernetes\" {\n name = \"${var.prefix}-network\"\n ip_range = var.private_network_cidr\n}\n\nresource \"hcloud_network_subnet\" \"kubernetes\" {\n type = \"cloud\"\n network_id = hcloud_network.kubernetes.id\n network_zone = var.network_zone\n ip_range = var.private_subnet_cidr\n}\n\nresource \"hcloud_ssh_key\" \"first\" {\n name = var.prefix\n public_key = var.ssh_public_keys.0\n}\n\nresource \"hcloud_server\" \"machine\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n }\n\n name = \"${var.prefix}-${each.key}\"\n ssh_keys = [hcloud_ssh_key.first.id]\n # boot into rescue OS\n rescue = \"linux64\"\n # dummy value for the OS because Flatcar is not available\n image = each.value.image\n server_type = each.value.size\n location = var.zone\n connection {\n host = self.ipv4_address\n timeout = \"5m\"\n private_key = file(var.ssh_private_key_path)\n }\n firewall_ids = each.value.node_type == \"master\" ? [hcloud_firewall.master.id] : [hcloud_firewall.worker.id]\n provisioner \"file\" {\n content = data.ct_config.machine-ignitions[each.key].rendered\n destination = \"/root/ignition.json\"\n }\n\n provisioner \"remote-exec\" {\n inline = [\n \"set -ex\",\n \"apt update\",\n \"apt install -y gawk\",\n \"curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install\",\n \"chmod +x flatcar-install\",\n \"./flatcar-install -s -i /root/ignition.json -C stable\",\n \"shutdown -r +1\",\n ]\n }\n\n # optional:\n provisioner \"remote-exec\" {\n connection {\n host = self.ipv4_address\n private_key = file(var.ssh_private_key_path)\n timeout = \"3m\"\n user = var.user_flatcar\n }\n\n inline = [\n \"sudo hostnamectl set-hostname ${self.name}\",\n ]\n }\n}\n\nresource \"hcloud_server_network\" \"machine\" {\n for_each = {\n for name, machine in var.machines :\n name => hcloud_server.machine[name]\n }\n server_id = each.value.id\n subnet_id = hcloud_network_subnet.kubernetes.id\n}\n\ndata \"ct_config\" \"machine-ignitions\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n }\n\n strict = false\n content = templatefile(\n \"${path.module}/templates/machine.yaml.tmpl\",\n {\n ssh_keys = jsonencode(var.ssh_public_keys)\n user_flatcar = var.user_flatcar\n name = each.key\n }\n )\n}\n\nresource \"hcloud_firewall\" \"master\" {\n name = \"${var.prefix}-master-firewall\"\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"22\"\n source_ips = var.ssh_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"6443\"\n source_ips = var.api_server_whitelist\n }\n}\n\nresource \"hcloud_firewall\" \"worker\" {\n name = \"${var.prefix}-worker-firewall\"\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"22\"\n source_ips = var.ssh_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"80\"\n source_ips = var.ingress_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"443\"\n source_ips = var.ingress_whitelist\n }\n\n rule {\n direction = \"in\"\n protocol = \"tcp\"\n port = \"30000-32767\"\n source_ips = var.nodeport_whitelist\n }\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf", "content": "output \"master_ip_addresses\" {\n value = {\n for name, machine in var.machines :\n name => {\n \"private_ip\" = hcloud_server_network.machine[name].ip\n \"public_ip\" = hcloud_server.machine[name].ipv4_address\n }\n if machine.node_type == \"master\"\n }\n}\n\noutput \"worker_ip_addresses\" {\n value = {\n for name, machine in var.machines :\n name => {\n \"private_ip\" = hcloud_server_network.machine[name].ip\n \"public_ip\" = hcloud_server.machine[name].ipv4_address\n }\n if machine.node_type == \"worker\"\n }\n}\n\noutput \"cluster_private_network_cidr\" {\n value = var.private_subnet_cidr\n}\n\noutput \"network_id\" {\n value = hcloud_network.kubernetes.id\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl", "content": "variant: flatcar\nversion: 1.0.0\n\npasswd:\n users:\n - name: ${user_flatcar}\n ssh_authorized_keys: ${ssh_keys}\n\nstorage:\n files:\n - path: /home/core/works\n filesystem: root\n mode: 0755\n contents:\n inline: |\n #!/bin/bash\n set -euo pipefail\n hostname=\"$(hostname)\"\n echo My name is ${name} and the hostname is $${hostname}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf", "content": "\nvariable \"zone\" {\n type = string\n default = \"fsn1\"\n}\n\nvariable \"prefix\" {\n default = \"k8s\"\n}\n\nvariable \"user_flatcar\" {\n type = string\n default = \"core\"\n}\n\nvariable \"machines\" {\n type = map(object({\n node_type = string\n size = string\n image = string\n }))\n}\n\n\n\nvariable \"ssh_public_keys\" {\n type = list(string)\n}\n\nvariable \"ssh_private_key_path\" {\n type = string\n default = \"~/.ssh/id_rsa\"\n}\n\nvariable \"ssh_whitelist\" {\n type = list(string)\n}\n\nvariable \"api_server_whitelist\" {\n type = list(string)\n}\n\nvariable \"nodeport_whitelist\" {\n type = list(string)\n}\n\nvariable \"ingress_whitelist\" {\n type = list(string)\n}\n\nvariable \"private_network_cidr\" {\n default = \"10.0.0.0/16\"\n}\n\nvariable \"private_subnet_cidr\" {\n default = \"10.0.10.0/24\"\n}\nvariable \"network_zone\" {\n default = \"eu-central\"\n}\n" }, { "path": "contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf", "content": "terraform {\n required_providers {\n hcloud = {\n source = \"hetznercloud/hcloud\"\n }\n ct = {\n source = \"poseidon/ct\"\n version = \"0.11.0\"\n }\n null = {\n source = \"hashicorp/null\"\n }\n }\n}\n" }, { "path": "contrib/terraform/hetzner/output.tf", "content": "output \"master_ips\" {\n value = module.kubernetes.master_ip_addresses\n}\n\noutput \"worker_ips\" {\n value = module.kubernetes.worker_ip_addresses\n}\n" }, { "path": "contrib/terraform/hetzner/sample-inventory/cluster.tfvars", "content": "prefix = \"default\"\nzone = \"hel1\"\nnetwork_zone = \"eu-central\"\ninventory_file = \"inventory.ini\"\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\nssh_private_key_path = \"~/.ssh/id_rsa\"\n\nmachines = {\n \"master-0\" : {\n \"node_type\" : \"master\",\n \"size\" : \"cx21\",\n \"image\" : \"ubuntu-22.04\",\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"cx21\",\n \"image\" : \"ubuntu-22.04\",\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n \"size\" : \"cx21\",\n \"image\" : \"ubuntu-22.04\",\n }\n}\n\nnodeport_whitelist = [\n \"0.0.0.0/0\"\n]\n\ningress_whitelist = [\n \"0.0.0.0/0\"\n]\n\nssh_whitelist = [\n \"0.0.0.0/0\"\n]\n\napi_server_whitelist = [\n \"0.0.0.0/0\"\n]\n" }, { "path": "contrib/terraform/hetzner/templates/inventory.tpl", "content": "[all]\n${connection_strings_master}\n${connection_strings_worker}\n\n[kube_control_plane]\n${list_master}\n\n[etcd]\n${list_master}\n\n[kube_node]\n${list_worker}\n\n[k8s_cluster:children]\nkube_control_plane\nkube_node\n\n[k8s_cluster:vars]\nnetwork_id=${network_id}\n" }, { "path": "contrib/terraform/hetzner/variables.tf", "content": "variable \"zone\" {\n description = \"The zone where to run the cluster\"\n}\nvariable \"network_zone\" {\n description = \"The network zone where the cluster is running\"\n default = \"eu-central\"\n}\n\nvariable \"prefix\" {\n description = \"Prefix for resource names\"\n default = \"default\"\n}\n\nvariable \"machines\" {\n description = \"Cluster machines\"\n type = map(object({\n node_type = string\n size = string\n image = string\n }))\n}\n\nvariable \"ssh_public_keys\" {\n description = \"Public SSH key which are injected into the VMs.\"\n type = list(string)\n}\n\nvariable \"ssh_private_key_path\" {\n description = \"Private SSH key which connect to the VMs.\"\n type = string\n default = \"~/.ssh/id_rsa\"\n}\n\nvariable \"ssh_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for ssh\"\n type = list(string)\n}\n\nvariable \"api_server_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for kubernetes api server\"\n type = list(string)\n}\n\nvariable \"nodeport_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for kubernetes nodeports\"\n type = list(string)\n}\n\nvariable \"ingress_whitelist\" {\n description = \"List of IP ranges (CIDR) to whitelist for HTTP\"\n type = list(string)\n}\n\nvariable \"inventory_file\" {\n description = \"Where to store the generated inventory file\"\n}\n" }, { "path": "contrib/terraform/hetzner/versions.tf", "content": "terraform {\n required_providers {\n hcloud = {\n source = \"hetznercloud/hcloud\"\n version = \"1.38.2\"\n }\n null = {\n source = \"hashicorp/null\"\n }\n }\n required_version = \">= 0.14\"\n}\n" }, { "path": "contrib/terraform/openstack/.gitignore", "content": ".terraform\n*.tfvars\n!sample-inventory/cluster.tfvars\n*.tfstate\n*.tfstate.backup\n" }, { "path": "contrib/terraform/openstack/README.md", "content": "# Kubernetes on OpenStack with Terraform\n\nProvision a Kubernetes cluster with [Terraform](https://www.terraform.io) on\nOpenStack.\n\n## Status\n\nThis will install a Kubernetes cluster on an OpenStack Cloud. It should work on\nmost modern installs of OpenStack that support the basic services.\n\n### Known compatible public clouds\n\n- [Auro](https://auro.io/)\n- [Betacloud](https://www.betacloud.io/)\n- [CityCloud](https://www.citycloud.com/)\n- [DreamHost](https://www.dreamhost.com/cloud/computing/)\n- [ELASTX](https://elastx.se/)\n- [EnterCloudSuite](https://www.entercloudsuite.com/)\n- [FugaCloud](https://fuga.cloud/)\n- [Open Telekom Cloud](https://cloud.telekom.de/)\n- [OVH](https://www.ovh.com/)\n- [Rackspace](https://www.rackspace.com/)\n- [Safespring](https://www.safespring.com)\n- [Ultimum](https://ultimum.io/)\n- [VexxHost](https://vexxhost.com/)\n- [Zetta](https://www.zetta.io/)\n- [Cloudify](https://www.cloudify.ro/en)\n\n## Approach\n\nThe terraform configuration inspects variables found in\n[variables.tf](variables.tf) to create resources in your OpenStack cluster.\nThere is a [python script](../terraform.py) that reads the generated`.tfstate`\nfile to generate a dynamic inventory that is consumed by the main ansible script\nto actually install kubernetes and stand up the cluster.\n\n### Networking\n\nThe configuration includes creating a private subnet with a router to the\nexternal net. It will allocate floating IPs from a pool and assign them to the\nhosts where that makes sense. You have the option of creating bastion hosts\ninside the private subnet to access the nodes there. Alternatively, a node with\na floating IP can be used as a jump host to nodes without.\n\n#### Using an existing router\n\nIt is possible to use an existing router instead of creating one. To use an\nexisting router set the router\\_id variable to the uuid of the router you wish\nto use.\n\nFor example:\n\n```ShellSession\nrouter_id = \"00c542e7-6f46-4535-ae95-984c7f0391a3\"\n```\n\n### Kubernetes Nodes\n\nYou can create many different kubernetes topologies by setting the number of\ndifferent classes of hosts. For each class there are options for allocating\nfloating IP addresses or not.\n\n- Control plane nodes with etcd\n- Control plane nodes without etcd\n- Standalone etcd hosts\n- Kubernetes worker nodes\n\nNote that the Ansible script will report an invalid configuration if you wind up\nwith an even number of etcd instances since that is not a valid configuration. This\nrestriction includes standalone etcd nodes that are deployed in a cluster along with\ncontrol plane nodes with etcd replicas. As an example, if you have three control plane\nnodes with etcd replicas and three standalone etcd nodes, the script will fail since\nthere are now six total etcd replicas.\n\n### GlusterFS shared file system\n\nThe Terraform configuration supports provisioning of an optional GlusterFS\nshared file system based on a separate set of VMs. To enable this, you need to\nspecify:\n\n- the number of Gluster hosts (minimum 2)\n- Size of the non-ephemeral volumes to be attached to store the GlusterFS bricks\n- Other properties related to provisioning the hosts\n\nEven if you are using Flatcar Container Linux by Kinvolk for your cluster, you will still\nneed the GlusterFS VMs to be based on either Debian or RedHat based images.\nFlatcar Container Linux by Kinvolk cannot serve GlusterFS, but can connect to it through\nbinaries available on hyperkube v1.4.3_coreos.0 or higher.\n\n## Requirements\n\n- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.14 or later\n- [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html)\n- you already have a suitable OS image in Glance\n- you already have a floating IP pool created\n- you have security groups enabled\n- you have a pair of keys generated that can be used to secure the new hosts\n\n## Module Architecture\n\nThe configuration is divided into four modules:\n\n- Network\n- Loadbalancer\n- IPs\n- Compute\n\nThe main reason for splitting the configuration up in this way is to easily\naccommodate situations where floating IPs are limited by a quota or if you have\nany external references to the floating IP (e.g. DNS) that would otherwise have\nto be updated.\n\nYou can force your existing IPs by modifying the compute variables in\n`kubespray.tf` as follows:\n\n```ini\nk8s_master_fips = [\"151.101.129.67\"]\nk8s_node_fips = [\"151.101.129.68\"]\n```\n\n## Terraform\n\nTerraform will be used to provision all of the OpenStack resources with base software as appropriate.\n\n### Configuration\n\n#### Inventory files\n\nCreate an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state):\n\n```ShellSession\ncp -LRp contrib/terraform/openstack/sample-inventory inventory/$CLUSTER\ncd inventory/$CLUSTER\nln -s ../../contrib/terraform/openstack/hosts\nln -s ../../contrib\n```\n\nThis will be the base for subsequent Terraform commands.\n\n#### OpenStack access and credentials\n\nNo provider variables are hardcoded inside `variables.tf` because Terraform\nsupports various authentication methods for OpenStack: the older script and\nenvironment method (using `openrc`) as well as a newer declarative method, and\ndifferent OpenStack environments may support Identity API version 2 or 3.\n\nThese are examples and may vary depending on your OpenStack cloud provider,\nfor an exhaustive list on how to authenticate on OpenStack with Terraform\nplease read the [OpenStack provider documentation](https://www.terraform.io/docs/providers/openstack/).\n\n##### Declarative method (recommended)\n\nThe recommended authentication method is to describe credentials in a YAML file `clouds.yaml` that can be stored in:\n\n- the current directory\n- `~/.config/openstack`\n- `/etc/openstack`\n\n`clouds.yaml`:\n\n```yaml\nclouds:\n mycloud:\n auth:\n auth_url: https://openstack:5000/v3\n username: \"username\"\n project_name: \"projectname\"\n project_id: projectid\n user_domain_name: \"Default\"\n password: \"password\"\n region_name: \"RegionOne\"\n interface: \"public\"\n identity_api_version: 3\n```\n\nIf you have multiple clouds defined in your `clouds.yaml` file you can choose\nthe one you want to use with the environment variable `OS_CLOUD`:\n\n```ShellSession\nexport OS_CLOUD=mycloud\n```\n\n##### Openrc method\n\nWhen using classic environment variables, Terraform uses default `OS_*`\nenvironment variables. A script suitable for your environment may be available\nfrom Horizon under *Project* -> *Compute* -> *Access & Security* -> *API Access*.\n\nWith identity v2:\n\n```ShellSession\nsource openrc\n\nenv | grep OS\n\nOS_AUTH_URL=https://openstack:5000/v2.0\nOS_PROJECT_ID=projectid\nOS_PROJECT_NAME=projectname\nOS_USERNAME=username\nOS_PASSWORD=password\nOS_REGION_NAME=RegionOne\nOS_INTERFACE=public\nOS_IDENTITY_API_VERSION=2\n```\n\nWith identity v3:\n\n```ShellSession\nsource openrc\n\nenv | grep OS\n\nOS_AUTH_URL=https://openstack:5000/v3\nOS_PROJECT_ID=projectid\nOS_PROJECT_NAME=username\nOS_PROJECT_DOMAIN_ID=default\nOS_USERNAME=username\nOS_PASSWORD=password\nOS_REGION_NAME=RegionOne\nOS_INTERFACE=public\nOS_IDENTITY_API_VERSION=3\nOS_USER_DOMAIN_NAME=Default\n```\n\nTerraform does not support a mix of DomainName and DomainID, choose one or the other:\n\n- provider.openstack: You must provide exactly one of DomainID or DomainName to authenticate by Username\n\n```ShellSession\nunset OS_USER_DOMAIN_NAME\nexport OS_USER_DOMAIN_ID=default\n```\n\nor\n\n```ShellSession\nunset OS_PROJECT_DOMAIN_ID\nset OS_PROJECT_DOMAIN_NAME=Default\n```\n\n#### Cluster variables\n\nThe construction of the cluster is driven by values found in\n[variables.tf](variables.tf).\n\nFor your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.\n\n|Variable | Description |\n|---------|-------------|\n|`cluster_name` | All OpenStack resources will use the Terraform variable`cluster_name` (default`example`) in their name to make it easier to track. For example the first compute resource will be named`example-kubernetes-1`. |\n|`az_list` | List of Availability Zones available in your OpenStack cluster. |\n|`network_name` | The name to be given to the internal network that will be generated |\n|`use_existing_network`| Use an existing network with the name of `network_name`. `false` by default |\n|`network_dns_domain` | (Optional) The dns_domain for the internal network that will be generated |\n|`dns_nameservers`| An array of DNS name server names to be used by hosts in the internal subnet. |\n|`floatingip_pool` | Name of the pool from which floating IPs will be allocated |\n|`k8s_master_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to master nodes instead of creating new random floating IPs. |\n|`bastion_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to bastion node instead of creating new random floating IPs. |\n|`external_net` | UUID of the external network that will be routed to |\n|`flavor_k8s_master`,`flavor_k8s_node`,`flavor_etcd`, `flavor_bastion`,`flavor_gfs_node` | Flavor depends on your openstack installation, you can get available flavor IDs through `openstack flavor list` |\n|`image`,`image_gfs`, `image_master` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. |\n|`image_uuid`,`image_gfs_uuid`, `image_master_uuid` | UUID of the image to use in provisioning the compute resources. Should already be loaded into glance. |\n|`ssh_user`,`ssh_user_gfs` | The username to ssh into the image with. This usually depends on the image you have selected |\n|`public_key_path` | Path on your local workstation to the public key file you wish to use in creating the key pairs |\n|`number_of_k8s_masters`, `number_of_k8s_masters_no_floating_ip` | Number of nodes that serve as both master and etcd. These can be provisioned with or without floating IP addresses|\n|`number_of_k8s_masters_no_etcd`, `number_of_k8s_masters_no_floating_ip_no_etcd` | Number of nodes that serve as just master with no etcd. These can be provisioned with or without floating IP addresses |\n|`number_of_etcd` | Number of pure etcd nodes |\n|`number_of_k8s_nodes`, `number_of_k8s_nodes_no_floating_ip` | Kubernetes worker nodes. These can be provisioned with or without floating ip addresses. |\n|`number_of_bastions` | Number of bastion hosts to create. Scripts assume this is really just zero or one |\n|`number_of_gfs_nodes_no_floating_ip` | Number of gluster servers to provision. |\n| `gfs_volume_size_in_gb` | Size of the non-ephemeral volumes to be attached to store the GlusterFS bricks |\n|`supplementary_master_groups` | To add ansible groups to the masters, such as `kube_node` for tainting them as nodes, empty by default. |\n|`supplementary_node_groups` | To add ansible groups to the nodes, such as `kube_ingress` for running ingress controller pods, empty by default. |\n|`bastion_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, `[\"0.0.0.0/0\"]` by default |\n|`bastion_allowed_remote_ipv6_ips` | List of IPv6 CIDR allowed to initiate a SSH connection, `[\"::/0\"]` by default |\n|`master_allowed_remote_ips` | List of CIDR blocks allowed to initiate an API connection, `[\"0.0.0.0/0\"]` by default |\n|`master_allowed_remote_ipv6_ips` | List of IPv6 CIDR blocks allowed to initiate an API connection, `[\"::/0\"]` by default |\n|`bastion_allowed_ports` | List of ports to open on bastion node, `[]` by default |\n|`bastion_allowed_ports_ipv6` | List of ports to open on bastion node for IPv6 CIDR blocks, `[]` by default |\n|`k8s_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, empty by default |\n|`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default |\n|`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `[\"::/0\"]` by default |\n|`worker_allowed_ports` | List of ports to open on worker nodes, `[{ \"protocol\" = \"tcp\", \"port_range_min\" = 30000, \"port_range_max\" = 32767, \"remote_ip_prefix\" = \"0.0.0.0/0\"}]` by default |\n|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ \"protocol\" = \"tcp\", \"port_range_min\" = 30000, \"port_range_max\" = 32767, \"remote_ip_prefix\" = \"::/0\"}, { \"protocol\" = \"ipv6-icmp\", \"port_range_min\" = 0, \"port_range_max\" = 0, \"remote_ip_prefix\" = \"::/0\"}]` by default |\n|`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ \"protocol\" = \"tcp\", \"port_range_min\" = 443, \"port_range_max\" = 443, \"remote_ip_prefix\" = \"0.0.0.0/0\"}]`, empty by default |\n|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, `[{ \"protocol\" = \"ipv6-icmp\", \"port_range_min\" = 0, \"port_range_max\" = 0, \"remote_ip_prefix\" = \"::/0\"}]` by default |\n|`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |\n|`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |\n|`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |\n|`node_volume_type` | Volume type of the root volume for nodes, 'Default' by default |\n|`gfs_root_volume_size_in_gb` | Size of the root volume for gluster, 0 to use ephemeral storage |\n|`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage |\n|`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage |\n|`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: \"\" (disabled) |\n|`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: \"\" (disabled) |\n|`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: \"\" (disabled) |\n|`additional_server_groups` | Extra server groups to create. Set \"policy\" to the policy for the group, expected format is `{\"new-server-group\" = {\"policy\" = \"anti-affinity\"}}`, default: {} (to not create any extra groups) |\n|`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. |\n|`port_security_enabled` | Allow to disable port security by setting this to `false`. `true` by default |\n|`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default |\n|`k8s_nodes` | Map containing worker node definition, see explanation below |\n|`k8s_masters` | Map containing master node definition, see explanation for k8s_nodes and `sample-inventory/cluster.tfvars` |\n|`k8s_master_loadbalancer_enabled` | Enable and use an Octavia load balancer for the K8s master nodes |\n|`k8s_master_loadbalancer_listener_port` | Define via which port the K8s Api should be exposed. `6443` by default |\n|`k8s_master_loadbalancer_server_port` | Define via which port the K8S api is available on the master nodes. `6443` by default |\n|`k8s_master_loadbalancer_public_ip` | Specify if an existing floating IP should be used for the load balancer. A new floating IP is assigned by default |\n\n##### k8s_nodes\n\nAllows a custom definition of worker nodes giving the operator full control over individual node flavor and availability zone placement.\nTo enable the use of this mode set the `number_of_k8s_nodes` and `number_of_k8s_nodes_no_floating_ip` variables to 0.\nThen define your desired worker node configuration using the `k8s_nodes` variable.\nThe `az`, `flavor` and `floating_ip` parameters are mandatory.\nThe optional parameter `extra_groups` (a comma-delimited string) can be used to define extra inventory group memberships for specific nodes.\n\n```yaml\nk8s_nodes:\n node-name:\n az: string # Name of the AZ\n flavor: string # Flavor ID to use\n floating_ip: bool # If floating IPs should be used or not\n reserved_floating_ip: string # If floating_ip is true use existing floating IP, if reserved_floating_ip is an empty string and floating_ip is true, a new floating IP will be created\n extra_groups: string # (optional) Additional groups to add for kubespray, defaults to no groups\n image_id: string # (optional) Image ID to use, defaults to var.image_id or var.image\n root_volume_size_in_gb: number # (optional) Size of the block storage to use as root disk, defaults to var.node_root_volume_size_in_gb or to use volume from flavor otherwise\n volume_type: string # (optional) Volume type to use, defaults to var.node_volume_type\n network_id: string # (optional) Use this network_id for the node, defaults to either var.network_id or ID of var.network_name\n server_group: string # (optional) Server group to add this node to. If set, this has to be one specified in additional_server_groups, defaults to use the server group specified in node_server_group_policy\n cloudinit: # (optional) Options for cloud-init\n extra_partitions: # List of extra partitions (other than the root partition) to setup during creation\n volume_path: string # Path to the volume to create partition for (e.g. /dev/vda )\n partition_path: string # Path to the partition (e.g. /dev/vda2 )\n mount_path: string # Path to where the partition should be mounted\n partition_start: string # Where the partition should start (e.g. 10GB ). Note, if you set the partition_start to 0 there will be no space left for the root partition\n partition_end: string # Where the partition should end (e.g. 10GB or -1 for end of volume)\n netplan_critical_dhcp_interface: string # Name of interface to set the dhcp flag critical = true, to circumvent [this issue](https://bugs.launchpad.net/ubuntu/+source/systemd/+bug/1776013).\n```\n\nFor example:\n\n```ini\nk8s_nodes = {\n \"1\" = {\n \"az\" = \"sto1\"\n \"flavor\" = \"83d8b44a-26a0-4f02-a981-079446926445\"\n \"floating_ip\" = true\n },\n \"2\" = {\n \"az\" = \"sto2\"\n \"flavor\" = \"83d8b44a-26a0-4f02-a981-079446926445\"\n \"floating_ip\" = true\n },\n \"3\" = {\n \"az\" = \"sto3\"\n \"flavor\" = \"83d8b44a-26a0-4f02-a981-079446926445\"\n \"floating_ip\" = true\n \"extra_groups\" = \"calico_rr\"\n }\n}\n```\n\nWould result in the same configuration as:\n\n```ini\nnumber_of_k8s_nodes = 3\nflavor_k8s_node = \"83d8b44a-26a0-4f02-a981-079446926445\"\naz_list = [\"sto1\", \"sto2\", \"sto3\"]\n```\n\nAnd:\n\n```ini\nk8s_nodes = {\n \"ing-1\" = {\n \"az\" = \"sto1\"\n \"flavor\" = \"83d8b44a-26a0-4f02-a981-079446926445\"\n \"floating_ip\" = true\n },\n \"ing-2\" = {\n \"az\" = \"sto2\"\n \"flavor\" = \"83d8b44a-26a0-4f02-a981-079446926445\"\n \"floating_ip\" = true\n },\n \"ing-3\" = {\n \"az\" = \"sto3\"\n \"flavor\" = \"83d8b44a-26a0-4f02-a981-079446926445\"\n \"floating_ip\" = true\n },\n \"big-1\" = {\n \"az\" = \"sto1\"\n \"flavor\" = \"3f73fc93-ec61-4808-88df-2580d94c1a9b\"\n \"floating_ip\" = false\n },\n \"big-2\" = {\n \"az\" = \"sto2\"\n \"flavor\" = \"3f73fc93-ec61-4808-88df-2580d94c1a9b\"\n \"floating_ip\" = false\n },\n \"big-3\" = {\n \"az\" = \"sto3\"\n \"flavor\" = \"3f73fc93-ec61-4808-88df-2580d94c1a9b\"\n \"floating_ip\" = false\n },\n \"small-1\" = {\n \"az\" = \"sto1\"\n \"flavor\" = \"7a6a998f-ac7f-4fb8-a534-2175b254f75e\"\n \"floating_ip\" = false\n },\n \"small-2\" = {\n \"az\" = \"sto2\"\n \"flavor\" = \"7a6a998f-ac7f-4fb8-a534-2175b254f75e\"\n \"floating_ip\" = false\n },\n \"small-3\" = {\n \"az\" = \"sto3\"\n \"flavor\" = \"7a6a998f-ac7f-4fb8-a534-2175b254f75e\"\n \"floating_ip\" = false\n }\n}\n```\n\nWould result in three nodes in each availability zone each with their own separate naming,\nflavor and floating ip configuration.\n\nThe \"schema\":\n\n```ini\nk8s_nodes = {\n \"key | node name suffix, must be unique\" = {\n \"az\" = string\n \"flavor\" = string\n \"floating_ip\" = bool\n },\n}\n```\n\nAll values are required.\n\n#### Terraform state files\n\nIn the cluster's inventory folder, the following files might be created (either by Terraform\nor manually), to prevent you from pushing them accidentally they are in a\n`.gitignore` file in the `terraform/openstack` directory :\n\n- `.terraform`\n- `.tfvars`\n- `.tfstate`\n- `.tfstate.backup`\n\nYou can still add them manually if you want to.\n\n### Initialization\n\nBefore Terraform can operate on your cluster you need to install the required\nplugins. This is accomplished as follows:\n\n```ShellSession\ncd inventory/$CLUSTER\nterraform -chdir=\"../../contrib/terraform/openstack\" init\n```\n\nThis should finish fairly quickly telling you Terraform has successfully initialized and loaded necessary modules.\n\n### Customizing with cloud-init\n\nYou can apply cloud-init based customization for the openstack instances before provisioning your cluster.\nOne common template is used for all instances. Adjust the file shown below:\n`contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml.tmpl`\nFor example, to enable openstack novnc access and ansible_user=root SSH access:\n\n```ShellSession\n#cloud-config\n## in some cases novnc console access is required\n## it requires ssh password to be set\nssh_pwauth: yes\nchpasswd:\n list: |\n root:secret\n expire: False\n\n## in some cases direct root ssh access via ssh key is required\ndisable_root: false\n```\n\n### Provisioning cluster\n\nYou can apply the Terraform configuration to your cluster with the following command\nissued from your cluster's inventory directory (`inventory/$CLUSTER`):\n\n```ShellSession\nterraform -chdir=\"../../contrib/terraform/openstack\" apply -var-file=cluster.tfvars\n```\n\nif you chose to create a bastion host, this script will create\n`contrib/terraform/openstack/k8s_cluster.yml` with an ssh command for Ansible to\nbe able to access your machines tunneling through the bastion's IP address. If\nyou want to manually handle the ssh tunneling to these machines, please delete\nor move that file. If you want to use this, just leave it there, as ansible will\npick it up automatically.\n\n### Destroying cluster\n\nYou can destroy your new cluster with the following command issued from the cluster's inventory directory:\n\n```ShellSession\nterraform -chdir=\"../../contrib/terraform/openstack\" destroy -var-file=cluster.tfvars\n```\n\nIf you've started the Ansible run, it may also be a good idea to do some manual cleanup:\n\n- remove SSH keys from the destroyed cluster from your `~/.ssh/known_hosts` file\n- clean up any temporary cache files: `rm /tmp/$CLUSTER-*`\n\n### Debugging\n\nYou can enable debugging output from Terraform by setting\n`OS_DEBUG` to 1 and`TF_LOG` to`DEBUG` before running the Terraform command.\n\n### Terraform output\n\nTerraform can output values that are useful for configure Neutron/Octavia LBaaS or Cinder persistent volume provisioning as part of your Kubernetes deployment:\n\n- `private_subnet_id`: the subnet where your instances are running is used for `openstack_lbaas_subnet_id`\n- `floating_network_id`: the network_id where the floating IP are provisioned is used for `openstack_lbaas_floating_network_id`\n\n## Ansible\n\n### Node access\n\n#### SSH\n\nEnsure your local ssh-agent is running and your ssh key has been added. This\nstep is required by the terraform provisioner:\n\n```ShellSession\neval $(ssh-agent -s)\nssh-add ~/.ssh/id_rsa\n```\n\nIf you have deployed and destroyed a previous iteration of your cluster, you will need to clear out any stale keys from your SSH \"known hosts\" file ( `~/.ssh/known_hosts`).\n\n#### Metadata variables\n\nThe [python script](../terraform.py) that reads the\ngenerated`.tfstate` file to generate a dynamic inventory recognizes\nsome variables within a \"metadata\" block, defined in a \"resource\"\nblock (example):\n\n```ini\nresource \"openstack_compute_instance_v2\" \"example\" {\n ...\n metadata {\n ssh_user = \"ubuntu\"\n prefer_ipv6 = true\n python_bin = \"/usr/bin/python3\"\n }\n ...\n}\n```\n\nAs the example shows, these let you define the SSH username for\nAnsible, a Python binary which is needed by Ansible if\n`/usr/bin/python` doesn't exist, and whether the IPv6 address of the\ninstance should be preferred over IPv4.\n\n#### Bastion host\n\nBastion access will be determined by:\n\n- Your choice on the amount of bastion hosts (set by `number_of_bastions` terraform variable).\n- The existence of nodes/masters with floating IPs (set by `number_of_k8s_masters`, `number_of_k8s_nodes`, `number_of_k8s_masters_no_etcd` terraform variables).\n\nIf you have a bastion host, your ssh traffic will be directly routed through it. This is regardless of whether you have masters/nodes with a floating IP assigned.\nIf you don't have a bastion host, but at least one of your masters/nodes have a floating IP, then ssh traffic will be tunneled by one of these machines.\n\nSo, either a bastion host, or at least master/node with a floating IP are required.\n\n#### Test access\n\nMake sure you can connect to the hosts. Note that Flatcar Container Linux by Kinvolk will have a state `FAILED` due to Python not being present. This is okay, because Python will be installed during bootstrapping, so long as the hosts are not `UNREACHABLE`.\n\n```ShellSession\n$ ansible -i inventory/$CLUSTER/hosts -m ping all\nexample-k8s_node-1 | SUCCESS => {\n \"changed\": false,\n \"ping\": \"pong\"\n}\nexample-etcd-1 | SUCCESS => {\n \"changed\": false,\n \"ping\": \"pong\"\n}\nexample-k8s-master-1 | SUCCESS => {\n \"changed\": false,\n \"ping\": \"pong\"\n}\n```\n\nIf it fails try to connect manually via SSH. It could be something as simple as a stale host key.\n\n### Configure cluster variables\n\nEdit `inventory/$CLUSTER/group_vars/all/all.yml`:\n\n- **bin_dir**:\n\n```yml\n# Directory where the binaries will be installed\n# Default:\n# bin_dir: /usr/local/bin\n# For Flatcar Container Linux by Kinvolk:\nbin_dir: /opt/bin\n```\n\n- and **cloud_provider**:\n\n```yml\ncloud_provider: openstack\n```\n\nEdit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:\n\n- Set variable **kube_network_plugin** to your desired networking plugin.\n - **flannel** works out-of-the-box\n - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets\n\n```yml\n# Choose network plugin (calico or flannel)\n# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing\nkube_network_plugin: flannel\n```\n\n- Set variable **resolvconf_mode**\n\n```yml\n# Can be docker_dns, host_resolvconf or none\n# Default:\n# resolvconf_mode: docker_dns\n# For Flatcar Container Linux by Kinvolk:\nresolvconf_mode: host_resolvconf\n```\n\n- Set max amount of attached cinder volume per host (default 256)\n\n```yml\nnode_volume_attach_limit: 26\n```\n\n### Deploy Kubernetes\n\n```ShellSession\nansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml\n```\n\nThis will take some time as there are many tasks to run.\n\n## Kubernetes\n\n### Set up kubectl\n\n1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your workstation\n2. Add a route to the internal IP of a master node (if needed):\n\n```ShellSession\nsudo route add [master-internal-ip] gw [router-ip]\n```\n\nor\n\n```ShellSession\nsudo route add -net [internal-subnet]/24 gw [router-ip]\n```\n\n1. List Kubernetes certificates & keys:\n\n```ShellSession\nssh [os-user]@[master-ip] sudo ls /etc/kubernetes/ssl/\n```\n\n1. Get `admin`'s certificates and keys:\n\n```ShellSession\nssh [os-user]@[master-ip] sudo cat /etc/kubernetes/ssl/admin-kube-master-1-key.pem > admin-key.pem\nssh [os-user]@[master-ip] sudo cat /etc/kubernetes/ssl/admin-kube-master-1.pem > admin.pem\nssh [os-user]@[master-ip] sudo cat /etc/kubernetes/ssl/ca.pem > ca.pem\n```\n\n1. Configure kubectl:\n\n```ShellSession\n$ kubectl config set-cluster default-cluster --server=https://[master-internal-ip]:6443 \\\n --certificate-authority=ca.pem\n\n$ kubectl config set-credentials default-admin \\\n --certificate-authority=ca.pem \\\n --client-key=admin-key.pem \\\n --client-certificate=admin.pem\n\n$ kubectl config set-context default-system --cluster=default-cluster --user=default-admin\n$ kubectl config use-context default-system\n```\n\n1. Check it:\n\n```ShellSession\nkubectl version\n```\n\n## GlusterFS\n\nGlusterFS is not deployed by the standard `cluster.yml` playbook, see the\n[GlusterFS playbook documentation](../../network-storage/glusterfs/README.md)\nfor instructions.\n\nBasically you will install Gluster as\n\n```ShellSession\nansible-playbook --become -i inventory/$CLUSTER/hosts ./contrib/network-storage/glusterfs/glusterfs.yml\n```\n\n## What's next\n\nTry out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).\n\n## Appendix\n\n### Migration from `number_of_k8s_nodes*` to `k8s_nodes`\n\nIf you currently have a cluster defined using the `number_of_k8s_nodes*` variables and wish\nto migrate to the `k8s_nodes` style you can do it like so:\n\n```ShellSession\n$ terraform state list\nmodule.compute.data.openstack_images_image_v2.gfs_image\nmodule.compute.data.openstack_images_image_v2.vm_image\nmodule.compute.openstack_compute_floatingip_associate_v2.k8s_master[0]\nmodule.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]\nmodule.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]\nmodule.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]\nmodule.compute.openstack_compute_instance_v2.k8s_master[0]\nmodule.compute.openstack_compute_instance_v2.k8s_node[0]\nmodule.compute.openstack_compute_instance_v2.k8s_node[1]\nmodule.compute.openstack_compute_instance_v2.k8s_node[2]\nmodule.compute.openstack_compute_keypair_v2.k8s\nmodule.compute.openstack_compute_servergroup_v2.k8s_etcd[0]\nmodule.compute.openstack_compute_servergroup_v2.k8s_master[0]\nmodule.compute.openstack_compute_servergroup_v2.k8s_node[0]\nmodule.compute.openstack_networking_secgroup_rule_v2.bastion[0]\nmodule.compute.openstack_networking_secgroup_rule_v2.egress[0]\nmodule.compute.openstack_networking_secgroup_rule_v2.k8s\nmodule.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[0]\nmodule.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[1]\nmodule.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[2]\nmodule.compute.openstack_networking_secgroup_rule_v2.k8s_master[0]\nmodule.compute.openstack_networking_secgroup_rule_v2.worker[0]\nmodule.compute.openstack_networking_secgroup_rule_v2.worker[1]\nmodule.compute.openstack_networking_secgroup_rule_v2.worker[2]\nmodule.compute.openstack_networking_secgroup_rule_v2.worker[3]\nmodule.compute.openstack_networking_secgroup_rule_v2.worker[4]\nmodule.compute.openstack_networking_secgroup_v2.bastion[0]\nmodule.compute.openstack_networking_secgroup_v2.k8s\nmodule.compute.openstack_networking_secgroup_v2.k8s_master\nmodule.compute.openstack_networking_secgroup_v2.worker\nmodule.ips.null_resource.dummy_dependency\nmodule.ips.openstack_networking_floatingip_v2.k8s_master[0]\nmodule.ips.openstack_networking_floatingip_v2.k8s_node[0]\nmodule.ips.openstack_networking_floatingip_v2.k8s_node[1]\nmodule.ips.openstack_networking_floatingip_v2.k8s_node[2]\nmodule.network.openstack_networking_network_v2.k8s[0]\nmodule.network.openstack_networking_router_interface_v2.k8s[0]\nmodule.network.openstack_networking_router_v2.k8s[0]\nmodule.network.openstack_networking_subnet_v2.k8s[0]\n$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"1\"]'\nMove \"module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]\" to \"module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\\\"1\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"2\"]'\nMove \"module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]\" to \"module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\\\"2\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"3\"]'\nMove \"module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]\" to \"module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\\\"3\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[0]' 'module.compute.openstack_compute_instance_v2.k8s_node[\"1\"]'\nMove \"module.compute.openstack_compute_instance_v2.k8s_node[0]\" to \"module.compute.openstack_compute_instance_v2.k8s_node[\\\"1\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[1]' 'module.compute.openstack_compute_instance_v2.k8s_node[\"2\"]'\nMove \"module.compute.openstack_compute_instance_v2.k8s_node[1]\" to \"module.compute.openstack_compute_instance_v2.k8s_node[\\\"2\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[2]' 'module.compute.openstack_compute_instance_v2.k8s_node[\"3\"]'\nMove \"module.compute.openstack_compute_instance_v2.k8s_node[2]\" to \"module.compute.openstack_compute_instance_v2.k8s_node[\\\"3\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[0]' 'module.ips.openstack_networking_floatingip_v2.k8s_node[\"1\"]'\nMove \"module.ips.openstack_networking_floatingip_v2.k8s_node[0]\" to \"module.ips.openstack_networking_floatingip_v2.k8s_node[\\\"1\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[1]' 'module.ips.openstack_networking_floatingip_v2.k8s_node[\"2\"]'\nMove \"module.ips.openstack_networking_floatingip_v2.k8s_node[1]\" to \"module.ips.openstack_networking_floatingip_v2.k8s_node[\\\"2\\\"]\"\nSuccessfully moved 1 object(s).\n$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[2]' 'module.ips.openstack_networking_floatingip_v2.k8s_node[\"3\"]'\nMove \"module.ips.openstack_networking_floatingip_v2.k8s_node[2]\" to \"module.ips.openstack_networking_floatingip_v2.k8s_node[\\\"3\\\"]\"\nSuccessfully moved 1 object(s).\n```\n\nOf course for nodes without floating ips those steps can be omitted.\n" }, { "path": "contrib/terraform/openstack/kubespray.tf", "content": "module \"network\" {\n source = \"./modules/network\"\n\n external_net = var.external_net\n network_name = var.network_name\n subnet_cidr = var.subnet_cidr\n cluster_name = var.cluster_name\n dns_nameservers = var.dns_nameservers\n network_dns_domain = var.network_dns_domain\n use_neutron = var.use_neutron\n port_security_enabled = var.port_security_enabled\n router_id = var.router_id\n}\n\nmodule \"ips\" {\n source = \"./modules/ips\"\n\n number_of_k8s_masters = var.number_of_k8s_masters\n number_of_k8s_masters_no_etcd = var.number_of_k8s_masters_no_etcd\n number_of_k8s_nodes = var.number_of_k8s_nodes\n floatingip_pool = var.floatingip_pool\n number_of_bastions = var.number_of_bastions\n external_net = var.external_net\n network_name = var.network_name\n router_id = module.network.router_id\n k8s_nodes = var.k8s_nodes\n k8s_masters = var.k8s_masters\n k8s_master_fips = var.k8s_master_fips\n bastion_fips = var.bastion_fips\n router_internal_port_id = module.network.router_internal_port_id\n}\n\nmodule \"compute\" {\n source = \"./modules/compute\"\n\n cluster_name = var.cluster_name\n az_list = var.az_list\n az_list_node = var.az_list_node\n number_of_k8s_masters = var.number_of_k8s_masters\n number_of_k8s_masters_no_etcd = var.number_of_k8s_masters_no_etcd\n number_of_etcd = var.number_of_etcd\n number_of_k8s_masters_no_floating_ip = var.number_of_k8s_masters_no_floating_ip\n number_of_k8s_masters_no_floating_ip_no_etcd = var.number_of_k8s_masters_no_floating_ip_no_etcd\n number_of_k8s_nodes = var.number_of_k8s_nodes\n number_of_bastions = var.number_of_bastions\n number_of_k8s_nodes_no_floating_ip = var.number_of_k8s_nodes_no_floating_ip\n number_of_gfs_nodes_no_floating_ip = var.number_of_gfs_nodes_no_floating_ip\n k8s_masters = var.k8s_masters\n k8s_nodes = var.k8s_nodes\n bastion_root_volume_size_in_gb = var.bastion_root_volume_size_in_gb\n etcd_root_volume_size_in_gb = var.etcd_root_volume_size_in_gb\n master_root_volume_size_in_gb = var.master_root_volume_size_in_gb\n node_root_volume_size_in_gb = var.node_root_volume_size_in_gb\n gfs_root_volume_size_in_gb = var.gfs_root_volume_size_in_gb\n gfs_volume_size_in_gb = var.gfs_volume_size_in_gb\n master_volume_type = var.master_volume_type\n node_volume_type = var.node_volume_type\n public_key_path = var.public_key_path\n image = var.image\n image_uuid = var.image_uuid\n image_gfs = var.image_gfs\n image_master = var.image_master\n image_master_uuid = var.image_master_uuid\n image_gfs_uuid = var.image_gfs_uuid\n ssh_user = var.ssh_user\n ssh_user_gfs = var.ssh_user_gfs\n flavor_k8s_master = var.flavor_k8s_master\n flavor_k8s_node = var.flavor_k8s_node\n flavor_etcd = var.flavor_etcd\n flavor_gfs_node = var.flavor_gfs_node\n network_name = var.network_name\n flavor_bastion = var.flavor_bastion\n k8s_master_fips = module.ips.k8s_master_fips\n k8s_master_no_etcd_fips = module.ips.k8s_master_no_etcd_fips\n k8s_masters_fips = module.ips.k8s_masters_fips\n k8s_node_fips = module.ips.k8s_node_fips\n k8s_nodes_fips = module.ips.k8s_nodes_fips\n bastion_fips = module.ips.bastion_fips\n bastion_allowed_remote_ips = var.bastion_allowed_remote_ips\n bastion_allowed_remote_ipv6_ips = var.bastion_allowed_remote_ipv6_ips\n master_allowed_remote_ips = var.master_allowed_remote_ips\n master_allowed_remote_ipv6_ips = var.master_allowed_remote_ipv6_ips\n k8s_allowed_remote_ips = var.k8s_allowed_remote_ips\n k8s_allowed_remote_ips_ipv6 = var.k8s_allowed_remote_ips_ipv6\n k8s_allowed_egress_ips = var.k8s_allowed_egress_ips\n k8s_allowed_egress_ipv6_ips = var.k8s_allowed_egress_ipv6_ips\n supplementary_master_groups = var.supplementary_master_groups\n supplementary_node_groups = var.supplementary_node_groups\n master_allowed_ports = var.master_allowed_ports\n master_allowed_ports_ipv6 = var.master_allowed_ports_ipv6\n worker_allowed_ports = var.worker_allowed_ports\n worker_allowed_ports_ipv6 = var.worker_allowed_ports_ipv6\n bastion_allowed_ports = var.bastion_allowed_ports\n bastion_allowed_ports_ipv6 = var.bastion_allowed_ports_ipv6\n use_access_ip = var.use_access_ip\n master_server_group_policy = var.master_server_group_policy\n node_server_group_policy = var.node_server_group_policy\n etcd_server_group_policy = var.etcd_server_group_policy\n extra_sec_groups = var.extra_sec_groups\n extra_sec_groups_name = var.extra_sec_groups_name\n group_vars_path = var.group_vars_path\n port_security_enabled = var.port_security_enabled\n force_null_port_security = var.force_null_port_security\n network_router_id = module.network.router_id\n network_id = module.network.network_id\n use_existing_network = var.use_existing_network\n private_subnet_id = module.network.subnet_id\n additional_server_groups = var.additional_server_groups\n\n depends_on = [\n module.network.subnet_id\n ]\n}\n\nmodule \"loadbalancer\" {\n source = \"./modules/loadbalancer\"\n\n cluster_name = var.cluster_name\n subnet_id = module.network.subnet_id\n floatingip_pool = var.floatingip_pool\n k8s_master_ips = module.compute.k8s_master_ips\n k8s_master_loadbalancer_enabled = var.k8s_master_loadbalancer_enabled\n k8s_master_loadbalancer_listener_port = var.k8s_master_loadbalancer_listener_port\n k8s_master_loadbalancer_server_port = var.k8s_master_loadbalancer_server_port\n k8s_master_loadbalancer_public_ip = var.k8s_master_loadbalancer_public_ip\n\n depends_on = [\n module.compute.k8s_master\n ]\n}\n\n\noutput \"private_subnet_id\" {\n value = module.network.subnet_id\n}\n\noutput \"floating_network_id\" {\n value = var.external_net\n}\n\noutput \"router_id\" {\n value = module.network.router_id\n}\n\noutput \"k8s_master_fips\" {\n value = var.number_of_k8s_masters + var.number_of_k8s_masters_no_etcd > 0 ? concat(module.ips.k8s_master_fips, module.ips.k8s_master_no_etcd_fips) : [for key, value in module.ips.k8s_masters_fips : value.address]\n}\n\noutput \"k8s_node_fips\" {\n value = var.number_of_k8s_nodes > 0 ? module.ips.k8s_node_fips : [for key, value in module.ips.k8s_nodes_fips : value.address]\n}\n\noutput \"bastion_fips\" {\n value = module.ips.bastion_fips\n}\n" }, { "path": "contrib/terraform/openstack/modules/compute/ansible_bastion_template.txt", "content": "ansible_ssh_common_args: \"-o ProxyCommand='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q USER@BASTION_ADDRESS {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}'\"\n" }, { "path": "contrib/terraform/openstack/modules/compute/main.tf", "content": "data \"openstack_images_image_v2\" \"vm_image\" {\n count = var.image_uuid == \"\" ? 1 : 0\n most_recent = true\n name = var.image\n}\n\ndata \"openstack_images_image_v2\" \"gfs_image\" {\n count = var.image_gfs_uuid == \"\" ? var.image_uuid == \"\" ? 1 : 0 : 0\n most_recent = true\n name = var.image_gfs == \"\" ? var.image : var.image_gfs\n}\n\ndata \"openstack_images_image_v2\" \"image_master\" {\n count = var.image_master_uuid == \"\" ? var.image_uuid == \"\" ? 1 : 0 : 0\n name = var.image_master == \"\" ? var.image : var.image_master\n}\n\ndata \"cloudinit_config\" \"cloudinit\" {\n part {\n content_type = \"text/cloud-config\"\n content = templatefile(\"${path.module}/templates/cloudinit.yaml.tmpl\", {\n extra_partitions = [],\n netplan_critical_dhcp_interface = \"\"\n })\n }\n}\n\ndata \"openstack_networking_network_v2\" \"k8s_network\" {\n count = var.use_existing_network ? 1 : 0\n name = var.network_name\n}\n\nresource \"openstack_compute_keypair_v2\" \"k8s\" {\n name = \"kubernetes-${var.cluster_name}\"\n public_key = chomp(file(var.public_key_path))\n}\n\nresource \"openstack_networking_secgroup_v2\" \"k8s_master\" {\n name = \"${var.cluster_name}-k8s-master\"\n description = \"${var.cluster_name} - Kubernetes Master\"\n delete_default_rules = true\n}\n\nresource \"openstack_networking_secgroup_v2\" \"k8s_master_extra\" {\n count = \"%{if var.extra_sec_groups}1%{else}0%{endif}\"\n name = \"${var.cluster_name}-k8s-master-${var.extra_sec_groups_name}\"\n description = \"${var.cluster_name} - Kubernetes Master nodes - rules not managed by terraform\"\n delete_default_rules = true\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_master\" {\n count = length(var.master_allowed_remote_ips)\n direction = \"ingress\"\n ethertype = \"IPv4\"\n protocol = \"tcp\"\n port_range_min = \"6443\"\n port_range_max = \"6443\"\n remote_ip_prefix = var.master_allowed_remote_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s_master.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_master_ports\" {\n count = length(var.master_allowed_ports)\n direction = \"ingress\"\n ethertype = \"IPv4\"\n protocol = lookup(var.master_allowed_ports[count.index], \"protocol\", \"tcp\")\n port_range_min = lookup(var.master_allowed_ports[count.index], \"port_range_min\")\n port_range_max = lookup(var.master_allowed_ports[count.index], \"port_range_max\")\n remote_ip_prefix = lookup(var.master_allowed_ports[count.index], \"remote_ip_prefix\", \"0.0.0.0/0\")\n security_group_id = openstack_networking_secgroup_v2.k8s_master.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_master_ipv6_ingress\" {\n count = length(var.master_allowed_remote_ipv6_ips)\n direction = \"ingress\"\n ethertype = \"IPv6\"\n protocol = \"tcp\"\n port_range_min = \"6443\"\n port_range_max = \"6443\"\n remote_ip_prefix = var.master_allowed_remote_ipv6_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s_master.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_master_ports_ipv6_ingress\" {\n count = length(var.master_allowed_ports_ipv6)\n direction = \"ingress\"\n ethertype = \"IPv6\"\n protocol = lookup(var.master_allowed_ports_ipv6[count.index], \"protocol\", \"tcp\")\n port_range_min = lookup(var.master_allowed_ports_ipv6[count.index], \"port_range_min\")\n port_range_max = lookup(var.master_allowed_ports_ipv6[count.index], \"port_range_max\")\n remote_ip_prefix = lookup(var.master_allowed_ports_ipv6[count.index], \"remote_ip_prefix\", \"::/0\")\n security_group_id = openstack_networking_secgroup_v2.k8s_master.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"master_egress_ipv6\" {\n count = length(var.k8s_allowed_egress_ipv6_ips)\n direction = \"egress\"\n ethertype = \"IPv6\"\n remote_ip_prefix = var.k8s_allowed_egress_ipv6_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s_master.id\n}\n\nresource \"openstack_networking_secgroup_v2\" \"bastion\" {\n name = \"${var.cluster_name}-bastion\"\n count = var.number_of_bastions != \"\" ? 1 : 0\n description = \"${var.cluster_name} - Bastion Server\"\n delete_default_rules = true\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"bastion\" {\n count = var.number_of_bastions != \"\" ? length(var.bastion_allowed_remote_ips) : 0\n direction = \"ingress\"\n ethertype = \"IPv4\"\n protocol = \"tcp\"\n port_range_min = \"22\"\n port_range_max = \"22\"\n remote_ip_prefix = var.bastion_allowed_remote_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.bastion[0].id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_bastion_ports\" {\n count = length(var.bastion_allowed_ports)\n direction = \"ingress\"\n ethertype = \"IPv4\"\n protocol = lookup(var.bastion_allowed_ports[count.index], \"protocol\", \"tcp\")\n port_range_min = lookup(var.bastion_allowed_ports[count.index], \"port_range_min\")\n port_range_max = lookup(var.bastion_allowed_ports[count.index], \"port_range_max\")\n remote_ip_prefix = lookup(var.bastion_allowed_ports[count.index], \"remote_ip_prefix\", \"0.0.0.0/0\")\n security_group_id = openstack_networking_secgroup_v2.bastion[0].id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"bastion_ipv6_ingress\" {\n count = var.number_of_bastions != \"\" ? length(var.bastion_allowed_remote_ipv6_ips) : 0\n direction = \"ingress\"\n ethertype = \"IPv6\"\n protocol = \"tcp\"\n port_range_min = \"22\"\n port_range_max = \"22\"\n remote_ip_prefix = var.bastion_allowed_remote_ipv6_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.bastion[0].id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_bastion_ports_ipv6_ingress\" {\n count = length(var.bastion_allowed_ports_ipv6)\n direction = \"ingress\"\n ethertype = \"IPv6\"\n protocol = lookup(var.bastion_allowed_ports_ipv6[count.index], \"protocol\", \"tcp\")\n port_range_min = lookup(var.bastion_allowed_ports_ipv6[count.index], \"port_range_min\")\n port_range_max = lookup(var.bastion_allowed_ports_ipv6[count.index], \"port_range_max\")\n remote_ip_prefix = lookup(var.bastion_allowed_ports_ipv6[count.index], \"remote_ip_prefix\", \"::/0\")\n security_group_id = openstack_networking_secgroup_v2.bastion[0].id\n}\n\nresource \"openstack_networking_secgroup_v2\" \"k8s\" {\n name = \"${var.cluster_name}-k8s\"\n description = \"${var.cluster_name} - Kubernetes\"\n delete_default_rules = true\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s\" {\n direction = \"ingress\"\n ethertype = \"IPv4\"\n remote_group_id = openstack_networking_secgroup_v2.k8s.id\n security_group_id = openstack_networking_secgroup_v2.k8s.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_ipv6\" {\n direction = \"ingress\"\n ethertype = \"IPv6\"\n remote_group_id = openstack_networking_secgroup_v2.k8s.id\n security_group_id = openstack_networking_secgroup_v2.k8s.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_allowed_remote_ips\" {\n count = length(var.k8s_allowed_remote_ips)\n direction = \"ingress\"\n ethertype = \"IPv4\"\n protocol = \"tcp\"\n port_range_min = \"22\"\n port_range_max = \"22\"\n remote_ip_prefix = var.k8s_allowed_remote_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"k8s_allowed_remote_ips_ipv6\" {\n count = length(var.k8s_allowed_remote_ips_ipv6)\n direction = \"ingress\"\n ethertype = \"IPv6\"\n protocol = \"tcp\"\n port_range_min = \"22\"\n port_range_max = \"22\"\n remote_ip_prefix = var.k8s_allowed_remote_ips_ipv6[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"egress\" {\n count = length(var.k8s_allowed_egress_ips)\n direction = \"egress\"\n ethertype = \"IPv4\"\n remote_ip_prefix = var.k8s_allowed_egress_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"egress_ipv6\" {\n count = length(var.k8s_allowed_egress_ipv6_ips)\n direction = \"egress\"\n ethertype = \"IPv6\"\n remote_ip_prefix = var.k8s_allowed_egress_ipv6_ips[count.index]\n security_group_id = openstack_networking_secgroup_v2.k8s.id\n}\n\nresource \"openstack_networking_secgroup_v2\" \"worker\" {\n name = \"${var.cluster_name}-k8s-worker\"\n description = \"${var.cluster_name} - Kubernetes worker nodes\"\n delete_default_rules = true\n}\n\nresource \"openstack_networking_secgroup_v2\" \"worker_extra\" {\n count = \"%{if var.extra_sec_groups}1%{else}0%{endif}\"\n name = \"${var.cluster_name}-k8s-worker-${var.extra_sec_groups_name}\"\n description = \"${var.cluster_name} - Kubernetes worker nodes - rules not managed by terraform\"\n delete_default_rules = true\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"worker\" {\n count = length(var.worker_allowed_ports)\n direction = \"ingress\"\n ethertype = \"IPv4\"\n protocol = lookup(var.worker_allowed_ports[count.index], \"protocol\", \"tcp\")\n port_range_min = lookup(var.worker_allowed_ports[count.index], \"port_range_min\")\n port_range_max = lookup(var.worker_allowed_ports[count.index], \"port_range_max\")\n remote_ip_prefix = lookup(var.worker_allowed_ports[count.index], \"remote_ip_prefix\", \"0.0.0.0/0\")\n security_group_id = openstack_networking_secgroup_v2.worker.id\n}\n\nresource \"openstack_networking_secgroup_rule_v2\" \"worker_ipv6_ingress\" {\n count = length(var.worker_allowed_ports_ipv6)\n direction = \"ingress\"\n ethertype = \"IPv6\"\n protocol = lookup(var.worker_allowed_ports_ipv6[count.index], \"protocol\", \"tcp\")\n port_range_min = lookup(var.worker_allowed_ports_ipv6[count.index], \"port_range_min\")\n port_range_max = lookup(var.worker_allowed_ports_ipv6[count.index], \"port_range_max\")\n remote_ip_prefix = lookup(var.worker_allowed_ports_ipv6[count.index], \"remote_ip_prefix\", \"::/0\")\n security_group_id = openstack_networking_secgroup_v2.worker.id\n}\n\nresource \"openstack_compute_servergroup_v2\" \"k8s_master\" {\n count = var.master_server_group_policy != \"\" ? 1 : 0\n name = \"k8s-master-srvgrp\"\n policies = [var.master_server_group_policy]\n}\n\nresource \"openstack_compute_servergroup_v2\" \"k8s_node\" {\n count = var.node_server_group_policy != \"\" ? 1 : 0\n name = \"k8s-node-srvgrp\"\n policies = [var.node_server_group_policy]\n}\n\nresource \"openstack_compute_servergroup_v2\" \"k8s_etcd\" {\n count = var.etcd_server_group_policy != \"\" ? 1 : 0\n name = \"k8s-etcd-srvgrp\"\n policies = [var.etcd_server_group_policy]\n}\n\nresource \"openstack_compute_servergroup_v2\" \"k8s_node_additional\" {\n for_each = var.additional_server_groups\n name = \"k8s-${each.key}-srvgrp\"\n policies = [each.value.policy]\n}\n\nlocals {\n# master groups\n master_sec_groups = compact([\n openstack_networking_secgroup_v2.k8s_master.id,\n openstack_networking_secgroup_v2.k8s.id,\n var.extra_sec_groups ?openstack_networking_secgroup_v2.k8s_master_extra[0].id : \"\",\n ])\n# worker groups\n worker_sec_groups = compact([\n openstack_networking_secgroup_v2.k8s.id,\n openstack_networking_secgroup_v2.worker.id,\n var.extra_sec_groups ? openstack_networking_secgroup_v2.worker_extra[0].id : \"\",\n ])\n# bastion groups\n bastion_sec_groups = compact(concat([\n openstack_networking_secgroup_v2.k8s.id,\n openstack_networking_secgroup_v2.bastion[0].id,\n ]))\n# etcd groups\n etcd_sec_groups = compact([openstack_networking_secgroup_v2.k8s.id])\n# glusterfs groups\n gfs_sec_groups = compact([openstack_networking_secgroup_v2.k8s.id])\n\n# Image uuid\n image_to_use_node = var.image_uuid != \"\" ? var.image_uuid : data.openstack_images_image_v2.vm_image[0].id\n# Image_gfs uuid\n image_to_use_gfs = var.image_gfs_uuid != \"\" ? var.image_gfs_uuid : var.image_uuid != \"\" ? var.image_uuid : data.openstack_images_image_v2.gfs_image[0].id\n# image_master uuidimage_gfs_uuid\n image_to_use_master = var.image_master_uuid != \"\" ? var.image_master_uuid : var.image_uuid != \"\" ? var.image_uuid : data.openstack_images_image_v2.image_master[0].id\n\n k8s_nodes_settings = {\n for name, node in var.k8s_nodes :\n name => {\n \"use_local_disk\" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb) == 0,\n \"image_id\" = node.image_id != null ? node.image_id : local.image_to_use_node,\n \"volume_size\" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb,\n \"volume_type\" = node.volume_type != null ? node.volume_type : var.node_volume_type,\n \"network_id\" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id)\n \"server_group\" = node.server_group != null ? [openstack_compute_servergroup_v2.k8s_node_additional[node.server_group].id] : (var.node_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : [])\n }\n }\n\n k8s_masters_settings = {\n for name, node in var.k8s_masters :\n name => {\n \"use_local_disk\" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb) == 0,\n \"image_id\" = node.image_id != null ? node.image_id : local.image_to_use_master,\n \"volume_size\" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb,\n \"volume_type\" = node.volume_type != null ? node.volume_type : var.master_volume_type,\n \"network_id\" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id)\n }\n }\n}\n\nresource \"openstack_networking_port_v2\" \"bastion_port\" {\n count = var.number_of_bastions\n name = \"${var.cluster_name}-bastion-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.bastion_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"bastion\" {\n name = \"${var.cluster_name}-bastion-${count.index + 1}\"\n count = var.number_of_bastions\n image_id = var.bastion_root_volume_size_in_gb == 0 ? local.image_to_use_node : null\n flavor_id = var.flavor_bastion\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n dynamic \"block_device\" {\n for_each = var.bastion_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : []\n content {\n uuid = local.image_to_use_node\n source_type = \"image\"\n volume_size = var.bastion_root_volume_size_in_gb\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.bastion_port.*.id, count.index)\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"bastion\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n\n provisioner \"local-exec\" {\n command = \"sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${var.bastion_fips[0]}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml\"\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_master_port\" {\n count = var.number_of_k8s_masters\n name = \"${var.cluster_name}-k8s-master-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.master_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_master\" {\n name = \"${var.cluster_name}-k8s-master-${count.index + 1}\"\n count = var.number_of_k8s_masters\n availability_zone = element(var.az_list, count.index)\n image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null\n flavor_id = var.flavor_k8s_master\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n\n dynamic \"block_device\" {\n for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []\n content {\n uuid = local.image_to_use_master\n source_type = \"image\"\n volume_size = var.master_root_volume_size_in_gb\n volume_type = var.master_volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.master_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_master[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"etcd,kube_control_plane,${var.supplementary_master_groups},k8s_cluster\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n\n provisioner \"local-exec\" {\n command = \"sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml\"\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_masters_port\" {\n for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}\n name = \"${var.cluster_name}-k8s-${each.key}\"\n network_id = local.k8s_masters_settings[each.key].network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.master_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_masters\" {\n for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}\n name = \"${var.cluster_name}-k8s-${each.key}\"\n availability_zone = each.value.az\n image_id = local.k8s_masters_settings[each.key].use_local_disk ? local.k8s_masters_settings[each.key].image_id : null\n flavor_id = each.value.flavor\n key_pair = openstack_compute_keypair_v2.k8s.name\n\n dynamic \"block_device\" {\n for_each = !local.k8s_masters_settings[each.key].use_local_disk ? [local.k8s_masters_settings[each.key].image_id] : []\n content {\n uuid = block_device.value\n source_type = \"image\"\n volume_size = local.k8s_masters_settings[each.key].volume_size\n volume_type = local.k8s_masters_settings[each.key].volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = openstack_networking_port_v2.k8s_masters_port[each.key].id\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.master_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_master[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"%{if each.value.etcd == true}etcd,%{endif}kube_control_plane,${var.supplementary_master_groups},k8s_cluster%{if each.value.floating_ip == false},no_floating%{endif}\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n\n provisioner \"local-exec\" {\n command = \"%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.module}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_masters_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}\"\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_master_no_etcd_port\" {\n count = var.number_of_k8s_masters_no_etcd\n name = \"${var.cluster_name}-k8s-master-ne-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.master_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_master_no_etcd\" {\n name = \"${var.cluster_name}-k8s-master-ne-${count.index + 1}\"\n count = var.number_of_k8s_masters_no_etcd\n availability_zone = element(var.az_list, count.index)\n image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null\n flavor_id = var.flavor_k8s_master\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n\n dynamic \"block_device\" {\n for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []\n content {\n uuid = local.image_to_use_master\n source_type = \"image\"\n volume_size = var.master_root_volume_size_in_gb\n volume_type = var.master_volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.k8s_master_no_etcd_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.master_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_master[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"kube_control_plane,${var.supplementary_master_groups},k8s_cluster\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n\n provisioner \"local-exec\" {\n command = \"sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml\"\n }\n}\n\nresource \"openstack_networking_port_v2\" \"etcd_port\" {\n count = var.number_of_etcd\n name = \"${var.cluster_name}-etcd-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.etcd_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"etcd\" {\n name = \"${var.cluster_name}-etcd-${count.index + 1}\"\n count = var.number_of_etcd\n availability_zone = element(var.az_list, count.index)\n image_id = var.etcd_root_volume_size_in_gb == 0 ? local.image_to_use_master : null\n flavor_id = var.flavor_etcd\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n dynamic \"block_device\" {\n for_each = var.etcd_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []\n content {\n uuid = local.image_to_use_master\n source_type = \"image\"\n volume_size = var.etcd_root_volume_size_in_gb\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.etcd_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.etcd_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_etcd[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_etcd[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"etcd,no_floating\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_master_no_floating_ip_port\" {\n count = var.number_of_k8s_masters_no_floating_ip\n name = \"${var.cluster_name}-k8s-master-nf-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.master_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_master_no_floating_ip\" {\n name = \"${var.cluster_name}-k8s-master-nf-${count.index + 1}\"\n count = var.number_of_k8s_masters_no_floating_ip\n availability_zone = element(var.az_list, count.index)\n image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null\n flavor_id = var.flavor_k8s_master\n key_pair = openstack_compute_keypair_v2.k8s.name\n\n dynamic \"block_device\" {\n for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []\n content {\n uuid = local.image_to_use_master\n source_type = \"image\"\n volume_size = var.master_root_volume_size_in_gb\n volume_type = var.master_volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.k8s_master_no_floating_ip_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.master_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_master[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"etcd,kube_control_plane,${var.supplementary_master_groups},k8s_cluster,no_floating\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_master_no_floating_ip_no_etcd_port\" {\n count = var.number_of_k8s_masters_no_floating_ip_no_etcd\n name = \"${var.cluster_name}-k8s-master-ne-nf-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.master_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_master_no_floating_ip_no_etcd\" {\n name = \"${var.cluster_name}-k8s-master-ne-nf-${count.index + 1}\"\n count = var.number_of_k8s_masters_no_floating_ip_no_etcd\n availability_zone = element(var.az_list, count.index)\n image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null\n flavor_id = var.flavor_k8s_master\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n dynamic \"block_device\" {\n for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []\n content {\n uuid = local.image_to_use_master\n source_type = \"image\"\n volume_size = var.master_root_volume_size_in_gb\n volume_type = var.master_volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.k8s_master_no_floating_ip_no_etcd_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.master_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_master[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_master[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"kube_control_plane,${var.supplementary_master_groups},k8s_cluster,no_floating\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_node_port\" {\n count = var.number_of_k8s_nodes\n name = \"${var.cluster_name}-k8s-node-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_node\" {\n name = \"${var.cluster_name}-k8s-node-${count.index + 1}\"\n count = var.number_of_k8s_nodes\n availability_zone = element(var.az_list_node, count.index)\n image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null\n flavor_id = var.flavor_k8s_node\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n dynamic \"block_device\" {\n for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : []\n content {\n uuid = local.image_to_use_node\n source_type = \"image\"\n volume_size = var.node_root_volume_size_in_gb\n volume_type = var.node_volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index)\n }\n\n\n dynamic \"scheduler_hints\" {\n for_each = var.node_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_node[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"kube_node,k8s_cluster,${var.supplementary_node_groups}\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n\n provisioner \"local-exec\" {\n command = \"sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml\"\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_node_no_floating_ip_port\" {\n count = var.number_of_k8s_nodes_no_floating_ip\n name = \"${var.cluster_name}-k8s-node-nf-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_node_no_floating_ip\" {\n name = \"${var.cluster_name}-k8s-node-nf-${count.index + 1}\"\n count = var.number_of_k8s_nodes_no_floating_ip\n availability_zone = element(var.az_list_node, count.index)\n image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null\n flavor_id = var.flavor_k8s_node\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = data.cloudinit_config.cloudinit.rendered\n\n dynamic \"block_device\" {\n for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : []\n content {\n uuid = local.image_to_use_node\n source_type = \"image\"\n volume_size = var.node_root_volume_size_in_gb\n volume_type = var.node_volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.k8s_node_no_floating_ip_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.node_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : []\n content {\n group = scheduler_hints.value\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"kube_node,k8s_cluster,no_floating,${var.supplementary_node_groups}\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n}\n\nresource \"openstack_networking_port_v2\" \"k8s_nodes_port\" {\n for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {}\n name = \"${var.cluster_name}-k8s-node-${each.key}\"\n network_id = local.k8s_nodes_settings[each.key].network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n lifecycle {\n ignore_changes = [ allowed_address_pairs ]\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"k8s_nodes\" {\n for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {}\n name = \"${var.cluster_name}-k8s-node-${each.key}\"\n availability_zone = each.value.az\n image_id = local.k8s_nodes_settings[each.key].use_local_disk ? local.k8s_nodes_settings[each.key].image_id : null\n flavor_id = each.value.flavor\n key_pair = openstack_compute_keypair_v2.k8s.name\n user_data = each.value.cloudinit != null ? templatefile(\"${path.module}/templates/cloudinit.yaml.tmpl\", {\n extra_partitions = each.value.cloudinit.extra_partitions,\n netplan_critical_dhcp_interface = each.value.cloudinit.netplan_critical_dhcp_interface,\n }) : data.cloudinit_config.cloudinit.rendered\n\n dynamic \"block_device\" {\n for_each = !local.k8s_nodes_settings[each.key].use_local_disk ? [local.k8s_nodes_settings[each.key].image_id] : []\n content {\n uuid = block_device.value\n source_type = \"image\"\n volume_size = local.k8s_nodes_settings[each.key].volume_size\n volume_type = local.k8s_nodes_settings[each.key].volume_type\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = openstack_networking_port_v2.k8s_nodes_port[each.key].id\n }\n\n dynamic \"scheduler_hints\" {\n for_each = local.k8s_nodes_settings[each.key].server_group\n content {\n group = scheduler_hints.value\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user\n kubespray_groups = \"kube_node,k8s_cluster,%{if !each.value.floating_ip}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? \",${each.value.extra_groups}\" : \"\"}\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n\n provisioner \"local-exec\" {\n command = \"%{if each.value.floating_ip}sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}\"\n }\n}\n\nresource \"openstack_networking_port_v2\" \"glusterfs_node_no_floating_ip_port\" {\n count = var.number_of_gfs_nodes_no_floating_ip\n name = \"${var.cluster_name}-gfs-node-nf-${count.index + 1}\"\n network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id\n admin_state_up = \"true\"\n port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled\n security_group_ids = var.port_security_enabled ? local.gfs_sec_groups : null\n no_security_groups = var.port_security_enabled ? null : false\n dynamic \"fixed_ip\" {\n for_each = var.private_subnet_id == \"\" ? [] : [true]\n content {\n subnet_id = var.private_subnet_id\n }\n }\n\n depends_on = [\n var.network_router_id\n ]\n}\n\nresource \"openstack_compute_instance_v2\" \"glusterfs_node_no_floating_ip\" {\n name = \"${var.cluster_name}-gfs-node-nf-${count.index + 1}\"\n count = var.number_of_gfs_nodes_no_floating_ip\n availability_zone = element(var.az_list, count.index)\n image_id = var.gfs_root_volume_size_in_gb == 0 ? local.image_to_use_gfs : null\n flavor_id = var.flavor_gfs_node\n key_pair = openstack_compute_keypair_v2.k8s.name\n\n dynamic \"block_device\" {\n for_each = var.gfs_root_volume_size_in_gb > 0 ? [local.image_to_use_gfs] : []\n content {\n uuid = local.image_to_use_gfs\n source_type = \"image\"\n volume_size = var.gfs_root_volume_size_in_gb\n boot_index = 0\n destination_type = \"volume\"\n delete_on_termination = true\n }\n }\n\n network {\n port = element(openstack_networking_port_v2.glusterfs_node_no_floating_ip_port.*.id, count.index)\n }\n\n dynamic \"scheduler_hints\" {\n for_each = var.node_server_group_policy != \"\" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []\n content {\n group = openstack_compute_servergroup_v2.k8s_node[0].id\n }\n }\n\n metadata = {\n ssh_user = var.ssh_user_gfs\n kubespray_groups = \"gfs-cluster,network-storage,no_floating\"\n depends_on = var.network_router_id\n use_access_ip = var.use_access_ip\n }\n}\n\nresource \"openstack_networking_floatingip_associate_v2\" \"bastion\" {\n count = var.number_of_bastions\n floating_ip = var.bastion_fips[count.index]\n port_id = element(openstack_networking_port_v2.bastion_port.*.id, count.index)\n}\n\n\nresource \"openstack_networking_floatingip_associate_v2\" \"k8s_master\" {\n count = var.number_of_k8s_masters\n floating_ip = var.k8s_master_fips[count.index]\n port_id = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index)\n}\n\nresource \"openstack_networking_floatingip_associate_v2\" \"k8s_masters\" {\n for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {}\n floating_ip = var.k8s_masters_fips[each.key].address\n port_id = openstack_networking_port_v2.k8s_masters_port[each.key].id\n}\n\nresource \"openstack_networking_floatingip_associate_v2\" \"k8s_master_no_etcd\" {\n count = var.master_root_volume_size_in_gb == 0 ? var.number_of_k8s_masters_no_etcd : 0\n floating_ip = var.k8s_master_no_etcd_fips[count.index]\n port_id = element(openstack_networking_port_v2.k8s_master_no_etcd_port.*.id, count.index)\n}\n\nresource \"openstack_networking_floatingip_associate_v2\" \"k8s_node\" {\n count = var.node_root_volume_size_in_gb == 0 ? var.number_of_k8s_nodes : 0\n floating_ip = var.k8s_node_fips[count.index]\n port_id = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index)\n}\n\nresource \"openstack_networking_floatingip_associate_v2\" \"k8s_nodes\" {\n for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {}\n floating_ip = var.k8s_nodes_fips[each.key].address\n port_id = openstack_networking_port_v2.k8s_nodes_port[each.key].id\n}\n\nresource \"openstack_blockstorage_volume_v3\" \"glusterfs_volume\" {\n name = \"${var.cluster_name}-glusterfs_volume-${count.index + 1}\"\n count = var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0\n description = \"Non-ephemeral volume for GlusterFS\"\n size = var.gfs_volume_size_in_gb\n}\n\nresource \"openstack_compute_volume_attach_v2\" \"glusterfs_volume\" {\n count = var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0\n instance_id = element(openstack_compute_instance_v2.glusterfs_node_no_floating_ip.*.id, count.index)\n volume_id = element(openstack_blockstorage_volume_v3.glusterfs_volume.*.id, count.index)\n}\n" }, { "path": "contrib/terraform/openstack/modules/compute/outputs.tf", "content": "output \"k8s_master_ips\" {\n value = concat(openstack_compute_instance_v2.k8s_master_no_floating_ip.*, openstack_compute_instance_v2.k8s_master_no_floating_ip_no_etcd.*)\n}\n" }, { "path": "contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml.tmpl", "content": "%{~ if length(extra_partitions) > 0 || netplan_critical_dhcp_interface != \"\" }\n#cloud-config\nbootcmd:\n%{~ for idx, partition in extra_partitions }\n- [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, ${partition.volume_path} ]\n- [ cloud-init-per, once, create-part-${idx}, parted, --script, ${partition.volume_path}, 'mkpart extended ext4 ${partition.partition_start} ${partition.partition_end}' ]\n- [ cloud-init-per, once, create-fs-part-${idx}, mkfs.ext4, ${partition.partition_path} ]\n%{~ endfor }\n\nruncmd:\n%{~ if netplan_critical_dhcp_interface != \"\" }\n - netplan apply\n%{~ endif }\n%{~ for idx, partition in extra_partitions }\n - mkdir -p ${partition.mount_path}\n - chown nobody:nogroup ${partition.mount_path}\n - mount ${partition.partition_path} ${partition.mount_path}\n%{~ endfor ~}\n\n%{~ if netplan_critical_dhcp_interface != \"\" }\nwrite_files:\n - path: /etc/netplan/90-critical-dhcp.yaml\n content: |\n network:\n version: 2\n ethernets:\n ${ netplan_critical_dhcp_interface }:\n dhcp4: true\n critical: true\n%{~ endif }\n\nmounts:\n%{~ for idx, partition in extra_partitions }\n - [ ${partition.partition_path}, ${partition.mount_path} ]\n%{~ endfor }\n%{~ else ~}\n# yamllint disable rule:comments\n#cloud-config\n## in some cases novnc console access is required\n## it requires ssh password to be set\n#ssh_pwauth: yes\n#chpasswd:\n# list: |\n# root:secret\n# expire: False\n\n## in some cases direct root ssh access via ssh key is required\n#disable_root: false\n\n## in some cases additional CA certs are required\n#ca-certs:\n# trusted: |\n# -----BEGIN CERTIFICATE-----\n%{~ endif }\n" }, { "path": "contrib/terraform/openstack/modules/compute/variables.tf", "content": "variable \"cluster_name\" {}\n\nvariable \"az_list\" {\n type = list(string)\n}\n\nvariable \"az_list_node\" {\n type = list(string)\n}\n\nvariable \"number_of_k8s_masters\" {}\n\nvariable \"number_of_k8s_masters_no_etcd\" {}\n\nvariable \"number_of_etcd\" {}\n\nvariable \"number_of_k8s_masters_no_floating_ip\" {}\n\nvariable \"number_of_k8s_masters_no_floating_ip_no_etcd\" {}\n\nvariable \"number_of_k8s_nodes\" {}\n\nvariable \"number_of_k8s_nodes_no_floating_ip\" {}\n\nvariable \"number_of_bastions\" {}\n\nvariable \"number_of_gfs_nodes_no_floating_ip\" {}\n\nvariable \"bastion_root_volume_size_in_gb\" {}\n\nvariable \"etcd_root_volume_size_in_gb\" {}\n\nvariable \"master_root_volume_size_in_gb\" {}\n\nvariable \"node_root_volume_size_in_gb\" {}\n\nvariable \"gfs_root_volume_size_in_gb\" {}\n\nvariable \"gfs_volume_size_in_gb\" {}\n\nvariable \"master_volume_type\" {}\n\nvariable \"node_volume_type\" {}\n\nvariable \"public_key_path\" {}\n\nvariable \"image\" {}\n\nvariable \"image_gfs\" {}\n\nvariable \"ssh_user\" {}\n\nvariable \"ssh_user_gfs\" {}\n\nvariable \"flavor_k8s_master\" {}\n\nvariable \"flavor_k8s_node\" {}\n\nvariable \"flavor_etcd\" {}\n\nvariable \"flavor_gfs_node\" {}\n\nvariable \"network_name\" {}\n\nvariable \"flavor_bastion\" {}\n\nvariable \"network_id\" {\n default = \"\"\n}\n\nvariable \"use_existing_network\" {\n type = bool\n}\n\nvariable \"network_router_id\" {\n default = \"\"\n}\n\nvariable \"k8s_master_fips\" {\n type = list\n}\n\nvariable \"k8s_master_no_etcd_fips\" {\n type = list\n}\n\nvariable \"k8s_node_fips\" {\n type = list\n}\n\nvariable \"k8s_masters_fips\" {\n type = map(object({\n address = string\n }))\n}\n\nvariable \"k8s_nodes_fips\" {\n type = map(object({\n address = string\n }))\n}\n\nvariable \"bastion_fips\" {\n type = list\n}\n\nvariable \"bastion_allowed_remote_ips\" {\n type = list\n}\n\nvariable \"bastion_allowed_remote_ipv6_ips\" {\n type = list\n}\n\nvariable \"master_allowed_remote_ips\" {\n type = list\n}\n\nvariable \"master_allowed_remote_ipv6_ips\" {\n type = list\n}\n\nvariable \"k8s_allowed_remote_ips\" {\n type = list\n}\n\nvariable \"k8s_allowed_remote_ips_ipv6\" {\n type = list\n}\n\nvariable \"k8s_allowed_egress_ips\" {\n type = list\n}\n\nvariable \"k8s_allowed_egress_ipv6_ips\" {\n type = list\n}\n\nvariable \"k8s_masters\" {\n type = map(object({\n az = string\n flavor = string\n etcd = bool\n floating_ip = bool\n reserved_floating_ip = optional(string)\n image_id = optional(string)\n root_volume_size_in_gb = optional(number)\n volume_type = optional(string)\n network_id = optional(string)\n }))\n}\n\nvariable \"k8s_nodes\" {\n type = map(object({\n az = string\n flavor = string\n floating_ip = bool\n reserved_floating_ip = optional(string)\n extra_groups = optional(string)\n image_id = optional(string)\n root_volume_size_in_gb = optional(number)\n volume_type = optional(string)\n network_id = optional(string)\n additional_server_groups = optional(list(string))\n server_group = optional(string)\n cloudinit = optional(object({\n extra_partitions = optional(list(object({\n volume_path = string\n partition_path = string\n partition_start = string\n partition_end = string\n mount_path = string\n })), [])\n netplan_critical_dhcp_interface = optional(string, \"\")\n }))\n }))\n}\n\nvariable \"additional_server_groups\" {\n type = map(object({\n policy = string\n }))\n}\n\nvariable \"supplementary_master_groups\" {\n default = \"\"\n}\n\nvariable \"supplementary_node_groups\" {\n default = \"\"\n}\n\nvariable \"master_allowed_ports\" {\n type = list\n}\n\nvariable \"master_allowed_ports_ipv6\" {\n type = list\n}\n\nvariable \"worker_allowed_ports\" {\n type = list\n}\n\nvariable \"worker_allowed_ports_ipv6\" {\n type = list\n}\n\nvariable \"bastion_allowed_ports\" {\n type = list\n}\n\nvariable \"bastion_allowed_ports_ipv6\" {\n type = list\n}\n\nvariable \"use_access_ip\" {}\n\nvariable \"master_server_group_policy\" {\n type = string\n}\n\nvariable \"node_server_group_policy\" {\n type = string\n}\n\nvariable \"etcd_server_group_policy\" {\n type = string\n}\n\nvariable \"extra_sec_groups\" {\n type = bool\n}\n\nvariable \"extra_sec_groups_name\" {\n type = string\n}\n\nvariable \"image_uuid\" {\n type = string\n}\n\nvariable \"image_gfs_uuid\" {\n type = string\n}\n\nvariable \"image_master\" {\n type = string\n}\n\nvariable \"image_master_uuid\" {\n type = string\n}\n\nvariable \"group_vars_path\" {\n type = string\n}\n\nvariable \"port_security_enabled\" {\n type = bool\n}\n\nvariable \"force_null_port_security\" {\n type = bool\n}\n\nvariable \"private_subnet_id\" {\n type = string\n}\n" }, { "path": "contrib/terraform/openstack/modules/compute/versions.tf", "content": "terraform {\n required_providers {\n openstack = {\n source = \"terraform-provider-openstack/openstack\"\n }\n }\n required_version = \">= 1.3.0\"\n}\n" }, { "path": "contrib/terraform/openstack/modules/ips/main.tf", "content": "resource \"null_resource\" \"dummy_dependency\" {\n triggers = {\n dependency_id = var.router_id\n }\n depends_on = [\n var.router_internal_port_id\n ]\n}\n\n# If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones.\nresource \"openstack_networking_floatingip_v2\" \"k8s_master\" {\n count = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters\n pool = var.floatingip_pool\n depends_on = [null_resource.dummy_dependency]\n}\n\nresource \"openstack_networking_floatingip_v2\" \"k8s_masters\" {\n for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, \"reserved_floating_ip\", \"\") == \"\") } : tomap({})\n pool = var.floatingip_pool\n depends_on = [null_resource.dummy_dependency]\n}\n\n# If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones.\nresource \"openstack_networking_floatingip_v2\" \"k8s_master_no_etcd\" {\n count = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters_no_etcd\n pool = var.floatingip_pool\n depends_on = [null_resource.dummy_dependency]\n}\n\nresource \"openstack_networking_floatingip_v2\" \"k8s_node\" {\n count = var.number_of_k8s_nodes\n pool = var.floatingip_pool\n depends_on = [null_resource.dummy_dependency]\n}\n\nresource \"openstack_networking_floatingip_v2\" \"bastion\" {\n count = length(var.bastion_fips) > 0 ? 0 : var.number_of_bastions\n pool = var.floatingip_pool\n depends_on = [null_resource.dummy_dependency]\n}\n\nresource \"openstack_networking_floatingip_v2\" \"k8s_nodes\" {\n for_each = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, \"reserved_floating_ip\", \"\") == \"\") } : tomap({})\n pool = var.floatingip_pool\n depends_on = [null_resource.dummy_dependency]\n}\n" }, { "path": "contrib/terraform/openstack/modules/ips/outputs.tf", "content": "locals {\n k8s_masters_reserved_fips = {\n for key, value in var.k8s_masters : key => {\n address = value.reserved_floating_ip\n } if value.floating_ip && (lookup(value, \"reserved_floating_ip\", \"\") != \"\")\n }\n k8s_masters_create_fips = {\n for key, value in openstack_networking_floatingip_v2.k8s_masters : key => {\n address = value.address\n }\n }\n k8s_nodes_reserved_fips = {\n for key, value in var.k8s_nodes : key => {\n address = value.reserved_floating_ip\n } if value.floating_ip && (lookup(value, \"reserved_floating_ip\", \"\") != \"\")\n }\n k8s_nodes_create_fips = {\n for key, value in openstack_networking_floatingip_v2.k8s_nodes : key => {\n address = value.address\n }\n }\n}\n\n# If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created.\noutput \"k8s_master_fips\" {\n value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master[*].address\n}\n\noutput \"k8s_masters_fips\" {\n value = merge(local.k8s_masters_create_fips, local.k8s_masters_reserved_fips)\n}\n\n# If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created.\noutput \"k8s_master_no_etcd_fips\" {\n value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master_no_etcd[*].address\n}\n\noutput \"k8s_node_fips\" {\n value = openstack_networking_floatingip_v2.k8s_node[*].address\n}\n\noutput \"k8s_nodes_fips\" {\n value = merge(local.k8s_nodes_create_fips, local.k8s_nodes_reserved_fips)\n}\n\noutput \"bastion_fips\" {\n value = length(var.bastion_fips) > 0 ? var.bastion_fips : openstack_networking_floatingip_v2.bastion[*].address\n}\n" }, { "path": "contrib/terraform/openstack/modules/ips/variables.tf", "content": "variable \"number_of_k8s_masters\" {}\n\nvariable \"number_of_k8s_masters_no_etcd\" {}\n\nvariable \"number_of_k8s_nodes\" {}\n\nvariable \"floatingip_pool\" {}\n\nvariable \"number_of_bastions\" {}\n\nvariable \"external_net\" {}\n\nvariable \"network_name\" {}\n\nvariable \"router_id\" {\n default = \"\"\n}\n\nvariable \"k8s_masters\" {}\n\nvariable \"k8s_nodes\" {}\n\nvariable \"k8s_master_fips\" {}\n\nvariable \"bastion_fips\" {}\n\nvariable \"router_internal_port_id\" {}\n" }, { "path": "contrib/terraform/openstack/modules/ips/versions.tf", "content": "terraform {\n required_providers {\n null = {\n source = \"hashicorp/null\"\n }\n openstack = {\n source = \"terraform-provider-openstack/openstack\"\n }\n }\n required_version = \">= 0.12.26\"\n}\n" }, { "path": "contrib/terraform/openstack/modules/loadbalancer/main.tf", "content": "resource \"openstack_lb_loadbalancer_v2\" \"k8s_lb\" {\n count = var.k8s_master_loadbalancer_enabled ? 1 : 0\n name = \"${var.cluster_name}-api-loadbalancer\"\n vip_subnet_id = var.subnet_id\n}\n\nresource \"openstack_lb_listener_v2\" \"api_listener\"{\n count = var.k8s_master_loadbalancer_enabled ? 1 : 0\n name = \"api-listener\"\n protocol = \"TCP\"\n protocol_port = var.k8s_master_loadbalancer_listener_port\n loadbalancer_id = openstack_lb_loadbalancer_v2.k8s_lb[0].id\n depends_on = [ openstack_lb_loadbalancer_v2.k8s_lb ]\n}\n\nresource \"openstack_lb_pool_v2\" \"api_pool\" {\n count = var.k8s_master_loadbalancer_enabled ? 1 : 0\n name = \"api-pool\"\n protocol = \"TCP\"\n lb_method = \"ROUND_ROBIN\"\n listener_id = openstack_lb_listener_v2.api_listener[0].id\n depends_on = [ openstack_lb_listener_v2.api_listener ]\n}\n\nresource \"openstack_lb_member_v2\" \"lb_member\" {\n count = var.k8s_master_loadbalancer_enabled ? length(var.k8s_master_ips) : 0\n name = var.k8s_master_ips[count.index].name\n pool_id = openstack_lb_pool_v2.api_pool[0].id\n address = var.k8s_master_ips[count.index].access_ip_v4\n protocol_port = var.k8s_master_loadbalancer_server_port\n depends_on = [ openstack_lb_pool_v2.api_pool ]\n}\n\nresource \"openstack_lb_monitor_v2\" \"monitor\" {\n count = var.k8s_master_loadbalancer_enabled ? 1 : 0\n name = \"Api Monitor\"\n pool_id = openstack_lb_pool_v2.api_pool[0].id\n type = \"TCP\"\n delay = 10\n timeout = 5\n max_retries = 5\n}\n\nresource \"openstack_networking_floatingip_v2\" \"floatip_1\" {\n count = var.k8s_master_loadbalancer_enabled && var.k8s_master_loadbalancer_public_ip == \"\" ? 1 : 0\n pool = var.floatingip_pool\n}\n\nresource \"openstack_networking_floatingip_associate_v2\" \"public_ip\" {\n count = var.k8s_master_loadbalancer_enabled ? 1 : 0\n floating_ip = var.k8s_master_loadbalancer_public_ip != \"\" ? var.k8s_master_loadbalancer_public_ip : openstack_networking_floatingip_v2.floatip_1[0].address\n port_id = openstack_lb_loadbalancer_v2.k8s_lb[0].vip_port_id\n depends_on = [ openstack_lb_loadbalancer_v2.k8s_lb ]\n}\n" }, { "path": "contrib/terraform/openstack/modules/loadbalancer/variables.tf", "content": "variable \"cluster_name\" {}\n\nvariable \"subnet_id\" {}\n\nvariable \"floatingip_pool\" {}\n\nvariable \"k8s_master_ips\" {}\n\nvariable \"k8s_master_loadbalancer_enabled\" {}\n\nvariable \"k8s_master_loadbalancer_listener_port\" {}\n\nvariable \"k8s_master_loadbalancer_server_port\" {}\n\nvariable \"k8s_master_loadbalancer_public_ip\" {}\n" }, { "path": "contrib/terraform/openstack/modules/loadbalancer/versions.tf", "content": "terraform {\n required_providers {\n openstack = {\n source = \"terraform-provider-openstack/openstack\"\n }\n }\n required_version = \">= 0.12.26\"\n}\n" }, { "path": "contrib/terraform/openstack/modules/network/main.tf", "content": "resource \"openstack_networking_router_v2\" \"k8s\" {\n name = \"${var.cluster_name}-router\"\n count = var.use_neutron == 1 && var.router_id == null ? 1 : 0\n admin_state_up = \"true\"\n external_network_id = var.external_net\n}\n\ndata \"openstack_networking_router_v2\" \"k8s\" {\n router_id = var.router_id\n count = var.use_neutron == 1 && var.router_id != null ? 1 : 0\n}\n\nresource \"openstack_networking_network_v2\" \"k8s\" {\n name = var.network_name\n count = var.use_neutron\n dns_domain = var.network_dns_domain != null ? var.network_dns_domain : null\n admin_state_up = \"true\"\n port_security_enabled = var.port_security_enabled\n}\n\nresource \"openstack_networking_subnet_v2\" \"k8s\" {\n name = \"${var.cluster_name}-internal-network\"\n count = var.use_neutron\n network_id = openstack_networking_network_v2.k8s[count.index].id\n cidr = var.subnet_cidr\n ip_version = 4\n dns_nameservers = var.dns_nameservers\n}\n\nresource \"openstack_networking_router_interface_v2\" \"k8s\" {\n count = var.use_neutron\n router_id = \"%{if openstack_networking_router_v2.k8s != []}${openstack_networking_router_v2.k8s[count.index].id}%{else}${var.router_id}%{endif}\"\n subnet_id = openstack_networking_subnet_v2.k8s[count.index].id\n}\n" }, { "path": "contrib/terraform/openstack/modules/network/outputs.tf", "content": "output \"router_id\" {\n value = \"%{if var.use_neutron == 1} ${var.router_id == null ? element(concat(openstack_networking_router_v2.k8s.*.id, [\"\"]), 0) : var.router_id} %{else} %{endif}\"\n}\n\noutput \"network_id\" {\n value = element(concat(openstack_networking_network_v2.k8s.*.id, [\"\"]),0)\n}\n\noutput \"router_internal_port_id\" {\n value = element(concat(openstack_networking_router_interface_v2.k8s.*.id, [\"\"]), 0)\n}\n\noutput \"subnet_id\" {\n value = element(concat(openstack_networking_subnet_v2.k8s.*.id, [\"\"]), 0)\n}\n" }, { "path": "contrib/terraform/openstack/modules/network/variables.tf", "content": "variable \"external_net\" {}\n\nvariable \"network_name\" {}\n\nvariable \"network_dns_domain\" {}\n\nvariable \"cluster_name\" {}\n\nvariable \"dns_nameservers\" {\n type = list\n}\n\nvariable \"port_security_enabled\" {\n type = bool\n}\n\nvariable \"subnet_cidr\" {}\n\nvariable \"use_neutron\" {}\n\nvariable \"router_id\" {}\n" }, { "path": "contrib/terraform/openstack/modules/network/versions.tf", "content": "terraform {\n required_providers {\n openstack = {\n source = \"terraform-provider-openstack/openstack\"\n }\n }\n required_version = \">= 0.12.26\"\n}\n" }, { "path": "contrib/terraform/openstack/variables.tf", "content": "variable \"cluster_name\" {\n default = \"example\"\n}\n\nvariable \"az_list\" {\n description = \"List of Availability Zones to use for masters in your OpenStack cluster\"\n type = list(string)\n default = [\"nova\"]\n}\n\nvariable \"az_list_node\" {\n description = \"List of Availability Zones to use for nodes in your OpenStack cluster\"\n type = list(string)\n default = [\"nova\"]\n}\n\nvariable \"number_of_bastions\" {\n default = 1\n}\n\nvariable \"number_of_k8s_masters\" {\n default = 2\n}\n\nvariable \"number_of_k8s_masters_no_etcd\" {\n default = 2\n}\n\nvariable \"number_of_etcd\" {\n default = 2\n}\n\nvariable \"number_of_k8s_masters_no_floating_ip\" {\n default = 2\n}\n\nvariable \"number_of_k8s_masters_no_floating_ip_no_etcd\" {\n default = 2\n}\n\nvariable \"number_of_k8s_nodes\" {\n default = 1\n}\n\nvariable \"number_of_k8s_nodes_no_floating_ip\" {\n default = 1\n}\n\nvariable \"number_of_gfs_nodes_no_floating_ip\" {\n default = 0\n}\n\nvariable \"bastion_root_volume_size_in_gb\" {\n default = 0\n}\n\nvariable \"etcd_root_volume_size_in_gb\" {\n default = 0\n}\n\nvariable \"master_root_volume_size_in_gb\" {\n default = 0\n}\n\nvariable \"node_root_volume_size_in_gb\" {\n default = 0\n}\n\nvariable \"gfs_root_volume_size_in_gb\" {\n default = 0\n}\n\nvariable \"gfs_volume_size_in_gb\" {\n default = 75\n}\n\nvariable \"master_volume_type\" {\n default = \"Default\"\n}\n\nvariable \"node_volume_type\" {\n default = \"Default\"\n}\n\nvariable \"public_key_path\" {\n description = \"The path of the ssh pub key\"\n default = \"~/.ssh/id_rsa.pub\"\n}\n\nvariable \"image\" {\n description = \"the image to use\"\n default = \"\"\n}\n\nvariable \"image_gfs\" {\n description = \"Glance image to use for GlusterFS\"\n default = \"\"\n}\n\nvariable \"ssh_user\" {\n description = \"used to fill out tags for ansible inventory\"\n default = \"ubuntu\"\n}\n\nvariable \"ssh_user_gfs\" {\n description = \"used to fill out tags for ansible inventory\"\n default = \"ubuntu\"\n}\n\nvariable \"flavor_bastion\" {\n description = \"Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs\"\n default = 3\n}\n\nvariable \"flavor_k8s_master\" {\n description = \"Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs\"\n default = 3\n}\n\nvariable \"flavor_k8s_node\" {\n description = \"Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs\"\n default = 3\n}\n\nvariable \"flavor_etcd\" {\n description = \"Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs\"\n default = 3\n}\n\nvariable \"flavor_gfs_node\" {\n description = \"Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs\"\n default = 3\n}\n\nvariable \"network_name\" {\n description = \"name of the internal network to use\"\n default = \"internal\"\n}\n\nvariable \"use_existing_network\" {\n description = \"Use an existing network\"\n type = bool\n default = \"false\"\n}\n\nvariable \"network_dns_domain\" {\n description = \"dns_domain for the internal network\"\n type = string\n default = null\n}\n\nvariable \"use_neutron\" {\n description = \"Use neutron\"\n default = 1\n}\n\nvariable \"port_security_enabled\" {\n description = \"Enable port security on the internal network\"\n type = bool\n default = \"true\"\n}\n\nvariable \"force_null_port_security\" {\n description = \"Force port security to be null. Some providers does not allow setting port security\"\n type = bool\n default = \"false\"\n}\n\nvariable \"subnet_cidr\" {\n description = \"Subnet CIDR block.\"\n type = string\n default = \"10.0.0.0/24\"\n}\n\nvariable \"dns_nameservers\" {\n description = \"An array of DNS name server names used by hosts in this subnet.\"\n type = list(string)\n default = []\n}\n\nvariable \"k8s_master_fips\" {\n description = \"specific pre-existing floating IPs to use for master nodes\"\n type = list(string)\n default = []\n}\n\nvariable \"bastion_fips\" {\n description = \"specific pre-existing floating IPs to use for bastion node\"\n type = list(string)\n default = []\n}\n\nvariable \"floatingip_pool\" {\n description = \"name of the floating ip pool to use\"\n default = \"external\"\n}\n\nvariable \"wait_for_floatingip\" {\n description = \"Terraform will poll the instance until the floating IP has been associated.\"\n default = \"false\"\n}\n\nvariable \"external_net\" {\n description = \"uuid of the external/public network\"\n}\n\nvariable \"supplementary_master_groups\" {\n description = \"supplementary kubespray ansible groups for masters, such kube_node\"\n default = \"\"\n}\n\nvariable \"supplementary_node_groups\" {\n description = \"supplementary kubespray ansible groups for worker nodes, such as kube_ingress\"\n default = \"\"\n}\n\nvariable \"bastion_allowed_remote_ips\" {\n description = \"An array of CIDRs allowed to SSH to hosts\"\n type = list(string)\n default = [\"0.0.0.0/0\"]\n}\n\nvariable \"bastion_allowed_remote_ipv6_ips\" {\n description = \"An array of IPv6 CIDRs allowed to SSH to hosts\"\n type = list(string)\n default = [\"::/0\"]\n}\n\nvariable \"master_allowed_remote_ips\" {\n description = \"An array of CIDRs allowed to access API of masters\"\n type = list(string)\n default = [\"0.0.0.0/0\"]\n}\n\nvariable \"master_allowed_remote_ipv6_ips\" {\n description = \"An array of IPv6 CIDRs allowed to access API of masters\"\n type = list(string)\n default = [\"::/0\"]\n}\n\nvariable \"k8s_allowed_remote_ips\" {\n description = \"An array of CIDRs allowed to SSH to hosts\"\n type = list(string)\n default = []\n}\n\nvariable \"k8s_allowed_remote_ips_ipv6\" {\n description = \"An array of IPv6 CIDRs allowed to SSH to hosts\"\n type = list(string)\n default = []\n}\n\nvariable \"k8s_allowed_egress_ips\" {\n description = \"An array of CIDRs allowed for egress traffic\"\n type = list(string)\n default = [\"0.0.0.0/0\"]\n}\n\nvariable \"k8s_allowed_egress_ipv6_ips\" {\n description = \"An array of CIDRs allowed for egress IPv6 traffic\"\n type = list(string)\n default = [\"::/0\"]\n}\n\nvariable \"master_allowed_ports\" {\n type = list(any)\n\n default = []\n}\n\nvariable \"master_allowed_ports_ipv6\" {\n type = list(any)\n\n default = [\n {\n \"protocol\" = \"ipv6-icmp\"\n \"port_range_min\" = 0\n \"port_range_max\" = 0\n \"remote_ip_prefix\" = \"::/0\"\n },\n ]\n}\n\nvariable \"worker_allowed_ports\" {\n type = list(any)\n\n default = [\n {\n \"protocol\" = \"tcp\"\n \"port_range_min\" = 30000\n \"port_range_max\" = 32767\n \"remote_ip_prefix\" = \"0.0.0.0/0\"\n },\n ]\n}\n\nvariable \"worker_allowed_ports_ipv6\" {\n type = list(any)\n\n default = [\n {\n \"protocol\" = \"tcp\"\n \"port_range_min\" = 30000\n \"port_range_max\" = 32767\n \"remote_ip_prefix\" = \"::/0\"\n },\n {\n \"protocol\" = \"ipv6-icmp\"\n \"port_range_min\" = 0\n \"port_range_max\" = 0\n \"remote_ip_prefix\" = \"::/0\"\n },\n ]\n}\n\nvariable \"bastion_allowed_ports\" {\n type = list(any)\n\n default = []\n}\n\nvariable \"bastion_allowed_ports_ipv6\" {\n type = list(any)\n\n default = []\n}\n\nvariable \"use_access_ip\" {\n default = 1\n}\n\nvariable \"master_server_group_policy\" {\n description = \"desired server group policy, e.g. anti-affinity\"\n default = \"\"\n}\n\nvariable \"node_server_group_policy\" {\n description = \"desired server group policy, e.g. anti-affinity\"\n default = \"\"\n}\n\nvariable \"etcd_server_group_policy\" {\n description = \"desired server group policy, e.g. anti-affinity\"\n default = \"\"\n}\n\nvariable \"router_id\" {\n description = \"uuid of an externally defined router to use\"\n default = null\n}\n\nvariable \"router_internal_port_id\" {\n description = \"uuid of the port connection our router to our network\"\n default = null\n}\n\nvariable \"k8s_masters\" {\n default = {}\n}\n\nvariable \"k8s_nodes\" {\n default = {}\n}\n\nvariable \"additional_server_groups\" {\n default = {}\n type = map(object({\n policy = string\n }))\n}\n\nvariable \"extra_sec_groups\" {\n default = false\n}\n\nvariable \"extra_sec_groups_name\" {\n default = \"custom\"\n}\n\nvariable \"image_uuid\" {\n description = \"uuid of image inside openstack to use\"\n default = \"\"\n}\n\nvariable \"image_gfs_uuid\" {\n description = \"uuid of image to be used on gluster fs nodes. If empty defaults to image_uuid\"\n default = \"\"\n}\n\nvariable \"image_master\" {\n description = \"uuid of image inside openstack to use\"\n default = \"\"\n}\n\nvariable \"image_master_uuid\" {\n description = \"uuid of image to be used on master nodes. If empty defaults to image_uuid\"\n default = \"\"\n}\n\nvariable \"group_vars_path\" {\n description = \"path to the inventory group vars directory\"\n type = string\n default = \"./group_vars\"\n}\n\nvariable \"k8s_master_loadbalancer_enabled\" {\n type = bool\n default = \"false\"\n}\n\nvariable \"k8s_master_loadbalancer_listener_port\" {\n type = string\n default = \"6443\"\n}\n\nvariable \"k8s_master_loadbalancer_server_port\" {\n type = string\n default = 6443\n}\n\nvariable \"k8s_master_loadbalancer_public_ip\" {\n type = string\n default = \"\"\n}\n" }, { "path": "contrib/terraform/openstack/versions.tf", "content": "terraform {\n required_providers {\n openstack = {\n source = \"terraform-provider-openstack/openstack\"\n version = \"~> 1.17\"\n }\n }\n required_version = \">= 1.3.0\"\n}\n" }, { "path": "contrib/terraform/terraform.py", "content": "#!/usr/bin/env python3\n#\n# Copyright 2015 Cisco Systems, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# original: https://github.com/CiscoCloud/terraform.py\n\n\"\"\"\\\nDynamic inventory for Terraform - finds all `.tfstate` files below the working\ndirectory and generates an inventory based on them.\n\"\"\"\nimport argparse\nfrom collections import defaultdict\nimport random\nfrom functools import wraps\nimport json\nimport os\nimport re\n\nVERSION = '0.4.0pre'\n\n\ndef tfstates(root=None):\n root = root or os.getcwd()\n for dirpath, _, filenames in os.walk(root):\n for name in filenames:\n if os.path.splitext(name)[-1] == '.tfstate':\n yield os.path.join(dirpath, name)\n\ndef convert_to_v3_structure(attributes, prefix=''):\n \"\"\" Convert the attributes from v4 to v3\n Receives a dict and return a dictionary \"\"\"\n result = {}\n if isinstance(attributes, str):\n # In the case when we receive a string (e.g. values for security_groups)\n return {'{}{}'.format(prefix, random.randint(1,10**10)): attributes}\n for key, value in attributes.items():\n if isinstance(value, list):\n if len(value):\n result['{}{}.#'.format(prefix, key, hash)] = len(value)\n for i, v in enumerate(value):\n result.update(convert_to_v3_structure(v, '{}{}.{}.'.format(prefix, key, i)))\n elif isinstance(value, dict):\n result['{}{}.%'.format(prefix, key)] = len(value)\n for k, v in value.items():\n result['{}{}.{}'.format(prefix, key, k)] = v\n else:\n result['{}{}'.format(prefix, key)] = value\n return result\n\ndef iterresources(filenames):\n for filename in filenames:\n with open(filename, 'r') as json_file:\n state = json.load(json_file)\n tf_version = state['version']\n if tf_version == 3:\n for module in state['modules']:\n name = module['path'][-1]\n for key, resource in module['resources'].items():\n yield name, key, resource\n elif tf_version == 4:\n # In version 4 the structure changes so we need to iterate\n # each instance inside the resource branch.\n for resource in state['resources']:\n name = resource['provider'].split('.')[-1]\n for instance in resource['instances']:\n key = \"{}.{}\".format(resource['type'], resource['name'])\n if 'index_key' in instance:\n key = \"{}.{}\".format(key, instance['index_key'])\n data = {}\n data['type'] = resource['type']\n data['provider'] = resource['provider']\n data['depends_on'] = instance.get('depends_on', [])\n data['primary'] = {'attributes': convert_to_v3_structure(instance['attributes'])}\n if 'id' in instance['attributes']:\n data['primary']['id'] = instance['attributes']['id']\n data['primary']['meta'] = instance['attributes'].get('meta',{})\n yield name, key, data\n else:\n raise KeyError('tfstate version %d not supported' % tf_version)\n\n\n## READ RESOURCES\nPARSERS = {}\n\n\ndef _clean_dc(dcname):\n # Consul DCs are strictly alphanumeric with underscores and hyphens -\n # ensure that the consul_dc attribute meets these requirements.\n return re.sub(r'[^\\w_\\-]', '-', dcname)\n\n\ndef iterhosts(resources):\n '''yield host tuples of (name, attributes, groups)'''\n for module_name, key, resource in resources:\n resource_type, name = key.split('.', 1)\n try:\n parser = PARSERS[resource_type]\n except KeyError:\n continue\n\n yield parser(resource, module_name)\n\n\ndef iterips(resources):\n '''yield ip tuples of (port_id, ip)'''\n for module_name, key, resource in resources:\n resource_type, name = key.split('.', 1)\n if resource_type == 'openstack_networking_floatingip_associate_v2':\n yield openstack_floating_ips(resource)\n\n\ndef parses(prefix):\n def inner(func):\n PARSERS[prefix] = func\n return func\n\n return inner\n\n\ndef calculate_mantl_vars(func):\n \"\"\"calculate Mantl vars\"\"\"\n\n @wraps(func)\n def inner(*args, **kwargs):\n name, attrs, groups = func(*args, **kwargs)\n\n # attrs\n if attrs.get('role', '') == 'control':\n attrs['consul_is_server'] = True\n else:\n attrs['consul_is_server'] = False\n\n # groups\n if attrs.get('publicly_routable', False):\n groups.append('publicly_routable')\n\n return name, attrs, groups\n\n return inner\n\n\ndef _parse_prefix(source, prefix, sep='.'):\n for compkey, value in list(source.items()):\n try:\n curprefix, rest = compkey.split(sep, 1)\n except ValueError:\n continue\n\n if curprefix != prefix or rest == '#':\n continue\n\n yield rest, value\n\n\ndef parse_attr_list(source, prefix, sep='.'):\n attrs = defaultdict(dict)\n for compkey, value in _parse_prefix(source, prefix, sep):\n idx, key = compkey.split(sep, 1)\n attrs[idx][key] = value\n\n return list(attrs.values())\n\n\ndef parse_dict(source, prefix, sep='.'):\n return dict(_parse_prefix(source, prefix, sep))\n\n\ndef parse_list(source, prefix, sep='.'):\n return [value for _, value in _parse_prefix(source, prefix, sep)]\n\n\ndef parse_bool(string_form):\n if type(string_form) is bool:\n return string_form\n\n token = string_form.lower()[0]\n\n if token == 't':\n return True\n elif token == 'f':\n return False\n else:\n raise ValueError('could not convert %r to a bool' % string_form)\n\ndef sanitize_groups(groups):\n _groups = []\n chars_to_replace = ['+', '-', '=', '.', '/', ' ']\n for i in groups:\n _i = i\n for char in chars_to_replace:\n _i = _i.replace(char, '_')\n _groups.append(_i)\n groups.clear()\n groups.extend(_groups)\n\n@parses('equinix_metal_device')\ndef equinix_metal_device(resource, tfvars=None):\n raw_attrs = resource['primary']['attributes']\n name = raw_attrs['hostname']\n groups = []\n\n attrs = {\n 'id': raw_attrs['id'],\n 'facilities': parse_list(raw_attrs, 'facilities'),\n 'hostname': raw_attrs['hostname'],\n 'operating_system': raw_attrs['operating_system'],\n 'locked': parse_bool(raw_attrs['locked']),\n 'tags': parse_list(raw_attrs, 'tags'),\n 'plan': raw_attrs['plan'],\n 'project_id': raw_attrs['project_id'],\n 'state': raw_attrs['state'],\n # ansible\n 'ansible_host': raw_attrs['network.0.address'],\n 'ansible_ssh_user': 'root', # Use root by default in metal\n # generic\n 'ipv4_address': raw_attrs['network.0.address'],\n 'public_ipv4': raw_attrs['network.0.address'],\n 'ipv6_address': raw_attrs['network.1.address'],\n 'public_ipv6': raw_attrs['network.1.address'],\n 'private_ipv4': raw_attrs['network.2.address'],\n 'provider': 'equinix',\n }\n\n if raw_attrs['operating_system'] == 'flatcar_stable':\n # For Flatcar set the ssh_user to core\n attrs.update({'ansible_ssh_user': 'core'})\n\n # add groups based on attrs\n groups.append('equinix_metal_operating_system_%s' % attrs['operating_system'])\n groups.append('equinix_metal_locked_%s' % attrs['locked'])\n groups.append('equinix_metal_state_%s' % attrs['state'])\n groups.append('equinix_metal_plan_%s' % attrs['plan'])\n\n # groups specific to kubespray\n groups = groups + attrs['tags']\n sanitize_groups(groups)\n\n return name, attrs, groups\n\n\ndef openstack_floating_ips(resource):\n raw_attrs = resource['primary']['attributes']\n attrs = {\n 'ip': raw_attrs['floating_ip'],\n 'port_id': raw_attrs['port_id'],\n }\n return attrs\n\ndef openstack_floating_ips(resource):\n raw_attrs = resource['primary']['attributes']\n return raw_attrs['port_id'], raw_attrs['floating_ip']\n\n@parses('openstack_compute_instance_v2')\n@calculate_mantl_vars\ndef openstack_host(resource, module_name):\n raw_attrs = resource['primary']['attributes']\n name = raw_attrs['name']\n groups = []\n\n attrs = {\n 'access_ip_v4': raw_attrs['access_ip_v4'],\n 'access_ip_v6': raw_attrs['access_ip_v6'],\n 'access_ip': raw_attrs['access_ip_v4'],\n 'access_ip6': raw_attrs['access_ip_v6'],\n 'ip': raw_attrs['network.0.fixed_ip_v4'],\n 'flavor': parse_dict(raw_attrs, 'flavor',\n sep='_'),\n 'id': raw_attrs['id'],\n 'image': parse_dict(raw_attrs, 'image',\n sep='_'),\n 'key_pair': raw_attrs['key_pair'],\n 'metadata': parse_dict(raw_attrs, 'metadata'),\n 'network': parse_attr_list(raw_attrs, 'network'),\n 'region': raw_attrs.get('region', ''),\n 'security_groups': parse_list(raw_attrs, 'security_groups'),\n # workaround for an OpenStack bug where hosts have a different domain\n # after they're restarted\n 'host_domain': 'novalocal',\n 'use_host_domain': True,\n # generic\n 'public_ipv4': raw_attrs['access_ip_v4'],\n 'private_ipv4': raw_attrs['access_ip_v4'],\n 'port_id' : raw_attrs['network.0.port'],\n 'provider': 'openstack',\n }\n\n if 'floating_ip' in raw_attrs:\n attrs['private_ipv4'] = raw_attrs['network.0.fixed_ip_v4']\n\n if 'metadata.use_access_ip' in raw_attrs and raw_attrs['metadata.use_access_ip'] == \"0\":\n attrs.pop('access_ip')\n\n try:\n if 'metadata.prefer_ipv6' in raw_attrs and raw_attrs['metadata.prefer_ipv6'] == \"1\":\n attrs.update({\n 'ansible_host': re.sub(r\"[\\[\\]]\", \"\", raw_attrs['access_ip_v6']),\n 'publicly_routable': True,\n })\n else:\n attrs.update({\n 'ansible_host': raw_attrs['access_ip_v4'],\n 'publicly_routable': True,\n })\n except (KeyError, ValueError):\n attrs.update({'ansible_host': '', 'publicly_routable': False})\n\n # Handling of floating IPs has changed: https://github.com/terraform-providers/terraform-provider-openstack/blob/master/CHANGELOG.md#010-june-21-2017\n\n # attrs specific to Ansible\n if 'metadata.ssh_user' in raw_attrs:\n attrs['ansible_user'] = raw_attrs['metadata.ssh_user']\n if 'metadata.ssh_port' in raw_attrs:\n attrs['ansible_port'] = raw_attrs['metadata.ssh_port']\n\n if 'volume.#' in list(raw_attrs.keys()) and int(raw_attrs['volume.#']) > 0:\n device_index = 1\n for key, value in list(raw_attrs.items()):\n match = re.search(\"^volume.*.device$\", key)\n if match:\n attrs['disk_volume_device_'+str(device_index)] = value\n device_index += 1\n\n\n # attrs specific to Mantl\n attrs.update({\n 'role': attrs['metadata'].get('role', 'none')\n })\n\n # add groups based on attrs\n groups.append('os_image=' + str(attrs['image']['id']))\n groups.append('os_flavor=' + str(attrs['flavor']['name']))\n groups.extend('os_metadata_%s=%s' % item\n for item in list(attrs['metadata'].items()))\n groups.append('os_region=' + str(attrs['region']))\n\n # groups specific to kubespray\n for group in attrs['metadata'].get('kubespray_groups', \"\").split(\",\"):\n groups.append(group)\n\n sanitize_groups(groups)\n\n return name, attrs, groups\n\n\ndef iter_host_ips(hosts, ips):\n '''Update hosts that have an entry in the floating IP list'''\n for host in hosts:\n port_id = host[1]['port_id']\n\n if port_id in ips:\n ip = ips[port_id]\n\n host[1].update({\n 'access_ip_v4': ip,\n 'access_ip': ip,\n 'public_ipv4': ip,\n 'ansible_host': ip,\n })\n\n if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == \"0\" and 'access_ip' in host[1]:\n host[1].pop('access_ip')\n\n yield host\n\n\n## QUERY TYPES\ndef query_host(hosts, target):\n for name, attrs, _ in hosts:\n if name == target:\n return attrs\n\n return {}\n\n\ndef query_list(hosts):\n groups = defaultdict(dict)\n meta = {}\n\n for name, attrs, hostgroups in hosts:\n for group in set(hostgroups):\n # Ansible 2.6.2 stopped supporting empty group names: https://github.com/ansible/ansible/pull/42584/commits/d4cd474b42ed23d8f8aabb2a7f84699673852eaf\n # Empty group name defaults to \"all\" in Ansible < 2.6.2 so we alter empty group names to \"all\"\n if not group: group = \"all\"\n\n groups[group].setdefault('hosts', [])\n groups[group]['hosts'].append(name)\n\n meta[name] = attrs\n\n groups['_meta'] = {'hostvars': meta}\n return groups\n\n\ndef query_hostfile(hosts):\n out = ['## begin hosts generated by terraform.py ##']\n out.extend(\n '{}\\t{}'.format(attrs['ansible_host'].ljust(16), name)\n for name, attrs, _ in hosts\n )\n\n out.append('## end hosts generated by terraform.py ##')\n return '\\n'.join(out)\n\n\ndef main():\n parser = argparse.ArgumentParser(\n __file__, __doc__,\n formatter_class=argparse.ArgumentDefaultsHelpFormatter, )\n modes = parser.add_mutually_exclusive_group(required=True)\n modes.add_argument('--list',\n action='store_true',\n help='list all variables')\n modes.add_argument('--host', help='list variables for a single host')\n modes.add_argument('--version',\n action='store_true',\n help='print version and exit')\n modes.add_argument('--hostfile',\n action='store_true',\n help='print hosts as a /etc/hosts snippet')\n parser.add_argument('--pretty',\n action='store_true',\n help='pretty-print output JSON')\n parser.add_argument('--nometa',\n action='store_true',\n help='with --list, exclude hostvars')\n default_root = os.environ.get('TERRAFORM_STATE_ROOT',\n os.path.abspath(os.path.join(os.path.dirname(__file__),\n '..', '..', )))\n parser.add_argument('--root',\n default=default_root,\n help='custom root to search for `.tfstate`s in')\n\n args = parser.parse_args()\n\n if args.version:\n print('%s %s' % (__file__, VERSION))\n parser.exit()\n\n hosts = iterhosts(iterresources(tfstates(args.root)))\n\n # Perform a second pass on the file to pick up floating_ip entries to update the ip address of referenced hosts\n ips = dict(iterips(iterresources(tfstates(args.root))))\n\n if ips:\n hosts = iter_host_ips(hosts, ips)\n\n if args.list:\n output = query_list(hosts)\n if args.nometa:\n del output['_meta']\n print(json.dumps(output, indent=4 if args.pretty else None))\n elif args.host:\n output = query_host(hosts, args.host)\n print(json.dumps(output, indent=4 if args.pretty else None))\n elif args.hostfile:\n output = query_hostfile(hosts)\n print(output)\n\n parser.exit()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "contrib/terraform/upcloud/README.md", "content": "# Kubernetes on UpCloud with Terraform\n\nProvision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray\n\n## Requirements\n\n* Terraform 0.13.0 or newer\n\n## Quickstart\n\nNOTE: Assumes you are at the root of the kubespray repo.\n\nFor authentication in your cluster you can use the environment variables.\n\n```bash\nexport TF_VAR_UPCLOUD_USERNAME=username\nexport TF_VAR_UPCLOUD_PASSWORD=password\n```\n\nTo allow API access to your UpCloud account, you need to allow API connections by visiting [Account-page](https://hub.upcloud.com/account) in your UpCloud Hub.\n\nCopy the cluster configuration file.\n\n```bash\nCLUSTER=my-upcloud-cluster\ncp -r inventory/sample inventory/$CLUSTER\ncp contrib/terraform/upcloud/cluster-settings.tfvars inventory/$CLUSTER/\nexport ANSIBLE_CONFIG=ansible.cfg\ncd inventory/$CLUSTER\n```\n\nEdit `cluster-settings.tfvars` to match your requirement.\n\nRun Terraform to create the infrastructure.\n\n```bash\nterraform init ../../contrib/terraform/upcloud\nterraform apply --var-file cluster-settings.tfvars \\\n -state=tfstate-$CLUSTER.tfstate \\\n ../../contrib/terraform/upcloud/\n```\n\nYou should now have a inventory file named `inventory.ini` that you can use with kubespray.\nYou can use the inventory file with kubespray to set up a cluster.\n\nIt is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:\n\n```bash\nansible -i inventory.ini -m ping all\n```\n\nYou can setup Kubernetes with kubespray using the generated inventory:\n\n```bash\nansible-playbook -i inventory.ini ../../cluster.yml -b -v\n```\n\n## Teardown\n\nYou can teardown your infrastructure using the following Terraform command:\n\n```bash\nterraform destroy --var-file cluster-settings.tfvars \\\n -state=tfstate-$CLUSTER.tfstate \\\n ../../contrib/terraform/upcloud/\n```\n\n## Variables\n\n* `prefix`: Prefix to add to all resources, if set to \"\" don't set any prefix\n* `template_name`: The name or UUID of a base image\n* `username`: a user to access the nodes, defaults to \"ubuntu\"\n* `private_network_cidr`: CIDR to use for the private network, defaults to \"172.16.0.0/24\"\n* `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]`\n* `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true`\n* `ssh_public_keys`: List of public SSH keys to install on all machines\n* `zone`: The zone where to run the cluster\n* `machines`: Machines to provision. Key of this object will be used as the name of the machine\n * `node_type`: The role of this node *(master|worker)*\n * `plan`: Preconfigured cpu/mem plan to use (disables `cpu` and `mem` attributes below)\n * `cpu`: number of cpu cores\n * `mem`: memory size in MB\n * `disk_size`: The size of the storage in GB\n * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false`\n * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null`\n * `additional_disks`: Additional disks to attach to the node.\n * `size`: The size of the additional disk in GB\n * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)\n* `firewall_enabled`: Enable firewall rules\n* `firewall_default_deny_in`: Set the firewall to deny inbound traffic by default. Automatically adds UpCloud DNS server and NTP port allowlisting.\n* `firewall_default_deny_out`: Set the firewall to deny outbound traffic by default.\n* `master_allowed_remote_ips`: List of IP ranges that should be allowed to access API of masters\n * `start_address`: Start of address range to allow\n * `end_address`: End of address range to allow\n* `k8s_allowed_remote_ips`: List of IP ranges that should be allowed SSH access to all nodes\n * `start_address`: Start of address range to allow\n * `end_address`: End of address range to allow\n* `master_allowed_ports`: List of port ranges that should be allowed to access the masters\n * `protocol`: Protocol *(tcp|udp|icmp)*\n * `port_range_min`: Start of port range to allow\n * `port_range_max`: End of port range to allow\n * `start_address`: Start of address range to allow\n * `end_address`: End of address range to allow\n* `worker_allowed_ports`: List of port ranges that should be allowed to access the workers\n * `protocol`: Protocol *(tcp|udp|icmp)*\n * `port_range_min`: Start of port range to allow\n * `port_range_max`: End of port range to allow\n * `start_address`: Start of address range to allow\n * `end_address`: End of address range to allow\n* `loadbalancer_enabled`: Enable managed load balancer\n* `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)*\n* `loadbalancer_legacy_network`: If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false (default value)\n* `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends\n * `port`: Port to load balance.\n * `target_port`: Port to the backend servers.\n * `backend_servers`: List of servers that traffic to the port should be forwarded to.\n * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol.\n* `router_enable`: If a router should be connected to the private network or not\n* `gateways`: Gateways that should be connected to the router, requires router_enable is set to true\n * `features`: List of features for the gateway\n * `plan`: Plan to use for the gateway\n * `connections`: The connections and tunnel to create for the gateway\n * `type`: What type of connection\n * `local_routes`: Map of local routes for the connection\n * `type`: Type of route\n * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix\n * `remote_routes`: Map of local routes for the connection\n * `type`: Type of route\n * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix\n * `tunnels`: The tunnels to create for this connection\n * `remote_address`: The remote address for the tunnel\n * `ipsec_properties`: Set properties of IPSec, if not set, defaults will be used\n * `child_rekey_time`: IKE child SA rekey time in seconds\n * `dpd_delay`: Delay before sending Dead Peer Detection packets if no traffic is detected, in seconds\n * `dpd_timeout`: Timeout period for DPD reply before considering the peer to be dead, in seconds\n * `ike_lifetime`: Maximum IKE SA lifetime in seconds()\n * `rekey_time`: IKE SA rekey time in seconds\n * `phase1_algorithms`: List of Phase 1: Proposal algorithms\n * `phase1_dh_group_numbers`: List of Phase 1 Diffie-Hellman group numbers\n * `phase1_integrity_algorithms`: List of Phase 1 integrity algorithms\n * `phase2_algorithms`: List of Phase 2: Security Association algorithms\n * `phase2_dh_group_numbers`: List of Phase 2 Diffie-Hellman group numbers\n * `phase2_integrity_algorithms`: List of Phase 2 integrity algorithms\n* `gateway_vpn_psks`: Separate variable for providing psks for connection tunnels. Environment variable can be exported in the following format `export TF_VAR_gateway_vpn_psks='{\"${gateway-name}-${connecton-name}-tunnel\":{psk:\"...\"}}'`\n* `static_routes`: Static routes to apply to the router, requires `router_enable` is set to true\n* `network_peerings`: Other UpCloud private networks to peer with, requires `router_enable` is set to true\n* `server_groups`: Group servers together\n * `servers`: The servers that should be included in the group.\n * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to \"strict\" or yes\" will result in all servers in the group being placed on separate compute hosts. The value can be \"strict\", \"yes\" or \"no\". \"strict\" refers to strict policy doesn't allow servers in the same server group to be on the same host. \"yes\" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.\n\n## Migration\n\nWhen `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state.\nThe error messages you'll see if you encounter this is:\n\n```text\nError: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider \"registry.terraform.io/hashicorp/null\" to obtain schema: unavailable provider \"registry.terraform.io/hashicorp/null\"\nError: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider \"registry.terraform.io/hashicorp/template\" to obtain schema: unavailable provider \"registry.terraform.io/hashicorp/template\"\n```\n\nThis can be fixed with the following lines\n\n```bash\nterraform state rm -state=terraform.tfstate null_resource.inventories\nterraform state rm -state=terraform.tfstate data.template_file.inventory\n```\n\n### Public to Private only migration\n\nSince there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs.\nThe way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`.\nAfter that you can add new nodes without `force_public_ip` and `dns_servers` set and create them.\nAdd the new nodes into the cluster and when all of them are added, remove the old nodes.\n" }, { "path": "contrib/terraform/upcloud/cluster-settings.tfvars", "content": "# See: https://developers.upcloud.com/1.3/5-zones/\nzone = \"fi-hel1\"\nprivate_cloud = false\n\n# Only used if private_cloud = true, public zone equivalent\n# For example use finnish public zone for finnish private zone\npublic_zone = \"fi-hel2\"\n\nusername = \"ubuntu\"\n\n# Prefix to use for all resources to separate them from other resources\nprefix = \"kubespray\"\n\ninventory_file = \"inventory.ini\"\n\n# Set the operating system using UUID or exact name\ntemplate_name = \"Ubuntu Server 20.04 LTS (Focal Fossa)\"\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa public key 1\",\n \"ssh-rsa public key 2\",\n]\n\n# check list of available plan https://developers.upcloud.com/1.3/7-plans/\nmachines = {\n \"control-plane-0\" : {\n \"node_type\" : \"master\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {}\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {\n # \"some-disk-name-1\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # },\n # \"some-disk-name-2\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # }\n }\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {\n # \"some-disk-name-1\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # },\n # \"some-disk-name-2\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # }\n }\n },\n \"worker-2\" : {\n \"node_type\" : \"worker\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {\n # \"some-disk-name-1\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # },\n # \"some-disk-name-2\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # }\n }\n }\n}\n\nfirewall_enabled = false\nfirewall_default_deny_in = false\nfirewall_default_deny_out = false\n\nmaster_allowed_remote_ips = [\n {\n \"start_address\" : \"0.0.0.0\"\n \"end_address\" : \"255.255.255.255\"\n }\n]\n\nk8s_allowed_remote_ips = [\n {\n \"start_address\" : \"0.0.0.0\"\n \"end_address\" : \"255.255.255.255\"\n }\n]\n\nmaster_allowed_ports = []\nworker_allowed_ports = []\n\nloadbalancer_enabled = false\nloadbalancer_plan = \"development\"\nloadbalancers = {\n # \"http\" : {\n # \"proxy_protocol\" : false\n # \"port\" : 80,\n # \"target_port\" : 80,\n # \"backend_servers\" : [\n # \"worker-0\",\n # \"worker-1\",\n # \"worker-2\"\n # ]\n # }\n}\n\nserver_groups = {\n # \"control-plane\" = {\n # servers = [\n # \"control-plane-0\"\n # ]\n # anti_affinity_policy = \"strict\"\n # },\n # \"workers\" = {\n # servers = [\n # \"worker-0\",\n # \"worker-1\",\n # \"worker-2\"\n # ]\n # anti_affinity_policy = \"yes\"\n # }\n}\n\nrouter_enable = false\ngateways = {\n # \"gateway\" : {\n # features: [ \"vpn\" ]\n # plan = \"production\"\n # connections = {\n # \"connection\" = {\n # name = \"connection\"\n # type = \"ipsec\"\n # remote_routes = {\n # \"them\" = {\n # type = \"static\"\n # static_network = \"1.2.3.4/24\"\n # }\n # }\n # local_routes = {\n # \"me\" = {\n # type = \"static\"\n # static_network = \"4.3.2.1/24\"\n # }\n # }\n # tunnels = {\n # \"tunnel1\" = {\n # remote_address = \"1.2.3.4\"\n # }\n # }\n # }\n # }\n # }\n}\n# gateway_vpn_psks = {} # Should be loaded as an environment variable\nstatic_routes = {\n # \"route\": {\n # route: \"1.2.3.4/24\"\n # nexthop: \"4.3.2.1\"\n # }\n}\nnetwork_peerings = {\n # \"peering\": {\n # remote_network: \"uuid\"\n # }\n}\n" }, { "path": "contrib/terraform/upcloud/main.tf", "content": "\nterraform {\n required_version = \">= 0.13.0\"\n}\nprovider \"upcloud\" {\n # Your UpCloud credentials are read from environment variables:\n username = var.UPCLOUD_USERNAME\n password = var.UPCLOUD_PASSWORD\n}\n\nmodule \"kubernetes\" {\n source = \"./modules/kubernetes-cluster\"\n\n prefix = var.prefix\n zone = var.zone\n private_cloud = var.private_cloud\n public_zone = var.public_zone\n\n template_name = var.template_name\n username = var.username\n\n private_network_cidr = var.private_network_cidr\n dns_servers = var.dns_servers\n use_public_ips = var.use_public_ips\n\n machines = var.machines\n\n ssh_public_keys = var.ssh_public_keys\n\n firewall_enabled = var.firewall_enabled\n firewall_default_deny_in = var.firewall_default_deny_in\n firewall_default_deny_out = var.firewall_default_deny_out\n master_allowed_remote_ips = var.master_allowed_remote_ips\n k8s_allowed_remote_ips = var.k8s_allowed_remote_ips\n bastion_allowed_remote_ips = var.bastion_allowed_remote_ips\n master_allowed_ports = var.master_allowed_ports\n worker_allowed_ports = var.worker_allowed_ports\n\n loadbalancer_enabled = var.loadbalancer_enabled\n loadbalancer_plan = var.loadbalancer_plan\n loadbalancer_legacy_network = var.loadbalancer_legacy_network\n loadbalancers = var.loadbalancers\n\n router_enable = var.router_enable\n gateways = var.gateways\n gateway_vpn_psks = var.gateway_vpn_psks\n static_routes = var.static_routes\n network_peerings = var.network_peerings\n\n server_groups = var.server_groups\n}\n\n#\n# Generate ansible inventory\n#\n\nresource \"local_file\" \"inventory\" {\n content = templatefile(\"${path.module}/templates/inventory.tpl\", {\n master_ip = module.kubernetes.master_ip\n worker_ip = module.kubernetes.worker_ip\n bastion_ip = module.kubernetes.bastion_ip\n username = var.username\n })\n filename = var.inventory_file\n}\n" }, { "path": "contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf", "content": "locals {\n # Create a list of all disks to create\n disks = flatten([\n for node_name, machine in var.machines : [\n for disk_name, disk in machine.additional_disks : {\n disk = disk\n disk_name = disk_name\n node_name = node_name\n }\n ]\n ])\n\n lb_backend_servers = flatten([\n for lb_name, loadbalancer in var.loadbalancers : [\n for backend_server in loadbalancer.backend_servers : {\n port = loadbalancer.target_port\n lb_name = lb_name\n server_name = backend_server\n }\n ]\n ])\n\n gateway_connections = flatten([\n for gateway_name, gateway in var.gateways : [\n for connection_name, connection in gateway.connections : {\n \"gateway_id\" = upcloud_gateway.gateway[gateway_name].id\n \"gateway_name\" = gateway_name\n \"connection_name\" = connection_name\n \"type\" = connection.type\n \"local_routes\" = connection.local_routes\n \"remote_routes\" = connection.remote_routes\n }\n ]\n ])\n\n gateway_connection_tunnels = flatten([\n for gateway_name, gateway in var.gateways : [\n for connection_name, connection in gateway.connections : [\n for tunnel_name, tunnel in connection.tunnels : {\n \"gateway_id\" = upcloud_gateway.gateway[gateway_name].id\n \"gateway_name\" = gateway_name\n \"connection_id\" = upcloud_gateway_connection.gateway_connection[\"${gateway_name}-${connection_name}\"].id\n \"connection_name\" = connection_name\n \"tunnel_name\" = tunnel_name\n \"local_address_name\" = tolist(upcloud_gateway.gateway[gateway_name].address).0.name\n \"remote_address\" = tunnel.remote_address\n \"ipsec_properties\" = tunnel.ipsec_properties\n }\n ]\n ]\n ])\n\n # If prefix is set, all resources will be prefixed with \"${var.prefix}-\"\n # Else don't prefix with anything\n resource-prefix = \"%{if var.prefix != \"\"}${var.prefix}-%{endif}\"\n\n master_ip = {\n for instance in upcloud_server.master :\n instance.hostname => {\n for nic in instance.network_interface :\n nic.type => nic.ip_address\n if nic.ip_address != null\n }\n }\n worker_ip = {\n for instance in upcloud_server.worker :\n instance.hostname => {\n for nic in instance.network_interface :\n nic.type => nic.ip_address\n if nic.ip_address != null\n }\n }\n\n bastion_ip = {\n for instance in upcloud_server.bastion :\n instance.hostname => {\n for nic in instance.network_interface :\n nic.type => nic.ip_address\n if nic.ip_address != null\n }\n }\n\n node_user_data = {\n for name, machine in var.machines :\n name => < 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}\n#!/bin/bash\necho -e \"[Resolve]\\nDNS=${ join(\" \", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }\" > /etc/systemd/resolved.conf\n\nsystemctl restart systemd-resolved\n%{ endif ~}\nEOF\n }\n}\n\nresource \"upcloud_network\" \"private\" {\n name = \"${local.resource-prefix}k8s-network\"\n zone = var.zone\n\n ip_network {\n address = var.private_network_cidr\n dhcp_default_route = var.router_enable\n # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.\n # See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562\n # dhcp_dns = length(var.private_network_dns) > 0 ? var.private_network_dns : null\n dhcp = true\n family = \"IPv4\"\n }\n\n router = var.router_enable ? upcloud_router.router[0].id : null\n}\n\nresource \"upcloud_storage\" \"additional_disks\" {\n for_each = {\n for disk in local.disks : \"${disk.node_name}_${disk.disk_name}\" => disk.disk\n }\n\n size = each.value.size\n tier = each.value.tier\n title = \"${local.resource-prefix}${each.key}\"\n zone = var.zone\n}\n\nresource \"upcloud_server\" \"master\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"master\"\n }\n\n hostname = \"${local.resource-prefix}${each.key}\"\n plan = each.value.plan\n cpu = each.value.cpu\n mem = each.value.mem\n zone = var.zone\n server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id\n\n template {\n storage = var.template_name\n size = each.value.disk_size\n }\n\n dynamic \"network_interface\" {\n for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []\n\n content {\n type = \"public\"\n }\n }\n\n # Private network interface\n network_interface {\n type = \"private\"\n network = upcloud_network.private.id\n }\n\n # Ignore volumes created by csi-driver\n lifecycle {\n ignore_changes = [storage_devices]\n }\n\n firewall = var.firewall_enabled\n\n dynamic \"storage_devices\" {\n for_each = {\n for disk_key_name, disk in upcloud_storage.additional_disks :\n disk_key_name => disk\n # Only add the disk if it matches the node name in the start of its name\n if length(regexall(\"^${each.key}_.+\", disk_key_name)) > 0\n }\n\n content {\n storage = storage_devices.value.id\n }\n }\n\n # Include at least one public SSH key\n login {\n user = var.username\n keys = var.ssh_public_keys\n create_password = false\n }\n\n metadata = local.node_user_data[each.key] != \"\" ? true : null\n user_data = local.node_user_data[each.key] != \"\" ? local.node_user_data[each.key] : null\n}\n\nresource \"upcloud_server\" \"worker\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"worker\"\n }\n\n hostname = \"${local.resource-prefix}${each.key}\"\n plan = each.value.plan\n cpu = each.value.cpu\n mem = each.value.mem\n zone = var.zone\n server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id\n\n\n template {\n storage = var.template_name\n size = each.value.disk_size\n }\n\n dynamic \"network_interface\" {\n for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []\n\n content {\n type = \"public\"\n }\n }\n\n # Private network interface\n network_interface {\n type = \"private\"\n network = upcloud_network.private.id\n }\n\n # Ignore volumes created by csi-driver\n lifecycle {\n ignore_changes = [storage_devices]\n }\n\n firewall = var.firewall_enabled\n\n dynamic \"storage_devices\" {\n for_each = {\n for disk_key_name, disk in upcloud_storage.additional_disks :\n disk_key_name => disk\n # Only add the disk if it matches the node name in the start of its name\n if length(regexall(\"^${each.key}_.+\", disk_key_name)) > 0\n }\n\n content {\n storage = storage_devices.value.id\n }\n }\n\n # Include at least one public SSH key\n login {\n user = var.username\n keys = var.ssh_public_keys\n create_password = false\n }\n\n metadata = local.node_user_data[each.key] != \"\" ? true : null\n user_data = local.node_user_data[each.key] != \"\" ? local.node_user_data[each.key] : null\n}\n\nresource \"upcloud_server\" \"bastion\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"bastion\"\n }\n\n hostname = \"${local.resource-prefix}${each.key}\"\n plan = each.value.plan\n cpu = each.value.cpu\n mem = each.value.mem\n zone = var.zone\n server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id\n\n\n template {\n storage = var.template_name\n size = each.value.disk_size\n }\n\n # Private network interface\n network_interface {\n type = \"private\"\n network = upcloud_network.private.id\n }\n\n # Private network interface\n network_interface {\n type = \"public\"\n }\n\n firewall = var.firewall_enabled\n\n dynamic \"storage_devices\" {\n for_each = {\n for disk_key_name, disk in upcloud_storage.additional_disks :\n disk_key_name => disk\n # Only add the disk if it matches the node name in the start of its name\n if length(regexall(\"^${each.key}_.+\", disk_key_name)) > 0\n }\n\n content {\n storage = storage_devices.value.id\n }\n }\n\n # Include at least one public SSH key\n login {\n user = var.username\n keys = var.ssh_public_keys\n create_password = false\n }\n}\n\nresource \"upcloud_firewall_rules\" \"master\" {\n for_each = upcloud_server.master\n server_id = each.value.id\n\n dynamic \"firewall_rule\" {\n for_each = var.master_allowed_remote_ips\n\n content {\n action = \"accept\"\n comment = \"Allow master API access from this network\"\n destination_port_end = \"6443\"\n destination_port_start = \"6443\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = firewall_rule.value.end_address\n source_address_start = firewall_rule.value.start_address\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = length(var.master_allowed_remote_ips) > 0 ? [1] : []\n\n content {\n action = \"drop\"\n comment = \"Deny master API access from other networks\"\n destination_port_end = \"6443\"\n destination_port_start = \"6443\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = \"255.255.255.255\"\n source_address_start = \"0.0.0.0\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.k8s_allowed_remote_ips\n\n content {\n action = \"accept\"\n comment = \"Allow SSH from this network\"\n destination_port_end = \"22\"\n destination_port_start = \"22\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = firewall_rule.value.end_address\n source_address_start = firewall_rule.value.start_address\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []\n\n content {\n action = \"drop\"\n comment = \"Deny SSH from other networks\"\n destination_port_end = \"22\"\n destination_port_start = \"22\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = \"255.255.255.255\"\n source_address_start = \"0.0.0.0\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.master_allowed_ports\n\n content {\n action = \"accept\"\n comment = \"Allow access on this port\"\n destination_port_end = firewall_rule.value.port_range_max\n destination_port_start = firewall_rule.value.port_range_min\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value.protocol\n source_address_end = firewall_rule.value.end_address\n source_address_start = firewall_rule.value.start_address\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value\n source_address_end = \"94.237.40.9\"\n source_address_start = \"94.237.40.9\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value\n source_address_end = \"94.237.127.9\"\n source_address_start = \"94.237.127.9\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv6\"\n protocol = firewall_rule.value\n source_address_end = \"2a04:3540:53::1\"\n source_address_start = \"2a04:3540:53::1\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv6\"\n protocol = firewall_rule.value\n source_address_end = \"2a04:3544:53::1\"\n source_address_start = \"2a04:3544:53::1\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"NTP Port\"\n source_port_end = \"123\"\n source_port_start = \"123\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value\n source_address_end = \"255.255.255.255\"\n source_address_start = \"0.0.0.0\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"NTP Port\"\n source_port_end = \"123\"\n source_port_start = \"123\"\n direction = \"in\"\n family = \"IPv6\"\n protocol = firewall_rule.value\n }\n }\n\n firewall_rule {\n action = var.firewall_default_deny_in ? \"drop\" : \"accept\"\n direction = \"in\"\n }\n\n firewall_rule {\n action = var.firewall_default_deny_out ? \"drop\" : \"accept\"\n direction = \"out\"\n }\n}\n\nresource \"upcloud_firewall_rules\" \"k8s\" {\n for_each = upcloud_server.worker\n server_id = each.value.id\n\n dynamic \"firewall_rule\" {\n for_each = var.k8s_allowed_remote_ips\n\n content {\n action = \"accept\"\n comment = \"Allow SSH from this network\"\n destination_port_end = \"22\"\n destination_port_start = \"22\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = firewall_rule.value.end_address\n source_address_start = firewall_rule.value.start_address\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []\n\n content {\n action = \"drop\"\n comment = \"Deny SSH from other networks\"\n destination_port_end = \"22\"\n destination_port_start = \"22\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = \"255.255.255.255\"\n source_address_start = \"0.0.0.0\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.worker_allowed_ports\n\n content {\n action = \"accept\"\n comment = \"Allow access on this port\"\n destination_port_end = firewall_rule.value.port_range_max\n destination_port_start = firewall_rule.value.port_range_min\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value.protocol\n source_address_end = firewall_rule.value.end_address\n source_address_start = firewall_rule.value.start_address\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value\n source_address_end = \"94.237.40.9\"\n source_address_start = \"94.237.40.9\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value\n source_address_end = \"94.237.127.9\"\n source_address_start = \"94.237.127.9\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv6\"\n protocol = firewall_rule.value\n source_address_end = \"2a04:3540:53::1\"\n source_address_start = \"2a04:3540:53::1\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"tcp\", \"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"UpCloud DNS\"\n source_port_end = \"53\"\n source_port_start = \"53\"\n direction = \"in\"\n family = \"IPv6\"\n protocol = firewall_rule.value\n source_address_end = \"2a04:3544:53::1\"\n source_address_start = \"2a04:3544:53::1\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"NTP Port\"\n source_port_end = \"123\"\n source_port_start = \"123\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = firewall_rule.value\n source_address_end = \"255.255.255.255\"\n source_address_start = \"0.0.0.0\"\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = var.firewall_default_deny_in ? [\"udp\"] : []\n\n content {\n action = \"accept\"\n comment = \"NTP Port\"\n source_port_end = \"123\"\n source_port_start = \"123\"\n direction = \"in\"\n family = \"IPv6\"\n protocol = firewall_rule.value\n }\n }\n\n firewall_rule {\n action = var.firewall_default_deny_in ? \"drop\" : \"accept\"\n direction = \"in\"\n }\n\n firewall_rule {\n action = var.firewall_default_deny_out ? \"drop\" : \"accept\"\n direction = \"out\"\n }\n}\n\nresource \"upcloud_firewall_rules\" \"bastion\" {\n for_each = upcloud_server.bastion\n server_id = each.value.id\n\n dynamic \"firewall_rule\" {\n for_each = var.bastion_allowed_remote_ips\n\n content {\n action = \"accept\"\n comment = \"Allow bastion SSH access from this network\"\n destination_port_end = \"22\"\n destination_port_start = \"22\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = firewall_rule.value.end_address\n source_address_start = firewall_rule.value.start_address\n }\n }\n\n dynamic \"firewall_rule\" {\n for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []\n\n content {\n action = \"drop\"\n comment = \"Drop bastion SSH access from other networks\"\n destination_port_end = \"22\"\n destination_port_start = \"22\"\n direction = \"in\"\n family = \"IPv4\"\n protocol = \"tcp\"\n source_address_end = \"255.255.255.255\"\n source_address_start = \"0.0.0.0\"\n }\n }\n\n firewall_rule {\n action = var.firewall_default_deny_in ? \"drop\" : \"accept\"\n direction = \"in\"\n }\n\n firewall_rule {\n action = var.firewall_default_deny_out ? \"drop\" : \"accept\"\n direction = \"out\"\n }\n}\n\nresource \"upcloud_loadbalancer\" \"lb\" {\n count = var.loadbalancer_enabled ? 1 : 0\n configured_status = \"started\"\n name = \"${local.resource-prefix}lb\"\n plan = var.loadbalancer_plan\n zone = var.private_cloud ? var.public_zone : var.zone\n network = var.loadbalancer_legacy_network ? upcloud_network.private.id : null\n\n dynamic \"networks\" {\n for_each = var.loadbalancer_legacy_network ? [] : [1]\n\n content {\n name = \"Private-Net\"\n type = \"private\"\n family = \"IPv4\"\n network = upcloud_network.private.id\n }\n }\n\n dynamic \"networks\" {\n for_each = var.loadbalancer_legacy_network ? [] : [1]\n\n content {\n name = \"Public-Net\"\n type = \"public\"\n family = \"IPv4\"\n }\n }\n\n lifecycle {\n ignore_changes = [ maintenance_dow, maintenance_time ]\n }\n}\n\nresource \"upcloud_loadbalancer_backend\" \"lb_backend\" {\n for_each = var.loadbalancer_enabled ? var.loadbalancers : {}\n\n loadbalancer = upcloud_loadbalancer.lb[0].id\n name = \"lb-backend-${each.key}\"\n properties {\n outbound_proxy_protocol = each.value.proxy_protocol ? \"v2\" : \"\"\n }\n}\n\nresource \"upcloud_loadbalancer_frontend\" \"lb_frontend\" {\n for_each = var.loadbalancer_enabled ? var.loadbalancers : {}\n\n loadbalancer = upcloud_loadbalancer.lb[0].id\n name = \"lb-frontend-${each.key}\"\n mode = \"tcp\"\n port = each.value.port\n default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name\n\n dynamic \"networks\" {\n for_each = var.loadbalancer_legacy_network ? [] : [1]\n\n content {\n name = \"Public-Net\"\n }\n }\n\n dynamic \"networks\" {\n for_each = each.value.allow_internal_frontend ? [1] : []\n\n content{\n name = \"Private-Net\"\n }\n }\n}\n\nresource \"upcloud_loadbalancer_static_backend_member\" \"lb_backend_member\" {\n for_each = {\n for be_server in local.lb_backend_servers :\n \"${be_server.server_name}-lb-backend-${be_server.lb_name}\" => be_server\n if var.loadbalancer_enabled\n }\n\n backend = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id\n name = \"${local.resource-prefix}${each.key}\"\n ip = merge(local.master_ip, local.worker_ip)[\"${local.resource-prefix}${each.value.server_name}\"].private\n port = each.value.port\n weight = 100\n max_sessions = var.loadbalancer_plan == \"production-small\" ? 50000 : 1000\n enabled = true\n}\n\nresource \"upcloud_server_group\" \"server_groups\" {\n for_each = var.server_groups\n title = each.key\n anti_affinity_policy = each.value.anti_affinity_policy\n labels = {}\n # Managed upstream via upcloud_server resource\n members = []\n lifecycle {\n ignore_changes = [members]\n }\n}\n\nresource \"upcloud_router\" \"router\" {\n count = var.router_enable ? 1 : 0\n\n name = \"${local.resource-prefix}router\"\n\n dynamic \"static_route\" {\n for_each = var.static_routes\n\n content {\n name = static_route.key\n\n nexthop = static_route.value[\"nexthop\"]\n route = static_route.value[\"route\"]\n }\n }\n\n}\n\nresource \"upcloud_gateway\" \"gateway\" {\n for_each = var.router_enable ? var.gateways : {}\n name = \"${local.resource-prefix}${each.key}-gateway\"\n zone = var.private_cloud ? var.public_zone : var.zone\n\n features = each.value.features\n plan = each.value.plan\n\n router {\n id = upcloud_router.router[0].id\n }\n}\n\nresource \"upcloud_gateway_connection\" \"gateway_connection\" {\n for_each = {\n for gc in local.gateway_connections : \"${gc.gateway_name}-${gc.connection_name}\" => gc\n }\n\n gateway = each.value.gateway_id\n name = \"${local.resource-prefix}${each.key}-gateway-connection\"\n type = each.value.type\n\n dynamic \"local_route\" {\n for_each = each.value.local_routes\n\n content {\n name = local_route.key\n type = local_route.value[\"type\"]\n static_network = local_route.value[\"static_network\"]\n }\n }\n\n dynamic \"remote_route\" {\n for_each = each.value.remote_routes\n\n content {\n name = remote_route.key\n type = remote_route.value[\"type\"]\n static_network = remote_route.value[\"static_network\"]\n }\n }\n}\n\nresource \"upcloud_gateway_connection_tunnel\" \"gateway_connection_tunnel\" {\n for_each = {\n for gct in local.gateway_connection_tunnels : \"${gct.gateway_name}-${gct.connection_name}-${gct.tunnel_name}-tunnel\" => gct\n }\n\n connection_id = each.value.connection_id\n name = each.key\n local_address_name = each.value.local_address_name\n remote_address = each.value.remote_address\n\n ipsec_auth_psk {\n psk = var.gateway_vpn_psks[each.key].psk\n }\n\n dynamic \"ipsec_properties\" {\n for_each = each.value.ipsec_properties != null ? { \"ip\": each.value.ipsec_properties } : {}\n\n content {\n child_rekey_time = ipsec_properties.value[\"child_rekey_time\"]\n dpd_delay = ipsec_properties.value[\"dpd_delay\"]\n dpd_timeout = ipsec_properties.value[\"dpd_timeout\"]\n ike_lifetime = ipsec_properties.value[\"ike_lifetime\"]\n rekey_time = ipsec_properties.value[\"rekey_time\"]\n phase1_algorithms = ipsec_properties.value[\"phase1_algorithms\"]\n phase1_dh_group_numbers = ipsec_properties.value[\"phase1_dh_group_numbers\"]\n phase1_integrity_algorithms = ipsec_properties.value[\"phase1_integrity_algorithms\"]\n phase2_algorithms = ipsec_properties.value[\"phase2_algorithms\"]\n phase2_dh_group_numbers = ipsec_properties.value[\"phase2_dh_group_numbers\"]\n phase2_integrity_algorithms = ipsec_properties.value[\"phase2_integrity_algorithms\"]\n }\n }\n}\n\nresource \"upcloud_network_peering\" \"peering\" {\n for_each = var.network_peerings\n\n name = \"${local.resource-prefix}${each.key}\"\n\n network {\n uuid = upcloud_network.private.id\n }\n\n peer_network {\n uuid = each.value.remote_network\n }\n}\n" }, { "path": "contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf", "content": "output \"master_ip\" {\n value = local.master_ip\n}\n\noutput \"worker_ip\" {\n value = local.worker_ip\n}\n\noutput \"bastion_ip\" {\n value = local.bastion_ip\n}\n\noutput \"loadbalancer_domain\" {\n value = var.loadbalancer_enabled ? upcloud_loadbalancer.lb[0].dns_name : null\n}\n" }, { "path": "contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf", "content": "variable \"prefix\" {\n type = string\n}\n\nvariable \"zone\" {\n type = string\n}\n\nvariable \"private_cloud\" {\n type = bool\n}\n\nvariable \"public_zone\" {\n type = string\n}\n\nvariable \"template_name\" {}\n\nvariable \"username\" {}\n\nvariable \"private_network_cidr\" {}\n\nvariable \"dns_servers\" {}\n\nvariable \"use_public_ips\" {}\n\nvariable \"machines\" {\n description = \"Cluster machines\"\n type = map(object({\n node_type = string\n plan = string\n cpu = optional(number)\n mem = optional(number)\n disk_size = number\n server_group : string\n force_public_ip : optional(bool, false)\n dns_servers : optional(set(string))\n additional_disks = map(object({\n size = number\n tier = string\n }))\n }))\n}\n\nvariable \"ssh_public_keys\" {\n type = list(string)\n}\n\nvariable \"firewall_enabled\" {\n type = bool\n}\n\nvariable \"master_allowed_remote_ips\" {\n type = list(object({\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"k8s_allowed_remote_ips\" {\n type = list(object({\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"bastion_allowed_remote_ips\" {\n type = list(object({\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"master_allowed_ports\" {\n type = list(object({\n protocol = string\n port_range_min = number\n port_range_max = number\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"worker_allowed_ports\" {\n type = list(object({\n protocol = string\n port_range_min = number\n port_range_max = number\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"firewall_default_deny_in\" {\n type = bool\n}\n\nvariable \"firewall_default_deny_out\" {\n type = bool\n}\n\nvariable \"loadbalancer_enabled\" {\n type = bool\n}\n\nvariable \"loadbalancer_plan\" {\n type = string\n}\n\nvariable \"loadbalancer_legacy_network\" {\n type = bool\n default = false\n}\n\nvariable \"loadbalancers\" {\n description = \"Load balancers\"\n\n type = map(object({\n proxy_protocol = bool\n port = number\n target_port = number\n allow_internal_frontend = optional(bool)\n backend_servers = list(string)\n }))\n}\n\nvariable \"server_groups\" {\n description = \"Server groups\"\n\n type = map(object({\n anti_affinity_policy = string\n }))\n}\n\nvariable \"router_enable\" {\n description = \"If a router should be enabled and connected to the private network or not\"\n\n type = bool\n}\n\nvariable \"gateways\" {\n description = \"Gateways that should be connected to the router, requires router_enable is set to true\"\n\n type = map(object({\n features = list(string)\n plan = optional(string)\n connections = optional(map(object({\n type = string\n local_routes = optional(map(object({\n type = string\n static_network = string\n })))\n remote_routes = optional(map(object({\n type = string\n static_network = string\n })))\n tunnels = optional(map(object({\n remote_address = string\n ipsec_properties = optional(object({\n child_rekey_time = number\n dpd_delay = number\n dpd_timeout = number\n ike_lifetime = number\n rekey_time = number\n phase1_algorithms = set(string)\n phase1_dh_group_numbers = set(string)\n phase1_integrity_algorithms = set(string)\n phase2_algorithms = set(string)\n phase2_dh_group_numbers = set(string)\n phase2_integrity_algorithms = set(string)\n }))\n })))\n })))\n }))\n}\n\nvariable \"gateway_vpn_psks\" {\n description = \"Separate variable for providing psks for connection tunnels\"\n\n type = map(object({\n psk = string\n }))\n default = {}\n sensitive = true\n}\n\nvariable \"static_routes\" {\n description = \"Static routes to apply to the router, requires router_enable is set to true\"\n\n type = map(object({\n nexthop = string\n route = string\n }))\n}\n\nvariable \"network_peerings\" {\n description = \"Other UpCloud private networks to peer with, requires router_enable is set to true\"\n\n type = map(object({\n remote_network = string\n }))\n}\n" }, { "path": "contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf", "content": "\nterraform {\n required_providers {\n upcloud = {\n source = \"UpCloudLtd/upcloud\"\n version = \"~>5.29.1\"\n }\n }\n required_version = \">= 0.13\"\n}\n" }, { "path": "contrib/terraform/upcloud/output.tf", "content": "\noutput \"master_ip\" {\n value = module.kubernetes.master_ip\n}\n\noutput \"worker_ip\" {\n value = module.kubernetes.worker_ip\n}\n\noutput \"bastion_ip\" {\n value = module.kubernetes.bastion_ip\n}\n\noutput \"loadbalancer_domain\" {\n value = module.kubernetes.loadbalancer_domain\n}\n" }, { "path": "contrib/terraform/upcloud/sample-inventory/cluster.tfvars", "content": "# See: https://developers.upcloud.com/1.3/5-zones/\nzone = \"fi-hel1\"\nusername = \"ubuntu\"\n\n# Prefix to use for all resources to separate them from other resources\nprefix = \"kubespray\"\n\ninventory_file = \"inventory.ini\"\n\n# Set the operating system using UUID or exact name\ntemplate_name = \"Ubuntu Server 20.04 LTS (Focal Fossa)\"\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\n# check list of available plan https://developers.upcloud.com/1.3/7-plans/\nmachines = {\n \"control-plane-0\" : {\n \"node_type\" : \"master\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {}\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {\n # \"some-disk-name-1\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # },\n # \"some-disk-name-2\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # }\n }\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {\n # \"some-disk-name-1\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # },\n # \"some-disk-name-2\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # }\n }\n },\n \"worker-2\" : {\n \"node_type\" : \"worker\",\n # plan to use instead of custom cpu/mem\n \"plan\" : null,\n #number of cpu cores\n \"cpu\" : \"2\",\n #memory size in MB\n \"mem\" : \"4096\"\n # The size of the storage in GB\n \"disk_size\" : 250\n \"additional_disks\" : {\n # \"some-disk-name-1\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # },\n # \"some-disk-name-2\": {\n # \"size\": 100,\n # \"tier\": \"maxiops\",\n # }\n }\n }\n}\n\nfirewall_enabled = false\nfirewall_default_deny_in = false\nfirewall_default_deny_out = false\n\n\nmaster_allowed_remote_ips = [\n {\n \"start_address\" : \"0.0.0.0\"\n \"end_address\" : \"255.255.255.255\"\n }\n]\n\nk8s_allowed_remote_ips = [\n {\n \"start_address\" : \"0.0.0.0\"\n \"end_address\" : \"255.255.255.255\"\n }\n]\n\nmaster_allowed_ports = []\nworker_allowed_ports = []\n\nloadbalancer_enabled = false\nloadbalancer_plan = \"development\"\nloadbalancers = {\n # \"http\" : {\n # \"port\" : 80,\n # \"target_port\" : 80,\n # \"backend_servers\" : [\n # \"worker-0\",\n # \"worker-1\",\n # \"worker-2\"\n # ]\n # }\n}\n\nserver_groups = {\n # \"control-plane\" = {\n # servers = [\n # \"control-plane-0\"\n # ]\n # anti_affinity_policy = \"strict\"\n # },\n # \"workers\" = {\n # servers = [\n # \"worker-0\",\n # \"worker-1\",\n # \"worker-2\"\n # ]\n # anti_affinity_policy = \"yes\"\n # }\n}\n" }, { "path": "contrib/terraform/upcloud/templates/inventory.tpl", "content": "[all]\n%{ for name, ips in master_ip ~}\n${name} ansible_user=${username} ansible_host=${lookup(ips, \"public\", ips.private)} ip=${ips.private}\n%{ endfor ~}\n%{ for name, ips in worker_ip ~}\n${name} ansible_user=${username} ansible_host=${lookup(ips, \"public\", ips.private)} ip=${ips.private}\n%{ endfor ~}\n\n[kube_control_plane]\n%{ for name, ips in master_ip ~}\n${name}\n%{ endfor ~}\n\n[etcd]\n%{ for name, ips in master_ip ~}\n${name}\n%{ endfor ~}\n\n[kube_node]\n%{ for name, ips in worker_ip ~}\n${name}\n%{ endfor ~}\n\n[k8s_cluster:children]\nkube_control_plane\nkube_node\n\n%{ if length(bastion_ip) > 0 ~}\n[bastion]\n%{ for name, ips in bastion_ip ~}\nbastion ansible_user=${username} ansible_host=${ips.public}\n%{ endfor ~}\n%{ endif ~}\n" }, { "path": "contrib/terraform/upcloud/variables.tf", "content": "variable \"prefix\" {\n type = string\n default = \"kubespray\"\n\n description = \"Prefix that is used to distinguish these resources from others\"\n}\n\nvariable \"zone\" {\n description = \"The zone where to run the cluster\"\n}\n\nvariable \"private_cloud\" {\n description = \"Whether the environment is in the private cloud region\"\n default = false\n}\n\nvariable \"public_zone\" {\n description = \"The public zone equivalent if the cluster is running in a private cloud zone\"\n}\n\nvariable \"template_name\" {\n description = \"Block describing the preconfigured operating system\"\n}\n\nvariable \"username\" {\n description = \"The username to use for the nodes\"\n default = \"ubuntu\"\n}\n\nvariable \"private_network_cidr\" {\n description = \"CIDR to use for the private network\"\n default = \"172.16.0.0/24\"\n}\n\nvariable \"dns_servers\" {\n description = \"DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved\"\n\n type = set(string)\n default = []\n}\n\nvariable \"use_public_ips\" {\n description = \"If all nodes should get a public IP\"\n type = bool\n default = true\n}\n\nvariable \"machines\" {\n description = \"Cluster machines\"\n\n type = map(object({\n node_type = string\n plan = string\n cpu = optional(number)\n mem = optional(number)\n disk_size = number\n server_group : string\n force_public_ip : optional(bool, false)\n dns_servers : optional(set(string))\n additional_disks = map(object({\n size = number\n tier = string\n }))\n }))\n}\n\nvariable \"ssh_public_keys\" {\n description = \"List of public SSH keys which are injected into the VMs.\"\n type = list(string)\n}\n\nvariable \"inventory_file\" {\n description = \"Where to store the generated inventory file\"\n}\n\nvariable \"UPCLOUD_USERNAME\" {\n description = \"UpCloud username with API access\"\n}\n\nvariable \"UPCLOUD_PASSWORD\" {\n description = \"Password for UpCloud API user\"\n}\n\nvariable \"firewall_enabled\" {\n description = \"Enable firewall rules\"\n default = false\n}\n\nvariable \"master_allowed_remote_ips\" {\n description = \"List of IP start/end addresses allowed to access API of masters\"\n type = list(object({\n start_address = string\n end_address = string\n }))\n default = []\n}\n\nvariable \"k8s_allowed_remote_ips\" {\n description = \"List of IP start/end addresses allowed to SSH to hosts\"\n type = list(object({\n start_address = string\n end_address = string\n }))\n default = []\n}\n\nvariable \"bastion_allowed_remote_ips\" {\n description = \"List of IP start/end addresses allowed to SSH to bastion\"\n type = list(object({\n start_address = string\n end_address = string\n }))\n default = []\n}\n\nvariable \"master_allowed_ports\" {\n description = \"List of ports to allow on masters\"\n type = list(object({\n protocol = string\n port_range_min = number\n port_range_max = number\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"worker_allowed_ports\" {\n description = \"List of ports to allow on workers\"\n type = list(object({\n protocol = string\n port_range_min = number\n port_range_max = number\n start_address = string\n end_address = string\n }))\n}\n\nvariable \"firewall_default_deny_in\" {\n description = \"Add firewall policies that deny all inbound traffic by default\"\n default = false\n}\n\nvariable \"firewall_default_deny_out\" {\n description = \"Add firewall policies that deny all outbound traffic by default\"\n default = false\n}\n\nvariable \"loadbalancer_enabled\" {\n description = \"Enable load balancer\"\n default = false\n}\n\nvariable \"loadbalancer_plan\" {\n description = \"Load balancer plan (development/production-small)\"\n default = \"development\"\n}\n\nvariable \"loadbalancer_legacy_network\" {\n description = \"If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false\"\n\n type = bool\n default = false\n}\n\nvariable \"loadbalancers\" {\n description = \"Load balancers\"\n\n type = map(object({\n proxy_protocol = bool\n port = number\n target_port = number\n allow_internal_frontend = optional(bool, false)\n backend_servers = list(string)\n }))\n default = {}\n}\n\nvariable \"server_groups\" {\n description = \"Server groups\"\n\n type = map(object({\n anti_affinity_policy = string\n }))\n\n default = {}\n}\n\nvariable \"router_enable\" {\n description = \"If a router should be enabled and connected to the private network or not\"\n\n type = bool\n default = false\n}\n\nvariable \"gateways\" {\n description = \"Gateways that should be connected to the router, requires router_enable is set to true\"\n\n type = map(object({\n features = list(string)\n plan = optional(string)\n connections = optional(map(object({\n type = string\n local_routes = optional(map(object({\n type = string\n static_network = string\n })), {})\n remote_routes = optional(map(object({\n type = string\n static_network = string\n })), {})\n tunnels = optional(map(object({\n remote_address = string\n ipsec_properties = optional(object({\n child_rekey_time = number\n dpd_delay = number\n dpd_timeout = number\n ike_lifetime = number\n rekey_time = number\n phase1_algorithms = set(string)\n phase1_dh_group_numbers = set(string)\n phase1_integrity_algorithms = set(string)\n phase2_algorithms = set(string)\n phase2_dh_group_numbers = set(string)\n phase2_integrity_algorithms = set(string)\n }))\n })), {})\n })), {})\n }))\n default = {}\n}\n\nvariable \"gateway_vpn_psks\" {\n description = \"Separate variable for providing psks for connection tunnels\"\n\n type = map(object({\n psk = string\n }))\n default = {}\n sensitive = true\n}\n\nvariable \"static_routes\" {\n description = \"Static routes to apply to the router, requires router_enable is set to true\"\n\n type = map(object({\n nexthop = string\n route = string\n }))\n default = {}\n}\n\nvariable \"network_peerings\" {\n description = \"Other UpCloud private networks to peer with, requires router_enable is set to true\"\n\n type = map(object({\n remote_network = string\n }))\n default = {}\n}\n" }, { "path": "contrib/terraform/upcloud/versions.tf", "content": "\nterraform {\n required_providers {\n upcloud = {\n source = \"UpCloudLtd/upcloud\"\n version = \"~>5.29.1\"\n }\n }\n required_version = \">= 0.13\"\n}\n" }, { "path": "contrib/terraform/vsphere/README.md", "content": "# Kubernetes on vSphere with Terraform\n\nProvision a Kubernetes cluster on [vSphere](https://www.vmware.com/products/vsphere.html) using Terraform and Kubespray.\n\n## Overview\n\nThe setup looks like following.\n\n```text\n Kubernetes cluster\n+-----------------------+\n| +--------------+ |\n| | +--------------+ |\n| | | | |\n| | | Master/etcd | |\n| | | node(s) | |\n| +-+ | |\n| +--------------+ |\n| ^ |\n| | |\n| v |\n| +--------------+ |\n| | +--------------+ |\n| | | | |\n| | | Worker | |\n| | | node(s) | |\n| +-+ | |\n| +--------------+ |\n+-----------------------+\n```\n\n## Warning\n\nThis setup assumes that the DHCP is disabled in the vSphere cluster and IP addresses have to be provided in the configuration file.\n\n## Requirements\n\n* Terraform 0.13.0 or newer (0.12 also works if you modify the provider block to include version and remove all `versions.tf` files)\n\n## Quickstart\n\nNOTE: *Assumes you are at the root of the kubespray repo*\n\nCopy the sample inventory for your cluster and copy the default terraform variables.\n\n```bash\nCLUSTER=my-vsphere-cluster\ncp -r inventory/sample inventory/$CLUSTER\ncp contrib/terraform/vsphere/default.tfvars inventory/$CLUSTER/\ncd inventory/$CLUSTER\n```\n\nEdit `default.tfvars` to match your setup. You MUST set values specific for you network and vSphere cluster.\n\n```bash\n# Ensure $EDITOR points to your favorite editor, e.g., vim, emacs, VS Code, etc.\n$EDITOR default.tfvars\n```\n\nFor authentication in your vSphere cluster you can use the environment variables.\n\n```bash\nexport TF_VAR_vsphere_user=username\nexport TF_VAR_vsphere_password=password\n```\n\nRun Terraform to create the infrastructure.\n\n```bash\nterraform init ../../contrib/terraform/vsphere\nterraform apply \\\n -var-file default.tfvars \\\n -state=tfstate-$CLUSTER.tfstate \\\n ../../contrib/terraform/vsphere\n```\n\nYou should now have a inventory file named `inventory.ini` that you can use with kubespray.\nYou can now copy your inventory file and use it with kubespray to set up a cluster.\nYou can type `terraform output` to find out the IP addresses of the nodes.\n\nIt is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:\n\n```bash\nansible -i inventory.ini -m ping all\n```\n\nExample to use this with the default sample inventory:\n\n```bash\nansible-playbook -i inventory.ini ../../cluster.yml -b -v\n```\n\n## Variables\n\n### Required\n\n* `machines`: Machines to provision. Key of this object will be used as the name of the machine\n * `node_type`: The role of this node *(master|worker)*\n * `ip`: The IP address of the machine\n * `netmask`: The netmask to use (to be used on the right hand side in CIDR notation, e.g., `24`)\n* `network`: The name of the network to attach the machines to\n* `gateway`: The IP address of the network gateway\n* `vsphere_datacenter`: The identifier of vSphere data center\n* `vsphere_compute_cluster`: The identifier of vSphere compute cluster\n* `vsphere_datastore`: The identifier of vSphere data store\n* `vsphere_server`: This is the vCenter server name or address for vSphere API operations.\n* `ssh_public_keys`: List of public SSH keys to install on all machines\n* `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand)\n\n### Optional\n\n* `folder`: Name of the folder to put all machines in (default: `\"\"`)\n* `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `\"k8s\"`)\n* `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`)\n* `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`)\n* `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`)\n* `firmware`: Firmware to use (default: `bios`)\n* `hardware_version`: The version of the hardware (default: `15`)\n* `master_cores`: The number of CPU cores for the master nodes (default: 4)\n* `master_memory`: The amount of RAM for the master nodes in MB (default: 4096)\n* `master_disk_size`: The amount of disk space for the master nodes in GB (default: 20)\n* `worker_cores`: The number of CPU cores for the worker nodes (default: 16)\n* `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192)\n* `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100)\n* `vapp`: Boolean to set the template type to vapp. (Default: false)\n* `interface_name`: Name of the interface to configure. (Default: ens192)\n\nAn example variables file can be found `default.tfvars`\n" }, { "path": "contrib/terraform/vsphere/default.tfvars", "content": "prefix = \"k8s\"\n\ninventory_file = \"inventory.ini\"\n\nnetwork = \"VM Network\"\n\nmachines = {\n \"master-0\" : {\n \"node_type\" : \"master\",\n \"ip\" : \"i-did-not-read-the-docs\", # e.g. 192.168.0.10\n \"netmask\" : \"24\"\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n \"ip\" : \"i-did-not-read-the-docs\", # e.g. 192.168.0.20\n \"netmask\" : \"24\"\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n \"ip\" : \"i-did-not-read-the-docs\", # e.g. 192.168.0.21\n \"netmask\" : \"24\"\n }\n}\n\ngateway = \"i-did-not-read-the-docs\" # e.g. 192.168.0.1\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\nvsphere_datacenter = \"i-did-not-read-the-docs\"\nvsphere_compute_cluster = \"i-did-not-read-the-docs\" # e.g. Cluster\nvsphere_datastore = \"i-did-not-read-the-docs\" # e.g. ssd-000000\nvsphere_server = \"i-did-not-read-the-docs\" # e.g. vsphere.server.com\n\ntemplate_name = \"i-did-not-read-the-docs\" # e.g. ubuntu-bionic-18.04-cloudimg\n" }, { "path": "contrib/terraform/vsphere/main.tf", "content": "provider \"vsphere\" {\n # Username and password set through env vars VSPHERE_USER and VSPHERE_PASSWORD\n user = var.vsphere_user\n password = var.vsphere_password\n\n vsphere_server = var.vsphere_server\n\n # If you have a self-signed cert\n allow_unverified_ssl = true\n}\n\ndata \"vsphere_datacenter\" \"dc\" {\n name = var.vsphere_datacenter\n}\n\ndata \"vsphere_datastore\" \"datastore\" {\n name = var.vsphere_datastore\n datacenter_id = data.vsphere_datacenter.dc.id\n}\n\ndata \"vsphere_network\" \"network\" {\n name = var.network\n datacenter_id = data.vsphere_datacenter.dc.id\n}\n\ndata \"vsphere_virtual_machine\" \"template\" {\n name = var.template_name\n datacenter_id = data.vsphere_datacenter.dc.id\n}\n\ndata \"vsphere_compute_cluster\" \"compute_cluster\" {\n name = var.vsphere_compute_cluster\n datacenter_id = data.vsphere_datacenter.dc.id\n}\n\nresource \"vsphere_resource_pool\" \"pool\" {\n name = \"${var.prefix}-cluster-pool\"\n parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id\n}\n\nmodule \"kubernetes\" {\n source = \"./modules/kubernetes-cluster\"\n\n prefix = var.prefix\n\n machines = var.machines\n\n ## Master ##\n master_cores = var.master_cores\n master_memory = var.master_memory\n master_disk_size = var.master_disk_size\n\n ## Worker ##\n worker_cores = var.worker_cores\n worker_memory = var.worker_memory\n worker_disk_size = var.worker_disk_size\n\n ## Global ##\n\n gateway = var.gateway\n dns_primary = var.dns_primary\n dns_secondary = var.dns_secondary\n\n pool_id = vsphere_resource_pool.pool.id\n datastore_id = data.vsphere_datastore.datastore.id\n\n folder = var.folder\n guest_id = data.vsphere_virtual_machine.template.guest_id\n scsi_type = data.vsphere_virtual_machine.template.scsi_type\n network_id = data.vsphere_network.network.id\n adapter_type = data.vsphere_virtual_machine.template.network_interface_types[0]\n interface_name = var.interface_name\n firmware = var.firmware\n hardware_version = var.hardware_version\n disk_thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned\n\n template_id = data.vsphere_virtual_machine.template.id\n vapp = var.vapp\n\n ssh_public_keys = var.ssh_public_keys\n}\n\n#\n# Generate ansible inventory\n#\n\nresource \"local_file\" \"inventory\" {\n content = templatefile(\"${path.module}/templates/inventory.tpl\", {\n connection_strings_master = join(\"\\n\", formatlist(\"%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d\",\n keys(module.kubernetes.master_ip),\n values(module.kubernetes.master_ip),\n range(1, length(module.kubernetes.master_ip) + 1))),\n connection_strings_worker = join(\"\\n\", formatlist(\"%s ansible_user=ubuntu ansible_host=%s\",\n keys(module.kubernetes.worker_ip),\n values(module.kubernetes.worker_ip))),\n list_master = join(\"\\n\", formatlist(\"%s\", keys(module.kubernetes.master_ip))),\n list_worker = join(\"\\n\", formatlist(\"%s\", keys(module.kubernetes.worker_ip)))\n })\n filename = var.inventory_file\n}\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf", "content": "resource \"vsphere_virtual_machine\" \"worker\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"worker\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n\n resource_pool_id = var.pool_id\n datastore_id = var.datastore_id\n\n num_cpus = var.worker_cores\n memory = var.worker_memory\n memory_reservation = var.worker_memory\n guest_id = var.guest_id\n enable_disk_uuid = \"true\" # needed for CSI provider\n scsi_type = var.scsi_type\n folder = var.folder\n firmware = var.firmware\n hardware_version = var.hardware_version\n\n wait_for_guest_net_routable = false\n wait_for_guest_net_timeout = 0\n\n network_interface {\n network_id = var.network_id\n adapter_type = var.adapter_type\n }\n\n disk {\n label = \"disk0\"\n size = var.worker_disk_size\n thin_provisioned = var.disk_thin_provisioned\n }\n\n lifecycle {\n ignore_changes = [disk]\n }\n\n clone {\n template_uuid = var.template_id\n }\n\n cdrom {\n client_device = true\n }\n\n dynamic \"vapp\" {\n for_each = var.vapp ? [1] : []\n\n content {\n properties = {\n \"user-data\" = base64encode(templatefile(\"${path.module}/templates/vapp-cloud-init.tpl\", { ssh_public_keys = var.ssh_public_keys }))\n }\n }\n }\n\n extra_config = {\n \"isolation.tools.copy.disable\" = \"FALSE\"\n \"isolation.tools.paste.disable\" = \"FALSE\"\n \"isolation.tools.setGUIOptions.enable\" = \"TRUE\"\n \"guestinfo.userdata\" = base64encode(templatefile(\"${path.module}/templates/cloud-init.tpl\", { ssh_public_keys = var.ssh_public_keys }))\n \"guestinfo.userdata.encoding\" = \"base64\"\n \"guestinfo.metadata\" = base64encode(templatefile(\"${path.module}/templates/metadata.tpl\", { hostname = \"${var.prefix}-${each.key}\",\n interface_name = var.interface_name\n ip = each.value.ip,\n netmask = each.value.netmask,\n gw = var.gateway,\n dns = var.dns_primary,\n ssh_public_keys = var.ssh_public_keys }))\n \"guestinfo.metadata.encoding\" = \"base64\"\n }\n}\n\nresource \"vsphere_virtual_machine\" \"master\" {\n for_each = {\n for name, machine in var.machines :\n name => machine\n if machine.node_type == \"master\"\n }\n\n name = \"${var.prefix}-${each.key}\"\n\n resource_pool_id = var.pool_id\n datastore_id = var.datastore_id\n\n num_cpus = var.master_cores\n memory = var.master_memory\n memory_reservation = var.master_memory\n guest_id = var.guest_id\n enable_disk_uuid = \"true\" # needed for CSI provider\n scsi_type = var.scsi_type\n folder = var.folder\n firmware = var.firmware\n hardware_version = var.hardware_version\n\n wait_for_guest_net_routable = false\n wait_for_guest_net_timeout = 0\n\n network_interface {\n network_id = var.network_id\n adapter_type = var.adapter_type\n }\n\n disk {\n label = \"disk0\"\n size = var.master_disk_size\n thin_provisioned = var.disk_thin_provisioned\n }\n\n lifecycle {\n ignore_changes = [disk]\n }\n\n clone {\n template_uuid = var.template_id\n }\n\n cdrom {\n client_device = true\n }\n\n dynamic \"vapp\" {\n for_each = var.vapp ? [1] : []\n\n content {\n properties = {\n \"user-data\" = base64encode(templatefile(\"${path.module}/templates/vapp-cloud-init.tpl\", { ssh_public_keys = var.ssh_public_keys }))\n }\n }\n }\n\n extra_config = {\n \"isolation.tools.copy.disable\" = \"FALSE\"\n \"isolation.tools.paste.disable\" = \"FALSE\"\n \"isolation.tools.setGUIOptions.enable\" = \"TRUE\"\n \"guestinfo.userdata\" = base64encode(templatefile(\"${path.module}/templates/cloud-init.tpl\", { ssh_public_keys = var.ssh_public_keys }))\n \"guestinfo.userdata.encoding\" = \"base64\"\n \"guestinfo.metadata\" = base64encode(templatefile(\"${path.module}/templates/metadata.tpl\", { hostname = \"${var.prefix}-${each.key}\",\n interface_name = var.interface_name\n ip = each.value.ip,\n netmask = each.value.netmask,\n gw = var.gateway,\n dns = var.dns_primary,\n ssh_public_keys = var.ssh_public_keys }))\n \"guestinfo.metadata.encoding\" = \"base64\"\n }\n}\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf", "content": "output \"master_ip\" {\n value = {\n for name, machine in var.machines :\n \"${var.prefix}-${name}\" => machine.ip\n if machine.node_type == \"master\"\n }\n}\n\noutput \"worker_ip\" {\n value = {\n for name, machine in var.machines :\n \"${var.prefix}-${name}\" => machine.ip\n if machine.node_type == \"worker\"\n }\n}\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tpl", "content": "#cloud-config\n\nssh_authorized_keys:\n%{ for ssh_public_key in ssh_public_keys ~}\n - ${ssh_public_key}\n%{ endfor ~}\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/templates/metadata.tpl", "content": "instance-id: ${hostname}\nlocal-hostname: ${hostname}\nnetwork:\n version: 2\n ethernets:\n ${interface_name}:\n match:\n name: ${interface_name}\n dhcp4: false\n addresses:\n - ${ip}/${netmask}\n gateway4: ${gw}\n nameservers:\n addresses: [${dns}]\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/templates/vapp-cloud-init.tpl", "content": "#cloud-config\n\nssh_authorized_keys:\n%{ for ssh_public_key in ssh_public_keys ~}\n - ${ssh_public_key}\n%{ endfor ~}\n\nwrite_files:\n - path: /etc/netplan/10-user-network.yaml\n content: |.\n network:\n version: 2\n ethernets:\n ${interface_name}:\n dhcp4: false #true to use dhcp\n addresses:\n - ${ip}/${netmask}\n gateway4: ${gw} # Set gw here\n nameservers:\n addresses:\n - ${dns} # Set DNS ip address here\n\nruncmd:\n - netplan apply\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf", "content": "## Global ##\nvariable \"prefix\" {}\n\nvariable \"machines\" {\n description = \"Cluster machines\"\n type = map(object({\n node_type = string\n ip = string\n netmask = string\n }))\n}\n\nvariable \"gateway\" {}\nvariable \"dns_primary\" {}\nvariable \"dns_secondary\" {}\nvariable \"pool_id\" {}\nvariable \"datastore_id\" {}\nvariable \"guest_id\" {}\nvariable \"scsi_type\" {}\nvariable \"network_id\" {}\nvariable \"interface_name\" {}\nvariable \"adapter_type\" {}\nvariable \"disk_thin_provisioned\" {}\nvariable \"template_id\" {}\nvariable \"vapp\" {\n type = bool\n}\nvariable \"firmware\" {}\nvariable \"folder\" {}\nvariable \"ssh_public_keys\" {\n type = list(string)\n}\nvariable \"hardware_version\" {}\n\n## Master ##\nvariable \"master_cores\" {}\nvariable \"master_memory\" {}\nvariable \"master_disk_size\" {}\n\n## Worker ##\nvariable \"worker_cores\" {}\nvariable \"worker_memory\" {}\nvariable \"worker_disk_size\" {}\n" }, { "path": "contrib/terraform/vsphere/modules/kubernetes-cluster/versions.tf", "content": "terraform {\n required_providers {\n vsphere = {\n source = \"hashicorp/vsphere\"\n version = \">= 1.24.3\"\n }\n }\n required_version = \">= 0.13\"\n}\n" }, { "path": "contrib/terraform/vsphere/output.tf", "content": "output \"master_ip_addresses\" {\n value = module.kubernetes.master_ip\n}\n\noutput \"worker_ip_addresses\" {\n value = module.kubernetes.worker_ip\n}\n\noutput \"vsphere_datacenter\" {\n value = var.vsphere_datacenter\n}\n\noutput \"vsphere_server\" {\n value = var.vsphere_server\n}\n\noutput \"vsphere_datastore\" {\n value = var.vsphere_datastore\n}\n\noutput \"vsphere_network\" {\n value = var.network\n}\n\noutput \"vsphere_folder\" {\n value = var.folder\n}\n\noutput \"vsphere_pool\" {\n value = \"${terraform.workspace}-cluster-pool\"\n}\n" }, { "path": "contrib/terraform/vsphere/sample-inventory/cluster.tfvars", "content": "prefix = \"default\"\n\ninventory_file = \"inventory.ini\"\n\nmachines = {\n \"master-0\" : {\n \"node_type\" : \"master\",\n \"ip\" : \"i-did-not-read-the-docs\" # e.g. 192.168.0.2/24\n },\n \"worker-0\" : {\n \"node_type\" : \"worker\",\n \"ip\" : \"i-did-not-read-the-docs\" # e.g. 192.168.0.2/24\n },\n \"worker-1\" : {\n \"node_type\" : \"worker\",\n \"ip\" : \"i-did-not-read-the-docs\" # e.g. 192.168.0.2/24\n }\n}\n\ngateway = \"i-did-not-read-the-docs\" # e.g. 192.168.0.2\n\nssh_public_keys = [\n # Put your public SSH key here\n \"ssh-rsa I-did-not-read-the-docs\",\n \"ssh-rsa I-did-not-read-the-docs 2\",\n]\n\nvsphere_datacenter = \"i-did-not-read-the-docs\"\nvsphere_compute_cluster = \"i-did-not-read-the-docs\" # e.g. Cluster\nvsphere_datastore = \"i-did-not-read-the-docs\" # e.g. ssd-000000\nvsphere_server = \"i-did-not-read-the-docs\" # e.g. vsphere.server.com\n\ntemplate_name = \"i-did-not-read-the-docs\" # e.g. ubuntu-bionic-18.04-cloudimg\n" }, { "path": "contrib/terraform/vsphere/templates/inventory.tpl", "content": "\n[all]\n${connection_strings_master}\n${connection_strings_worker}\n\n[kube_control_plane]\n${list_master}\n\n[etcd]\n${list_master}\n\n[kube_node]\n${list_worker}\n\n[k8s_cluster:children]\nkube_control_plane\nkube_node\n" }, { "path": "contrib/terraform/vsphere/variables.tf", "content": "## Global ##\n\n# Required variables\n\nvariable \"machines\" {\n description = \"Cluster machines\"\n type = map(object({\n node_type = string\n ip = string\n netmask = string\n }))\n}\n\nvariable \"network\" {}\n\nvariable \"gateway\" {}\n\nvariable \"vsphere_datacenter\" {}\n\nvariable \"vsphere_compute_cluster\" {}\n\nvariable \"vsphere_datastore\" {}\n\nvariable \"vsphere_user\" {}\n\nvariable \"vsphere_password\" {\n sensitive = true\n}\n\nvariable \"vsphere_server\" {}\n\nvariable \"ssh_public_keys\" {\n description = \"List of public SSH keys which are injected into the VMs.\"\n type = list(string)\n}\n\nvariable \"template_name\" {}\n\n# Optional variables (ones where reasonable defaults exist)\nvariable \"vapp\" {\n default = false\n}\n\nvariable \"interface_name\" {\n default = \"ens192\"\n}\n\nvariable \"folder\" {\n default = \"\"\n}\n\nvariable \"prefix\" {\n default = \"k8s\"\n}\n\nvariable \"inventory_file\" {\n default = \"inventory.ini\"\n}\n\nvariable \"dns_primary\" {\n default = \"8.8.4.4\"\n}\n\nvariable \"dns_secondary\" {\n default = \"8.8.8.8\"\n}\n\nvariable \"firmware\" {\n default = \"bios\"\n}\n\nvariable \"hardware_version\" {\n default = \"15\"\n}\n\n## Master ##\n\nvariable \"master_cores\" {\n default = 4\n}\n\nvariable \"master_memory\" {\n default = 4096\n}\n\nvariable \"master_disk_size\" {\n default = \"20\"\n}\n\n## Worker ##\n\nvariable \"worker_cores\" {\n default = 16\n}\n\nvariable \"worker_memory\" {\n default = 8192\n}\nvariable \"worker_disk_size\" {\n default = \"100\"\n}\n" }, { "path": "contrib/terraform/vsphere/versions.tf", "content": "terraform {\n required_providers {\n vsphere = {\n source = \"hashicorp/vsphere\"\n version = \">= 1.24.3\"\n }\n }\n required_version = \">= 0.13\"\n}\n" }, { "path": "docs/CNI/calico.md", "content": "# Calico\n\nCheck if the calico-node container is running\n\n```ShellSession\ndocker ps | grep calico\n```\n\nThe **calicoctl.sh** is wrap script with configured access credentials for command calicoctl allows to check the status of the network workloads.\n\n* Check the status of Calico nodes\n\n```ShellSession\ncalicoctl.sh node status\n```\n\n* Show the configured network subnet for containers\n\n```ShellSession\ncalicoctl.sh get ippool -o wide\n```\n\n* Show the workloads (ip addresses of containers and their location)\n\n```ShellSession\ncalicoctl.sh get workloadEndpoint -o wide\n```\n\nand\n\n```ShellSession\ncalicoctl.sh get hostEndpoint -o wide\n```\n\n## Configuration\n\n### Optional : Define datastore type\n\nThe default datastore, Kubernetes API datastore is recommended for on-premises deployments, and supports only Kubernetes workloads; etcd is the best datastore for hybrid deployments.\n\nAllowed values are `kdd` (default) and `etcd`.\n\nNote: using kdd and more than 50 nodes, consider using the `typha` daemon to provide scaling.\n\nTo re-define you need to edit the inventory and add a group variable `calico_datastore`\n\n```yml\ncalico_datastore: kdd\n```\n\n### Optional : Define network backend\n\nIn some cases you may want to define Calico network backend. Allowed values are `bird`, `vxlan` or `none`. `vxlan` is the default value.\n\nTo re-define you need to edit the inventory and add a group variable `calico_network_backend`\n\n```yml\ncalico_network_backend: none\n```\n\n### Optional : Define the default pool CIDRs\n\nBy default, `kube_pods_subnet` is used as the IP range CIDR for the default IP Pool, and `kube_pods_subnet_ipv6` for IPv6.\nIn some cases you may want to add several pools and not have them considered by Kubernetes as external (which means that they must be within or equal to the range defined in `kube_pods_subnet` and `kube_pods_subnet_ipv6` ), it starts with the default IP Pools of which IP range CIDRs can by defined in group_vars (k8s_cluster/k8s-net-calico.yml):\n\n```ShellSession\ncalico_pool_cidr: 10.233.64.0/20\ncalico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n```\n\n### Optional : BGP Peering with border routers\n\nIn some cases you may want to route the pods subnet and so NAT is not needed on the nodes.\nFor instance if you have a cluster spread on different locations and you want your pods to talk each other no matter where they are located.\nThe following variables need to be set as follow:\n\n```yml\npeer_with_router: true # enable the peering with the datacenter's border router (default value: false).\nnat_outgoing: false # (optional) NAT outgoing (default value: true).\n```\n\nAnd you'll need to edit the inventory and add a hostvar `local_as` by node.\n\n```ShellSession\nnode1 ansible_ssh_host=95.54.0.12 local_as=xxxxxx\n```\n\n### Optional : Defining BGP peers\n\nPeers can be defined using the `peers` variable (see docs/calico_peer_example examples).\nIn order to define global peers, the `peers` variable can be defined in group_vars with the \"scope\" attribute of each global peer set to \"global\".\nIn order to define peers on a per node basis, the `peers` variable must be defined in hostvars or group_vars with the \"scope\" attribute unset or set to \"node\".\n\nNB: Ansible's `hash_behaviour` is by default set to \"replace\", thus defining both global and per node peers would end up with having only per node peers. If having both global and per node peers defined was meant to happen, global peers would have to be defined in hostvars for each host (as well as per node peers)\n\nNB²: Peers definition at node scope can be customized with additional fields `filters`, `sourceAddress` and `numAllowedLocalASNumbers` (see for details)\n\nSince calico 3.4, Calico supports advertising Kubernetes service cluster IPs over BGP, just as it advertises pod IPs.\nThis can be enabled by setting the following variable as follow in group_vars (k8s_cluster/k8s-net-calico.yml)\n\n```yml\ncalico_advertise_cluster_ips: true\n```\n\nSince calico 3.10, Calico supports advertising Kubernetes service ExternalIPs over BGP in addition to cluster IPs advertising.\nThis can be enabled by setting the following variable in group_vars (k8s_cluster/k8s-net-calico.yml)\n\n```yml\ncalico_advertise_service_external_ips:\n- x.x.x.x/24\n- y.y.y.y/32\n```\n\n### Optional : Define global AS number\n\nOptional parameter `global_as_num` defines Calico global AS number (`/calico/bgp/v1/global/as_num` etcd key).\nIt defaults to \"64512\".\n\n### Optional : BGP Peering with route reflectors\n\nAt large scale you may want to disable full node-to-node mesh in order to\noptimize your BGP topology and improve `calico-node` containers' start times.\n\nTo do so you can deploy BGP route reflectors and peer `calico-node` with them as\nrecommended here:\n\n* \n* \n\nYou need to edit your inventory and add:\n\n* `calico_rr` group with nodes in it. `calico_rr` can be combined with\n `kube_node` and/or `kube_control_plane`.\n* `cluster_id` by route reflector node/group (see details [here](https://hub.docker.com/r/calico/routereflector/))\n\nHere's an example of Kubespray inventory with standalone route reflectors:\n\n```ini\n[all]\nrr0 ansible_ssh_host=10.210.1.10 ip=10.210.1.10\nrr1 ansible_ssh_host=10.210.1.11 ip=10.210.1.11\nnode2 ansible_ssh_host=10.210.1.12 ip=10.210.1.12\nnode3 ansible_ssh_host=10.210.1.13 ip=10.210.1.13\nnode4 ansible_ssh_host=10.210.1.14 ip=10.210.1.14\nnode5 ansible_ssh_host=10.210.1.15 ip=10.210.1.15\n\n[kube_control_plane]\nnode2\nnode3\n\n[etcd]\nnode2\nnode3\nnode4\n\n[kube_node]\nnode2\nnode3\nnode4\nnode5\n\n[calico_rr]\nrr0\nrr1\n\n[rack0]\nrr0\nrr1\nnode2\nnode3\nnode4\nnode5\n\n[rack0:vars]\ncluster_id=\"1.0.0.1\"\ncalico_rr_id=rr1\ncalico_group_id=rr1\n```\n\nThe inventory above will deploy the following topology assuming that calico's\n`global_as_num` is set to `65400`:\n\n![Image](../figures/kubespray-calico-rr.png?raw=true)\n\n### Optional : Define default endpoint to host action\n\nBy default Calico blocks traffic from endpoints to the host itself by using an iptables DROP action. When using it in kubernetes the action has to be changed to RETURN (default in kubespray) or ACCEPT (see ) Otherwise all network packets from pods (with hostNetwork=False) to services endpoints (with hostNetwork=True) within the same node are dropped.\n\nTo re-define default action please set the following variable in your inventory:\n\n```yml\ncalico_endpoint_to_host_action: \"ACCEPT\"\n```\n\n### Optional : Define address on which Felix will respond to health requests\n\nSince Calico 3.2.0, HealthCheck default behavior changed from listening on all interfaces to just listening on localhost.\n\nTo re-define health host please set the following variable in your inventory:\n\n```yml\ncalico_healthhost: \"0.0.0.0\"\n```\n\n### Optional : Configure VXLAN hardware Offload\n\nThe VXLAN Offload is disable by default. It can be configured like this to enabled it:\n\n```yml\ncalico_feature_detect_override: \"ChecksumOffloadBroken=false\" # The vxlan offload will enabled (It may cause problem on buggy NIC driver)\n```\n\n### Optional : Configure Calico Node probe timeouts\n\nUnder certain conditions a deployer may need to tune the Calico liveness and readiness probes timeout settings. These can be configured like this:\n\n```yml\ncalico_node_livenessprobe_timeout: 10\ncalico_node_readinessprobe_timeout: 10\n```\n\n### Optional : Enable NAT with IPv6\n\nTo allow outgoing IPv6 traffic going from pods to the Internet, enable the following:\n\n```yml\nnat_outgoing_ipv6: true # NAT outgoing ipv6 (default value: false).\n```\n\n## Config encapsulation for cross server traffic\n\nCalico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is the more mature implementation and enabled by default, please check your environment if you need *IP in IP* encapsulation.\n\n*IP in IP* and *VXLAN* is mutually exclusive modes.\n\nKubespray defaults have changed after version 2.18 from auto-enabling `ipip` mode to auto-enabling `vxlan`. This was done to facilitate wider deployment scenarios including those where vxlan acceleration is provided by the underlying network devices.\n\nIf you are running your cluster with the default calico settings and are upgrading to a release post 2.18.x (i.e. 2.19 and later or `master` branch) then you have two options:\n\n* perform a manual migration to vxlan before upgrading kubespray (see migrating from IP in IP to VXLAN below)\n* pin the pre-2.19 settings in your ansible inventory (see IP in IP mode settings below)\n\n**Note:**: Vxlan in ipv6 only supported when kernel >= 3.12. So if your kernel version < 3.12, Please don't set `calico_vxlan_mode_ipv6: Always`. More details see [#Issue 6877](https://github.com/projectcalico/calico/issues/6877).\n\n### IP in IP mode\n\nTo configure Ip in Ip mode you need to use the bird network backend.\n\n```yml\ncalico_ipip_mode: 'Always' # Possible values is `Always`, `CrossSubnet`, `Never`\ncalico_vxlan_mode: 'Never'\ncalico_network_backend: 'bird'\n```\n\n### BGP mode\n\nTo enable BGP no-encapsulation mode:\n\n```yml\ncalico_ipip_mode: 'Never'\ncalico_vxlan_mode: 'Never'\ncalico_network_backend: 'bird'\n```\n\n### Migrating from IP in IP to VXLAN\n\nIf you would like to migrate from the old IP in IP with `bird` network backends default to the new VXLAN based encapsulation you need to perform this change before running an upgrade of your cluster; the `cluster.yml` and `upgrade-cluster.yml` playbooks will refuse to continue if they detect incompatible settings.\n\nExecute the following steps on one of the control plane nodes, ensure the cluster in healthy before proceeding.\n\n```shell\ncalicoctl.sh patch felixconfig default -p '{\"spec\":{\"vxlanEnabled\":true}}'\ncalicoctl.sh patch ippool default-pool -p '{\"spec\":{\"ipipMode\":\"Never\", \"vxlanMode\":\"Always\"}}'\n```\n\n**Note:** if you created multiple ippools you will need to patch all of them individually to change their encapsulation. The kubespray playbooks only handle the default ippool created by kubespray.\n\nWait for the `vxlan.calico` interfaces to be created on all cluster nodes and traffic to be routed through it then you can disable `ipip`.\n\n```shell\ncalicoctl.sh patch felixconfig default -p '{\"spec\":{\"ipipEnabled\":false}}'\n```\n\n## Configuring interface MTU\n\nThis is an advanced topic and should usually not be modified unless you know exactly what you are doing. Calico is smart enough to deal with the defaults and calculate the proper MTU. If you do need to set up a custom MTU you can change `calico_veth_mtu` as follows:\n\n* If Wireguard is enabled, subtract 60 from your network MTU (i.e. 1500-60=1440)\n* If using VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450)\n* If using IPIP, subtract 20 from your network MTU (i.e. 1500-20=1480)\n* if not using any encapsulation, set to your network MTU (i.e. 1500 or 9000)\n\n```yaml\ncalico_veth_mtu: 1440\n```\n\n## Cloud providers configuration\n\nPlease refer to the official documentation, for example [GCE configuration](http://docs.projectcalico.org/v1.5/getting-started/docker/installation/gce) requires a security rule for calico ip-ip tunnels. Note, calico is always configured with ``calico_ipip_mode: Always`` if the cloud provider was defined.\n\n### Optional : Ignore kernel's RPF check setting\n\nBy default the felix agent(calico-node) will abort if the Kernel RPF setting is not 'strict'. If you want Calico to ignore the Kernel setting:\n\n```yml\ncalico_node_ignorelooserpf: true\n```\n\nNote that in OpenStack you must allow `ipip` traffic in your security groups,\notherwise you will experience timeouts.\nTo do this you must add a rule which allows it, for example:\n\n### Optional : Felix configuration via extraenvs of calico node\n\nPossible environment variable parameters for [configuring Felix](https://docs.projectcalico.org/reference/felix/configuration)\n\n```yml\ncalico_node_extra_envs:\n FELIX_DEVICEROUTESOURCEADDRESS: 172.17.0.1\n```\n\n```ShellSession\nneutron security-group-rule-create --protocol 4 --direction egress k8s-a0tp4t\nneutron security-group-rule-create --protocol 4 --direction igress k8s-a0tp4t\n```\n\n### Optional : Use Calico CNI host-local IPAM plugin\n\nCalico currently supports two types of CNI IPAM plugins, `host-local` and `calico-ipam` (default).\n\nTo allow Calico to determine the subnet to use from the Kubernetes API based on the `Node.podCIDR` field, enable the following setting.\n\n```yml\ncalico_ipam_host_local: true\n```\n\nRefer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information.\n\n### Optional : Disable CNI logging to disk\n\nCalico CNI plugin logs to /var/log/calico/cni/cni.log and to stderr.\nstderr of CNI plugins can be found in the logs of container runtime.\n\nYou can disable Calico CNI logging to disk by setting `calico_cni_log_file_path: false`.\n\n## eBPF Support\n\nCalico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information.\n\nNote that it is advisable to always use the latest version of Calico when using the eBPF dataplane.\n\n### Enabling eBPF support\n\nTo enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system.\n\n```yaml\ncalico_bpf_enabled: true\n```\n\n**NOTE:** there is known incompatibility in using the `kernel-kvm` kernel package on Ubuntu OSes because it is missing support for `CONFIG_NET_SCHED` which is a requirement for Calico eBPF support. When using Calico eBPF with Ubuntu ensure you run the `-generic` kernel.\n\n### Cleaning up after kube-proxy\n\nCalico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned.\n\nTo check that kube-proxy was running in ipvs mode:\n\n```ShellSession\n# ipvsadm -l\n```\n\nTo clean up any ipvs leftovers:\n\n```ShellSession\n# ipvsadm -C\n```\n\n### Calico access to the kube-api\n\nCalico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.\n\nKubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).\n\nIf no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.\n\n### Tunneled versus Direct Server Return\n\nBy default Calico uses Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.\n\nTo configure DSR:\n\n```yaml\ncalico_bpf_service_mode: \"DSR\"\n```\n\n### eBPF Logging and Troubleshooting\n\nIn order to enable Calico eBPF mode logging:\n\n```yaml\ncalico_bpf_log_level: \"Debug\"\n```\n\nTo view the logs you need to use the `tc` command to read the kernel trace buffer:\n\n```ShellSession\ntc exec bpf debug\n```\n\nPlease see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs).\n\n## Wireguard Encryption\n\nCalico supports using Wireguard for encryption. Please see the docs on [encrypt cluster pod traffic](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic).\n\nTo enable wireguard support:\n\n```yaml\ncalico_wireguard_enabled: true\n```\n\nThe following OSes will require enabling the EPEL repo in order to bring in wireguard tools:\n\n* CentOS 8\n* AlmaLinux 8\n* Rocky Linux 8\n* Amazon Linux 2\n\n```yaml\nepel_enabled: true\n```\n" }, { "path": "docs/CNI/cilium.md", "content": "# Cilium\n\n## Unprivileged agent configuration\n\nBy default, Cilium is installed with `securityContext.privileged: false`. You need to set the `kube_owner` variable to `root` in the inventory:\n\n```yml\nkube_owner: root\n```\n\n## IP Address Management (IPAM)\n\nIP Address Management (IPAM) is responsible for the allocation and management of IP addresses used by network endpoints (container and others) managed by Cilium. The default mode is \"Cluster Scope\".\n\nYou can set the following parameters, for example: cluster-pool, kubernetes:\n\n```yml\ncilium_ipam_mode: cluster-pool\n```\n\n### Set the cluster Pod CIDRs\n\nCluster Pod CIDRs use the kube_pods_subnet value by default.\nIf your node network is in the same range you will lose connectivity to other nodes.\nDefaults to kube_pods_subnet if not set.\nYou can set the following parameters:\n\n```yml\ncilium_pool_cidr: 10.233.64.0/18\n```\n\nWhen cilium_enable_ipv6 is used. Defaults to kube_pods_subnet_ipv6 if not set.\nyou need to set the IPV6 value:\n\n```yml\ncilium_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n```\n\n### Set the Pod CIDR size of a node\n\nWhen cilium IPAM uses the \"Cluster Scope\" mode, it will pre-allocate a segment of IP to each node,\nschedule the Pod to this node, and then allocate IP from here. cilium_pool_mask_size Specifies\nthe size allocated from cluster Pod CIDR to node.ipam.podCIDRs.\nDefaults to kube_network_node_prefix if not set.\n\n```yml\ncilium_pool_mask_size: \"24\"\n```\n\ncilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cluster Pod IPV6 CIDR. Defaults to kube_network_node_prefix_ipv6 if not set.\n\n```yml\ncilium_pool_mask_size_ipv6: \"120\"\n```\n\n### IP Load Balancer Pools\n\nCilium's IP Load Balancer Pools can be configured with the `cilium_loadbalancer_ip_pools` variable:\n\n```yml\ncilium_loadbalancer_ip_pools:\n - name: \"blue-pool\"\n cidrs:\n - \"10.0.10.0/24\"\n ranges:\n - start: \"20.0.20.100\"\n stop: \"20.0.20.200\"\n - start: \"1.2.3.4\"\n```\n\nFor further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/)\n\n### BGP Control Plane\n\nCilium's BGP Control Plane can be enabled by setting `cilium_enable_bgp_control_plane` to `true`.:\n\n```yml\ncilium_enable_bgp_control_plane: true\n```\n\nFor further information, check [BGP Peering Policy documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/)\n\n### BGP Control Plane Resources (New bgpv2 API v1.16+)\n\nCilium BGP control plane is managed by a set of custom resources which provide a flexible way to configure BGP peers, policies, and advertisements.\n\nCilium's BGP Instances can be configured with the `cilium_bgp_cluster_configs` variable:\n\n```yml\ncilium_bgp_cluster_configs:\n - name: \"cilium-bgp\"\n spec:\n bgpInstances:\n - name: \"instance-64512\"\n localASN: 64512\n peers:\n - name: \"peer-64512-tor1\"\n peerASN: 64512\n peerAddress: '10.47.1.1'\n peerConfigRef:\n name: \"cilium-peer\"\n nodeSelector:\n matchExpressions:\n - {key: somekey, operator: NotIn, values: ['never-used-value']}\n```\n\nCillium's BGP Peers can be configured with the `cilium_bgp_peer_configs` variable:\n\n```yml\ncilium_bgp_peer_configs:\n - name: cilium-peer\n spec:\n # authSecretRef: bgp-auth-secret\n gracefulRestart:\n enabled: true\n restartTimeSeconds: 15\n families:\n - afi: ipv4\n safi: unicast\n advertisements:\n matchLabels:\n advertise: \"bgp\"\n - afi: ipv6\n safi: unicast\n advertisements:\n matchLabels:\n advertise: \"bgp\"\n```\n\nCillium's BGP Advertisements can be configured with the `cilium_bgp_advertisements` variable:\n\n```yml\ncilium_bgp_advertisements:\n - name: bgp-advertisements\n labels:\n advertise: bgp\n spec:\n advertisements:\n - advertisementType: \"PodCIDR\"\n attributes:\n communities:\n standard: [ \"64512:99\" ]\n - advertisementType: \"Service\"\n service:\n addresses:\n - ClusterIP\n - ExternalIP\n - LoadBalancerIP\n selector:\n matchExpressions:\n - {key: somekey, operator: NotIn, values: ['never-used-value']}\n```\n\nCillium's BGP Node Config Overrides can be configured with the `cilium_bgp_node_config_overrides` variable:\n\n```yml\ncilium_bgp_node_config_overrides:\n - name: bgpv2-cplane-dev-multi-homing-worker\n spec:\n bgpInstances:\n - name: \"instance-65000\"\n routerID: \"192.168.10.1\"\n localPort: 1790\n peers:\n - name: \"peer-65000-tor1\"\n localAddress: fd00:10:0:2::2\n - name: \"peer-65000-tor2\"\n localAddress: fd00:11:0:2::2\n```\n\nFor further information, check [BGP Control Plane Resources documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/)\n\n### BGP Peering Policies (Legacy < v1.16)\n\nCilium's BGP Peering Policies can be configured with the `cilium_bgp_peering_policies` variable:\n\n```yml\ncilium_bgp_peering_policies:\n - name: \"01-bgp-peering-policy\"\n spec:\n virtualRouters:\n - localASN: 64512\n exportPodCIDR: false\n neighbors:\n - peerAddress: '10.47.1.1/24'\n peerASN: 64512\n serviceSelector:\n matchExpressions:\n - {key: somekey, operator: NotIn, values: ['never-used-value']}\n```\n\nFor further information, check [BGP Peering Policy documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/#bgp-peering-policy-legacy)\n\n## Kube-proxy replacement with Cilium\n\nCilium can run without kube-proxy by setting `cilium_kube_proxy_replacement`\nto `strict` (< v1.16) or `true` (Cilium v1.16+ no longer accepts `strict`, however this is converted to `true` by kubespray when running v1.16+).\n\nWithout kube-proxy, cilium needs to know the address of the kube-apiserver\nand this must be set globally for all Cilium components (agents and operators).\nWe can only use the localhost apiserver loadbalancer in this mode\nwhenever it uses the same port as the kube-apiserver (by default it does).\n\n## Cilium Operator\n\nUnlike some operators, Cilium Operator does not exist for installation purposes.\n> The Cilium Operator is responsible for managing duties in the cluster which should logically be handled once for the entire cluster, rather than once for each node in the cluster.\n\n### Adding custom flags to the Cilium Operator\n\nYou can set additional cilium-operator container arguments using `cilium_operator_custom_args`.\nThis is an advanced option, and you should only use it if you know what you are doing.\n\nAccepts an array or a string.\n\n```yml\ncilium_operator_custom_args: [\"--foo=bar\", \"--baz=qux\"]\n```\n\nor\n\n```yml\ncilium_operator_custom_args: \"--foo=bar\"\n```\n\nYou do not need to add a custom flag to enable debugging. Instead, feel free to use the `CILIUM_DEBUG` variable.\n\n### Adding extra volumes and mounting them\n\nYou can use `cilium_operator_extra_volumes` to add extra volumes to the Cilium Operator, and use `cilium_operator_extra_volume_mounts` to mount those volumes.\nThis is an advanced option, and you should only use it if you know what you are doing.\n\n```yml\ncilium_operator_extra_volumes:\n - configMap:\n name: foo\n name: foo-mount-path\n\ncilium_operator_extra_volume_mounts:\n - mountPath: /tmp/foo/bar\n name: foo-mount-path\n readOnly: true\n```\n\n## Choose Cilium version\n\n```yml\ncilium_version: \"1.19.1\"\n```\n\n## Add variable to config\n\nUse following variables:\n\nExample:\n\n```yml\ncilium_config_extra_vars:\n enable-endpoint-routes: true\n```\n\n## Change Identity Allocation Mode\n\nCilium assigns an identity for each endpoint. This identity is used to enforce basic connectivity between endpoints.\n\nCilium currently supports two different identity allocation modes:\n\n- \"crd\" stores identities in kubernetes as CRDs (custom resource definition).\n - These can be queried with `kubectl get ciliumid`\n- \"kvstore\" stores identities in an etcd kvstore.\n\n## Enable Transparent Encryption\n\nCilium supports the transparent encryption of Cilium-managed host traffic and\ntraffic between Cilium-managed endpoints either using IPsec or Wireguard.\n\nWireguard option is only available in Cilium 1.10.0 and newer.\n\n### IPsec Encryption\n\nFor further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/)\n\nTo enable IPsec encryption, you just need to set three variables.\n\n```yml\ncilium_encryption_enabled: true\ncilium_encryption_type: \"ipsec\"\n```\n\nThe third variable is `cilium_ipsec_key`. You need to create a secret key string for this variable.\nKubespray does not automate this process.\nCilium documentation currently recommends creating a key using the following command:\n\n```shell\necho \"3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128\"\n```\n\nNote that Kubespray handles secret creation. So you only need to pass the key as the `cilium_ipsec_key` variable, base64 encoded:\n\n```shell\necho \"cilium_ipsec_key: \"$(echo -n \"3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128\" | base64 -w0)\n```\n\n### Wireguard Encryption\n\nFor further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/security/network/encryption-wireguard/)\n\nTo enable Wireguard encryption, you just need to set two variables.\n\n```yml\ncilium_encryption_enabled: true\ncilium_encryption_type: \"wireguard\"\n```\n\nKubespray currently supports Linux distributions with Wireguard Kernel mode on Linux 5.6 and newer.\n\n## Bandwidth Manager\n\nCilium's bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation.\n\nBandwidth enforcement currently does not work in combination with L7 Cilium Network Policies.\nIn case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods.\n\nBandwidth Manager requires a v5.1.x or more recent Linux kernel.\n\nFor further information, make sure to check the official [Cilium documentation](https://docs.cilium.io/en/latest/network/kubernetes/bandwidth-manager/)\n\nTo use this function, set the following parameters\n\n```yml\ncilium_enable_bandwidth_manager: true\n```\n\n## Host Firewall\n\nHost Firewall enforces security policies for Kubernetes nodes. It is disable by default, since it can break the cluster connectivity.\n\n```yaml\ncilium_enable_host_firewall: true\n```\n\nFor further information, check [host firewall documentation](https://docs.cilium.io/en/latest/security/host-firewall/)\n\n## Policy Audit Mode\n\nWhen _Policy Audit Mode_ is enabled, no network policy is enforced. This feature helps to validate the impact of host policies before enforcing them.\n\n```yaml\ncilium_policy_audit_mode: true\n```\n\nIt is disable by default, and should not be enabled in production.\n\n## Install Cilium Hubble\n\nk8s-net-cilium.yml:\n\n```yml\ncilium_enable_hubble: true ## enable support hubble in cilium\ncilium_hubble_install: true ## install hubble-relay, hubble-ui\ncilium_hubble_tls_generate: true ## install hubble-certgen and generate certificates\n```\n\nTo validate that Hubble UI is properly configured, set up a port forwarding for hubble-ui service:\n\n```shell script\nkubectl port-forward -n kube-system svc/hubble-ui 12000:80\n```\n\nand then open [http://localhost:12000/](http://localhost:12000/).\n\n## Hubble metrics\n\n```yml\ncilium_enable_hubble_metrics: true\ncilium_hubble_metrics:\n - dns\n - drop\n - tcp\n - flow\n - icmp\n - http\n```\n\n[More](https://docs.cilium.io/en/v1.9/operations/metrics/#hubble-exported-metrics)\n\n## Upgrade considerations\n\n### Rolling-restart timeouts\n\nCilium relies on the kernel's BPF support, which is extremely fast at runtime but incurs a compilation penalty on initialization and update.\n\nAs a result, the Cilium DaemonSet pods can take a significant time to start, which scales with the number of nodes and endpoints in your cluster.\n\nAs part of cluster.yml, this DaemonSet is restarted, and Kubespray's [default timeouts for this operation](../roles/network_plugin/cilium/defaults/main.yml)\nare not appropriate for large clusters.\n\nThis means that you will likely want to update these timeouts to a value more in-line with your cluster's number of nodes and their respective CPU performance.\nThis is configured by the following values:\n\n```yaml\n# Configure how long to wait for the Cilium DaemonSet to be ready again\ncilium_rolling_restart_wait_retries_count: 30\ncilium_rolling_restart_wait_retries_delay_seconds: 10\n```\n\nThe total time allowed (count * delay) should be at least `($number_of_nodes_in_cluster * $cilium_pod_start_time)` for successful rolling updates. There are no\ndrawbacks to making it higher and giving yourself a time buffer to accommodate transient slowdowns.\n\nNote: To find the `$cilium_pod_start_time` for your cluster, you can simply restart a Cilium pod on a node of your choice and look at how long it takes for it\nto become ready.\n\nNote 2: The default CPU requests/limits for Cilium pods is set to a very conservative 100m:500m which will likely yield very slow startup for Cilium pods. You\nprobably want to significantly increase the CPU limit specifically if short bursts of CPU from Cilium are acceptable to you.\n" }, { "path": "docs/CNI/cni.md", "content": "CNI\n==============\n\nThis network plugin only unpacks CNI plugins version `cni_version` into `/opt/cni/bin` and instructs implementation of container runtime cri to use cni.\n\nIt's intended usage is for custom CNI configuration, e.g. manual routing tables + bridge + loopback CNI plugin outside kubespray scope. Furthermore, it's used for non-kubespray supported CNI plugins which you can install afterward.\n\nYou are required to fill `/etc/cni/net.d` with valid CNI configuration after using kubespray.\n" }, { "path": "docs/CNI/flannel.md", "content": "# Flannel\n\nFlannel is a network fabric for containers, designed for Kubernetes\n\nSupported [backends](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard): `vxlan`, `host-gw` and `wireguard`\n\n**Warning:** You may encounter this [bug](https://github.com/coreos/flannel/pull/1282) with `VXLAN` backend, while waiting on a newer Flannel version the current workaround (`ethtool --offload flannel.1 rx off tx off`) is showcase in kubespray [networking test](tests/testcases/040_check-network-adv.yml:31).\n\n## Verifying flannel install\n\n* Flannel configuration file should have been created there\n\n```ShellSession\ncat /run/flannel/subnet.env\nFLANNEL_NETWORK=10.233.0.0/18\nFLANNEL_SUBNET=10.233.16.1/24\nFLANNEL_MTU=1450\nFLANNEL_IPMASQ=false\n```\n\n* Check if the network interface has been created\n\n```ShellSession\nip a show dev flannel.1\n4: flannel.1: mtu 1450 qdisc noqueue state UNKNOWN group default\n link/ether e2:f3:a7:0f:bf:cb brd ff:ff:ff:ff:ff:ff\n inet 10.233.16.0/18 scope global flannel.1\n valid_lft forever preferred_lft forever\n inet6 fe80::e0f3:a7ff:fe0f:bfcb/64 scope link\n valid_lft forever preferred_lft forever\n```\n\n* Try to run a container and check its ip address\n\n```ShellSession\nkubectl run test --image=busybox --command -- tail -f /dev/null\nreplicationcontroller \"test\" created\n\nkubectl describe po test-34ozs | grep ^IP\nIP: 10.233.16.2\n```\n\n```ShellSession\nkubectl exec test-34ozs -- ip a show dev eth0\n8: eth0@if9: mtu 1450 qdisc noqueue\n link/ether 02:42:0a:e9:2b:03 brd ff:ff:ff:ff:ff:ff\n inet 10.233.16.2/24 scope global eth0\n valid_lft forever preferred_lft forever\n inet6 fe80::42:aff:fee9:2b03/64 scope link tentative flags 08\n valid_lft forever preferred_lft forever\n```\n" }, { "path": "docs/CNI/kube-ovn.md", "content": "# Kube-OVN\n\nKube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.\n\nFor more information please check [Kube-OVN documentation](https://github.com/alauda/kube-ovn)\n\n**Warning:** Kernel version (`cat /proc/version`) needs to be different from `3.10.0-862` or kube-ovn won't start and will print this message:\n\n```bash\nkernel version 3.10.0-862 has a nat related bug that will affect ovs function, please update to a version greater than 3.10.0-898\n```\n\n## How to use it\n\nEnable kube-ovn in `group_vars/k8s_cluster/k8s_cluster.yml`\n\n```yml\n...\nkube_network_plugin: kube-ovn\n...\n```\n\n## Verifying kube-ovn install\n\nKube-OVN run ovn and controller in `kube-ovn` namespace\n\n* Check the status of kube-ovn pods\n\n```ShellSession\n# From the CLI\nkubectl get pod -n kube-ovn\n\n# Output\nNAME READY STATUS RESTARTS AGE\nkube-ovn-cni-49lsm 1/1 Running 0 2d20h\nkube-ovn-cni-9db8f 1/1 Running 0 2d20h\nkube-ovn-cni-wftdk 1/1 Running 0 2d20h\nkube-ovn-controller-68d7bb48bd-7tnvg 1/1 Running 0 2d21h\novn-central-6675dbb7d9-d7z8m 1/1 Running 0 4d16h\novs-ovn-hqn8p 1/1 Running 0 4d16h\novs-ovn-hvpl8 1/1 Running 0 4d16h\novs-ovn-r5frh 1/1 Running 0 4d16h\n```\n\n* Check the default and node subnet\n\n```ShellSession\n# From the CLI\nkubectl get subnet\n\n# Output\nNAME PROTOCOL CIDR PRIVATE NAT\njoin IPv4 100.64.0.0/16 false false\novn-default IPv4 10.16.0.0/16 false true\n```\n" }, { "path": "docs/CNI/kube-router.md", "content": "# Kube-router\n\nKube-router is a L3 CNI provider, as such it will setup IPv4 routing between\nnodes to provide Pods' networks reachability.\n\nSee [kube-router documentation](https://www.kube-router.io/).\n\n## Verifying kube-router install\n\nKube-router runs its pods as a `DaemonSet` in the `kube-system` namespace:\n\n* Check the status of kube-router pods\n\n```ShellSession\n# From the CLI\nkubectl get pod --namespace=kube-system -l k8s-app=kube-router -owide\n\n# output\nNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE\nkube-router-4f679 1/1 Running 0 2d 192.168.186.4 mykube-k8s-node-nf-2 \nkube-router-5slf8 1/1 Running 0 2d 192.168.186.11 mykube-k8s-node-nf-3 \nkube-router-lb6k2 1/1 Running 0 20h 192.168.186.14 mykube-k8s-node-nf-6 \nkube-router-rzvrb 1/1 Running 0 20h 192.168.186.17 mykube-k8s-node-nf-4 \nkube-router-v6n56 1/1 Running 0 2d 192.168.186.6 mykube-k8s-node-nf-1 \nkube-router-wwhg8 1/1 Running 0 20h 192.168.186.16 mykube-k8s-node-nf-5 \nkube-router-x2xs7 1/1 Running 0 2d 192.168.186.10 mykube-k8s-master-1 \n```\n\n* Peek at kube-router container logs:\n\n```ShellSession\n# From the CLI\nkubectl logs --namespace=kube-system -l k8s-app=kube-router | grep Peer.Up\n\n# output\ntime=\"2018-09-17T16:47:14Z\" level=info msg=\"Peer Up\" Key=192.168.186.6 State=BGP_FSM_OPENCONFIRM Topic=Peer\ntime=\"2018-09-17T16:47:16Z\" level=info msg=\"Peer Up\" Key=192.168.186.11 State=BGP_FSM_OPENCONFIRM Topic=Peer\ntime=\"2018-09-17T16:47:46Z\" level=info msg=\"Peer Up\" Key=192.168.186.10 State=BGP_FSM_OPENCONFIRM Topic=Peer\ntime=\"2018-09-18T19:12:24Z\" level=info msg=\"Peer Up\" Key=192.168.186.14 State=BGP_FSM_OPENCONFIRM Topic=Peer\ntime=\"2018-09-18T19:12:28Z\" level=info msg=\"Peer Up\" Key=192.168.186.17 State=BGP_FSM_OPENCONFIRM Topic=Peer\ntime=\"2018-09-18T19:12:38Z\" level=info msg=\"Peer Up\" Key=192.168.186.16 State=BGP_FSM_OPENCONFIRM Topic=Peer\n[...]\n```\n\n## Gathering kube-router state\n\nKube-router Pods come bundled with a \"Pod Toolbox\" which provides very\nuseful internal state views for:\n\n* IPVS: via `ipvsadm`\n* BGP peering and routing info: via `gobgp`\n\nYou need to `kubectl exec -it ...` into a kube-router container to use these, see\n for details.\n\n## Kube-router configuration\n\nYou can change the default configuration by overriding `kube_router_...` variables\n(as found at `roles/network_plugin/kube-router/defaults/main.yml`),\nthese are named to follow `kube-router` command-line options as per\n.\n\n## Advanced BGP Capabilities\n\n\n\nIf you have other networking devices or SDN systems that talk BGP, kube-router will fit in perfectly.\nFrom a simple full node-to-node mesh to per-node peering configurations, most routing needs can be attained.\nThe configuration is Kubernetes native (annotations) just like the rest of kube-router.\n\nFor more details please refer to the .\n\nNext options will set up annotations for kube-router, using `kubectl annotate` command.\n\n```yml\nkube_router_annotations_master: []\nkube_router_annotations_node: []\nkube_router_annotations_all: []\n```\n" }, { "path": "docs/CNI/macvlan.md", "content": "# Macvlan\n\n## How to use it\n\n* Enable macvlan in `group_vars/k8s_cluster/k8s_cluster.yml`\n\n```yml\n...\nkube_network_plugin: macvlan\n...\n```\n\n* Adjust the `macvlan_interface` in `group_vars/k8s_cluster/k8s-net-macvlan.yml` or by host in the `host.yml` file:\n\n```yml\nall:\n hosts:\n node1:\n ip: 10.2.2.1\n access_ip: 10.2.2.1\n ansible_host: 10.2.2.1\n macvlan_interface: ens5\n```\n\n## Issue encountered\n\n* Service DNS\n\nreply from unexpected source:\n\nadd `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml`\n\n* Disable nodelocaldns\n\nThe nodelocal dns IP is not reachable.\n\nDisable it in `sample/group_vars/k8s_cluster/k8s_cluster.yml`\n\n```yml\nenable_nodelocaldns: false\n```\n" }, { "path": "docs/CNI/multus.md", "content": "# Multus\n\nMultus is a meta CNI plugin that provides multiple network interface support to\npods. For each interface, Multus delegates CNI calls to secondary CNI plugins\nsuch as Calico, macvlan, etc.\n\nSee [multus documentation](https://github.com/k8snetworkplumbingwg/multus-cni).\n\n## Multus installation\n\nSince Multus itself does not implement networking, it requires a master plugin, which is specified through the variable `kube_network_plugin`. To enable Multus an additional variable `kube_network_plugin_multus` must be set to `true`. For example,\n\n```yml\nkube_network_plugin: calico\nkube_network_plugin_multus: true\n```\n\nwill install Multus and Calico and configure Multus to use Calico as the primary network plugin.\n\nNamespace isolation enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. To enable namespace isolation:\n\n```yml\nmultus_namespace_isolation: true\n```\n\n### Cilium compatibility\n\nIf you are using `cilium` as the primary CNI you'll have to set `cilium_cni_exclusive` to `false` to avoid cillium reverting multus config.\n\n```yml\nkube_network_plugin: cilium\nkube_network_plugin_multus: true\ncilium_cni_exclusive: false\n```\n\n## Using Multus\n\nOnce Multus is installed, you can create CNI configurations (as a CRD objects) for additional networks, in this case a macvlan CNI configuration is defined. You may replace the config field with any valid CNI configuration where the CNI binary is available on the nodes.\n\n```ShellSession\ncat <\n" }, { "path": "docs/CRI/containerd.md", "content": "# containerd\n\n[containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability\nKubespray supports basic functionality for using containerd as the default container runtime in a cluster.\n\n_To use the containerd container runtime set the following variables:_\n\n## k8s_cluster.yml\n\nWhen kube_node contains etcd, you define your etcd cluster to be as well schedulable for Kubernetes workloads. Thus containerd and dockerd can not run at same time, must be set to bellow for running etcd cluster with only containerd.\n\n```yaml\ncontainer_manager: containerd\n```\n\n## etcd.yml\n\n```yaml\netcd_deployment_type: host\n```\n\n## Containerd config\n\nExample: define registry mirror for docker hub\n\n```yaml\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - host: https://registry-1.docker.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n```\n\ncontainerd falls back to `https://{{ prefix }}` when none of the mirrors have the image.\nThis can be changed with the [`server` field](https://github.com/containerd/containerd/blob/main/docs/hosts.md#server-field):\n\n```yaml\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - host: https://registry-1.docker.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n server: https://mirror.example.org\n```\n\nThe `containerd_registries` and `containerd_insecure_registries` configs are deprecated.\n\n### Containerd Runtimes\n\nContainerd supports multiple runtime configurations that can be used with\n[RuntimeClass] Kubernetes feature. See [runtime classes in containerd] for the\ndetails of containerd configuration.\n\nIn kubespray, the default runtime name is \"runc\", and it can be configured with the `containerd_runc_runtime` dictionary:\n\n```yaml\ncontainerd_runc_runtime:\n name: runc\n type: \"io.containerd.runc.v2\"\n options:\n Root: \"\"\n SystemdCgroup: \"false\"\n BinaryName: /usr/local/bin/my-runc\n base_runtime_spec: cri-base.json\n```\n\nFurther runtimes can be configured with `containerd_additional_runtimes`, which\nis a list of such dictionaries.\n\nDefault runtime can be changed by setting `containerd_default_runtime`.\n\n#### Base runtime specs and limiting number of open files\n\n`base_runtime_spec` key in a runtime dictionary is used to explicitly\nspecify a runtime spec json file. `runc` runtime has it set to `cri-base.json`,\nwhich is generated with `ctr oci spec > /etc/containerd/cri-base.json` and\nupdated to include a custom setting for maximum number of file descriptors per\ncontainer.\n\nYou can change maximum number of file descriptors per container for the default\n`runc` runtime by setting the `containerd_base_runtime_spec_rlimit_nofile`\nvariable.\n\nYou can tune many more [settings][runtime-spec] by supplying your own file name and content with `containerd_base_runtime_specs`:\n\n```yaml\ncontainerd_base_runtime_specs:\n cri-spec-custom.json: |\n {\n \"ociVersion\": \"1.1.0\",\n \"process\": {\n \"user\": {\n \"uid\": 0,\n ...\n```\n\nThe files in this dict will be placed in containerd config directory,\n`/etc/containerd` by default. The files can then be referenced by filename in a\nruntime:\n\n```yaml\ncontainerd_runc_runtime:\n name: runc\n base_runtime_spec: cri-spec-custom.json\n ...\n```\n\nConfig insecure-registry access to self hosted registries.\n\n```yaml\ncontainerd_registries_mirrors:\n - prefix: test.registry.io\n mirrors:\n - host: http://test.registry.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: true\n - prefix: 172.19.16.11:5000\n mirrors:\n - host: http://172.19.16.11:5000\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: true\n - prefix: repo:5000\n mirrors:\n - host: http://repo:5000\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: true\n```\n\n[containerd]: https://containerd.io/\n[RuntimeClass]: https://kubernetes.io/docs/concepts/containers/runtime-class/\n[runtime classes in containerd]: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#runtime-classes\n[runtime-spec]: https://github.com/opencontainers/runtime-spec\n\n### Optional : NRI\n\n[Node Resource Interface](https://github.com/containerd/nri) (NRI) is disabled by default for the containerd. If you\nare using contained version v1.7.0 or above, then you can enable it with the\nfollowing configuration:\n\n```yaml\nnri_enabled: true\n```\n\n### Optional : Static Binary\n\nTo ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary.\n\n```yaml\ncontainerd_static_binary: true\n```\n" }, { "path": "docs/CRI/cri-o.md", "content": "# CRI-O\n\n[CRI-O] is a lightweight container runtime for Kubernetes.\nKubespray supports basic functionality for using CRI-O as the default container runtime in a cluster.\n\n* Kubernetes supports CRI-O on v1.11.1 or later.\n* etcd: configure either kubeadm managed etcd or host deployment\n\n_To use the CRI-O container runtime set the following variables:_\n\n## all/all.yml\n\n```yaml\ndownload_container: false\nskip_downloads: false\netcd_deployment_type: host # optionally kubeadm\n```\n\n## k8s_cluster/k8s_cluster.yml\n\n```yaml\ncontainer_manager: crio\n```\n\n## all/crio.yml\n\nEnable docker hub registry mirrors\n\n```yaml\ncrio_registries:\n - prefix: docker.io\n insecure: false\n blocked: false\n location: registry-1.docker.io\n unqualified: false\n mirrors:\n - location: 192.168.100.100:5000\n insecure: true\n - location: mirror.gcr.io\n insecure: false\n```\n\n[CRI-O]: https://cri-o.io/\n\nThe following is a method to enable insecure registries.\n\n```yaml\ncrio_insecure_registries:\n - 10.0.0.2:5000\n```\n\nAnd you can config authentication for these registries after `crio_insecure_registries`.\n\n```yaml\ncrio_registry_auth:\n - registry: 10.0.0.2:5000\n username: user\n password: pass\n```\n\n## Note about user namespaces\n\nCRI-O has support for user namespaces. This feature is optional and can be enabled by setting the following two variables.\n\n```yaml\ncrio_runtimes:\n - name: runc\n path: /usr/bin/runc\n type: oci\n root: /run/runc\n allowed_annotations:\n - \"io.kubernetes.cri-o.userns-mode\"\n\ncrio_remap_enable: true\n```\n\nThe `allowed_annotations` configures `crio.conf` accordingly.\n\nThe `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to add an entry for the **containers** user.\nBy default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space.\n\nThe `crio_default_capabilities` configure the default containers capabilities for the crio.\nDefaults capabilities are:\n\n```yaml\ncrio_default_capabilities:\n - CHOWN\n - DAC_OVERRIDE\n - FSETID\n - FOWNER\n - SETGID\n - SETUID\n - SETPCAP\n - NET_BIND_SERVICE\n - KILL\n```\n\nYou can add MKNOD to the list for a rancher deployment\n\n## Optional : NRI\n\n[Node Resource Interface](https://github.com/containerd/nri) (NRI) is disabled by default for the CRI-O. If you\nare using CRI-O version v1.26.0 or above, then you can enable it with the\nfollowing configuration:\n\n```yaml\nnri_enabled: true\n```\n" }, { "path": "docs/CRI/docker.md", "content": "# Docker support\n\nThe docker runtime is supported by kubespray and while the `dockershim` is deprecated to be removed in kubernetes 1.24+ there are alternative ways to use docker such as through the [cri-dockerd](https://github.com/Mirantis/cri-dockerd) project supported by Mirantis.\n\nUsing the docker container manager:\n\n```yaml\ncontainer_manager: docker\n```\n\n*Note:* `cri-dockerd` has replaced `dockershim` across supported kubernetes version in kubespray 2.20.\n\nEnabling the `overlay2` graph driver:\n\n```yaml\ndocker_storage_options: -s overlay2\n```\n\nChanging the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`:\n\n```yaml\ndocker_cgroup_driver: systemd\n```\n\nIf you have more than 3 nameservers kubespray will only use the first 3 else it will fail. Set the `docker_dns_servers_strict` to `false` to prevent deployment failure.\n\n```yaml\ndocker_dns_servers_strict: false\n```\n\nSet the path used to store Docker data:\n\n```yaml\ndocker_daemon_graph: \"/var/lib/docker\"\n```\n\nChanging the docker daemon iptables support:\n\n```yaml\ndocker_iptables_enabled: \"false\"\n```\n\nDocker log options:\n\n```yaml\n# Rotate container stderr/stdout logs at 50m and keep last 5\ndocker_log_opts: \"--log-opt max-size=50m --log-opt max-file=5\"\n```\n\nChange the docker `bin_dir`, this should not be changed unless you use a custom docker package:\n\n```yaml\ndocker_bin_dir: \"/usr/bin\"\n```\n\nTo keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'.\nkubespray deletes the docker package on each run, so caching the package makes sense:\n\n```yaml\ndocker_rpm_keepcache: 1\n```\n\nAllowing insecure-registry access to self hosted registries. Can be ipaddress and domain_name.\n\n```yaml\n## example define 172.19.16.11 or mirror.registry.io\ndocker_insecure_registries:\n - mirror.registry.io\n - 172.19.16.11\n```\n\nAdding other registry, i.e. China registry mirror:\n\n```yaml\ndocker_registry_mirrors:\n - https://registry.docker-cn.com\n - https://mirror.aliyuncs.com\n```\n\nOverriding default system MountFlags value. This option takes a mount propagation flag: `shared`, `slave` or `private`, which control whether mounts in the file system namespace set up for docker will receive or propagate mounts and unmounts. Leave empty for system default:\n\n```yaml\ndocker_mount_flags:\n```\n\nAdding extra options to pass to the docker daemon:\n\n```yaml\n## This string should be exactly as you wish it to appear.\ndocker_options: \"\"\n```\n\nFor Debian based distributions, set the path to store the GPG key to avoid using the default one used in `apt_key` module (e.g. /etc/apt/trusted.gpg)\n\n```yaml\ndocker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg\n```\n" }, { "path": "docs/CRI/gvisor.md", "content": "# gVisor\n\n[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system.\n\ngVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers.\n\n## Usage\n\nTo enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`.\n\nContainerd support:\n\n```yaml\ncontainer_manager: containerd\ngvisor_enabled: true\n```\n" }, { "path": "docs/CRI/kata-containers.md", "content": "# Kata Containers\n\n[Kata Containers](https://katacontainers.io) is a secure container runtime with lightweight virtual machines that supports multiple hypervisor solutions.\n\n## Hypervisors\n\n_Qemu_ is the only hypervisor supported by Kubespray.\n\n## Installation\n\nTo enable Kata Containers, set the following variables:\n\n**k8s-cluster.yml**:\n\n```yaml\ncontainer_manager: containerd\nkata_containers_enabled: true\n```\n\n**etcd.yml**:\n\n```yaml\netcd_deployment_type: host\n```\n\n## Usage\n\nBy default, runc is used for pods.\nKubespray generates the runtimeClass kata-qemu, and it is necessary to specify it as\nthe runtimeClassName of a pod spec to use Kata Containers:\n\n```shell\n$ kubectl get runtimeclass\nNAME HANDLER AGE\nkata-qemu kata-qemu 3m34s\n$\n$ cat nginx.yaml\napiVersion: v1\nkind: Pod\nmetadata:\n name: mypod\nspec:\n runtimeClassName: kata-qemu\n containers:\n - name: nginx\n image: nginx:1.14.2\n$\n$ kubectl apply -f nginx.yaml\n```\n\n## Configuration\n\n### Recommended : Pod Overhead\n\n[Pod Overhead](https://kubernetes.io/docs/concepts/configuration/pod-overhead/) is a feature for accounting for the resources consumed by the Runtime Class used by the Pod.\n\nWhen this feature is enabled, Kubernetes will count the fixed amount of CPU and memory set in the configuration as used by the virtual machine and not by the containers running in the Pod.\n\nPod Overhead is mandatory if you run Pods with Kata Containers that use [resources limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits).\n\n**Set cgroup driver**:\n\nTo enable Pod Overhead feature you have to configure Kubelet with the appropriate cgroup driver, using the following configuration:\n\n`cgroupfs` works best:\n\n```yaml\nkubelet_cgroup_driver: cgroupfs\n```\n\n... but when using `cgroups v2` (see ) you can use systemd as well:\n\n```yaml\nkubelet_cgroup_driver: systemd\n```\n\n**Qemu hypervisor configuration**:\n\nThe configuration for the Qemu hypervisor uses the following values:\n\n```yaml\nkata_containers_qemu_overhead: true\nkata_containers_qemu_overhead_fixed_cpu: 10m\nkata_containers_qemu_overhead_fixed_memory: 290Mi\n```\n\n### Optional : Select Kata Containers version\n\nOptionally you can select the Kata Containers release version to be installed. The available releases are published in [GitHub](https://github.com/kata-containers/kata-containers/releases).\n\n```yaml\nkata_containers_version: 2.2.2\n```\n\n### Optional : Debug\n\nDebug is disabled by default for all the components of Kata Containers. You can change this behaviour with the following configuration:\n\n```yaml\nkata_containers_qemu_debug: 'false'\n```\n" }, { "path": "docs/CSI/aws-ebs-csi.md", "content": "# AWS EBS CSI Driver\n\nAWS EBS CSI driver allows you to provision EBS volumes for pods in EC2 instances. The old in-tree AWS cloud provider is deprecated and will be removed in future versions of Kubernetes. So transitioning to the CSI driver is advised.\n\nTo enable AWS EBS CSI driver, uncomment the `aws_ebs_csi_enabled` option in `group_vars/all/aws.yml` and set it to `true`.\n\nTo set the number of replicas for the AWS CSI controller, you can change `aws_ebs_csi_controller_replicas` option in `group_vars/all/aws.yml`.\n\nMake sure to add a role, for your EC2 instances hosting Kubernetes, that allows it to do the actions necessary to request a volume and attach it: [AWS CSI Policy](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json)\n\nIf you want to deploy the AWS EBS storage class used with the CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`.\n\nYou can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over AWS EC2 with EBS CSI Driver enabled.\n\n## Usage example\n\nTo check if AWS EBS CSI Driver is deployed properly, check that the ebs-csi pods are running:\n\n```ShellSession\n$ kubectl -n kube-system get pods | grep ebs\nebs-csi-controller-85d86bccc5-8gtq5 4/4 Running 4 40s\nebs-csi-node-n4b99 3/3 Running 3 40s\n```\n\nCheck the associated storage class (if you enabled persistent_volumes):\n\n```ShellSession\n$ kubectl get storageclass\nNAME PROVISIONER AGE\nebs-sc ebs.csi.aws.com 45s\n```\n\nYou can run a PVC and an example Pod using this file `ebs-pod.yml`:\n\n```yml\n--\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: ebs-claim\nspec:\n accessModes:\n - ReadWriteOnce\n storageClassName: ebs-sc\n resources:\n requests:\n storage: 1Gi\n---\napiVersion: v1\nkind: Pod\nmetadata:\n name: app\nspec:\n containers:\n - name: app\n image: centos\n command: [\"/bin/sh\"]\n args: [\"-c\", \"while true; do echo $(date -u) >> /data/out.txt; sleep 5; done\"]\n volumeMounts:\n - name: persistent-storage\n mountPath: /data\n volumes:\n - name: persistent-storage\n persistentVolumeClaim:\n claimName: ebs-claim\n```\n\nApply this conf to your cluster: ```kubectl apply -f ebs-pod.yml```\n\nYou should see the PVC provisioned and bound:\n\n```ShellSession\n$ kubectl get pvc\nNAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE\nebs-claim Bound pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c 1Gi RWO ebs-sc 50s\n```\n\nAnd the volume mounted to the example Pod (wait until the Pod is Running):\n\n```ShellSession\n$ kubectl exec -it app -- df -h | grep data\n/dev/nvme1n1 1014M 34M 981M 4% /data\n```\n\n## More info\n\nFor further information about the AWS EBS CSI Driver, you can refer to this page: [AWS EBS Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/).\n" }, { "path": "docs/CSI/azure-csi.md", "content": "# Azure Disk CSI Driver\n\nThe Azure Disk CSI driver allows you to provision volumes for pods with a Kubernetes deployment over Azure Cloud. The CSI driver replaces to volume provisioning done by the in-tree azure cloud provider which is deprecated.\n\nThis documentation is an updated version of the in-tree Azure cloud provider documentation (azure.md).\n\nTo deploy Azure Disk CSI driver, uncomment the `azure_csi_enabled` option in `group_vars/all/azure.yml` and set it to `true`.\n\n## Azure Disk CSI Storage Class\n\nIf you want to deploy the Azure Disk storage class to provision volumes dynamically, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`.\n\n## Parameters\n\nBefore creating the instances you must first set the `azure_csi_` variables in the `group_vars/all.yml` file.\n\nAll values can be retrieved using the azure cli tool which can be downloaded here: \n\nAfter installation you have to run `az login` to get access to your account.\n\n### azure\\_csi\\_tenant\\_id + azure\\_csi\\_subscription\\_id\n\nRun `az account show` to retrieve your subscription id and tenant id:\n`azure_csi_tenant_id` -> tenantId field\n`azure_csi_subscription_id` -> id field\n\n### azure\\_csi\\_location\n\nThe region your instances are located in, it can be something like `francecentral` or `norwayeast`. A full list of region names can be retrieved via `az account list-locations`\n\n### azure\\_csi\\_resource\\_group\n\nThe name of the resource group your instances are in, a list of your resource groups can be retrieved via `az group list`\n\nOr you can do `az vm list | grep resourceGroup` and get the resource group corresponding to the VMs of your cluster.\n\nThe resource group name is not case-sensitive.\n\n### azure\\_csi\\_vnet\\_name\n\nThe name of the virtual network your instances are in, can be retrieved via `az network vnet list`\n\n### azure\\_csi\\_vnet\\_resource\\_group\n\nThe name of the resource group your vnet is in, can be retrieved via `az network vnet list | grep resourceGroup` and get the resource group corresponding to the vnet of your cluster.\n\n### azure\\_csi\\_subnet\\_name\n\nThe name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME`\n\n### azure\\_csi\\_security\\_group\\_name\n\nThe name of the network security group your instances are in, can be retrieved via `az network nsg list`\n\n### azure\\_csi\\_aad\\_client\\_id + azure\\_csi\\_aad\\_client\\_secret\n\nThese will have to be generated first:\n\n- Create an Azure AD Application with:\n\n ```ShellSession\n az ad app create --display-name kubespray --identifier-uris http://kubespray --homepage http://kubespray.com --password CLIENT_SECRET\n ```\n\nDisplay name, identifier-uri, homepage and the password can be chosen\n\nNote the AppId in the output.\n\n- Create Service principal for the application with:\n\n ```ShellSession\n az ad sp create --id AppId\n ```\n\nThis is the AppId from the last command\n\n- Create the role assignment with:\n\n ```ShellSession\n az role assignment create --role \"Owner\" --assignee http://kubespray --subscription SUBSCRIPTION_ID\n ```\n\nazure\\_csi\\_aad\\_client\\_id must be set to the AppId, azure\\_csi\\_aad\\_client\\_secret is your chosen secret.\n\n### azure\\_csi\\_use\\_instance\\_metadata\n\nUse instance metadata service where possible. Boolean value.\n\n## Test the Azure Disk CSI driver\n\nTo test the dynamic provisioning using Azure CSI driver, make sure to have the storage class deployed (through persistent volumes), and apply the following manifest:\n\n```yml\n---\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: pvc-azuredisk\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 1Gi\n storageClassName: disk.csi.azure.com\n---\nkind: Pod\napiVersion: v1\nmetadata:\n name: nginx-azuredisk\nspec:\n nodeSelector:\n kubernetes.io/os: linux\n containers:\n - image: nginx\n name: nginx-azuredisk\n command:\n - \"/bin/sh\"\n - \"-c\"\n - while true; do echo $(date) >> /mnt/azuredisk/outfile; sleep 1; done\n volumeMounts:\n - name: azuredisk\n mountPath: \"/mnt/azuredisk\"\n volumes:\n - name: azuredisk\n persistentVolumeClaim:\n claimName: pvc-azuredisk\n```\n" }, { "path": "docs/CSI/cinder-csi.md", "content": "# Cinder CSI Driver\n\nCinder CSI driver allows you to provision volumes over an OpenStack deployment. The Kubernetes historic in-tree cloud provider is deprecated and will be removed in future versions.\n\nTo enable Cinder CSI driver, uncomment the `cinder_csi_enabled` option in `group_vars/all/openstack.yml` and set it to `true`.\n\nTo set the number of replicas for the Cinder CSI controller, you can change `cinder_csi_controller_replicas` option in `group_vars/all/openstack.yml`.\n\nYou need to source the OpenStack credentials you use to deploy your machines that will host Kubernetes: `source path/to/your/openstack-rc` or `. path/to/your/openstack-rc`.\n\nMake sure the hostnames in your `inventory` file are identical to your instance names in OpenStack. Otherwise [cinder](https://docs.openstack.org/cinder/latest/) won't work as expected.\n\nIf you want to deploy the cinder provisioner used with Cinder CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`.\n\nYou can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over OpenStack with Cinder CSI Driver enabled.\n\n## Usage example\n\nTo check if Cinder CSI Driver works properly, see first that the cinder-csi pods are running:\n\n```ShellSession\n$ kubectl -n kube-system get pods | grep cinder\ncsi-cinder-controllerplugin-7f8bf99785-cpb5v 5/5 Running 0 100m\ncsi-cinder-nodeplugin-rm5x2 2/2 Running 0 100m\n```\n\nCheck the associated storage class (if you enabled persistent_volumes):\n\n```ShellSession\n$ kubectl get storageclass\nNAME PROVISIONER AGE\ncinder-csi cinder.csi.openstack.org 100m\n```\n\nYou can run a PVC and an Nginx Pod using this file `nginx.yaml`:\n\n```yml\n---\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: csi-pvc-cinderplugin\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 1Gi\n storageClassName: cinder-csi\n\n---\napiVersion: v1\nkind: Pod\nmetadata:\n name: nginx\nspec:\n containers:\n - image: nginx\n imagePullPolicy: IfNotPresent\n name: nginx\n ports:\n - containerPort: 80\n protocol: TCP\n volumeMounts:\n - mountPath: /var/lib/www/html\n name: csi-data-cinderplugin\n volumes:\n - name: csi-data-cinderplugin\n persistentVolumeClaim:\n claimName: csi-pvc-cinderplugin\n readOnly: false\n```\n\nApply this conf to your cluster: ```kubectl apply -f nginx.yml```\n\nYou should see the PVC provisioned and bound:\n\n```ShellSession\n$ kubectl get pvc\nNAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE\ncsi-pvc-cinderplugin Bound pvc-f21ad0a1-5b7b-405e-a462-48da5cb76beb 1Gi RWO cinder-csi 8s\n```\n\nAnd the volume mounted to the Nginx Pod (wait until the Pod is Running):\n\n```ShellSession\nkubectl exec -it nginx -- df -h | grep /var/lib/www/html\n/dev/vdb 976M 2.6M 958M 1% /var/lib/www/html\n```\n\n## Compatibility with in-tree cloud provider\n\nIt is not necessary to enable OpenStack as a cloud provider for Cinder CSI Driver to work.\nThough, you can run both the in-tree openstack cloud provider and the Cinder CSI Driver at the same time. The storage class provisioners associated to each one of them are differently named.\n\n## Cinder v2 support\n\nFor the moment, only Cinder v3 is supported by the CSI Driver.\n\n## More info\n\nFor further information about the Cinder CSI Driver, you can refer to this page: [Cloud Provider OpenStack](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md).\n" }, { "path": "docs/CSI/gcp-pd-csi.md", "content": "# GCP Persistent Disk CSI Driver\n\nThe GCP Persistent Disk CSI driver allows you to provision volumes for pods with a Kubernetes deployment over Google Cloud Platform. The CSI driver replaces to volume provioning done by the in-tree azure cloud provider which is deprecated.\n\nTo deploy GCP Persistent Disk CSI driver, uncomment the `gcp_pd_csi_enabled` option in `group_vars/all/gcp.yml` and set it to `true`.\n\n## GCP Persistent Disk Storage Class\n\nIf you want to deploy the GCP Persistent Disk storage class to provision volumes dynamically, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`.\n\n## GCP credentials\n\nIn order for the CSI driver to provision disks, you need to create for it a service account on GCP with the appropriate permissions.\n\nFollow these steps to configure it:\n\n```ShellSession\n# This will open a web page for you to authenticate\ngcloud auth login\nexport PROJECT=nameofmyproject\ngcloud config set project $PROJECT\n\ngit clone https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver\n\nexport GCE_PD_SA_NAME=my-gce-pd-csi-sa\nexport GCE_PD_SA_DIR=/my/safe/credentials/directory\n\n./deploy/setup-project.sh\n```\n\nThe above will create a file named `cloud-sa.json` in the specified `GCE_PD_SA_DIR`. This file contains the service account with the appropriate credentials for the CSI driver to perform actions on GCP to request disks for pods.\n\nYou need to provide this file's path through the variable `gcp_pd_csi_sa_cred_file` in `inventory/mycluster/group_vars/all/gcp.yml`\n\nYou can now deploy Kubernetes with Kubespray over GCP.\n\n## GCP PD CSI Driver test\n\nTo test the dynamic provisioning using GCP PD CSI driver, make sure to have the storage class deployed (through persistent volumes), and apply the following manifest:\n\n```yml\n---\nkind: PersistentVolumeClaim\napiVersion: v1\nmetadata:\n name: podpvc\nspec:\n accessModes:\n - ReadWriteOnce\n storageClassName: csi-gce-pd\n resources:\n requests:\n storage: 1Gi\n\n---\napiVersion: v1\nkind: Pod\nmetadata:\n name: web-server\nspec:\n containers:\n - name: web-server\n image: nginx\n volumeMounts:\n - mountPath: /var/lib/www/html\n name: mypvc\n volumes:\n - name: mypvc\n persistentVolumeClaim:\n claimName: podpvc\n readOnly: false\n```\n\n## GCP PD documentation\n\nYou can find the official GCP Persistent Disk CSI driver installation documentation here: [GCP PD CSI Driver](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/docs/kubernetes/user-guides/driver-install.md\n)\n" }, { "path": "docs/CSI/vsphere-csi.md", "content": "# vSphere CSI Driver\n\nvSphere CSI driver allows you to provision volumes over a vSphere deployment. The Kubernetes historic in-tree cloud provider is deprecated and will be removed in future versions.\n\n## Prerequisites\n\nThe vSphere user for CSI driver requires a set of privileges to perform Cloud Native Storage operations. Follow the [official guide](https://vsphere-csi-driver.sigs.k8s.io/driver-deployment/prerequisites.html#roles_and_privileges) to configure those.\n\n## Kubespray configuration\n\nTo enable vSphere CSI driver, uncomment the `vsphere_csi_enabled` option in `group_vars/all/vsphere.yml` and set it to `true`.\n\nTo set the number of replicas for the vSphere CSI controller, you can change `vsphere_csi_controller_replicas` option in `group_vars/all/vsphere.yml`.\n\nYou need to source the vSphere credentials you use to deploy your machines that will host Kubernetes.\n\n| Variable | Required | Type | Choices | Default | Comment |\n|-------------------------------------------------|----------|---------|-----------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------|\n| external_vsphere_vcenter_ip | TRUE | string | | | IP/URL of the vCenter |\n| external_vsphere_vcenter_port | TRUE | string | | \"443\" | Port of the vCenter API |\n| external_vsphere_insecure | TRUE | string | \"true\", \"false\" | \"true\" | set to \"true\" if the host above uses a self-signed cert |\n| external_vsphere_user | TRUE | string | | | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) |\n| external_vsphere_password | TRUE | string | | | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable) |\n| external_vsphere_datacenter | TRUE | string | | | Datacenter name to use |\n| external_vsphere_kubernetes_cluster_id | TRUE | string | | \"kubernetes-cluster-id\" | Kubernetes cluster ID to use |\n| external_vsphere_version | TRUE | string | | \"7.0u1\" | Vmware Vsphere version where located all VMs |\n| external_vsphere_cloud_controller_image_tag | TRUE | string | | \"v1.31.0\" | CPI manager image tag to use |\n| vsphere_syncer_image_tag | TRUE | string | | \"v3.3.1\" | Syncer image tag to use |\n| vsphere_csi_attacher_image_tag | TRUE | string | | \"v4.3.0\" | CSI attacher image tag to use |\n| vsphere_csi_controller | TRUE | string | | \"v3.3.1\" | CSI controller image tag to use |\n| vsphere_csi_controller_replicas | TRUE | integer | | 1 | Number of pods Kubernetes should deploy for the CSI controller |\n| vsphere_csi_liveness_probe_image_tag | TRUE | string | | \"v2.10.0\" | CSI liveness probe image tag to use |\n| vsphere_csi_provisioner_image_tag | TRUE | string | | \"v2.1.0\" | CSI provisioner image tag to use |\n| vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | \"v3.5.0\" | CSI node driver registrar image tag to use |\n| vsphere_csi_driver_image_tag | TRUE | string | | \"v3.3.1\" | CSI driver image tag to use |\n| vsphere_csi_resizer_tag | TRUE | string | | \"v1.8.0\" | CSI resizer image tag to use |\n| vsphere_csi_aggressive_node_drain | FALSE | boolean | | false | Enable aggressive node drain strategy |\n| vsphere_csi_aggressive_node_unreachable_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in an unreachable state |\n| vsphere_csi_aggressive_node_not_ready_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in not-ready state |\n| vsphere_csi_namespace | TRUE | string | | \"kube-system\" | vSphere CSI namespace to use; kube-system for backward compatibility, should be change to vmware-system-csi on the long run |\n\n## Usage example\n\nTo test the dynamic provisioning using vSphere CSI driver, make sure to create a [storage policy](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#create-a-storage-policy) and [storage class](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#create-a-storageclass), then apply the following manifest:\n\n```yml\n---\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: csi-pvc-vsphere\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 1Gi\n storageClassName: mongodb-sc\n\n---\napiVersion: v1\nkind: Pod\nmetadata:\n name: nginx\nspec:\n containers:\n - image: nginx\n imagePullPolicy: IfNotPresent\n name: nginx\n ports:\n - containerPort: 80\n protocol: TCP\n volumeMounts:\n - mountPath: /usr/share/nginx/html\n name: csi-data-vsphere\n volumes:\n - name: csi-data-vsphere\n persistentVolumeClaim:\n claimName: csi-pvc-vsphere\n readOnly: false\n```\n\nApply this conf to your cluster: ```kubectl apply -f nginx.yml```\n\nYou should see the PVC provisioned and bound:\n\n```ShellSession\n$ kubectl get pvc\nNAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE\ncsi-pvc-vsphere Bound pvc-dc7b1d21-ee41-45e1-98d9-e877cc1533ac 1Gi RWO mongodb-sc 10s\n```\n\nAnd the volume mounted to the Nginx Pod (wait until the Pod is Running):\n\n```ShellSession\nkubectl exec -it nginx -- df -h | grep /usr/share/nginx/html\n/dev/sdb 976M 2.6M 907M 1% /usr/share/nginx/html\n```\n\n## More info\n\nFor further information about the vSphere CSI Driver, you can refer to the official [vSphere Cloud Provider documentation](https://cloud-provider-vsphere.sigs.k8s.io/container_storage_interface.html).\n" }, { "path": "docs/_sidebar.md", "content": "* [Readme](/)\n* Advanced\n * [Arch](/docs/advanced/arch.md)\n * [Cert Manager](/docs/advanced/cert_manager.md)\n * [Dns-stack](/docs/advanced/dns-stack.md)\n * [Downloads](/docs/advanced/downloads.md)\n * [Gcp-lb](/docs/advanced/gcp-lb.md)\n * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md)\n * [Netcheck](/docs/advanced/netcheck.md)\n * [Ntp](/docs/advanced/ntp.md)\n * [Proxy](/docs/advanced/proxy.md)\n * [Registry](/docs/advanced/registry.md)\n* Ansible\n * [Ansible](/docs/ansible/ansible.md)\n * [Ansible Collection](/docs/ansible/ansible_collection.md)\n * [Inventory](/docs/ansible/inventory.md)\n * [Vars](/docs/ansible/vars.md)\n* Cloud Controllers\n * [Openstack](/docs/cloud_controllers/openstack.md)\n * [Vsphere](/docs/cloud_controllers/vsphere.md)\n* Cloud Providers\n * [Aws](/docs/cloud_providers/aws.md)\n * [Azure](/docs/cloud_providers/azure.md)\n * [Cloud](/docs/cloud_providers/cloud.md)\n* CNI\n * [Calico](/docs/CNI/calico.md)\n * [Cilium](/docs/CNI/cilium.md)\n * [Cni](/docs/CNI/cni.md)\n * [Flannel](/docs/CNI/flannel.md)\n * [Kube-ovn](/docs/CNI/kube-ovn.md)\n * [Kube-router](/docs/CNI/kube-router.md)\n * [Macvlan](/docs/CNI/macvlan.md)\n * [Multus](/docs/CNI/multus.md)\n* CRI\n * [Containerd](/docs/CRI/containerd.md)\n * [Cri-o](/docs/CRI/cri-o.md)\n * [Docker](/docs/CRI/docker.md)\n * [Gvisor](/docs/CRI/gvisor.md)\n * [Kata-containers](/docs/CRI/kata-containers.md)\n* CSI\n * [Aws-ebs-csi](/docs/CSI/aws-ebs-csi.md)\n * [Azure-csi](/docs/CSI/azure-csi.md)\n * [Cinder-csi](/docs/CSI/cinder-csi.md)\n * [Gcp-pd-csi](/docs/CSI/gcp-pd-csi.md)\n * [Vsphere-csi](/docs/CSI/vsphere-csi.md)\n* Developers\n * [Ci-setup](/docs/developers/ci-setup.md)\n * [Ci](/docs/developers/ci.md)\n * [Test Cases](/docs/developers/test_cases.md)\n * [Vagrant](/docs/developers/vagrant.md)\n* External Storage Provisioners\n * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)\n * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)\n* Getting Started\n * [Comparisons](/docs/getting_started/comparisons.md)\n * [Getting-started](/docs/getting_started/getting-started.md)\n * [Setting-up-your-first-cluster](/docs/getting_started/setting-up-your-first-cluster.md)\n* Ingress\n * [Alb Ingress Controller](/docs/ingress/alb_ingress_controller.md)\n * [Kube-vip](/docs/ingress/kube-vip.md)\n * [Metallb](/docs/ingress/metallb.md)\n* Operating Systems\n * [Amazonlinux](/docs/operating_systems/amazonlinux.md)\n * [Bootstrap-os](/docs/operating_systems/bootstrap-os.md)\n * [Fcos](/docs/operating_systems/fcos.md)\n * [Flatcar](/docs/operating_systems/flatcar.md)\n * [Kylinlinux](/docs/operating_systems/kylinlinux.md)\n * [Openeuler](/docs/operating_systems/openeuler.md)\n * [Opensuse](/docs/operating_systems/opensuse.md)\n * [Rhel](/docs/operating_systems/rhel.md)\n * [Uoslinux](/docs/operating_systems/uoslinux.md)\n* Operations\n * [Cgroups](/docs/operations/cgroups.md)\n * [Encrypting-secret-data-at-rest](/docs/operations/encrypting-secret-data-at-rest.md)\n * [Etcd](/docs/operations/etcd.md)\n * [Ha-mode](/docs/operations/ha-mode.md)\n * [Hardening](/docs/operations/hardening.md)\n * [Integration](/docs/operations/integration.md)\n * [Kernel-requirements](/docs/operations/kernel-requirements.md)\n * [Large-deployments](/docs/operations/large-deployments.md)\n * [Mirror](/docs/operations/mirror.md)\n * [Nodes](/docs/operations/nodes.md)\n * [Offline-environment](/docs/operations/offline-environment.md)\n * [Port-requirements](/docs/operations/port-requirements.md)\n * [Recover-control-plane](/docs/operations/recover-control-plane.md)\n * [Upgrades](/docs/operations/upgrades.md)\n* Roadmap\n * [Roadmap](/docs/roadmap/roadmap.md)\n* Upgrades\n * [Migrate Docker2containerd](/docs/upgrades/migrate_docker2containerd.md)\n" }, { "path": "docs/advanced/arch.md", "content": "# Architecture compatibility\n\nThe following table shows the impact of the CPU architecture on compatible features:\n\n- amd64: Cluster using only x86/amd64 CPUs\n- arm64: Cluster using only arm64 CPUs\n- amd64 + arm64: Cluster with a mix of x86/amd64 and arm64 CPUs\n\n| kube_network_plugin | amd64 | arm64 | amd64 + arm64 |\n|---------------------|-------|-------|---------------|\n| Calico | Y | Y | Y |\n| Flannel | Y | N | N |\n| Canal | Y | N | N |\n| Cilium | Y | Y | N |\n| Contib | Y | N | N |\n| kube-router | Y | N | N |\n" }, { "path": "docs/advanced/cert_manager.md", "content": "# Installation Guide\n\n- [Installation Guide](#installation-guide)\n - [Kubernetes TLS Root CA Certificate/Key Secret](#kubernetes-tls-root-ca-certificatekey-secret)\n - [Securing Ingress Resources](#securing-ingress-resources)\n - [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key)\n - [Install Cloudflare PKI/TLS `cfssl` Toolkit.](#install-cloudflare-pkitls-cfssl-toolkit)\n - [Create Root Certificate Authority (CA) Configuration File](#create-root-certificate-authority-ca-configuration-file)\n - [Create Certificate Signing Request (CSR) Configuration File](#create-certificate-signing-request-csr-configuration-file)\n - [Create TLS Root CA Certificate and Key](#create-tls-root-ca-certificate-and-key)\n\nCert-Manager is a native Kubernetes certificate management controller. It can help with issuing certificates from a variety of sources, such as Let’s Encrypt, HashiCorp Vault, Venafi, a simple signing key pair, or self signed. It will ensure certificates are valid and up to date, and attempt to renew certificates at a configured time before expiry.\n\n## Kubernetes TLS Root CA Certificate/Key Secret\n\nIf you're planning to secure your ingress resources using TLS client certificates, you'll need to create and deploy the Kubernetes `ca-key-pair` secret consisting of the Root CA certificate and key to your K8s cluster.\n\nFor further information, read the official [Cert-Manager CA Configuration](https://cert-manager.io/docs/configuration/ca/) doc.\n\n`cert-manager` can now be enabled by editing your K8s cluster addons inventory e.g. `inventory\\sample\\group_vars\\k8s_cluster\\addons.yml` and setting `cert_manager_enabled` to true.\n\n```ini\n# Cert manager deployment\ncert_manager_enabled: true\n```\n\nIf you don't have a TLS Root CA certificate and key available, you can create these by following the steps outlined in section [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key) using the Cloudflare PKI/TLS `cfssl` toolkit. TLS Root CA certificates and keys can also be created using `ssh-keygen` and OpenSSL, if `cfssl` is not available.\n\n## Securing Ingress Resources\n\nA common use-case for cert-manager is requesting TLS signed certificates to secure your ingress resources. This can be done by simply adding annotations to your Ingress resources and cert-manager will facilitate creating the Certificate resource for you. A small sub-component of cert-manager, ingress-shim, is responsible for this.\n\nFor example, if you're using the Traefik ingress controller, you can secure the Prometheus ingress by adding the annotation `cert-manager.io/cluster-issuer: ca-issuer` and the `spec.tls` section to the `Ingress` resource definition.\n\n```yaml\napiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: prometheus-k8s\n namespace: monitoring\n labels:\n prometheus: k8s\n annotations:\n cert-manager.io/cluster-issuer: ca-issuer\nspec:\n ingressClassName: \"traefik\"\n tls:\n - hosts:\n - prometheus.example.com\n secretName: prometheus-dashboard-certs\n rules:\n - host: prometheus.example.com\n http:\n paths:\n - path: /\n pathType: ImplementationSpecific\n backend:\n service:\n name: prometheus-k8s\n port:\n name: web\n```\n\nOnce deployed to your K8s cluster, every 3 months cert-manager will automatically rotate the Prometheus `prometheus.example.com` TLS client certificate and key, and store these as the Kubernetes `prometheus-dashboard-certs` secret.\n\nPlease consult the official upstream documentation:\n\n- [cert-manager Ingress Usage](https://cert-manager.io/usage/ingress/)\n- [cert-manager Ingress Tutorial](https://cert-manager.io/tutorials/acme/ingress/#step-3-assign-a-dns-name)\n\n### ACME\n\nThe ACME Issuer type represents a single account registered with the Automated Certificate Management Environment (ACME) Certificate Authority server. When you create a new ACME Issuer, cert-manager will generate a private key which is used to identify you with the ACME server.\n\nCertificates issued by public ACME servers are typically trusted by client’s computers by default. This means that, for example, visiting a website that is backed by an ACME certificate issued for that URL, will be trusted by default by most client’s web browsers. ACME certificates are typically free.\n\n- [ACME Configuration](https://cert-manager.io/docs/configuration/acme/)\n- [ACME HTTP Validation](https://cert-manager.io/docs/tutorials/acme/http-validation/)\n - [HTTP01 Challenges](https://cert-manager.io/docs/configuration/acme/http01/)\n- [ACME DNS Validation](https://cert-manager.io/docs/tutorials/acme/dns-validation/)\n - [DNS01 Challenges](https://cert-manager.io/docs/configuration/acme/dns01/)\n- [ACME FAQ](https://cert-manager.io/docs/troubleshooting/acme/)\n\n#### ACME With An Internal Certificate Authority\n\nThe ACME Issuer with an internal certificate authority requires cert-manager to trust the certificate authority. This trust must be done at the cert-manager deployment level.\nTo add a trusted certificate authority to cert-manager, add it's certificate to `group_vars/k8s-cluster/addons.yml`:\n\n```yaml\ncert_manager_trusted_internal_ca: |\n -----BEGIN CERTIFICATE-----\n [REPLACE with your CA certificate]\n -----END CERTIFICATE-----\n```\n\nOnce the CA is trusted, you can define your issuer normally.\n\n### Create New TLS Root CA Certificate and Key\n\n#### Install Cloudflare PKI/TLS `cfssl` Toolkit\n\ne.g. For Ubuntu/Debian distributions, the toolkit is part of the `golang-cfssl` package.\n\n```shell\nsudo apt-get install -y golang-cfssl\n```\n\n#### Create Root Certificate Authority (CA) Configuration File\n\nThe default TLS certificate expiry time period is `8760h` which is 1 years from the date the certificate is created.\n\n```shell\n$ cat > ca-config.json < ca-csr.json < for details).\nThese are configurable in inventory in as a dictionary in the `dns_upstream_forward_extra_opts` variable.\nBy default, no other option than the ones hardcoded (see `roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2` and `roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2`).\n\n### coredns_kubernetes_extra_opts\n\nCustom options to be added to the kubernetes coredns plugin.\n\n### coredns_kubernetes_extra_domains\n\nExtra domains to be forwarded to the kubernetes coredns plugin.\n\n### coredns_additional_configs\n\nExtra configuration to be added to CoreDNS configuration\n\n### coredns_rewrite_block\n\n[Rewrite](https://coredns.io/plugins/rewrite/) plugin block to perform internal message rewriting.\n\n### coredns_external_zones\n\nArray of optional external zones to coredns forward queries to. It's injected into\n`coredns`' config file before default kubernetes zone. Use it as an optimization for well-known zones and/or internal-only\ndomains, i.e. VPN for internal networks (default is unset)\n\nExample:\n\n```yaml\ncoredns_external_zones:\n- zones:\n - example.com\n - example.io:1053\n nameservers:\n - 1.1.1.1\n - 2.2.2.2\n cache: 5\n- zones:\n - https://mycompany.local:4453\n nameservers:\n - 192.168.0.53\n cache: 0\n- zones:\n - mydomain.tld\n nameservers:\n - 10.233.0.3\n cache: 5\n rewrite:\n - name stop website.tld website.namespace.svc.cluster.local\n```\n\nor as INI\n\n```ini\ncoredns_external_zones='[{\"cache\": 30,\"zones\":[\"example.com\",\"example.io:453\"],\"nameservers\":[\"1.1.1.1\",\"2.2.2.2\"]}]'\n```\n\n### dns_etchosts (coredns)\n\nOptional hosts file content to coredns use as /etc/hosts file. This will also be used by nodelocaldns, if enabled.\n\nExample:\n\n```yaml\ndns_etchosts: |\n 192.168.0.100 api.example.com\n 192.168.0.200 ingress.example.com\n```\n\n### enable_coredns_reverse_dns_lookups\n\nWhether reverse DNS lookups are enabled in the coredns config. Defaults to `true`.\n\n### CoreDNS default zone cache plugin\n\nIf you wish to configure the caching behaviour of CoreDNS on the default zone, you can do so using the `coredns_default_zone_cache_block` string block.\n\nAn example value (more information on the [plugin's documentation](https://coredns.io/plugins/cache/)) to:\n\n* raise the max cache TTL to 3600 seconds\n* raise the max amount of success responses to cache to 3000\n* disable caching of denial responses altogether\n* enable pre-fetching of lookups with at least 10 lookups per minute before they expire\n\nWould be as follows:\n\n```yaml\ncoredns_default_zone_cache_block: |\n cache 3600 {\n success 3000\n denial 0\n prefetch 10 1m\n }\n```\n\n### Handle old/extra dns_domains\n\nIf you need to change the dns_domain of your cluster for whatever reason (switching to or from `cluster.local` for example),\nand you have workloads that embed it in their configuration you can use the variable `old_dns_domains`.\nThis will add some configuration to coredns and nodelocaldns to ensure the DNS requests using the old domain are handled correctly.\nExample:\n\n```yaml\nold_dns_domains:\n- example1.com\n- example2.com\ndns_domain: cluster.local\n```\n\nwill make `my-svc.my-ns.svc.example1.com`, `my-svc.my-ns.svc.example2.com` and `my-svc.my-ns.svc.cluster.local` have the same DNS answer.\n\n### systemd_resolved_disable_stub_listener\n\nWhether or not to set `DNSStubListener=no` when using systemd-resolved. Defaults to `true` on Flatcar.\nYou might need to set it to `true` if CoreDNS fails to start with `address already in use` errors.\n\n## DNS modes supported by Kubespray\n\nYou can modify how Kubespray sets up DNS for your cluster with the variables ``dns_mode`` and ``resolvconf_mode``.\n\n### dns_mode\n\n``dns_mode`` configures how Kubespray will setup cluster DNS. There are four modes available:\n\n#### dns_mode: coredns (default)\n\nThis installs CoreDNS as the default cluster DNS for all queries.\n\n#### dns_mode: coredns_dual\n\nThis installs CoreDNS as the default cluster DNS for all queries, plus a secondary CoreDNS stack.\n\n#### dns_mode: manual\n\nThis does not install coredns, but allows you to specify\n`manual_dns_server`, which will be configured on nodes for handling Pod DNS.\nUse this method if you plan to install your own DNS server in the cluster after\ninitial deployment.\n\n#### dns_mode: none\n\nThis does not install any of DNS solution at all. This basically disables cluster DNS completely and\nleaves you with a non functional cluster.\n\n## resolvconf_mode\n\n``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers.\nThere are three modes available:\n\n### resolvconf_mode: host_resolvconf (default)\n\nThis activates the classic Kubespray behavior that modifies the hosts ``/etc/resolv.conf`` file and dhclient\nconfiguration to point to the cluster dns server (either coredns or coredns_dual, depending on dns_mode).\n\nAs cluster DNS is not available on early deployment stage, this mode is split into 2 stages. In the first\nstage (``dns_early: true``), ``/etc/resolv.conf`` is configured to use the DNS servers found in ``upstream_dns_servers``\nand ``nameservers``. Later, ``/etc/resolv.conf`` is reconfigured to use the cluster DNS server first, leaving\nthe other nameservers as backups.\n\nAlso note, existing records will be purged from the `/etc/resolv.conf`,\nincluding resolvconf's base/head/cloud-init config files and those that come from dhclient.\n\n### resolvconf_mode: docker_dns\n\nThis sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags.\n\nThe following nameservers are added to the docker daemon (in the same order as listed here):\n\n* cluster nameserver (depends on dns_mode)\n* content of optional upstream_dns_servers variable\n* host system nameservers (read from hosts /etc/resolv.conf)\n\nThe following search domains are added to the docker daemon (in the same order as listed here):\n\n* cluster domains (``default.svc.{{ dns_domain }}``, ``svc.{{ dns_domain }}``)\n* content of optional searchdomains variable\n* host system search domains (read from hosts /etc/resolv.conf)\n\nThe following dns options are added to the docker daemon\n\n* ndots:{{ ndots }}\n* timeout:2\n* attempts:2\n\nThese dns options can be overridden by setting a different list:\n\n```yaml\ndocker_dns_options:\n- ndots:{{ ndots }}\n- timeout:2\n- attempts:2\n- rotate\n```\n\nFor normal PODs, k8s will ignore these options and setup its own DNS settings for the PODs, taking\nthe --cluster_dns (either coredns or coredns_dual, depending on dns_mode) kubelet option into account.\nFor ``hostNetwork: true`` PODs however, k8s will let docker setup DNS settings. Docker containers which\nare not started/managed by k8s will also use these docker options.\n\nThe host system name servers are added to ensure name resolution is also working while cluster DNS is not\nrunning yet. This is especially important in early stages of cluster deployment. In this early stage,\nDNS queries to the cluster DNS will timeout after a few seconds, resulting in the system nameserver being\nused as a backup nameserver. After cluster DNS is running, all queries will be answered by the cluster DNS\nservers, which in turn will forward queries to the system nameserver if required.\n\n### resolvconf_mode: none\n\nDoes nothing regarding ``/etc/resolv.conf``. This leaves you with a cluster that works as expected in most cases.\nThe only exception is that ``hostNetwork: true`` PODs and non-k8s managed containers will not be able to resolve\ncluster service names.\n\n## Nodelocal DNS cache\n\nSetting ``enable_nodelocaldns`` to ``true`` will make pods reach out to the dns (core-dns) caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query core-dns (depending on what main DNS plugin is configured in your cluster) for cache misses of cluster hostnames(cluster.local suffix by default).\n\nMore information on the rationale behind this implementation can be found [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/1024-nodelocal-cache-dns/README.md).\n\n**As per the 2.10 release, Nodelocal DNS cache is enabled by default.**\n\n### External zones\n\nIt's possible to extent the `nodelocaldns`' configuration by adding an array of external zones. For example:\n\n```yaml\nnodelocaldns_external_zones:\n- zones:\n - example.com\n - example.io:1053\n nameservers:\n - 1.1.1.1\n - 2.2.2.2\n cache: 5\n- zones:\n - https://mycompany.local:4453\n nameservers:\n - 192.168.0.53\n```\n\n### dns_etchosts (nodelocaldns)\n\nSee [dns_etchosts](#dns_etchosts-coredns) above.\n\n### nodelocaldns_additional_configs\n\nExtra configuration to be added to CoreDNS configuration\n\n### Nodelocal DNS HA\n\nUnder some circumstances the single POD nodelocaldns implementation may not be able to be replaced soon enough and a cluster upgrade or a nodelocaldns upgrade can cause DNS requests to time out for short intervals. If for any reason your applications cannot tolerate this behavior you can enable a redundant nodelocal DNS pod on each node:\n\n```yaml\nenable_nodelocaldns_secondary: true\n```\n\n**Note:** when the nodelocaldns secondary is enabled, the primary is instructed to no longer tear down the iptables rules it sets up to direct traffic to itself. In case both daemonsets have failing pods on the same node, this can cause a DNS blackout with traffic no longer being forwarded to the coredns central service as a fallback. Please ensure you account for this also if you decide to disable the nodelocaldns cache.\n\nThere is a time delta (in seconds) allowed for the secondary nodelocaldns to survive in case both primary and secondary daemonsets are updated at the same time. It is advised to tune this variable after you have performed some tests in your own environment.\n\n```yaml\nnodelocaldns_secondary_skew_seconds: 5\n```\n\n## Limitations\n\n* Kubespray has yet ways to configure Kubedns addon to forward requests SkyDns can\n not answer with authority to arbitrary recursive resolvers. This task is left\n for future. See [official SkyDns docs](https://github.com/skynetservices/skydns)\n for details.\n\n* There is\n [no way to specify a custom value](https://github.com/kubernetes/kubernetes/issues/33554)\n for the SkyDNS ``ndots`` param.\n\n* the ``searchdomains`` have a limitation of a 6 names and 256 chars\n length. Due to default ``svc, default.svc`` subdomains, the actual\n limits are a 4 names and 239 chars respectively. If `remove_default_searchdomains: true`\n added you are back to 6 names.\n\n* the ``nameservers`` have a limitation of a 3 servers, although there\n is a way to mitigate that with the ``upstream_dns_servers``,\n see below. Anyway, the ``nameservers`` can take no more than a two\n custom DNS servers because of one slot is reserved for a Kubernetes\n cluster needs.\n" }, { "path": "docs/advanced/downloads.md", "content": "# Downloading binaries and containers\n\nKubespray supports several download/upload modes. The default is:\n\n* Each node downloads binaries and container images on its own, which is ``download_run_once: False``.\n* For K8s apps, pull policy is ``k8s_image_pull_policy: IfNotPresent``.\n* For system managed containers, like kubelet or etcd, pull policy is ``download_always_pull: False``, which is pull if only the wanted repo and tag/sha256 digest differs from that the host has.\n\nThere is also a \"pull once, push many\" mode as well:\n\n* Setting ``download_run_once: True`` will make kubespray download container images and binaries only once and then push them to the cluster nodes. The default download delegate node is the first `kube_control_plane`.\n* Set ``download_localhost: True`` to make localhost the download delegate. This can be useful if cluster nodes cannot access external addresses. To use this requires that the container runtime is installed and running on the Ansible master and that the current user is either in the docker group or can do passwordless sudo, to be able to use the container runtime. Note: even if `download_localhost` is false, files will still be copied to the Ansible server (local host) from the delegated download node, and then distributed from the Ansible server to all cluster nodes.\n\nNOTE: When `download_run_once` is true and `download_localhost` is false, all downloads will be done on the delegate node, including downloads for container images that are not required on that node. As a consequence, the storage required on that node will probably be more than if download_run_once was false, because all images will be loaded into the storage of the container runtime on that node, instead of just the images required for that node.\n\nOn caching:\n\n* When `download_run_once` is `True`, all downloaded files will be cached locally in `download_cache_dir`, which defaults to `/tmp/kubespray_cache`. On subsequent provisioning runs, this local cache will be used to provision the nodes, minimizing bandwidth usage and improving provisioning time. Expect about 800MB of disk space to be used on the ansible node for the cache. Disk space required for the image cache on the kubernetes nodes is a much as is needed for the largest image, which is currently slightly less than 150MB.\n* By default, if `download_run_once` is false, kubespray will not retrieve the downloaded images and files from the download delegate node to the local cache, or use that cache to pre-provision those nodes. If you have a full cache with container images and files and you don’t need to download anything, but want to use a cache - set `download_force_cache` to `True`.\n* By default, cached images that are used to pre-provision the remote nodes will be deleted from the remote nodes after use, to save disk space. Setting `download_keep_remote_cache` will prevent the files from being deleted. This can be useful while developing kubespray, as it can decrease provisioning times. As a consequence, the required storage for images on the remote nodes will increase from 150MB to about 550MB, which is currently the combined size of all required container images.\n\nContainer images and binary files are described by the vars like ``foo_version``,\n``foo_download_url``, ``foo_checksum`` for binaries and ``foo_image_repo``,\n``foo_image_tag`` or optional ``foo_digest_checksum`` for containers.\n\nContainer images may be defined by its repo and tag, for example:\n`andyshinn/dnsmasq:2.72`. Or by repo and tag and sha256 digest:\n`andyshinn/dnsmasq@sha256:7c883354f6ea9876d176fe1d30132515478b2859d6fc0cbf9223ffdc09168193`.\n\nNote, the SHA256 digest and the image tag must be both specified and correspond\nto each other. The given example above is represented by the following vars:\n\n```yaml\ndnsmasq_digest_checksum: 7c883354f6ea9876d176fe1d30132515478b2859d6fc0cbf9223ffdc09168193\ndnsmasq_image_repo: andyshinn/dnsmasq\ndnsmasq_image_tag: '2.72'\n```\n\nThe full list of available vars may be found in the download's ansible role defaults. Those also allow to specify custom urls and local repositories for binaries and container\nimages as well. See also the DNS stack docs for the related intranet configuration,\nso the hosts can resolve those urls and repos.\n" }, { "path": "docs/advanced/gcp-lb.md", "content": "# GCP Load Balancers for type=LoadBalancer of Kubernetes Services\n\n> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)\n\nGoogle Cloud Platform can be used for creation of Kubernetes Service Load Balancer.\n\nThis feature is able to deliver by adding parameters to `kube-controller-manager` and `kubelet`. You need specify:\n\n```ShellSession\n --cloud-provider=gce\n --cloud-config=/etc/kubernetes/cloud-config\n```\n\nTo get working it in kubespray, you need to add tag to GCE instances and specify it in kubespray group vars and also set `cloud_provider` to `gce`. So for example, in file `group_vars/all/gcp.yml`:\n\n```yaml\n cloud_provider: gce\n gce_node_tags: k8s-lb\n```\n\nWhen you will setup it and create SVC in Kubernetes with `type=LoadBalancer`, cloud provider will create public IP and will set firewall.\nNote: Cloud provider run under VM service account, so this account needs to have correct permissions to be able to create all GCP resources.\n" }, { "path": "docs/advanced/kubernetes-reliability.md", "content": "# Overview\n\nDistributed system such as Kubernetes are designed to be resilient to the\nfailures. More details about Kubernetes High-Availability (HA) may be found at\n[Building High-Availability Clusters](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/)\n\nTo have a simple view the most of the parts of HA will be skipped to describe\nKubelet<->Controller Manager communication only.\n\nBy default the normal behavior looks like:\n\n1. Kubelet updates it status to apiserver periodically, as specified by\n `--node-status-update-frequency`. The default value is **10s**.\n\n2. Kubernetes controller manager checks the statuses of Kubelet every\n `–-node-monitor-period`. The default value is **5s**.\n\n3. In case the status is updated within `--node-monitor-grace-period` of time,\n Kubernetes controller manager considers healthy status of Kubelet. The\n default value is **40s**.\n\n> Kubernetes controller manager and Kubelet work asynchronously. It means that\n> the delay may include any network latency, API Server latency, etcd latency,\n> latency caused by load on one's control plane nodes and so on. So if\n> `--node-status-update-frequency` is set to 5s in reality it may appear in\n> etcd in 6-7 seconds or even longer when etcd cannot commit data to quorum\n> nodes.\n\n## Failure\n\nKubelet will try to make `nodeStatusUpdateRetry` post attempts. Currently\n`nodeStatusUpdateRetry` is constantly set to 5 in\n[kubelet.go](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet.go#L102).\n\nKubelet will try to update the status in\n[tryUpdateNodeStatus](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet_node_status.go#L312)\nfunction. Kubelet uses `http.Client()` Golang method, but has no specified\ntimeout. Thus there may be some glitches when API Server is overloaded while\nTCP connection is established.\n\nSo, there will be `nodeStatusUpdateRetry` * `--node-status-update-frequency`\nattempts to set a status of node.\n\nAt the same time Kubernetes controller manager will try to check\n`nodeStatusUpdateRetry` times every `--node-monitor-period` of time. After\n`--node-monitor-grace-period` it will consider node unhealthy. Pods will then be rescheduled based on the\n[Taint Based Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions)\ntimers that you set on them individually, or the API Server's global timers:`--default-not-ready-toleration-seconds` &\n``--default-unreachable-toleration-seconds``.\n\nKube proxy has a watcher over API. Once pods are evicted, Kube proxy will\nnotice and will update iptables of the node. It will remove endpoints from\nservices so pods from failed node won't be accessible anymore.\n\n## Recommendations for different cases\n\n## Fast Update and Fast Reaction\n\nIf `--node-status-update-frequency` is set to **4s** (10s is default).\n`--node-monitor-period` to **2s** (5s is default).\n`--node-monitor-grace-period` to **20s** (40s is default).\n`--default-not-ready-toleration-seconds` and ``--default-unreachable-toleration-seconds`` are set to **30**\n(300 seconds is default). Note these two values should be integers representing the number of seconds (\"s\" or \"m\" for\nseconds\\minutes are not specified).\n\nIn such scenario, pods will be evicted in **50s** because the node will be\nconsidered as down after **20s**, and `--default-not-ready-toleration-seconds` or\n``--default-unreachable-toleration-seconds`` occur after **30s** more. However, this scenario creates an overhead on\netcd as every node will try to update its status every 2 seconds.\n\nIf the environment has 1000 nodes, there will be 15000 node updates per\nminute which may require large etcd containers or even dedicated nodes for etcd.\n\n> If we calculate the number of tries, the division will give 5, but in reality\n> it will be from 3 to 5 with `nodeStatusUpdateRetry` attempts of each try. The\n> total number of attempts will vary from 15 to 25 due to latency of all\n> components.\n\n## Medium Update and Average Reaction\n\nLet's set `--node-status-update-frequency` to **20s**\n`--node-monitor-grace-period` to **2m** and `--default-not-ready-toleration-seconds` and\n``--default-unreachable-toleration-seconds`` to **60**.\nIn that case, Kubelet will try to update status every 20s. So, it will be 6 * 5\n= 30 attempts before Kubernetes controller manager will consider unhealthy\nstatus of node. After 1m it will evict all pods. The total time will be 3m\nbefore eviction process.\n\nSuch scenario is good for medium environments as 1000 nodes will require 3000\netcd updates per minute.\n\n> In reality, there will be from 4 to 6 node update tries. The total number of\n> of attempts will vary from 20 to 30.\n\n## Low Update and Slow reaction\n\nLet's set `--node-status-update-frequency` to **1m**.\n`--node-monitor-grace-period` will set to **5m** and `--default-not-ready-toleration-seconds` and\n``--default-unreachable-toleration-seconds`` to **60**. In this scenario, every kubelet will try to update the status\nevery minute. There will be 5 * 5 = 25 attempts before unhealthy status. After 5m,\nKubernetes controller manager will set unhealthy status. This means that pods\nwill be evicted after 1m after being marked unhealthy. (6m in total).\n\n> In reality, there will be from 3 to 5 tries. The total number of attempt will\n> vary from 15 to 25.\n\nThere can be different combinations such as Fast Update with Slow reaction to\nsatisfy specific cases.\n" }, { "path": "docs/advanced/netcheck.md", "content": "# Network Checker Application\n\nWith the ``deploy_netchecker`` var enabled (defaults to false), Kubespray deploys a\nNetwork Checker Application from the 3rd side `mirantis/k8s-netchecker` docker\nimages. It consists of the server and agents trying to reach the server by usual\nfor Kubernetes applications network connectivity meanings. Therefore, this\nautomatically verifies a pod to pod connectivity via the cluster IP and checks\nif DNS resolve is functioning as well.\n\nThe checks are run by agents on a periodic basis and cover standard and host network\npods as well. The history of performed checks may be found in the agents' application\nlogs.\n\nTo get the most recent and cluster-wide network connectivity report, run from\nany of the cluster nodes:\n\n```ShellSession\ncurl http://localhost:31081/api/v1/connectivity_check\n```\n\nNote that Kubespray does not invoke the check but only deploys the application, if\nrequested.\n\nThere are related application specific variables:\n\n```yml\nnetchecker_port: 31081\nagent_report_interval: 15\nnetcheck_namespace: default\n```\n\nNote that the application verifies DNS resolve for FQDNs comprising only the\ncombination of the ``netcheck_namespace.dns_domain`` vars, for example the\n``netchecker-service.default.svc.cluster.local``. If you want to deploy the application\nto the non default namespace, make sure as well to adjust the ``searchdomains`` var\nso the resulting search domain records to contain that namespace, like:\n\n```yml\nsearch: foospace.cluster.local default.cluster.local ...\nnameserver: ...\n```\n" }, { "path": "docs/advanced/ntp.md", "content": "# NTP synchronization\n\nThe Network Time Protocol (NTP) is a networking protocol for clock synchronization between computer systems. Time synchronization is important to Kubernetes and Etcd.\n\n## Enable the NTP\n\nTo start the ntpd(or chrony) service and enable it at system boot. There are related specific variables:\n\n```ShellSession\nntp_enabled: true\n```\n\nThe NTP service would be enabled and sync time automatically.\n\n## Customize the NTP configure file\n\nIn the Air-Gap environment, the node cannot access the NTP server by internet. So the node can use the customized ntp server by configuring ntp file.\n\n```ShellSession\nntp_enabled: true\nntp_manage_config: true\nntp_servers:\n - \"0.your-ntp-server.org iburst\"\n - \"1.your-ntp-server.org iburst\"\n - \"2.your-ntp-server.org iburst\"\n - \"3.your-ntp-server.org iburst\"\n```\n\n## Setting the TimeZone\n\nThe timezone can also be set by the `ntp_timezone` , eg: \"Etc/UTC\",\"Asia/Shanghai\". If not set, the timezone will not change.\n\n```ShellSession\nntp_enabled: true\nntp_timezone: Etc/UTC\n```\n\n## Advanced Configure\n\nEnable `tinker panic` is useful when running NTP in a VM environment to avoiding clock drift on VMs. It only takes effect when ntp_manage_config is true.\n\n```ShellSession\nntp_tinker_panic: true\n```\n\nForce sync time immediately by NTP after the ntp installed, which is useful in newly installed system.\n\n```ShellSession\nntp_force_sync_immediately: true\n```\n\nWhen using Ubuntu 24.04 or a distribution that already has `systemd-timesyncd` installed, use the `ntpsec` package.\n\n```ShellSession\nntp_package: ntpsec\n```\n" }, { "path": "docs/advanced/proxy.md", "content": "# Setting up Environment Proxy\n\nIf you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.\n\n## Set proxy for http and https\n\n `http_proxy:\"http://example.proxy.tld:port\"`\n `https_proxy:\"http://example.proxy.tld:port\"`\n\n## Set custom CA\n\nCA must be already on each target nodes\n\n `https_proxy_cert_file: /path/to/host/custom/ca.crt`\n\n## Set default no_proxy (this will override default no_proxy generation)\n\n`no_proxy: \"node1,node1_ip,node2,node2_ip...additional_host\"`\n\n## Set additional addresses to default no_proxy (all cluster nodes and loadbalancer)\n\n`additional_no_proxy: \"additional_host1,additional_host2\"`\n\n## Exclude workers from no_proxy\n\nSince workers are included in the no_proxy variable, by default, docker engine will be restarted on all nodes (all\npods will restart) when adding or removing workers. To override this behaviour by only including control plane nodes in the\nno_proxy variable, set:\n`no_proxy_exclude_workers: true`\n" }, { "path": "docs/advanced/registry.md", "content": "# Private Docker Registry in Kubernetes\n\nKubernetes offers an optional private Docker registry addon, which you can turn\non when you bring up a cluster or install later. This gives you a place to\nstore truly private Docker images for your cluster.\n\n## How it works\n\nThe private registry runs as a `Pod` in your cluster. It does not currently\nsupport SSL or authentication, which triggers Docker's \"insecure registry\"\nlogic. To work around this, we run a proxy on each node in the cluster,\nexposing a port onto the node (via a hostPort), which Docker accepts as\n\"secure\", since it is accessed by `localhost`.\n\n## Turning it on\n\nSome cluster installs (e.g. GCE) support this as a cluster-birth flag. The\n`ENABLE_CLUSTER_REGISTRY` variable in `cluster/gce/config-default.sh` governs\nwhether the registry is run or not. To set this flag, you can specify\n`KUBE_ENABLE_CLUSTER_REGISTRY=true` when running `kube-up.sh`. If your cluster\ndoes not include this flag, the following steps should work. Note that some of\nthis is cloud-provider specific, so you may have to customize it a bit.\n\n### Make some storage\n\nThe primary job of the registry is to store data. To do that we have to decide\nwhere to store it. For cloud environments that have networked storage, we can\nuse Kubernetes's `PersistentVolume` abstraction. The following template is\nexpanded by `salt` in the GCE cluster turnup, but can easily be adapted to\nother situations:\n\n```yaml\nkind: PersistentVolume\napiVersion: v1\nmetadata:\n name: kube-system-kube-registry-pv\nspec:\n{% if pillar.get('cluster_registry_disk_type', '') == 'gce' %}\n capacity:\n storage: {{ pillar['cluster_registry_disk_size'] }}\n accessModes:\n - ReadWriteOnce\n gcePersistentDisk:\n pdName: \"{{ pillar['cluster_registry_disk_name'] }}\"\n fsType: \"ext4\"\n{% endif %}\n```\n\nIf, for example, you wanted to use NFS you would just need to change the\n`gcePersistentDisk` block to `nfs`. See\n[here](https://kubernetes.io/docs/concepts/storage/volumes/) for more details on volumes.\n\nNote that in any case, the storage (in the case the GCE PersistentDisk) must be\ncreated independently - this is not something Kubernetes manages for you (yet).\n\n### I don't want or don't have persistent storage\n\nIf you are running in a place that doesn't have networked storage, or if you\njust want to kick the tires on this without committing to it, you can easily\nadapt the `ReplicationController` specification below to use a simple\n`emptyDir` volume instead of a `persistentVolumeClaim`.\n\n## Claim the storage\n\nNow that the Kubernetes cluster knows that some storage exists, you can put a\nclaim on that storage. As with the `PersistentVolume` above, you can start\nwith the `salt` template:\n\n```yaml\nkind: PersistentVolumeClaim\napiVersion: v1\nmetadata:\n name: kube-registry-pvc\n namespace: kube-system\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: {{ pillar['cluster_registry_disk_size'] }}\n```\n\nThis tells Kubernetes that you want to use storage, and the `PersistentVolume`\nyou created before will be bound to this claim (unless you have other\n`PersistentVolumes` in which case those might get bound instead). This claim\ngives you the right to use this storage until you release the claim.\n\n## Run the registry\n\nNow we can run a Docker registry:\n\n```yaml\napiVersion: v1\nkind: ReplicationController\nmetadata:\n name: kube-registry-v0\n namespace: kube-system\n labels:\n k8s-app: registry\n version: v0\nspec:\n replicas: 1\n selector:\n k8s-app: registry\n version: v0\n template:\n metadata:\n labels:\n k8s-app: registry\n version: v0\n spec:\n containers:\n - name: registry\n image: registry:2\n resources:\n limits:\n cpu: 100m\n memory: 100Mi\n env:\n - name: REGISTRY_HTTP_ADDR\n value: :5000\n - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY\n value: /var/lib/registry\n volumeMounts:\n - name: image-store\n mountPath: /var/lib/registry\n ports:\n - containerPort: 5000\n name: registry\n protocol: TCP\n volumes:\n - name: image-store\n persistentVolumeClaim:\n claimName: kube-registry-pvc\n```\n\n*Note:* that if you have set multiple replicas, make sure your CSI driver has support for the `ReadWriteMany` accessMode.\n\n## Expose the registry in the cluster\n\nNow that we have a registry `Pod` running, we can expose it as a Service:\n\n```yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: kube-registry\n namespace: kube-system\n labels:\n k8s-app: registry\n kubernetes.io/name: \"KubeRegistry\"\nspec:\n selector:\n k8s-app: registry\n ports:\n - name: registry\n port: 5000\n protocol: TCP\n```\n\n## Expose the registry on each node\n\nNow that we have a running `Service`, we need to expose it onto each Kubernetes\n`Node` so that Docker will see it as `localhost`. We can load a `Pod` on every\nnode by creating following daemonset.\n\n```yaml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: kube-registry-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-registry-proxy\n version: v0.4\nspec:\n template:\n metadata:\n labels:\n k8s-app: kube-registry-proxy\n kubernetes.io/name: \"kube-registry-proxy\"\n version: v0.4\n spec:\n containers:\n - name: kube-registry-proxy\n image: gcr.io/google_containers/kube-registry-proxy:0.4\n resources:\n limits:\n cpu: 100m\n memory: 50Mi\n env:\n - name: REGISTRY_HOST\n value: kube-registry.kube-system.svc.cluster.local\n - name: REGISTRY_PORT\n value: \"5000\"\n ports:\n - name: registry\n containerPort: 80\n hostPort: 5000\n```\n\nWhen modifying replication-controller, service and daemon-set definitions, take\ncare to ensure *unique* identifiers for the rc-svc couple and the daemon-set.\nFailing to do so will have register the localhost proxy daemon-sets to the\nupstream service. As a result they will then try to proxy themselves, which\nwill, for obvious reasons, not work.\n\nThis ensures that port 5000 on each node is directed to the registry `Service`.\nYou should be able to verify that it is running by hitting port 5000 with a web\nbrowser and getting a 404 error:\n\n```ShellSession\n$ curl localhost:5000\n404 page not found\n```\n\n## Using the registry\n\nTo use an image hosted by this registry, simply say this in your `Pod`'s\n`spec.containers[].image` field:\n\n```yaml\n image: localhost:5000/user/container\n```\n\nBefore you can use the registry, you have to be able to get images into it,\nthough. If you are building an image on your Kubernetes `Node`, you can spell\nout `localhost:5000` when you build and push. More likely, though, you are\nbuilding locally and want to push to your cluster.\n\nYou can use `kubectl` to set up a port-forward from your local node to a\nrunning Pod:\n\n```ShellSession\n$ POD=$(kubectl get pods --namespace kube-system -l k8s-app=registry \\\n -o template --template '{{range .items}}{{.metadata.name}} {{.status.phase}}{{\"\\n\"}}{{end}}' \\\n | grep Running | head -1 | cut -f1 -d' ')\n\n$ kubectl port-forward --namespace kube-system $POD 5000:5000 &\n```\n\nNow you can build and push images on your local computer as\n`localhost:5000/yourname/container` and those images will be available inside\nyour kubernetes cluster with the same name.\n" }, { "path": "docs/ansible/ansible.md", "content": "# Ansible\n\n## Installing Ansible\n\nKubespray supports multiple ansible versions and ships different `requirements.txt` files for them.\nDepending on your available python version you may be limited in choosing which ansible version to use.\n\nIt is recommended to deploy the ansible version used by kubespray into a python virtual environment.\n\n```ShellSession\nVENVDIR=kubespray-venv\nKUBESPRAYDIR=kubespray\npython3 -m venv $VENVDIR\nsource $VENVDIR/bin/activate\ncd $KUBESPRAYDIR\npip install -r requirements.txt\n```\n\nIn case you have a similar message when installing the requirements:\n\n```ShellSession\nERROR: Could not find a version that satisfies the requirement ansible==7.6.0 (from -r requirements.txt (line 1)) (from versions: [...], 6.7.0)\nERROR: No matching distribution found for ansible==7.6.0 (from -r requirements.txt (line 1))\n```\n\nIt means that the version of Python you are running is not compatible with the version of Ansible that Kubespray supports.\nIf the latest version supported according to pip is 6.7.0 it means you are running Python 3.8 or lower while you need at least Python 3.9 (see the table below).\n\n### Ansible Python Compatibility\n\nBased on the table below and the available python version for your ansible host you should choose the appropriate ansible version to use with kubespray.\n\n| Ansible Version | Python Version |\n|-------------------|----------------|\n| >=2.18.0, <2.19.0 | 3.11-3.13 |\n\n## Customize Ansible vars\n\nKubespray expects users to use one of the following variables sources for settings and customization:\n\n| Layer | Comment |\n|----------------------------------------|------------------------------------------------------------------------------|\n| inventory vars | |\n| - **inventory group_vars** | most used |\n| - inventory host_vars | host specific vars overrides, group_vars is usually more practical |\n| **extra vars** (always win precedence) | override with ``ansible-playbook -e @foo.yml`` |\n\n> Extra vars are best used to override kubespray internal variables, for instances, roles/vars/. Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray interface. Thus they can change, disappear, or break stuff unexpectedly.\n\n## Ansible tags\n\nThe following tags are defined in playbooks:\n\n| Tag name | Used for |\n|--------------------------------|-------------------------------------------------------|\n| annotate | Create kube-router annotation |\n| apps | K8s apps definitions |\n| asserts | Check tasks for download role |\n| aws-ebs-csi-driver | Configuring csi driver: aws-ebs |\n| azure-csi-driver | Configuring csi driver: azure |\n| bastion | Setup ssh config for bastion |\n| bootstrap_os | Anything related to host OS configuration |\n| calico | Network plugin Calico |\n| calico_rr | Configuring Calico route reflector |\n| cert-manager | Configuring certificate manager for K8s |\n| cilium | Network plugin Cilium |\n| cinder-csi-driver | Configuring csi driver: cinder |\n| client | Kubernetes clients role |\n| cloud-provider | Cloud-provider related tasks |\n| cluster-roles | Configuring cluster wide application (psp ...) |\n| cni | CNI plugins for Network Plugins |\n| containerd | Configuring containerd engine runtime for hosts |\n| container_engine_accelerator | Enable nvidia accelerator for runtimes |\n| container-engine | Configuring container engines |\n| container-runtimes | Configuring container runtimes |\n| control-plane | Configuring K8s control plane node role |\n| coredns | Configuring coredns deployment |\n| crio | Configuring crio container engine for hosts |\n| crun | Configuring crun runtime |\n| csi-driver | Configuring csi driver |\n| dns | Remove dns entries when resetting |\n| docker | Configuring docker engine runtime for hosts |\n| download | Fetching container images to a delegate host |\n| etcd | Configuring etcd cluster |\n| etcd-secrets | Configuring etcd certs/keys |\n| etchosts | Configuring /etc/hosts entries for hosts |\n| external-cloud-controller | Configure cloud controllers |\n| external-openstack | Cloud controller : openstack |\n| external-provisioner | Configure external provisioners |\n| external-vsphere | Cloud controller : vsphere |\n| facts | Gathering facts and misc check results |\n| files | Remove files when resetting |\n| flannel | Network plugin flannel |\n| gce | Cloud-provider GCP |\n| gcp-pd-csi-driver | Configuring csi driver: gcp-pd |\n| gvisor | Configuring gvisor runtime |\n| helm | Installing and configuring Helm |\n| ingress-controller | Configure ingress controllers |\n| ingress_alb | AWS ALB Ingress Controller |\n| init | Windows kubernetes init nodes |\n| iptables | Flush and clear iptable when resetting |\n| k8s-pre-upgrade | Upgrading K8s cluster |\n| kata-containers | Configuring kata-containers runtime |\n| kubeadm | Roles linked to kubeadm tasks |\n| kube-apiserver | Configuring static pod kube-apiserver |\n| kube-controller-manager | Configuring static pod kube-controller-manager |\n| kube-vip | Installing and configuring kube-vip |\n| kubectl | Installing kubectl and bash completion |\n| kubelet | Configuring kubelet service |\n| kube-ovn | Network plugin kube-ovn |\n| kube-router | Network plugin kube-router |\n| kube-proxy | Configuring static pod kube-proxy |\n| localhost | Special steps for the localhost (ansible runner) |\n| local-path-provisioner | Configure External provisioner: local-path |\n| local-volume-provisioner | Configure External provisioner: local-volume |\n| macvlan | Network plugin macvlan |\n| metallb | Installing and configuring metallb |\n| metrics_server | Configuring metrics_server |\n| netchecker | Installing netchecker K8s app |\n| network | Configuring networking plugins for K8s |\n| mounts | Umount kubelet dirs when resetting |\n| multus | Network plugin multus |\n| nginx | Configuring LB for kube-apiserver instances |\n| node | Configuring K8s minion (compute) node role |\n| nodelocaldns | Configuring nodelocaldns daemonset |\n| node-label | Tasks linked to labeling of nodes |\n| node-webhook | Tasks linked to webhook (granting access to resources)|\n| nvidia_gpu | Enable nvidia accelerator for runtimes |\n| oci | Cloud provider: oci |\n| persistent_volumes | Configure csi volumes |\n| persistent_volumes_aws_ebs_csi | Configuring csi driver: aws-ebs |\n| persistent_volumes_cinder_csi | Configuring csi driver: cinder |\n| persistent_volumes_gcp_pd_csi | Configuring csi driver: gcp-pd |\n| persistent_volumes_openstack | Configuring csi driver: openstack |\n| policy-controller | Configuring Calico policy controller |\n| post-remove | Tasks running post-remove operation |\n| post-upgrade | Tasks running post-upgrade operation |\n| pre-remove | Tasks running pre-remove operation |\n| pre-upgrade | Tasks running pre-upgrade operation |\n| preinstall | Preliminary configuration steps |\n| registry | Configuring local docker registry |\n| reset | Tasks running doing the node reset |\n| resolvconf | Configuring /etc/resolv.conf for hosts/apps |\n| services | Remove services (etcd, kubelet etc...) when resetting |\n| snapshot | Enabling csi snapshot |\n| snapshot-controller | Configuring csi snapshot controller |\n| system-packages | Install packages using OS package manager |\n| upgrade | Upgrading, f.e. container images/binaries |\n| upload | Distributing images/binaries across hosts |\n| vsphere-csi-driver | Configuring csi driver: vsphere |\n| win_nodes | Running windows specific tasks |\n| youki | Configuring youki runtime |\n\n## Example commands\n\nExample command to filter and apply only DNS configuration tasks and skip\neverything else related to host OS configuration and downloading images of containers:\n\n```ShellSession\nansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os\n```\n\nAnd this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:\n\n```ShellSession\nansible-playbook -i inventory/sample/hosts.ini -e dns_mode='none' cluster.yml --tags resolvconf\n```\n\nAnd this prepares all container images locally (at the ansible runner node) without installing\nor upgrading related stuff or trying to upload container to K8s cluster nodes:\n\n```ShellSession\nansible-playbook -i inventory/sample/hosts.ini cluster.yml \\\n -e download_run_once=true -e download_localhost=true \\\n --tags download --skip-tags upload,upgrade\n```\n\nNote: use `--tags` and `--skip-tags` wisely and only if you're 100% sure what you're doing.\n\n## Troubleshooting Ansible issues\n\nHaving the wrong version of ansible, ansible collections or python dependencies can cause issue.\nIn particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRARY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)\n\n```ShellSession\nexport ANSIBLE_LIBRARY=/library`\n```\n\nA simple way to ensure you get all the correct version of Ansible is to use\nthe [pre-built docker image from Quay](https://quay.io/repository/kubespray/kubespray?tab=tags).\nYou will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/)\nto access the inventory and SSH key in the container, like this:\n\n```ShellSession\ngit checkout v2.30.0\ndocker pull quay.io/kubespray/kubespray:v2.30.0\ndocker run --rm -it --mount type=bind,source=\"$(pwd)\"/inventory/sample,dst=/inventory \\\n --mount type=bind,source=\"${HOME}\"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \\\n quay.io/kubespray/kubespray:v2.30.0 bash\n# Inside the container you may now run the kubespray playbooks:\nansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml\n```\n" }, { "path": "docs/ansible/ansible_collection.md", "content": "# Ansible collection\n\nKubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html).\n\n## Usage\n\n1. Set up an inventory with the appropriate host groups and required group vars.\n See also the documentation on [kubespray inventories](./inventory.md) and the\n general [\"Getting started\" documentation](../getting_started/getting-started.md#building-your-own-inventory).\n\n2. Add Kubespray to your requirements.yml file\n\n ```yaml\n collections:\n - name: https://github.com/kubernetes-sigs/kubespray\n type: git\n version: master # use the appropriate tag or branch for the version you need\n ```\n\n3. Install your collection\n\n ```ShellSession\n ansible-galaxy install -r requirements.yml\n ```\n\n4. Create a playbook to install your Kubernetes cluster\n\n ```yaml\n - name: Install Kubernetes\n ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster\n ```\n\n5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray\n\n ```ShellSession\n ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK\n ```\n" }, { "path": "docs/ansible/inventory.md", "content": "# Inventory\n\nThe inventory is composed of 3 groups:\n\n* **kube_node** : list of kubernetes nodes where the pods will run.\n* **kube_control_plane** : list of servers where kubernetes control plane components (apiserver, scheduler, controller) will run.\n* **etcd**: list of servers to compose the etcd server. You should have at least 3 servers for failover purpose.\n\nWhen _kube_node_ contains _etcd_, you define your etcd cluster to be as well schedulable for Kubernetes workloads.\nIf you want it a standalone, make sure those groups do not intersect.\nIf you want the server to act both as control-plane and node, the server must be defined\non both groups _kube_control_plane_ and _kube_node_. If you want a standalone and\nunschedulable control plane, the server must be defined only in the _kube_control_plane_ and\nnot _kube_node_.\n\nThere are also two special groups:\n\n* **calico_rr** : explained for [advanced Calico networking cases](/docs/CNI/calico.md)\n* **bastion** : configure a bastion host if your nodes are not directly reachable\n\nLastly, the **k8s_cluster** is dynamically defined as the union of **kube_node**, **kube_control_plane** and **calico_rr**.\nThis is used internally and for the purpose of defining whole cluster variables (`/group_vars/k8s_cluster/*.yml`)\n\nBelow is a complete inventory example:\n\n```ini\n## Configure 'ip' variable to bind kubernetes services on a\n## different ip than the default iface\nnode1 ansible_host=95.54.0.12 ip=10.3.0.1\nnode2 ansible_host=95.54.0.13 ip=10.3.0.2\nnode3 ansible_host=95.54.0.14 ip=10.3.0.3\nnode4 ansible_host=95.54.0.15 ip=10.3.0.4\nnode5 ansible_host=95.54.0.16 ip=10.3.0.5\nnode6 ansible_host=95.54.0.17 ip=10.3.0.6\n\n[kube_control_plane]\nnode1\nnode2\n\n[etcd]\nnode1\nnode2\nnode3\n\n[kube_node]\nnode2\nnode3\nnode4\nnode5\nnode6\n```\n\n## Inventory customization\n\nSee [Customize Ansible vars](/docs/ansible/ansible.md#customize-ansible-vars)\nand [Ansible documentation on group_vars](https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html#assigning-a-variable-to-many-machines-group-variables)\n\n## Bastion host\n\nIf you prefer to not make your nodes publicly accessible (nodes with private IPs only),\nyou can use a so-called _bastion_ host to connect to your nodes. To specify and use a bastion,\nsimply add a line to your inventory, where you have to replace x.x.x.x with the public IP of the\nbastion host.\n\n```ShellSession\n[bastion]\nbastion ansible_host=x.x.x.x\n```\n\nFor more information about Ansible and bastion hosts, read\n[Running Ansible Through an SSH Bastion Host](https://blog.scottlowe.org/2015/12/24/running-ansible-through-ssh-bastion-host/)\n" }, { "path": "docs/ansible/vars.md", "content": "# Configurable Parameters in Kubespray\n\n## Generic Ansible variables\n\nYou can view facts gathered by Ansible automatically\n[here](https://docs.ansible.com/ansible/latest/user_guide/playbooks_vars_facts.html#ansible-facts).\n\nSome variables of note include:\n\n* *ansible_user*: user to connect to via SSH\n* *ansible_default_ipv4.address*: IP address Ansible automatically chooses.\n Generated based on the output from the command ``ip -4 route get 8.8.8.8``\n\n## Common vars that are used in Kubespray\n\n* *calico_version* - Specify version of Calico to use\n* *calico_cni_version* - Specify version of Calico CNI plugin to use\n* *docker_version* - Specify version of Docker to use (should be quoted\n string). Must match one of the keys defined for *docker_versioned_pkg*\n in `roles/container-engine/docker/vars/*.yml`.\n* *containerd_version* - Specify version of containerd to use when setting `container_manager` to `containerd`\n* *docker_containerd_version* - Specify which version of containerd to use when setting `container_manager` to `docker`\n* *etcd_version* - Specify version of ETCD to use\n* *calico_ipip_mode* - Configures Calico ipip encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Never')\n* *calico_vxlan_mode* - Configures Calico vxlan encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Always')\n* *calico_network_backend* - Configures Calico network backend - valid values are 'none', 'bird' and 'vxlan' (default 'vxlan')\n* *kube_network_plugin* - Sets k8s network plugin (default Calico)\n* *kube_proxy_mode* - Changes k8s proxy mode to iptables, ipvs, nftables mode\n* *kube_version* - Specify a given Kubernetes version\n* *searchdomains* - Array of DNS domains to search when looking up hostnames\n* *remove_default_searchdomains* - Boolean that removes the default searchdomain\n* *nameservers* - Array of nameservers to use for DNS lookup\n* *preinstall_selinux_state* - Set selinux state, permitted values are permissive, enforcing and disabled.\n\n## Addressing variables\n\n* *ip* - IP to use for binding services (host var). This would **usually** be the public ip.\n* *access_ip* - IP to use from other hosts to connect to this host. Often required when deploying\n from a cloud, such as OpenStack or GCE and you have separate public/floating and private IPs.\n This would **usually** be the private ip.\n* *ansible_default_ipv4.address* - Not Kubespray-specific, but it is used if ip\n and access_ip are undefined\n* *ip6* - IPv6 address to use for binding services. (host var)\n If *ipv6_stack*(*enable_dual_stack_networks* deprecated) is set to ``true`` and *ip6* is defined,\n kubelet's ``--node-ip`` and node's ``InternalIP`` will be the combination of *ip* and *ip6*.\n Similarly used for ipv6only scheme.\n* *access_ip6* - similarly ``access_ip`` but IPv6\n* *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6\n and access_ip6 are undefined\n* *loadbalancer_apiserver* - If defined, all hosts will connect to this\n address instead of localhost for kube_control_planes and kube_control_plane[0] for\n kube_nodes. See more details in the\n [HA guide](/docs/operations/ha-mode.md).\n* *loadbalancer_apiserver_localhost* - makes all hosts to connect to\n the apiserver internally load balanced endpoint. Mutual exclusive to the\n `loadbalancer_apiserver`. See more details in the\n [HA guide](/docs/operations/ha-mode.md).\n\n## Special network variables\n\nThese variables help avoid a large number of if/else constructs throughout the code associated with enabling different network stack.\nThese variables are used in all templates.\nBy default, only ipv4_stack is enabled, so it is given priority in dualstack mode.\nDon't change these variables if you don't understand what you're doing.\n\n* *main_access_ip* - equal to ``access_ip`` when ipv4_stack is enabled(even in case of dualstack),\n and ``access_ip6`` for IPv6 only clusters\n* *main_ip* - equal to ``ip`` when ipv4_stack is enabled(even in case of dualstack),\n and ``ip6`` for IPv6 only clusters\n* *main_access_ips* - list of ``access_ip`` and ``access_ip6`` for dualstack and one corresponding variable for single\n* *main_ips* - list of ``ip`` and ``ip6`` for dualstack and one corresponding variable for single\n\n## Cluster variables\n\nKubernetes needs some parameters in order to get deployed. These are the\nfollowing default cluster parameters:\n\n* *cluster_name* - Name of cluster (default is cluster.local)\n\n* *container_manager* - Container Runtime to install in the nodes (default is containerd)\n\n* *image_command_tool* - Tool used to pull images (default depends on `container_manager`\n and is `nerdctl` for `containerd`, `crictl` for `crio`, `docker` for `docker`)\n\n* *image_command_tool_on_localhost* - Tool used to pull images on localhost\n (default is equal to `image_command_tool`)\n\n* *dns_domain* - Name of cluster DNS domain (default is cluster.local)\n\n* *kube_network_plugin* - Plugin to use for container networking\n\n* *kube_service_addresses* - Subnet for cluster IPs (default is\n 10.233.0.0/18). Must not overlap with kube_pods_subnet\n\n* *kube_pods_subnet* - Subnet for Pod IPs (default is 10.233.64.0/18). Must not\n overlap with kube_service_addresses.\n\n* *kube_network_node_prefix* - Subnet allocated per-node for pod IPs. Remaining\n bits in kube_pods_subnet dictates how many kube_nodes can be in cluster. Setting this > 25 will\n raise an assertion in playbooks if the `kubelet_max_pods` var also isn't adjusted accordingly\n (assertion not applicable to calico which doesn't use this as a hard limit, see\n [Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes)).\n\n* *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.\n\n* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),\n for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).\n It is not recommended to change this variable directly.\n\n* *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.\n\n* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),\n for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).\n It is not recommended to change this variable directly.\n\n* *kube_network_node_prefix_ipv6* - Subnet allocated per-node for pod IPv6 IPs. Remaining bits in ``kube_pods_subnet_ipv6`` dictates how many kube_nodes can be in cluster.\n\n* *skydns_server* - Cluster IP for DNS (default is 10.233.0.3)\n\n* *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4)\n\n* *enable_coredns_k8s_external* - If enabled, it configures the [k8s_external plugin](https://coredns.io/plugins/k8s_external/)\n on the CoreDNS service.\n\n* *coredns_k8s_external_zone* - Zone that will be used when CoreDNS k8s_external plugin is enabled\n (default is k8s_external.local)\n\n* *enable_coredns_k8s_endpoint_pod_names* - If enabled, it configures endpoint_pod_names option for kubernetes plugin.\n on the CoreDNS service.\n\n* *cloud_provider* - The provider for cloud services. (default is unset, Set to `external` for running with an external cloud provider)\n\n* *kube_feature_gates* - A list of key=value pairs that describe feature gates for\n alpha/experimental Kubernetes features. (defaults is `[]`).\n Additionally, you can use also the following variables to individually customize your kubernetes components installation (they works exactly like `kube_feature_gates`):\n * *kube_apiserver_feature_gates*\n * *kube_controller_feature_gates*\n * *kube_scheduler_feature_gates*\n * *kube_proxy_feature_gates*\n * *kubelet_feature_gates*\n\n* *kubeadm_feature_gates* - A list of key=value pairs that describe feature gates for\n alpha/experimental Kubeadm features. (defaults is `[]`)\n\n* *authorization_modes* - A list of [authorization mode](\n https://kubernetes.io/docs/reference/access-authn-authz/authorization/#using-flags-for-your-authorization-module)\n that the cluster should be configured for. Defaults to `['Node', 'RBAC']`\n (Node and RBAC authorizers).\n Note: `Node` and `RBAC` are enabled by default. Previously deployed clusters can be\n converted to RBAC mode. However, your apps which rely on Kubernetes API will\n require a service account and cluster role bindings. You can override this\n setting by setting authorization_modes to `[]`.\n\n* *kube_apiserver_admission_control_config_file* - Enable configuration for `kube-apiserver` admission plugins.\n Currently this variable allow you to configure the `EventRateLimit` admission plugin.\n\n To configure the **EventRateLimit** plugin you have to define a data structure like this:\n\n```yml\nkube_apiserver_admission_event_rate_limits:\n limit_1:\n type: Namespace\n qps: 50\n burst: 100\n cache_size: 2000\n limit_2:\n type: User\n qps: 50\n burst: 100\n ...\n```\n\n* *kube_apiserver_service_account_lookup* - Enable validation service account before validating token. Default `true`.\n\nNote, if cloud providers have any use of the ``10.233.0.0/16``, like instances'\nprivate addresses, make sure to pick another values for ``kube_service_addresses``\nand ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.\n\n## Enabling Dual Stack (IPV4 + IPV6) or IPV6 only networking\n\nIPv4 stack enable by *ipv4_stack* is set to ``true``, by default.\nIPv6 stack enable by *ipv6_stack* is set to ``false`` by default.\nThis will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.\nSet both variables to ``true`` for Dual Stack mode.\nIPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).\nYou can also make IPv6 only clusters with ``false`` in *ipv4_stack*.\n\n## DNS variables\n\nBy default, hosts are set up with 8.8.8.8 as an upstream DNS server and all\nother settings from your existing /etc/resolv.conf are lost. Set the following\nvariables to match your requirements.\n\n* *upstream_dns_servers* - Array of upstream DNS servers configured on host in\n addition to Kubespray deployed DNS\n* *nameservers* - Array of DNS servers configured for use by hosts\n* *searchdomains* - Array of up to 4 search domains\n* *remove_default_searchdomains* - Boolean. If enabled, `searchdomains` variable can hold 6 search domains.\n* *dns_etchosts* - Content of hosts file for coredns and nodelocaldns\n* *dns_upstream_forward_extra_opts* - Options to add in the forward section of coredns/nodelocaldns related to upstream DNS servers\n\nFor more information, see [DNS\nStack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/advanced/dns-stack.md).\n\n## Other service variables\n\n* *docker_options* - Commonly used to set\n ``--insecure-registry=myregistry.mydomain:5000``\n\n* *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install.\n\n* *containerd_default_runtime* - If defined, changes the default Containerd runtime used by the Kubernetes CRI plugin.\n\n* *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin.\n [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overridden in inventory vars.\n\n* *crio_criu_support_enabled* - When set to `true`, enables the container checkpoint/restore in CRI-O. It's required to install [CRIU](https://criu.org/Installation) on the host when dumping/restoring checkpoints. And it's recommended to enable the feature gate `ContainerCheckpoint` so that the kubelet get a higher level API to simplify the operations (**Note**: It's still in experimental stage, just for container analytics so far). You can follow the [documentation](https://kubernetes.io/blog/2022/12/05/forensic-container-checkpointing-alpha/).\n\n* *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a\n proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames\n that correspond to each node.\n\n* *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet.\n By default autodetection is used to match container manager configuration.\n `systemd` is the preferred driver for `containerd` though it can have issues with `cgroups v1` and `kata-containers` in which case you may want to change to `cgroupfs`.\n\n* *kubelet_rotate_certificates* - Auto rotate the kubelet client certificates by requesting new certificates\n from the kube-apiserver when the certificate expiration approaches.\n\n* *kubelet_rotate_server_certificates* - Auto rotate the kubelet server certificates by requesting new certificates\n from the kube-apiserver when the certificate expiration approaches.\n Note that enabling this also activates *kubelet_csr_approver* which approves automatically the CSRs.\n To customize its behavior, you can override the Helm values via *kubelet_csr_approver_values*.\n See [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver) for more information.\n\n* *kubelet_streaming_connection_idle_timeout* - Set the maximum time a streaming connection can be idle before the connection is automatically closed.\n\n* *kubelet_image_gc_high_threshold* - Set the percent of disk usage after which image garbage collection is always run.\n The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive.\n When specified, the value must be greater than imageGCLowThresholdPercent. Default: 85\n\n* *kubelet_image_gc_low_threshold* - Set the percent of disk usage before which image garbage collection is never run.\n Lowest disk usage to garbage collect to.\n The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive.\n When specified, the value must be less than imageGCHighThresholdPercent. Default: 80\n\n* *kubelet_max_parallel_image_pulls* - Sets the maximum number of image pulls in parallel. The value is `1` by default which means the default is serial image pulling, set it to a integer great than `1` to enable image pulling in parallel.\n\n* *kubelet_make_iptables_util_chains* - If `true`, causes the kubelet ensures a set of `iptables` rules are present on host.\n\n* *kubelet_cpu_manager_policy* - If set to `static`, allows pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node. And it should be set with `kube_reserved` or `system-reserved`, enable this with the following guide:[Control CPU Management Policies on the Node](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/)\n\n* *kubelet_cpu_manager_policy_options* - A dictionary of cpuManagerPolicyOptions to enable. Keep in mind to enable the corresponding feature gates and make sure to pass the booleans as string (i.e. don't forget the quotes)!\n\n```yml\nkubelet_cpu_manager_policy_options:\n distribute-cpus-across-numa: \"true\"\n full-pcpus-only: \"true\"\n```\n\n* *kubelet_topology_manager_policy* - Control the behavior of the allocation of CPU and Memory from different [NUMA](https://en.wikipedia.org/wiki/Non-uniform_memory_access) Nodes. Enable this with the following guide: [Control Topology Management Policies on a node](https://kubernetes.io/docs/tasks/administer-cluster/topology-manager).\n\n* *kubelet_topology_manager_scope* - The Topology Manager can deal with the alignment of resources in a couple of distinct scopes: `container` and `pod`. See [Topology Manager Scopes](https://kubernetes.io/docs/tasks/administer-cluster/topology-manager/#topology-manager-scopes).\n\n* *kubelet_systemd_hardening* - If `true`, provides kubelet systemd service with security features for isolation.\n\n **N.B.** To enable this feature, ensure you are using the **`cgroup v2`** on your system. Check it out with command: `sudo ls -l /sys/fs/cgroup/*.slice`. If directory does not exist, enable this with the following guide: [enable cgroup v2](https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cgroup-v2).\n\n * *kubelet_secure_addresses* - By default *kubelet_systemd_hardening* set the **control plane** `ansible_host` IPs as the `kubelet_secure_addresses`. In case you have multiple interfaces in your control plane nodes and the `kube-apiserver` is not bound to the default interface, you can override them with this variable.\n Example:\n\n The **control plane** node may have 2 interfaces with the following IP addresses: `eth0:10.0.0.110`, `eth1:192.168.1.110`.\n\n By default the `kubelet_secure_addresses` is set with the `10.0.0.110` the ansible control host uses `eth0` to connect to the machine. In case you want to use `eth1` as the outgoing interface on which `kube-apiserver` connects to the `kubelet`s, you should override the variable in this way: `kubelet_secure_addresses: \"192.168.1.110\"`.\n\n* *kubelet_systemd_wants_dependencies* - List of kubelet service dependencies, other than container runtime.\n\n If you use nfs dynamically mounted volumes, sometimes rpc-statd does not start within the kubelet. You can fix it with this parameter : `kubelet_systemd_wants_dependencies: [\"rpc-statd.service\"]` This will add `Wants=rpc-statd.service` in `[Unit]` section of /etc/systemd/system/kubelet.service\n\n* *node_labels* - Labels applied to nodes via `kubectl label node`.\n For example, labels can be set in the inventory as variables or more widely in group_vars.\n *node_labels* can only be defined as a dict:\n\n```yml\nnode_labels:\n label1_name: label1_value\n label2_name: label2_value\n```\n\n* *node_taints* - Taints applied to nodes via `kubectl taint node`.\n For example, taints can be set in the inventory as variables or more widely in group_vars.\n *node_taints* has to be defined as a list of strings in format `key=value:effect`, e.g.:\n\n```yml\nnode_taints:\n - \"node.example.com/external=true:NoSchedule\"\n```\n\n* *kubernetes_audit* - When set to `true`, enables Auditing.\n The auditing parameters can be tuned via the following variables (which default values are shown below):\n * `audit_log_path`: /var/log/audit/kube-apiserver-audit.log\n * `audit_log_maxage`: 30\n * `audit_log_maxbackups`: 10\n * `audit_log_maxsize`: 100\n * `audit_policy_file`: \"{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml\"\n\n By default, the `audit_policy_file` contains [default rules](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2) that can be overridden with the `audit_policy_custom_rules` variable.\n* *kubernetes_audit_webhook* - When set to `true`, enables the webhook audit backend.\n The webhook parameters can be tuned via the following variables (which default values are shown below):\n * `audit_webhook_config_file`: \"{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml\"\n * `audit_webhook_server_url`: `\"https://audit.app\"`\n * `audit_webhook_server_extra_args`: {}\n * `audit_webhook_mode`: batch\n * `audit_webhook_batch_max_size`: 100\n * `audit_webhook_batch_max_wait`: 1s\n* *kubectl_alias* - Bash alias of kubectl to interact with Kubernetes cluster much easier.\n\n* *remove_anonymous_access* - When set to `true`, removes the `kubeadm:bootstrap-signer-clusterinfo` rolebinding created by kubeadm.\n By default, kubeadm creates a rolebinding in the `kube-public` namespace which grants permissions to anonymous users. This rolebinding allows kubeadm to discover and validate cluster information during the join phase.\n In a nutshell, this option removes the rolebinding after the init phase of the first control plane node and then configures kubeadm to use file discovery for the join phase of other nodes.\n This option does not remove the anonymous authentication feature of the API server.\n\n### Custom flags for Kube Components\n\nFor all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments.\n\nExtra flags for the kubelet can be specified using these variables, in the form of dicts of key-value pairs of\nconfiguration parameters that will be inserted into the kubelet YAML config file. Example:\n\n```yml\nkubelet_config_extra_args:\n evictionHard:\n memory.available: \"100Mi\"\n evictionSoftGracePeriod:\n memory.available: \"30s\"\n evictionSoft:\n memory.available: \"300Mi\"\n```\n\nThe possible vars are:\n\n* *kubelet_config_extra_args*\n\nPreviously, the same parameters could be passed as flags to kubelet binary with the following vars:\n\n* *kubelet_custom_flags*\n\n```yml\nkubelet_custom_flags:\n - \"--eviction-hard=memory.available<100Mi\"\n - \"--eviction-soft-grace-period=memory.available=30s\"\n - \"--eviction-soft=memory.available<300Mi\"\n```\n\nThis alternative is deprecated and will remain until the flags are completely removed from kubelet\n\nExtra flags for the API server, controller, and scheduler components can be specified using these variables,\nin the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubeadm YAML config file:\n\n* *kube_kubeadm_apiserver_extra_args*\n* *kube_kubeadm_controller_extra_args*\n* *kube_kubeadm_scheduler_extra_args*\n\n### Kubeadm patches\n\nWhen extra flags are not sufficient and there is a need to further customize kubernetes components,\n[kubeadm patches](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches)\ncan be used.\nYou should use the [`kubeadm_patches` variable](../../roles/kubernetes/kubeadm_common/defaults/main.yml) for that purpose.\n\n## App variables\n\n* *helm_version* - Only supports v3.x. Existing v2 installs (with Tiller) will not be modified and need to be removed manually.\n" }, { "path": "docs/calico_peer_example/new-york.yml", "content": "# ---\n# peers:\n# - router_id: \"10.99.0.34\"\n# as: \"65xxx\"\n# filters: []\n# numallowedlocalasnumbers: 0\n# sourceaddress: \"None\"\n# - router_id: \"10.99.0.35\"\n# as: \"65xxx\"\n# filters: []\n# numallowedlocalasnumbers: 0\n# sourceaddress: \"None\"\n\n# loadbalancer_apiserver:\n# address: \"10.99.0.44\"\n# port: \"8383\"\n" }, { "path": "docs/calico_peer_example/paris.yml", "content": "# ---\n# peers:\n# - router_id: \"10.99.0.2\"\n# as: \"65xxx\"\n# filters: []\n# numallowedlocalasnumbers: 0\n# sourceaddress: \"None\"\n# - router_id: \"10.99.0.3\"\n# as: \"65xxx\"\n# filters: []\n# numallowedlocalasnumbers: 0\n# sourceaddress: \"None\"\n\n# loadbalancer_apiserver:\n# address: \"10.99.0.21\"\n# port: \"8383\"\n" }, { "path": "docs/cloud_controllers/openstack.md", "content": "# OpenStack\n\n## Known compatible public clouds\n\nKubespray has been tested on a number of OpenStack Public Clouds including (in alphabetical order):\n\n- [Auro](https://auro.io/)\n- [Betacloud](https://www.betacloud.io/)\n- [CityCloud](https://www.citycloud.com/)\n- [DreamHost](https://www.dreamhost.com/cloud/computing/)\n- [ELASTX](https://elastx.se/)\n- [EnterCloudSuite](https://www.entercloudsuite.com/)\n- [FugaCloud](https://fuga.cloud/)\n- [Infomaniak](https://infomaniak.com)\n- [Open Telekom Cloud](https://cloud.telekom.de/) : requires to set the variable `wait_for_floatingip = \"true\"` in your cluster.tfvars\n- [OVHcloud](https://www.ovhcloud.com/)\n- [Rackspace](https://www.rackspace.com/)\n- [Ultimum](https://ultimum.io/)\n- [VexxHost](https://vexxhost.com/)\n- [Zetta](https://www.zetta.io/)\n\n## The OpenStack cloud provider\n\nThe cloud provider is configured to have Octavia by default in Kubespray.\n\n- Enable the external OpenStack cloud provider in `group_vars/all/all.yml`:\n\n ```yaml\n cloud_provider: external\n external_cloud_provider: openstack\n ```\n\n- Enable Cinder CSI in `group_vars/all/openstack.yml`:\n\n ```yaml\n cinder_csi_enabled: true\n ```\n\n- Enable topology support (optional), if your openstack provider has custom Zone names you can override the default \"nova\" zone by setting the variable `cinder_topology_zones`\n\n ```yaml\n cinder_topology: true\n ```\n\n- Enabling `cinder_csi_ignore_volume_az: true`, ignores volumeAZ and schedules on any of the available node AZ.\n\n ```yaml\n cinder_csi_ignore_volume_az: true\n ```\n\n- If you are using OpenStack loadbalancer(s) replace the `openstack_lbaas_subnet_id` with the new `external_openstack_lbaas_subnet_id`. **Note** The new cloud provider is using Octavia instead of Neutron LBaaS by default!\n\n- If you are in a case of a multi-nic OpenStack VMs (see [kubernetes/cloud-provider-openstack#407](https://github.com/kubernetes/cloud-provider-openstack/issues/407) and [#6083](https://github.com/kubernetes-sigs/kubespray/issues/6083) for explanation), you should override the default OpenStack networking configuration:\n\n ```yaml\n external_openstack_network_ipv6_disabled: false\n external_openstack_network_internal_networks: []\n external_openstack_network_public_networks: []\n ```\n\n- You can override the default OpenStack metadata configuration (see [#6338](https://github.com/kubernetes-sigs/kubespray/issues/6338) for explanation):\n\n ```yaml\n external_openstack_metadata_search_order: \"configDrive,metadataService\"\n ```\n\n- Available variables for configuring lbaas:\n\n ```yaml\n external_openstack_lbaas_enabled: true\n external_openstack_lbaas_floating_network_id: \"Neutron network ID to get floating IP from\"\n external_openstack_lbaas_floating_subnet_id: \"Neutron subnet ID to get floating IP from\"\n external_openstack_lbaas_method: ROUND_ROBIN\n external_openstack_lbaas_provider: amphora\n external_openstack_lbaas_subnet_id: \"Neutron subnet ID to create LBaaS VIP\"\n external_openstack_lbaas_member_subnet_id: \"Neutron subnet ID on which to create the members of the load balancer\"\n external_openstack_lbaas_network_id: \"Neutron network ID to create LBaaS VIP\"\n external_openstack_lbaas_manage_security_groups: false\n external_openstack_lbaas_create_monitor: false\n external_openstack_lbaas_monitor_delay: 5s\n external_openstack_lbaas_monitor_max_retries: 1\n external_openstack_lbaas_monitor_timeout: 3s\n external_openstack_lbaas_internal_lb: false\n\n ```\n\n- Run `source path/to/your/openstack-rc` to read your OpenStack credentials like `OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, etc. Those variables are used for accessing OpenStack from the external cloud provider.\n- Run the `cluster.yml` playbook\n\n## Additional step needed when using calico or kube-router\n\nBeing L3 CNI, calico and kube-router do not encapsulate all packages with the hosts' ip addresses. Instead the packets will be routed with the PODs ip addresses directly.\n\nOpenStack will filter and drop all packets from ips it does not know to prevent spoofing.\n\nIn order to make L3 CNIs work on OpenStack you will need to tell OpenStack to allow pods packets by allowing the network they use.\n\nFirst you will need the ids of your OpenStack instances that will run kubernetes:\n\n ```bash\n openstack server list --project YOUR_PROJECT\n +--------------------------------------+--------+----------------------------------+--------+-------------+\n | ID | Name | Tenant ID | Status | Power State |\n +--------------------------------------+--------+----------------------------------+--------+-------------+\n | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | k8s-1 | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running |\n | 725cd548-6ea3-426b-baaa-e7306d3c8052 | k8s-2 | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running |\n ```\n\nThen you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports (though they are now configured through using OpenStack):\n\n ```bash\n openstack port list -c id -c device_id --project YOUR_PROJECT\n +--------------------------------------+--------------------------------------+\n | id | device_id |\n +--------------------------------------+--------------------------------------+\n | 5662a4e0-e646-47f0-bf88-d80fbd2d99ef | e1f48aad-df96-4bce-bf61-62ae12bf3f95 |\n | e5ae2045-a1e1-4e99-9aac-4353889449a7 | 725cd548-6ea3-426b-baaa-e7306d3c8052 |\n ```\n\nGiven the port ids on the left, you can set the two `allowed-address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.)\n\n ```bash\n # allow kube_service_addresses and kube_pods_subnet network\n openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18\n openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18\n ```\n\nIf all the VMs in the tenant correspond to Kubespray deployment, you can \"sweep run\" above with:\n\n ```bash\n openstack port list --device-owner=compute:nova -c ID -f value | xargs -tI@ openstack port set @ --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18\n ```\n\nNow you can finally run the playbook.\n" }, { "path": "docs/cloud_controllers/vsphere.md", "content": "# vSphere\n\nKubespray can be deployed with vSphere as Cloud provider. This feature supports:\n\n- Volumes\n- Persistent Volumes\n- Storage Classes and provisioning of volumes\n- vSphere Storage Policy Based Management for Containers orchestrated by Kubernetes\n\n## Out-of-tree vSphere cloud provider\n\n### Prerequisites\n\nYou need at first to configure your vSphere environment by following the [official documentation](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#prerequisites).\n\nAfter this step you should have:\n\n- vSphere upgraded to 6.7 U3 or later\n- VM hardware upgraded to version 15 or higher\n- UUID activated for each VM where Kubernetes will be deployed\n\n### Kubespray configuration\n\nFirst in `inventory/sample/group_vars/all/all.yml` you must set the `cloud_provider` to `external` and `external_cloud_provider` to `vsphere`.\n\n```yml\ncloud_provider: \"external\"\nexternal_cloud_provider: \"vsphere\"\n```\n\nThen, `inventory/sample/group_vars/all/vsphere.yml`, you need to declare your vCenter credentials and enable the vSphere CSI following the description below.\n\n| Variable | Required | Type | Choices | Default | Comment |\n|----------------------------------------|----------|---------|----------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------|\n| external_vsphere_vcenter_ip | TRUE | string | | | IP/URL of the vCenter |\n| external_vsphere_vcenter_port | TRUE | string | | \"443\" | Port of the vCenter API |\n| external_vsphere_insecure | TRUE | string | \"true\", \"false\" | \"true\" | set to \"true\" if the host above uses a self-signed cert |\n| external_vsphere_user | TRUE | string | | | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) |\n| external_vsphere_password | TRUE | string | | | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable) |\n| external_vsphere_datacenter | TRUE | string | | | Datacenter name to use |\n| external_vsphere_kubernetes_cluster_id | TRUE | string | | \"kubernetes-cluster-id\" | Kubernetes cluster ID to use |\n| vsphere_csi_enabled | TRUE | boolean | | false | Enable vSphere CSI |\n\nExample configuration:\n\n```yml\nexternal_vsphere_vcenter_ip: \"myvcenter.domain.com\"\nexternal_vsphere_vcenter_port: \"443\"\nexternal_vsphere_insecure: \"true\"\nexternal_vsphere_user: \"administrator@vsphere.local\"\nexternal_vsphere_password: \"K8s_admin\"\nexternal_vsphere_datacenter: \"DATACENTER_name\"\nexternal_vsphere_kubernetes_cluster_id: \"kubernetes-cluster-id\"\nvsphere_csi_enabled: true\n```\n\nFor a more fine-grained CSI setup, refer to the [vsphere-csi](/docs/CSI/vsphere-csi.md) documentation.\n\n### Deployment\n\nOnce the configuration is set, you can execute the playbook again to apply the new configuration:\n\n```ShellSession\ncd kubespray\nansible-playbook -i inventory/sample/hosts.ini -b -v cluster.yml\n```\n\nYou'll find some useful examples [here](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#sample-manifests-to-test-csi-driver-functionality) to test your configuration.\n\n## In-tree vSphere cloud provider ([deprecated](https://cloud-provider-vsphere.sigs.k8s.io/concepts/in_tree_vs_out_of_tree.html))\n\n### Prerequisites (deprecated)\n\nYou need at first to configure your vSphere environment by following the [official documentation](https://kubernetes.io/docs/getting-started-guides/vsphere/#vsphere-cloud-provider).\n\nAfter this step you should have:\n\n- UUID activated for each VM where Kubernetes will be deployed\n- A vSphere account with required privileges\n\nIf you intend to leverage the [zone and region node labeling](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#failure-domain-beta-kubernetes-io-region), create a tag category for both the zone and region in vCenter. The tags can then be applied at the host, cluster, datacenter, or folder level, and the cloud provider will walk the hierarchy to extract and apply the labels to the Kubernetes nodes.\n\n### Kubespray configuration (deprecated)\n\nFirst you must define the cloud provider in `inventory/sample/group_vars/all.yml` and set it to `vsphere`.\n\n```yml\ncloud_provider: vsphere\n```\n\nThen, in the same file, you need to declare your vCenter credentials following the description below.\n\n| Variable | Required | Type | Choices | Default | Comment |\n|------------------------------|----------|---------|----------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| vsphere_vcenter_ip | TRUE | string | | | IP/URL of the vCenter |\n| vsphere_vcenter_port | TRUE | integer | | | Port of the vCenter API. Commonly 443 |\n| vsphere_insecure | TRUE | integer | 1, 0 | | set to 1 if the host above uses a self-signed cert |\n| vsphere_user | TRUE | string | | | User name for vCenter with required privileges |\n| vsphere_password | TRUE | string | | | Password for vCenter |\n| vsphere_datacenter | TRUE | string | | | Datacenter name to use |\n| vsphere_datastore | TRUE | string | | | Datastore name to use |\n| vsphere_working_dir | TRUE | string | | | Working directory from the view \"VMs and template\" in the vCenter where VM are placed |\n| vsphere_scsi_controller_type | TRUE | string | buslogic, pvscsi, parallel | pvscsi | SCSI controller name. Commonly \"pvscsi\". |\n| vsphere_vm_uuid | FALSE | string | | | VM Instance UUID of virtual machine that host K8s master. Can be retrieved from instanceUuid property in VmConfigInfo, or as vc.uuid in VMX file or in `/sys/class/dmi/id/product_serial` (Optional, only used for Kubernetes <= 1.9.2) |\n| vsphere_public_network | FALSE | string | | Blank | Name of the network the VMs are joined to |\n| vsphere_resource_pool | FALSE | string | | Blank | Name of the Resource pool where the VMs are located (Optional, only used for Kubernetes >= 1.9.2) |\n| vsphere_zone_category | FALSE | string | | | Name of the tag category used to set the `failure-domain.beta.kubernetes.io/zone` label on nodes (Optional, only used for Kubernetes >= 1.12.0) |\n| vsphere_region_category | FALSE | string | | | Name of the tag category used to set the `failure-domain.beta.kubernetes.io/region` label on nodes (Optional, only used for Kubernetes >= 1.12.0) |\n\nExample configuration:\n\n```yml\nvsphere_vcenter_ip: \"myvcenter.domain.com\"\nvsphere_vcenter_port: 443\nvsphere_insecure: 1\nvsphere_user: \"k8s@vsphere.local\"\nvsphere_password: \"K8s_admin\"\nvsphere_datacenter: \"DATACENTER_name\"\nvsphere_datastore: \"DATASTORE_name\"\nvsphere_working_dir: \"Docker_hosts\"\nvsphere_scsi_controller_type: \"pvscsi\"\nvsphere_resource_pool: \"K8s-Pool\"\n```\n\n### Deployment (deprecated)\n\nOnce the configuration is set, you can execute the playbook again to apply the new configuration:\n\n```ShellSession\ncd kubespray\nansible-playbook -i inventory/sample/hosts.ini -b -v cluster.yml\n```\n\nYou'll find some useful examples [here](https://github.com/kubernetes/examples/tree/master/staging/volumes/vsphere) to test your configuration.\n" }, { "path": "docs/cloud_providers/aws.md", "content": "# AWS\n\n> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)\n\nTo deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`. Refer to the [Kubespray Configuration](#kubespray-configuration) for customizing the provider.\n\nPrior to creating your instances, you **must** ensure that you have created IAM roles and policies for both \"kubernetes-master\" and \"kubernetes-node\". You can find the IAM policies [here](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/aws_iam/). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role.\n\nYou would also need to tag the resources in your VPC accordingly for the aws provider to utilize them. Tag the subnets, route tables and all instances that kubernetes will be run on with key `kubernetes.io/cluster/$cluster_name` (`$cluster_name` must be a unique identifier for the cluster). Tag the subnets that must be targeted by external ELBs with the key `kubernetes.io/role/elb` and internal ELBs with the key `kubernetes.io/role/internal-elb`.\n\nMake sure your VPC has both DNS Hostnames support and Private DNS enabled.\n\nThe next step is to make sure the hostnames in your `inventory` file are identical to your internal hostnames in AWS. This may look something like `ip-111-222-333-444.us-west-2.compute.internal`. You can then specify how Ansible connects to these instances with `ansible_ssh_host` and `ansible_ssh_user`.\n\nYou can now create your cluster!\n\n## Dynamic Inventory\n\nThere is also a dynamic inventory script for AWS that can be used if desired. However, be aware that it makes some certain assumptions about how you'll create your inventory. It also does not handle all use cases and groups that we may use as part of more advanced deployments. Additions welcome.\n\nThis will produce an inventory that is passed into Ansible that looks like the following:\n\n```json\n{\n \"_meta\": {\n \"hostvars\": {\n \"ip-172-31-3-xxx.us-east-2.compute.internal\": {\n \"ansible_ssh_host\": \"172.31.3.xxx\"\n },\n \"ip-172-31-8-xxx.us-east-2.compute.internal\": {\n \"ansible_ssh_host\": \"172.31.8.xxx\"\n }\n }\n },\n \"etcd\": [\n \"ip-172-31-3-xxx.us-east-2.compute.internal\"\n ],\n \"k8s_cluster\": {\n \"children\": [\n \"kube_control_plane\",\n \"kube_node\"\n ]\n },\n \"kube_control_plane\": [\n \"ip-172-31-3-xxx.us-east-2.compute.internal\"\n ],\n \"kube_node\": [\n \"ip-172-31-8-xxx.us-east-2.compute.internal\"\n ]\n}\n```\n\nGuide:\n\n- Create instances in AWS as needed.\n- Either during or after creation, add tags to the instances with a key of `kubespray-role` and a value of `kube_control_plane`, `etcd`, or `kube_node`. You can also share roles like `kube_control_plane, etcd`\n- Copy the `kubespray-aws-inventory.py` script from `kubespray/contrib/aws_inventory` to the `kubespray/inventory` directory.\n- Set the following AWS credentials and info as environment variables in your terminal:\n\n```ShellSession\nexport AWS_ACCESS_KEY_ID=\"xxxxx\"\nexport AWS_SECRET_ACCESS_KEY=\"yyyyy\"\nexport AWS_REGION=\"us-east-2\"\n```\n\n- We will now create our cluster. There will be either one or two small changes. The first is that we will specify `-i inventory/kubespray-aws-inventory.py` as our inventory script. The other is conditional. If your AWS instances are public facing, you can set the `VPC_VISIBILITY` variable to `public` and that will result in public IP and DNS names being passed into the inventory. This causes your cluster.yml command to look like `VPC_VISIBILITY=\"public\" ansible-playbook ... cluster.yml`\n\n**Optional** Using labels and taints\n\nTo add labels to your kubernetes node, add the following tag to your instance:\n\n- Key: `kubespray-node-labels`\n- Value: `node-role.kubernetes.io/ingress=`\n\nTo add taints to your kubernetes node, add the following tag to your instance:\n\n- Key: `kubespray-node-taints`\n- Value: `node-role.kubernetes.io/ingress=:NoSchedule`\n\n## Kubespray configuration\n\nDeclare the cloud config variables for the `aws` provider as follows. Setting these variables are optional and depend on your use case.\n\n| Variable | Type | Comment |\n|------------------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| aws_zone | string | Force set the AWS zone. Recommended to leave blank. |\n| aws_vpc | string | The AWS VPC flag enables the possibility to run the master components on a different aws account, on a different cloud provider or on-premise. If the flag is set also the KubernetesClusterTag must be provided |\n| aws_subnet_id | string | SubnetID enables using a specific subnet to use for ELB's |\n| aws_route_table_id | string | RouteTableID enables using a specific RouteTable |\n| aws_role_arn | string | RoleARN is the IAM role to assume when interaction with AWS APIs |\n| aws_kubernetes_cluster_tag | string | KubernetesClusterTag is the legacy cluster id we'll use to identify our cluster resources |\n| aws_kubernetes_cluster_id | string | KubernetesClusterID is the cluster id we'll use to identify our cluster resources |\n| aws_disable_security_group_ingress | bool | The aws provider creates an inbound rule per load balancer on the node security group. However, this can run into the AWS security group rule limit of 50 if many LoadBalancers are created. This flag disables the automatic ingress creation. It requires that the user has setup a rule that allows inbound traffic on kubelet ports from the local VPC subnet (so load balancers can access it). E.g. 10.82.0.0/16 30000-32000. |\n| aws_elb_security_group | string | Only in Kubelet version >= 1.7 : AWS has a hard limit of 500 security groups. For large clusters creating a security group for each ELB can cause the max number of security groups to be reached. If this is set instead of creating a new Security group for each ELB this security group will be used instead. |\n| aws_disable_strict_zone_check | bool | During the instantiation of an new AWS cloud provider, the detected region is validated against a known set of regions. In a non-standard, AWS like environment (e.g. Eucalyptus), this check may be undesirable. Setting this to true will disable the check and provide a warning that the check was skipped. Please note that this is an experimental feature and work-in-progress for the moment. |\n" }, { "path": "docs/cloud_providers/azure.md", "content": "# Azure\n\n> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)\n\nTo deploy Kubernetes on [Azure](https://azure.microsoft.com) uncomment the `cloud_provider` option in `group_vars/all/all.yml` and set it to `'azure'`.\n\nAll your instances are required to run in a resource group and a routing table has to be attached to the subnet your instances are in.\n\nNot all features are supported yet though, for a list of the current status have a look [here](https://github.com/Azure/AKS)\n\n## Parameters\n\nBefore creating the instances you must first set the `azure_` variables in the `group_vars/all/all.yml` file.\n\nAll values can be retrieved using the Azure CLI tool which can be downloaded here: \nAfter installation you have to run `az login` to get access to your account.\n\n### azure_cloud\n\nAzure Stack has different API endpoints, depending on the Azure Stack deployment. These need to be provided to the Azure SDK.\nPossible values are: `AzureChinaCloud`, `AzureGermanCloud`, `AzurePublicCloud` and `AzureUSGovernmentCloud`.\nThe full list of existing settings for the AzureChinaCloud, AzureGermanCloud, AzurePublicCloud and AzureUSGovernmentCloud\nis available in the source code [here](https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/docs/cloud-provider-config.md)\n\n### azure\\_tenant\\_id + azure\\_subscription\\_id\n\nrun `az account show` to retrieve your subscription id and tenant id:\n`azure_tenant_id` -> Tenant ID field\n`azure_subscription_id` -> ID field\n\n### azure\\_location\n\nThe region your instances are located, can be something like `westeurope` or `westcentralus`. A full list of region names can be retrieved via `az account list-locations`\n\n### azure\\_resource\\_group\n\nThe name of the resource group your instances are in, can be retrieved via `az group list`\n\n### azure\\_vmtype\n\nThe type of the vm. Supported values are `standard` or `vmss`. If vm is type of `Virtual Machines` then value is `standard`. If vm is part of `Virtual Machine Scale Sets` then value is `vmss`\n\n### azure\\_vnet\\_name\n\nThe name of the virtual network your instances are in, can be retrieved via `az network vnet list`\n\n### azure\\_vnet\\_resource\\_group\n\nThe name of the resource group that contains the vnet.\n\n### azure\\_subnet\\_name\n\nThe name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME`\n\n### azure\\_security\\_group\\_name\n\nThe name of the network security group your instances are in, can be retrieved via `az network nsg list`\n\n### azure\\_security\\_group\\_resource\\_group\n\nThe name of the resource group that contains the network security group. Defaults to `azure_vnet_resource_group`\n\n### azure\\_route\\_table\\_name\n\nThe name of the route table used with your instances.\n\n### azure\\_route\\_table\\_resource\\_group\n\nThe name of the resource group that contains the route table. Defaults to `azure_vnet_resource_group`\n\n### azure\\_aad\\_client\\_id + azure\\_aad\\_client\\_secret\n\nThese will have to be generated first:\n\n- Create an Azure AD Application with:\n\n ```ShellSession\n az ad app create --display-name kubernetes --identifier-uris http://kubernetes --homepage http://example.com --password CLIENT_SECRET\n ```\n\ndisplay name, identifier-uri, homepage and the password can be chosen\nNote the AppId in the output.\n\n- Create Service principal for the application with:\n\n ```ShellSession\n az ad sp create --id AppId\n ```\n\nThis is the AppId from the last command\n\n- Create the role assignment with:\n\n ```ShellSession\n az role assignment create --role \"Owner\" --assignee http://kubernetes --subscription SUBSCRIPTION_ID\n ```\n\nazure\\_aad\\_client\\_id must be set to the AppId, azure\\_aad\\_client\\_secret is your chosen secret.\n\n### azure\\_loadbalancer\\_sku\n\nSku of Load Balancer and Public IP. Candidate values are: basic and standard.\n\n### azure\\_exclude\\_master\\_from\\_standard\\_lb\n\nazure\\_exclude\\_master\\_from\\_standard\\_lb excludes master nodes from `standard` load balancer.\n\n### azure\\_disable\\_outbound\\_snat\n\nazure\\_disable\\_outbound\\_snat disables the outbound SNAT for public load balancer rules. It should only be set when azure\\_exclude\\_master\\_from\\_standard\\_lb is `standard`.\n\n### azure\\_primary\\_availability\\_set\\_name\n\n(Optional) The name of the availability set that should be used as the load balancer backend .If this is set, the Azure\ncloudprovider will only add nodes from that availability set to the load balancer backend pool. If this is not set, and\nmultiple agent pools (availability sets) are used, then the cloudprovider will try to add all nodes to a single backend\npool which is forbidden. In other words, if you use multiple agent pools (availability sets), you MUST set this field.\n\n### azure\\_use\\_instance\\_metadata\n\nUse instance metadata service where possible\n\n## Provisioning Azure with Resource Group Templates\n\nYou'll find Resource Group Templates and scripts to provision the required infrastructure to Azure in [*contrib/azurerm*](../contrib/azurerm/README.md)\n" }, { "path": "docs/cloud_providers/cloud.md", "content": "# Cloud providers\n\n> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)\n\n## Provisioning\n\nYou can deploy instances in your cloud environment in several ways. Examples include Terraform, Ansible (ec2 and gce modules), and manual creation.\n\n## Deploy kubernetes\n\nWith ansible-playbook command\n\n```ShellSession\nansible-playbook -u smana -e ansible_ssh_user=admin -e cloud_provider=[aws|gce] -b --become-user=root -i inventory/single.cfg cluster.yml\n```\n" }, { "path": "docs/developers/ci-setup.md", "content": "# CI Setup\n\n## Pipeline\n\nSee [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.\n\n## Runners\n\nKubespray has 2 types of GitLab runners, both deployed on the Kubespray CI cluster (hosted on Oracle Cloud Infrastructure):\n\n- pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)\n- vagrant: custom executor running in pods with access to the libvirt socket on the nodes\n\n## Vagrant\n\nVagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docker/Dockerfile) docker image with `/var/run/libvirt/libvirt-sock` exposed from the host, allowing the container to boot VMs on the host.\n\n## CI Variables\n\nIn CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on.\n\n## CI clusters\n\nDISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix.\n\nThe cluster is deployed with kubespray itself and maintained by the kubespray maintainers.\n\nThe following files are used for that inventory:\n\n### cluster.tfvars (OBSOLETE: this section is no longer accurate)\n\n```ini\n# your Kubernetes cluster name here\ncluster_name = \"ci\"\n\n# Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/\nequinix_metal_project_id = \"_redacted_\"\n\n# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned\n# leave this value blank if the public key is already setup in the Equinix Metal project\n# Terraform will complain if the public key is setup in Equinix Metal\npublic_key_path = \"~/.ssh/id_rsa.pub\"\n\n# cluster location\nmetro = \"da\"\n\n# standalone etcds\nnumber_of_etcd = 0\n\nplan_etcd = \"t1.small.x86\"\n\n# masters\nnumber_of_k8s_masters = 1\n\nnumber_of_k8s_masters_no_etcd = 0\n\nplan_k8s_masters = \"c3.small.x86\"\n\nplan_k8s_masters_no_etcd = \"t1.small.x86\"\n\n# nodes\nnumber_of_k8s_nodes = 1\n\nplan_k8s_nodes = \"c3.medium.x86\"\n```\n\n### group_vars/all/mirrors.yml\n\n```yaml\n---\ndocker_registry_mirrors:\n - \"https://mirror.gcr.io\"\n\ncontainerd_grpc_max_recv_message_size: 16777216\ncontainerd_grpc_max_send_message_size: 16777216\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - host: https://registry-1.docker.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n\ncontainerd_max_container_log_line_size: 16384\n\ncrio_registries_mirrors:\n - prefix: docker.io\n insecure: false\n blocked: false\n location: registry-1.docker.io\n mirrors:\n - location: mirror.gcr.io\n insecure: false\n\nnetcheck_agent_image_repo: \"{{ quay_image_repo }}/kubespray/k8s-netchecker-agent\"\nnetcheck_server_image_repo: \"{{ quay_image_repo }}/kubespray/k8s-netchecker-server\"\n\nnginx_image_repo: \"{{ quay_image_repo }}/kubespray/nginx\"\n```\n\n### group_vars/all/settings.yml\n\n```yaml\n---\n# Networking setting\nkube_service_addresses: 172.30.0.0/18\nkube_pods_subnet: 172.30.64.0/18\nkube_network_plugin: calico\n# avoid overlap with CI jobs deploying nodelocaldns\nnodelocaldns_ip: 169.254.255.100\n\n# ipip: False\ncalico_ipip_mode: \"Never\"\ncalico_vxlan_mode: \"Never\"\ncalico_network_backend: \"bird\"\ncalico_wireguard_enabled: True\n\n# Cluster settings\nupgrade_cluster_setup: True\nforce_certificate_regeneration: True\n\n# Etcd settings\netcd_deployment_type: \"host\"\n\n# Kubernetes settings\nkube_controller_terminated_pod_gc_threshold: 100\nkubelet_enforce_node_allocatable: pods\nkubelet_preferred_address_types: 'InternalIP,ExternalIP,Hostname'\nkubelet_custom_flags:\n - \"--serialize-image-pulls=true\"\n - \"--eviction-hard=memory.available<1Gi\"\n - \"--eviction-soft-grace-period=memory.available=30s\"\n - \"--eviction-soft=memory.available<2Gi\"\n - \"--system-reserved cpu=100m,memory=4Gi\"\n - \"--eviction-minimum-reclaim=memory.available=2Gi\"\n\n# DNS settings\nresolvconf_mode: none\ndns_min_replicas: 1\nupstream_dns_servers:\n - 1.1.1.1\n - 1.0.0.1\n\n# Extensions\nhelm_enabled: True\ncert_manager_enabled: True\nmetrics_server_enabled: True\n\n# Enable ZSWAP\nkubelet_fail_swap_on: False\nkube_feature_gates:\n - \"NodeSwap=True\"\n```\n\n## Additional files\n\nThis section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.\n\n### /root/path-calico.sh\n\n```bash\n#!/bin/bash\n\ncalicoctl patch felixconfig default -p '{\"spec\":{\"allowIPIPPacketsFromWorkloads\":true, \"allowVXLANPacketsFromWorkloads\": true}}'\n```\n\n### /root/kubevirt/kubevirt.sh\n\n```bash\n#!/bin/bash\n\nexport VERSION=$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases | grep tag_name | grep -v -- '-rc' | sort -r | head -1 | awk -F': ' '{print $2}' | sed 's/,//' | xargs)\necho $VERSION\nkubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml\nkubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml\n```\n\n### /root/kubevirt/virtctl.sh\n\n```bash\n#!/bin/bash\n\nVERSION=$(kubectl get kubevirt.kubevirt.io/kubevirt -n kubevirt -o=jsonpath=\"{.status.observedKubeVirtVersion}\")\nARCH=$(uname -s | tr A-Z a-z)-$(uname -m | sed 's/x86_64/amd64/') || windows-amd64.exe\necho ${ARCH}\ncurl -L -o virtctl https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/virtctl-${VERSION}-${ARCH}\nchmod +x virtctl\nsudo install virtctl /usr/local/bin\n```\n" }, { "path": "docs/developers/ci.md", "content": "# CI test coverage\n\nTo generate this Matrix run `./tests/scripts/md-table/main.py`\n\n## containerd\n\n| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |\n|---| --- | --- | --- | --- | --- | --- | --- |\nalmalinux9 | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |\namazon | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian11 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |\ndebian12 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |\ndebian13 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |\nfedora39 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |\nfedora40 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora41 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |\nfedora42 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nflatcar4081 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nopeneuler24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nrockylinux10 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |\nrockylinux9 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |\nubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nubuntu24 | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |\n\n## crio\n\n| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |\n|---| --- | --- | --- | --- | --- | --- | --- |\nalmalinux9 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\namazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian11 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian12 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian13 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora39 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora40 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora41 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora42 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nflatcar4081 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nopeneuler24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nrockylinux10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nrockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nubuntu24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\n\n## docker\n\n| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |\n|---| --- | --- | --- | --- | --- | --- | --- |\nalmalinux9 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\namazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian12 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\ndebian13 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora39 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora40 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora41 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nfedora42 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nflatcar4081 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nopeneuler24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nrockylinux10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nrockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: |\nubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\nubuntu24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |\n" }, { "path": "docs/developers/test_cases.md", "content": "# Node Layouts\n\nThere are five node layout types: `default`, `separate`, `ha`, `all-in-one`, and `node-etcd-client`.\n\n`default` is a non-HA two nodes setup with one separate `kube_node`\nand the `etcd` group merged with the `kube_control_plane`.\n\n`separate` layout is when there is only node of each type, which includes\n a kube_control_plane, kube_node, and etcd cluster member.\n\n`ha` layout consists of two etcd nodes, two control planes and a single worker node,\nwith role intersection.\n\n`all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged.\n\n`node-etcd-client` layout consists of a 4 nodes cluster, all of them in `kube_node`, first 3 in `etcd` and only one `kube_control_plane`.\nThis is necessary to tests setups requiring that nodes are etcd clients (use of cilium as `network_plugin` for instance)\n\nNote, the canal network plugin deploys flannel as well plus calico policy controller.\n\n## Test cases\n\nThe [CI Matrix](/docs/developers/ci.md) displays OS, Network Plugin and Container Manager tested.\n\nAll tests are breakdown into 3 \"stages\" (\"Stage\" means a build step of the build pipeline) as follows:\n\n- _unit_tests_: Linting, markdown, vagrant & terraform validation etc...\n- _part1_: Molecule and AIO tests\n- _part2_: Standard tests with different layouts and OS/Runtime/Network\n- _part3_: Upgrade jobs, terraform jobs and recover control plane tests\n- _special_: Other jobs (manuals)\n\nThe steps are ordered as `unit_tests->part1->part2->part3->special`.\n" }, { "path": "docs/developers/vagrant.md", "content": "# Vagrant\n\nAssuming you have Vagrant 2.0+ installed with virtualbox or libvirt/qemu\n(vmware may work, but is untested) you should be able to launch a 3 node\nKubernetes cluster by simply running `vagrant up`.\n\nThis will spin up 3 VMs and install kubernetes on them.\nOnce they are completed you can connect to any of them by running `vagrant ssh k8s-[1..3]`.\n\nTo give an estimate of the expected duration of a provisioning run:\nOn a dual core i5-6300u laptop with an SSD, provisioning takes around 13\nto 15 minutes, once the container images and other files are cached.\nNote that libvirt/qemu is recommended over virtualbox as it is quite a bit\nfaster, especially during boot-up time.\n\nFor proper performance a minimum of 12GB RAM is recommended.\nIt is possible to run a 3 node cluster on a laptop with 8GB of RAM using\nthe default Vagrantfile, provided you have 8GB zram swap configured and\nnot much more than a browser and a mail client running.\nIf you decide to run on such a machine, then also make sure that any tmpfs\ndevices, that are mounted, are mostly empty and disable any swapfiles\nmounted on HDD/SSD or you will be in for some serious swap-madness.\nThings can get a bit sluggish during provisioning, but when that's done,\nthe system will actually be able to perform quite well.\n\n## Customize Vagrant\n\nYou can override the default settings in the `Vagrantfile` either by\ndirectly modifying the `Vagrantfile` or through an override file.\nIn the same directory as the `Vagrantfile`, create a folder called\n`vagrant` and create `config.rb` file in it.\n\nExample:\n\n```ruby\n# vagrant/config.rb\n$instance_name_prefix = \"kub\"\n$vm_cpus = 1\n$num_instances = 3\n$os = \"centos8-bento\"\n$subnet = \"10.0.20\"\n$network_plugin = \"flannel\"\n\n$extra_vars = {\n dns_domain: my.custom.domain\n}\n# or\n$extra_vars = \"path/to/extra/vars/file.yml\"\n```\n\nFor all available options look at the Vagrantfile (search for \"CONFIG\")\n\n## Use alternative OS for Vagrant\n\nBy default, Vagrant uses Ubuntu 18.04 box to provision a local cluster.\nYou may use an alternative supported operating system for your local cluster.\n\nCustomize `$os` variable in `Vagrantfile` or as override, e.g.,:\n\n```ShellSession\necho '$os = \"flatcar-stable\"' >> vagrant/config.rb\n```\n\nThe supported operating systems for vagrant are defined in the `SUPPORTED_OS`\nconstant in the `Vagrantfile`.\n\n## File and image caching\n\nKubespray can take quite a while to start on a laptop. To improve provisioning\nspeed, the variable 'download_run_once' is set. This will make kubespray\ndownload all files and containers just once and then redistributes them to\nthe other nodes and as a bonus, also cache all downloads locally and re-use\nthem on the next provisioning run. For more information on download settings\nsee [download documentation](/docs/advanced/downloads.md).\n\n## Example use of Vagrant\n\nThe following is an example of setting up and running kubespray using `vagrant`.\nCustomize your settings as shown, above, then run the commands:\n\n```ShellSession\n# use virtualenv to install all python requirements\nVENVDIR=venv\n$ virtualenv --python=/usr/bin/python3.7 $VENVDIR\n$ source $VENVDIR/bin/activate\n$ pip install -r requirements.txt\n\n$ vagrant up\n\n# Access the cluster\n$ export INV=.vagrant/provisioners/ansible/inventory\n$ export KUBECONFIG=${INV}/artifacts/admin.conf\n# make the kubectl binary available\n$ export PATH=$PATH:$PWD/$INV/artifacts\n```\n\nIf a vagrant run failed and you've made some changes to fix the issue causing\nthe fail, here is how you would re-run ansible:\n\n```ShellSession\nvagrant provision\n```\n\nIf all went well, you check if it's all working as expected:\n\n```ShellSession\n$ kubectl get nodes\nNAME STATUS ROLES AGE VERSION\nkub-1 Ready control-plane,master 4m37s v1.22.5\nkub-2 Ready control-plane,master 4m7s v1.22.5\nkub-3 Ready 3m7s v1.22.5\n```\n\nAnother nice test is the following:\n\n```ShellSession\n$ kubectl get pods --all-namespaces -o wide\nNAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES\nkube-system coredns-8474476ff8-m2469 1/1 Running 0 2m45s 10.233.65.2 kub-2 \nkube-system coredns-8474476ff8-v5wzj 1/1 Running 0 2m41s 10.233.64.3 kub-1 \nkube-system dns-autoscaler-5ffdc7f89d-76tnv 1/1 Running 0 2m43s 10.233.64.2 kub-1 \nkube-system kube-apiserver-kub-1 1/1 Running 1 4m54s 10.0.20.101 kub-1 \nkube-system kube-apiserver-kub-2 1/1 Running 1 4m33s 10.0.20.102 kub-2 \nkube-system kube-controller-manager-kub-1 1/1 Running 1 5m1s 10.0.20.101 kub-1 \nkube-system kube-controller-manager-kub-2 1/1 Running 1 4m33s 10.0.20.102 kub-2 \nkube-system kube-flannel-9xgf5 1/1 Running 0 3m10s 10.0.20.102 kub-2 \nkube-system kube-flannel-l8jbl 1/1 Running 0 3m10s 10.0.20.101 kub-1 \nkube-system kube-flannel-zss4t 1/1 Running 0 3m10s 10.0.20.103 kub-3 \nkube-system kube-multus-ds-amd64-bhpc9 1/1 Running 0 3m2s 10.0.20.103 kub-3 \nkube-system kube-multus-ds-amd64-n6vl8 1/1 Running 0 3m2s 10.0.20.102 kub-2 \nkube-system kube-multus-ds-amd64-qttgs 1/1 Running 0 3m2s 10.0.20.101 kub-1 \nkube-system kube-proxy-2x4jl 1/1 Running 0 3m33s 10.0.20.101 kub-1 \nkube-system kube-proxy-d48r7 1/1 Running 0 3m33s 10.0.20.103 kub-3 \nkube-system kube-proxy-f45lp 1/1 Running 0 3m33s 10.0.20.102 kub-2 \nkube-system kube-scheduler-kub-1 1/1 Running 1 4m54s 10.0.20.101 kub-1 \nkube-system kube-scheduler-kub-2 1/1 Running 1 4m33s 10.0.20.102 kub-2 \nkube-system nginx-proxy-kub-3 1/1 Running 0 3m33s 10.0.20.103 kub-3 \nkube-system nodelocaldns-cg9tz 1/1 Running 0 2m41s 10.0.20.102 kub-2 \nkube-system nodelocaldns-htswt 1/1 Running 0 2m41s 10.0.20.103 kub-3 \nkube-system nodelocaldns-nsp7s 1/1 Running 0 2m41s 10.0.20.101 kub-1 \nlocal-path-storage local-path-provisioner-66df45bfdd-km4zg 1/1 Running 0 2m54s 10.233.66.2 kub-3 \n```\n" }, { "path": "docs/external_storage_provisioners/local_volume_provisioner.md", "content": "# Local Static Storage Provisioner\n\nThe [local static storage provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner)\nis NOT a dynamic storage provisioner as you would\nexpect from a cloud provider. Instead, it simply creates PersistentVolumes for\nall mounts under the `host_dir` of the specified storage class.\nThese storage classes are specified in the `local_volume_provisioner_storage_classes` nested dictionary.\n\nExample:\n\n```yaml\nlocal_volume_provisioner_storage_classes:\n local-storage:\n host_dir: /mnt/disks\n mount_dir: /mnt/disks\n fast-disks:\n host_dir: /mnt/fast-disks\n mount_dir: /mnt/fast-disks\n block_cleaner_command:\n - \"/scripts/shred.sh\"\n - \"2\"\n volume_mode: Filesystem\n fs_type: ext4\n```\n\nFor each key in `local_volume_provisioner_storage_classes` a \"storage class\" with\nthe same name is created in the entry `storageClassMap` of the ConfigMap `local-volume-provisioner`.\nThe subkeys of each storage class in `local_volume_provisioner_storage_classes`\nare converted to camelCase and added as attributes to the storage class in the\nConfigMap.\n\nThe result of the above example is:\n\n```yaml\ndata:\n storageClassMap: |\n local-storage:\n hostDir: /mnt/disks\n mountDir: /mnt/disks\n fast-disks:\n hostDir: /mnt/fast-disks\n mountDir: /mnt/fast-disks\n blockCleanerCommand:\n - \"/scripts/shred.sh\"\n - \"2\"\n volumeMode: Filesystem\n fsType: ext4\n```\n\nAdditionally, a StorageClass object (`storageclasses.storage.k8s.io`) is also\ncreated for each storage class:\n\n```bash\n$ kubectl get storageclasses.storage.k8s.io\nNAME PROVISIONER RECLAIMPOLICY\nfast-disks kubernetes.io/no-provisioner Delete\nlocal-storage kubernetes.io/no-provisioner Delete\n```\n\nThe default StorageClass is `local-storage` on `/mnt/disks`;\nthe rest of this documentation will use that path as an example.\n\n## Examples to create local storage volumes\n\n1. Using tmpfs\n\n ```bash\n for vol in vol1 vol2 vol3; do\n mkdir /mnt/disks/$vol\n mount -t tmpfs -o size=5G $vol /mnt/disks/$vol\n done\n ```\n\n The tmpfs method is not recommended for production because the mounts are not\n persistent and data will be deleted on reboot.\n\n1. Mount physical disks\n\n ```bash\n mkdir /mnt/disks/ssd1\n mount /dev/vdb1 /mnt/disks/ssd1\n ```\n\n Physical disks are recommended for production environments because it offers\n complete isolation in terms of I/O and capacity.\n\n1. Mount unpartitioned physical devices\n\n ```bash\n for disk in /dev/sdc /dev/sdd /dev/sde; do\n ln -s $disk /mnt/disks\n done\n ```\n\n This saves time of precreating filesystems. Note that your storageclass must have\n `volume_mode` set to `\"Filesystem\"` and `fs_type` defined. If either is not set, the\n disk will be added as a raw block device.\n\n1. PersistentVolumes with `volumeMode=\"Block\"`\n\n Just like above, you can create PersistentVolumes with volumeMode `Block`\n by creating a symbolic link under discovery directory to the block device on\n the node, if you set `volume_mode` to `\"Block\"`. This will create a volume\n presented into a Pod as a block device, without any filesystem on it.\n\n1. File-backed sparsefile method\n\n ```bash\n truncate /mnt/disks/disk5 --size 2G\n mkfs.ext4 /mnt/disks/disk5\n mkdir /mnt/disks/vol5\n mount /mnt/disks/disk5 /mnt/disks/vol5\n ```\n\n If you have a development environment and only one disk, this is the best way\n to limit the quota of persistent volumes.\n\n1. Simple directories\n\n In a development environment, using `mount --bind` works also, but there is no capacity\n management.\n\n## Usage notes\n\nMake sure to make any mounts persist via `/etc/fstab` or with systemd mounts (for\nFlatcar Container Linux or Fedora CoreOS). Pods with persistent volume claims will not be\nable to start if the mounts become unavailable.\n\n## Further reading\n\nRefer to the upstream docs here: \n" }, { "path": "docs/external_storage_provisioners/scheduler_plugins.md", "content": "# Scheduler plugins for Kubernetes\n\n[scheduler-plugins](https://github.com/kubernetes-sigs/scheduler-plugins) is out-of-tree scheduler plugins based on the [scheduler framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/).\n\nThe kube-scheduler binary includes a list of plugins:\n\n- [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta]\n- [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta]\n- [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta]\n- [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]\n- [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha]\n- [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha]\n- [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample]\n- [CrossNodePreemption](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/crossnodepreemption/README.md) [Sample]\n- [PodState](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/podstate/README.md) [Sample]\n- [QualityOfService](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/qos/README.md) [Sample]\n\nCurrently, we use [helm chart](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#installing-the-chart) to install the scheduler plugins, so that a second scheduler would be created and running. **Note that running multi-scheduler will inevitably encounter resource conflicts when the cluster is short of resources**.\n\n## Compatibility Matrix\n\nThere are requirements for the version of Kubernetes, please see [Compatibility Matrix\n](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master?tab=readme-ov-file#compatibility-matrix). It deserves our attention.\n\n| Scheduler Plugins | Compiled With K8s Version |\n| ----------------- | ------------------------- |\n| v0.28.9 | v1.28.9 |\n| v0.27.8 | v1.27.8 |\n\n## Turning it on\n\n The `scheduler_plugins_enabled` option is used to enable the installation of scheduler plugins.\n\n You can enable or disable some plugins by setting the `scheduler_plugins_enabled_plugins` or `scheduler_plugins_disabled_plugins` option. They must be in the list we mentioned above.\n\n In addition, to use custom plugin configuration, set a value for `scheduler_plugins_plugin_config` option.\n\n For example, for Coscheduling plugin, you want to customize the permit waiting timeout to 10 seconds:\n\n ```yaml\n scheduler_plugins_plugin_config:\n - name: Coscheduling\n args:\n permitWaitingTimeSeconds: 10 # default is 60\n ```\n\n## Leverage plugin\n\n Once the plugin is installed, we can apply CRs into cluster. For example, if using `CoScheduling`, we can apply the CR and test the deployment in the [example](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/doc/install.md#test-coscheduling).\n" }, { "path": "docs/getting_started/comparisons.md", "content": "# Comparison\n\n## Kubespray vs Kops\n\nKubespray runs on bare metal and most clouds, using Ansible as its substrate for\nprovisioning and orchestration. [Kops](https://github.com/kubernetes/kops) performs the provisioning and orchestration\nitself, and as such is less flexible in deployment platforms. For people with\nfamiliarity with Ansible, existing Ansible deployments or the desire to run a\nKubernetes cluster across multiple platforms, Kubespray is a good choice. Kops,\nhowever, is more tightly integrated with the unique features of the clouds it\nsupports so it could be a better choice if you know that you will only be using\none platform for the foreseeable future.\n\n## Kubespray vs Kubeadm\n\n[Kubeadm](https://github.com/kubernetes/kubeadm) provides domain Knowledge of Kubernetes clusters' life cycle\nmanagement, including self-hosted layouts, dynamic discovery services and so\non. Had it belonged to the new [operators world](https://coreos.com/blog/introducing-operators.html),\nit may have been named a \"Kubernetes cluster operator\". Kubespray however,\ndoes generic configuration management tasks from the \"OS operators\" ansible\nworld, plus some initial K8s clustering (with networking plugins included) and\ncontrol plane bootstrapping.\n\nKubespray has started using `kubeadm` internally for cluster creation since v2.3\nin order to consume life cycle management domain knowledge from it\nand offload generic OS configuration things from it, which hopefully benefits both sides.\n" }, { "path": "docs/getting_started/getting-started.md", "content": "# Getting started\n\n## Install ansible\n\nInstall Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible).\n\n## Building your own inventory\n\nAnsible inventory can be stored in 3 formats: YAML, JSON, or INI-like. See the\n[example inventory](/inventory/sample/inventory.ini)\nand [Ansible documentation on building your inventory](https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html),\nand [details on the inventory structure expected by Kubespray](/docs/ansible/inventory.md).\n\n```ShellSession\n inventory/mycluster/inventory.ini\n\n# Review and change parameters under ``inventory/mycluster/group_vars``\n inventory/mycluster/group_vars/all.yml # for every node, including etcd\n inventory/mycluster/group_vars/k8s_cluster.yml # for every node in the cluster (not etcd when it's separate)\n inventory/mycluster/group_vars/kube_control_plane.yml # for the control plane\n inventory/myclsuter/group_vars/kube_node.yml # for worker nodes\n```\n\n## Installing the cluster\n\n```ShellSession\nansible-playbook -i inventory/mycluster/ cluster.yml -b -v \\\n --private-key=~/.ssh/private_key\n```\n\n### Adding nodes\n\nYou may want to add worker, control plane or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your control planes. This is especially helpful when doing something like autoscaling your clusters.\n\n- Add the new worker node to your inventory in the appropriate group (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html)).\n- Run the ansible-playbook command, substituting `cluster.yml` for `scale.yml`:\n\n```ShellSession\nansible-playbook -i inventory/mycluster/hosts.yml scale.yml -b -v \\\n --private-key=~/.ssh/private_key\n```\n\n### Remove nodes\n\nYou may want to remove **control plane**, **worker**, or **etcd** nodes from your\nexisting cluster. This can be done by re-running the `remove-node.yml`\nplaybook. First, all specified nodes will be drained, then stop some\nkubernetes services and delete some certificates,\nand finally execute the kubectl command to delete these nodes.\nThis can be combined with the add node function. This is generally helpful\nwhen doing something like autoscaling your clusters. Of course, if a node\nis not working, you can remove the node and install it again.\n\nUse `--extra-vars \"node=,\"` to select the node(s) you want to delete.\n\n```ShellSession\nansible-playbook -i inventory/mycluster/hosts.yml remove-node.yml -b -v \\\n--private-key=~/.ssh/private_key \\\n--extra-vars \"node=nodename,nodename2\"\n```\n\n> Note: The playbook does not currently support the removal of the first control plane or etcd node. These nodes are essential for maintaining cluster operations and must remain intact.\n\nIf a node is completely unreachable by ssh, add `--extra-vars reset_nodes=false`\nto skip the node reset step. If one node is unavailable, but others you wish\nto remove are able to connect via SSH, you could set `reset_nodes=false` as a host\nvar in inventory.\n\n## Connecting to Kubernetes\n\nBy default, Kubespray configures kube_control_plane hosts with insecure access to\nkube-apiserver via port 8080. A kubeconfig file is not necessary in this case,\nbecause kubectl will use to connect. The kubeconfig files\ngenerated will point to localhost (on kube_control_planes) and kube_node hosts will\nconnect either to a localhost nginx proxy or to a loadbalancer if configured.\nMore details on this process are in the [HA guide](/docs/operations/ha-mode.md).\n\nKubespray permits connecting to the cluster remotely on any IP of any\nkube_control_plane host on port 6443 by default. However, this requires\nauthentication. One can get a kubeconfig from kube_control_plane hosts\n(see [below](#accessing-kubernetes-api)).\n\nFor more information on kubeconfig and accessing a Kubernetes cluster, refer to\nthe Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/).\n\n## Accessing Kubernetes API\n\nThe main client of Kubernetes is `kubectl`. It is installed on each kube_control_plane\nhost and can optionally be configured on your ansible host by setting\n`kubectl_localhost: true` and `kubeconfig_localhost: true` in the configuration:\n\n- If `kubectl_localhost` enabled, `kubectl` will download onto `/usr/local/bin/` and setup with bash completion. A helper script `inventory/mycluster/artifacts/kubectl.sh` also created for setup with below `admin.conf`.\n- If `kubeconfig_localhost` enabled `admin.conf` will appear in the `inventory/mycluster/artifacts/` directory after deployment.\n- The location where these files are downloaded to can be configured via the `artifacts_dir` variable.\n\nNOTE: The controller host name in the admin.conf file might be a private IP. If so, change it to use the controller's public IP or the cluster's load balancer.\n\nYou can see a list of nodes by running the following commands:\n\n```ShellSession\ncd inventory/mycluster/artifacts\n./kubectl.sh get nodes\n```\n\nIf desired, copy admin.conf to ~/.kube/config.\n\n## Setting up your first cluster\n\n[Setting up your first cluster](/docs/getting_started/setting-up-your-first-cluster.md) is an\n applied step-by-step guide for setting up your first cluster with Kubespray.\n" }, { "path": "docs/getting_started/setting-up-your-first-cluster.md", "content": "# Setting up your first cluster with Kubespray\n\nThis tutorial walks you through the detailed steps for setting up Kubernetes\nwith [Kubespray](https://kubespray.io/).\n\nThe guide is inspired on the tutorial [Kubernetes The Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way), with the\ndifference that here we want to showcase how to spin up a Kubernetes cluster\nin a more managed fashion with Kubespray.\n\n## Target Audience\n\nThe target audience for this tutorial is someone looking for a\nhands-on guide to get started with Kubespray.\n\n## Cluster Details\n\n* [kubespray](https://github.com/kubernetes-sigs/kubespray)\n* [kubernetes](https://github.com/kubernetes/kubernetes)\n\n## Prerequisites\n\n* Google Cloud Platform: This tutorial leverages the [Google Cloud Platform](https://cloud.google.com/) to streamline provisioning of the compute infrastructure required to bootstrap a Kubernetes cluster from the ground up. [Sign up](https://cloud.google.com/free/) for $300 in free credits.\n* Google Cloud Platform SDK: Follow the Google Cloud SDK [documentation](https://cloud.google.com/sdk/) to install and configure the `gcloud` command\n line utility. Make sure to set a default compute region and compute zone.\n* The [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) command line utility is used to interact with the Kubernetes\n API Server.\n* Linux or Mac environment with Python 3\n\n## Provisioning Compute Resources\n\nKubernetes requires a set of machines to host the Kubernetes control plane and the worker nodes where containers are ultimately run. In this lab you will provision the compute resources required for running a secure and highly available Kubernetes cluster across a single [compute zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones).\n\n### Networking\n\nThe Kubernetes [networking model](https://kubernetes.io/docs/concepts/cluster-administration/networking/#kubernetes-model) assumes a flat network in which containers and nodes can communicate with each other. In cases where this is not desired [network policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/) can limit how groups of containers are allowed to communicate with each other and external network endpoints.\n\n> Setting up network policies is out of scope for this tutorial.\n\n#### Virtual Private Cloud Network\n\nIn this section a dedicated [Virtual Private Cloud](https://cloud.google.com/compute/docs/networks-and-firewalls#networks) (VPC) network will be setup to host the Kubernetes cluster.\n\nCreate the `kubernetes-the-kubespray-way` custom VPC network:\n\n```ShellSession\ngcloud compute networks create kubernetes-the-kubespray-way --subnet-mode custom\n```\n\nA [subnet](https://cloud.google.com/compute/docs/vpc/#vpc_networks_and_subnets) must be provisioned with an IP address range large enough to assign a private IP address to each node in the Kubernetes cluster.\n\nCreate the `kubernetes` subnet in the `kubernetes-the-kubespray-way` VPC network:\n\n```ShellSession\ngcloud compute networks subnets create kubernetes \\\n --network kubernetes-the-kubespray-way \\\n --range 10.240.0.0/24\n ```\n\n> The `10.240.0.0/24` IP address range can host up to 254 compute instances.\n\n#### Firewall Rules\n\nCreate a firewall rule that allows internal communication across all protocols.\nIt is important to note that the vxlan (udp) protocol has to be allowed in order for\nthe calico (see later) networking plugin to work.\n\n```ShellSession\ngcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \\\n --allow tcp,udp,icmp \\\n --network kubernetes-the-kubespray-way \\\n --source-ranges 10.240.0.0/24\n```\n\nCreate a firewall rule that allows external SSH, ICMP, and HTTPS:\n\n```ShellSession\ngcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-external \\\n --allow tcp:80,tcp:6443,tcp:443,tcp:22,icmp \\\n --network kubernetes-the-kubespray-way \\\n --source-ranges 0.0.0.0/0\n```\n\nIt is not feasible to restrict the firewall to a specific IP address from\nwhere you are accessing the cluster as the nodes also communicate over the public internet and would otherwise run into\nthis firewall. Technically you could limit the firewall to the (fixed) IP\naddresses of the cluster nodes and the remote IP addresses for accessing the\ncluster.\n\n### Compute Instances\n\nThe compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04.\nEach compute instance will be provisioned with a fixed private IP address and\n a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)).\nUsing fixed public IP addresses has the advantage that our cluster node\nconfiguration does not need to be updated with new public IP addresses every\ntime the machines are shut down and later on restarted.\n\nCreate three compute instances which will host the Kubernetes control plane:\n\n```ShellSession\nfor i in 0 1 2; do\n gcloud compute instances create controller-${i} \\\n --async \\\n --boot-disk-size 200GB \\\n --can-ip-forward \\\n --image-family ubuntu-2404-lts-amd64 \\\n --image-project ubuntu-os-cloud \\\n --machine-type e2-standard-2 \\\n --private-network-ip 10.240.0.1${i} \\\n --scopes compute-rw,storage-ro,service-management,service-control,logging-write,monitoring \\\n --subnet kubernetes \\\n --tags kubernetes-the-kubespray-way,controller\ndone\n```\n\n> Do not forget to fix the IP addresses if you plan on re-using the cluster\nafter temporarily shutting down the VMs - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)\n\nCreate three compute instances which will host the Kubernetes worker nodes:\n\n```ShellSession\nfor i in 0 1 2; do\n gcloud compute instances create worker-${i} \\\n --async \\\n --boot-disk-size 200GB \\\n --can-ip-forward \\\n --image-family ubuntu-2404-lts-amd64 \\\n --image-project ubuntu-os-cloud \\\n --machine-type e2-standard-2 \\\n --private-network-ip 10.240.0.2${i} \\\n --scopes compute-rw,storage-ro,service-management,service-control,logging-write,monitoring \\\n --subnet kubernetes \\\n --tags kubernetes-the-kubespray-way,worker\ndone\n```\n\n> Do not forget to fix the IP addresses if you plan on re-using the cluster\nafter temporarily shutting down the VMs - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)\n\nList the compute instances in your default compute zone:\n\n```ShellSession\ngcloud compute instances list --filter=\"tags.items=kubernetes-the-kubespray-way\"\n```\n\n> Output\n\n```ShellSession\nNAME ZONE MACHINE_TYPE PREEMPTIBLE INTERNAL_IP EXTERNAL_IP STATUS\ncontroller-0 us-west1-c e2-standard-2 10.240.0.10 XX.XX.XX.XXX RUNNING\ncontroller-1 us-west1-c e2-standard-2 10.240.0.11 XX.XXX.XXX.XX RUNNING\ncontroller-2 us-west1-c e2-standard-2 10.240.0.12 XX.XXX.XX.XXX RUNNING\nworker-0 us-west1-c e2-standard-2 10.240.0.20 XX.XX.XXX.XXX RUNNING\nworker-1 us-west1-c e2-standard-2 10.240.0.21 XX.XX.XX.XXX RUNNING\nworker-2 us-west1-c e2-standard-2 10.240.0.22 XX.XXX.XX.XX RUNNING\n```\n\n### Configuring SSH Access\n\nKubespray is relying on SSH to configure the controller and worker instances.\n\nTest SSH access to the `controller-0` compute instance:\n\n```ShellSession\nIP_CONTROLLER_0=$(gcloud compute instances list --filter=\"tags.items=kubernetes-the-kubespray-way AND name:controller-0\" --format=\"value(EXTERNAL_IP)\")\nUSERNAME=$(whoami)\nssh $USERNAME@$IP_CONTROLLER_0\n```\n\nIf this is your first time connecting to a compute instance SSH keys will be\ngenerated for you. In this case you will need to enter a passphrase at the\nprompt to continue.\n\n> If you get a 'Remote host identification changed!' warning, you probably\nalready connected to that IP address in the past with another host key. You\ncan remove the old host key by running `ssh-keygen -R $IP_CONTROLLER_0`\n\nPlease repeat this procedure for all the controller and worker nodes, to\nensure that SSH access is properly functioning for all nodes.\n\n## Set-up Kubespray\n\nThe following set of instruction is based on the [Quick Start](https://github.com/kubernetes-sigs/kubespray) but slightly altered for our\nset-up.\n\nAs Ansible is a python application, we will create a fresh virtual\nenvironment to install the dependencies for the Kubespray playbook:\n\n```ShellSession\npython3 -m venv venv\nsource venv/bin/activate\n```\n\nNext, we will git clone the Kubespray code into our working directory:\n\n```ShellSession\ngit clone https://github.com/kubernetes-sigs/kubespray.git\ncd kubespray\ngit checkout release-2.17\n```\n\nNow we need to install the dependencies for Ansible to run the Kubespray\nplaybook:\n\n```ShellSession\npip install -r requirements.txt\n```\n\nCopy ``inventory/sample`` as ``inventory/mycluster``:\n\n```ShellSession\ncp -rfp inventory/sample inventory/mycluster\n```\n\nUpdate the sample Ansible inventory file with ip given by gcloud:\n\n```ShellSession\ngcloud compute instances list --filter=\"tags.items=kubernetes-the-kubespray-way\"\n```\n\nOpen `inventory/mycluster/inventory.ini` file and add it so\nthat controller-0, controller-1 and controller-2 in the `kube_control_plane` group and\nworker-0, worker-1 and worker-2 in the `kube_node` group. Add respective `ip` to the respective local VPC IP for each node.\n\nThe main configuration for the cluster is stored in\n`inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml`. In this file we\n will update the `supplementary_addresses_in_ssl_keys` with a list of the IP\n addresses of the controller nodes. In that way we can access the\n kubernetes API server as an administrator from outside the VPC network. You\n can also see that the `kube_network_plugin` is by default set to 'calico'.\n If you set this to 'cloud', it did not work on GCP at the time of testing.\n\nKubespray also offers to easily enable popular kubernetes add-ons. You can\nmodify the\nlist of add-ons in `inventory/mycluster/group_vars/k8s_cluster/addons.yml`.\nLet's enable the metrics server as this is a crucial monitoring element for\nthe kubernetes cluster, just change the 'false' to 'true' for\n`metrics_server_enabled`.\n\nNow we will deploy the configuration:\n\n```ShellSession\nansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa cluster.yml\n```\n\nAnsible will now execute the playbook, this can take up to 20 minutes.\n\n## Access the kubernetes cluster\n\nWe will leverage a kubeconfig file from one of the controller nodes to access\n the cluster as administrator from our local workstation.\n\n> In this simplified set-up, we did not include a load balancer that usually sits on top of the three controller nodes for a high available API server endpoint. In this simplified tutorial we connect directly to one of the three controllers.\n\nFirst, we need to edit the permission of the kubeconfig file on one of the\ncontroller nodes:\n\n```ShellSession\nssh $USERNAME@$IP_CONTROLLER_0\nUSERNAME=$(whoami)\nsudo chown -R $USERNAME:$USERNAME /etc/kubernetes/admin.conf\nexit\n```\n\nNow we will copy over the kubeconfig file:\n\n```ShellSession\nscp $USERNAME@$IP_CONTROLLER_0:/etc/kubernetes/admin.conf kubespray-do.conf\n```\n\nThis kubeconfig file uses the internal IP address of the controller node to\naccess the API server. This kubeconfig file will thus not work of from\noutside the VPC network. We will need to change the API server IP address\nto the controller node his external IP address. The external IP address will be\naccepted in the\nTLS negotiation as we added the controllers external IP addresses in the SSL\ncertificate configuration.\nOpen the file and modify the server IP address from the local IP to the\nexternal IP address of controller-0, as stored in $IP_CONTROLLER_0.\n\n> Example\n\n```ShellSession\napiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: XXX\n server: https://35.205.205.80:6443\n name: cluster.local\n...\n```\n\nNow, we load the configuration for `kubectl`:\n\n```ShellSession\nexport KUBECONFIG=$PWD/kubespray-do.conf\n```\n\nWe should be all set to communicate with our cluster from our local workstation:\n\n```ShellSession\nkubectl get nodes\n```\n\n> Output\n\n```ShellSession\nNAME STATUS ROLES AGE VERSION\ncontroller-0 Ready master 47m v1.17.9\ncontroller-1 Ready master 46m v1.17.9\ncontroller-2 Ready master 46m v1.17.9\nworker-0 Ready 45m v1.17.9\nworker-1 Ready 45m v1.17.9\nworker-2 Ready 45m v1.17.9\n```\n\n## Smoke tests\n\n### Metrics\n\nVerify if the metrics server addon was correctly installed and works:\n\n```ShellSession\nkubectl top nodes\n```\n\n> Output\n\n```ShellSession\nNAME CPU(cores) CPU% MEMORY(bytes) MEMORY%\ncontroller-0 191m 10% 1956Mi 26%\ncontroller-1 190m 10% 1828Mi 24%\ncontroller-2 182m 10% 1839Mi 24%\nworker-0 87m 4% 1265Mi 16%\nworker-1 102m 5% 1268Mi 16%\nworker-2 108m 5% 1299Mi 17%\n```\n\nPlease note that metrics might not be available at first and need a couple of\n minutes before you can actually retrieve them.\n\n### Network\n\nLet's verify if the network layer is properly functioning and pods can reach\neach other:\n\n```ShellSession\nkubectl run myshell1 -it --rm --image busybox -- sh\nhostname -i\n# launch myshell2 in separate terminal (see next code block) and ping the hostname of myshell2\nping \n```\n\n```ShellSession\nkubectl run myshell2 -it --rm --image busybox -- sh\nhostname -i\nping \n```\n\n> Output\n\n```ShellSession\nPING 10.233.108.2 (10.233.108.2): 56 data bytes\n64 bytes from 10.233.108.2: seq=0 ttl=62 time=2.876 ms\n64 bytes from 10.233.108.2: seq=1 ttl=62 time=0.398 ms\n64 bytes from 10.233.108.2: seq=2 ttl=62 time=0.378 ms\n^C\n--- 10.233.108.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.378/1.217/2.876 ms\n```\n\n### Deployments\n\nIn this section you will verify the ability to create and manage [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/).\n\nCreate a deployment for the [nginx](https://nginx.org/en/) web server:\n\n```ShellSession\nkubectl create deployment nginx --image=nginx\n```\n\nList the pod created by the `nginx` deployment:\n\n```ShellSession\nkubectl get pods -l app=nginx\n```\n\n> Output\n\n```ShellSession\nNAME READY STATUS RESTARTS AGE\nnginx-86c57db685-bmtt8 1/1 Running 0 18s\n```\n\n#### Port Forwarding\n\nIn this section you will verify the ability to access applications remotely using [port forwarding](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/).\n\nRetrieve the full name of the `nginx` pod:\n\n```ShellSession\nPOD_NAME=$(kubectl get pods -l app=nginx -o jsonpath=\"{.items[0].metadata.name}\")\n```\n\nForward port `8080` on your local machine to port `80` of the `nginx` pod:\n\n```ShellSession\nkubectl port-forward $POD_NAME 8080:80\n```\n\n> Output\n\n```ShellSession\nForwarding from 127.0.0.1:8080 -> 80\nForwarding from [::1]:8080 -> 80\n```\n\nIn a new terminal make an HTTP request using the forwarding address:\n\n```ShellSession\ncurl --head http://127.0.0.1:8080\n```\n\n> Output\n\n```ShellSession\nHTTP/1.1 200 OK\nServer: nginx/1.19.1\nDate: Thu, 13 Aug 2020 11:12:04 GMT\nContent-Type: text/html\nContent-Length: 612\nLast-Modified: Tue, 07 Jul 2020 15:52:25 GMT\nConnection: keep-alive\nETag: \"5f049a39-264\"\nAccept-Ranges: bytes\n```\n\nSwitch back to the previous terminal and stop the port forwarding to the `nginx` pod:\n\n```ShellSession\nForwarding from 127.0.0.1:8080 -> 80\nForwarding from [::1]:8080 -> 80\nHandling connection for 8080\n^C\n```\n\n#### Logs\n\nIn this section you will verify the ability to [retrieve container logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/).\n\nPrint the `nginx` pod logs:\n\n```ShellSession\nkubectl logs $POD_NAME\n```\n\n> Output\n\n```ShellSession\n...\n127.0.0.1 - - [13/Aug/2020:11:12:04 +0000] \"HEAD / HTTP/1.1\" 200 0 \"-\" \"curl/7.64.1\" \"-\"\n```\n\n#### Exec\n\nIn this section you will verify the ability to [execute commands in a container](https://kubernetes.io/docs/tasks/debug/debug-application/get-shell-running-container/#running-individual-commands-in-a-container).\n\nPrint the nginx version by executing the `nginx -v` command in the `nginx` container:\n\n```ShellSession\nkubectl exec -ti $POD_NAME -- nginx -v\n```\n\n> Output\n\n```ShellSession\nnginx version: nginx/1.19.1\n```\n\n### Kubernetes services\n\n#### Expose outside the cluster\n\nIn this section you will verify the ability to expose applications using a [Service](https://kubernetes.io/docs/concepts/services-networking/service/).\n\nExpose the `nginx` deployment using a [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) service:\n\n```ShellSession\nkubectl expose deployment nginx --port 80 --type NodePort\n```\n\n> The LoadBalancer service type can not be used because your cluster is not configured with [cloud provider integration](https://kubernetes.io/docs/getting-started-guides/scratch/#cloud-provider). Setting up cloud provider integration is out of scope for this tutorial.\n\nRetrieve the node port assigned to the `nginx` service:\n\n```ShellSession\nNODE_PORT=$(kubectl get svc nginx \\\n --output=jsonpath='{range .spec.ports[0]}{.nodePort}')\n```\n\nCreate a firewall rule that allows remote access to the `nginx` node port:\n\n```ShellSession\ngcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-nginx-service \\\n --allow=tcp:${NODE_PORT} \\\n --network kubernetes-the-kubespray-way\n```\n\nRetrieve the external IP address of a worker instance:\n\n```ShellSession\nEXTERNAL_IP=$(gcloud compute instances describe worker-0 \\\n --format 'value(networkInterfaces[0].accessConfigs[0].natIP)')\n```\n\nMake an HTTP request using the external IP address and the `nginx` node port:\n\n```ShellSession\ncurl -I http://${EXTERNAL_IP}:${NODE_PORT}\n```\n\n> Output\n\n```ShellSession\nHTTP/1.1 200 OK\nServer: nginx/1.19.1\nDate: Thu, 13 Aug 2020 11:15:02 GMT\nContent-Type: text/html\nContent-Length: 612\nLast-Modified: Tue, 07 Jul 2020 15:52:25 GMT\nConnection: keep-alive\nETag: \"5f049a39-264\"\nAccept-Ranges: bytes\n```\n\n#### Local DNS\n\nWe will now also verify that kubernetes built-in DNS works across namespaces.\nCreate a namespace:\n\n```ShellSession\nkubectl create namespace dev\n```\n\nCreate an nginx deployment and expose it within the cluster:\n\n```ShellSession\nkubectl create deployment nginx --image=nginx -n dev\nkubectl expose deployment nginx --port 80 --type ClusterIP -n dev\n```\n\nRun a temporary container to see if we can reach the service from the default\nnamespace:\n\n```ShellSession\nkubectl run curly -it --rm --image curlimages/curl:7.70.0 -- /bin/sh\ncurl --head http://nginx.dev:80\n```\n\n> Output\n\n```ShellSession\nHTTP/1.1 200 OK\nServer: nginx/1.19.1\nDate: Thu, 13 Aug 2020 11:15:59 GMT\nContent-Type: text/html\nContent-Length: 612\nLast-Modified: Tue, 07 Jul 2020 15:52:25 GMT\nConnection: keep-alive\nETag: \"5f049a39-264\"\nAccept-Ranges: bytes\n```\n\nType `exit` to leave the shell.\n\n## Cleaning Up\n\n### Kubernetes resources\n\nDelete the dev namespace, the nginx deployment and service:\n\n```ShellSession\nkubectl delete namespace dev\nkubectl delete deployment nginx\nkubectl delete svc/nginx\n```\n\n### Kubernetes state\n\nNote: you can skip this step if you want to entirely remove the machines.\n\nIf you want to keep the VMs and just remove the cluster state, you can simply\n run another Ansible playbook:\n\n```ShellSession\nansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa reset.yml\n```\n\nResetting the cluster to the VMs original state usually takes about a couple\nof minutes.\n\n### Compute instances\n\nDelete the controller and worker compute instances:\n\n```ShellSession\ngcloud -q compute instances delete \\\n controller-0 controller-1 controller-2 \\\n worker-0 worker-1 worker-2 \\\n --zone $(gcloud config get-value compute/zone)\n ```\n\n\n### Network\n\n\nDelete the fixed IP addresses (assuming you named them equal to the VM names),\nif any:\n\n```ShellSession\ngcloud -q compute addresses delete controller-0 controller-1 controller-2 \\\n worker-0 worker-1 worker-2\n```\n\nDelete the `kubernetes-the-kubespray-way` firewall rules:\n\n```ShellSession\ngcloud -q compute firewall-rules delete \\\n kubernetes-the-kubespray-way-allow-nginx-service \\\n kubernetes-the-kubespray-way-allow-internal \\\n kubernetes-the-kubespray-way-allow-external\n```\n\nDelete the `kubernetes-the-kubespray-way` network VPC:\n\n```ShellSession\ngcloud -q compute networks subnets delete kubernetes\ngcloud -q compute networks delete kubernetes-the-kubespray-way\n```\n" }, { "path": "docs/ingress/alb_ingress_controller.md", "content": "# AWS ALB Ingress Controller\n\n**NOTE:** The current image version is `v1.1.9`. Please file any issues you find and note the version used.\n\nThe AWS ALB Ingress Controller satisfies Kubernetes [ingress resources](https://kubernetes.io/docs/concepts/services-networking/ingress/) by provisioning [Application Load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html).\n\nThis project was originated by [Ticketmaster](https://github.com/ticketmaster) and [CoreOS](https://github.com/coreos) as part of Ticketmaster's move to AWS and CoreOS Tectonic. Learn more about Ticketmaster's Kubernetes initiative from Justin Dean's video at [Tectonic Summit](https://www.youtube.com/watch?v=wqXVKneP0Hg).\n\nThis project was donated to Kubernetes SIG-AWS to allow AWS, CoreOS, Ticketmaster and other SIG-AWS contributors to officially maintain the project. SIG-AWS reached this consensus on June 1, 2018.\n\n## Documentation\n\nCheckout our [Live Docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/#aws-alb-ingress-controller)!\n\n## Getting started\n\nTo get started with the controller, see our [walkthrough](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/guide/walkthrough/echoserver/).\n\n## Setup\n\n- See [controller setup](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/guide/controller/setup/) on how to install ALB ingress controller\n- See [external-dns setup](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/guide/external-dns/setup/) for how to setup the external-dns to manage route 53 records.\n\n## Building\n\nFor details on building this project, see our [building guide](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/BUILDING/).\n\n## Community, discussion, contribution, and support\n\nLearn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/).\n\nYou can reach the maintainers of this project at:\n\n- [Slack channel](https://kubernetes.slack.com/messages/sig-aws)\n- [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-aws)\n\n### Code of conduct\n\nParticipation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md).\n\n## License\n\n[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fcoreos%2Falb-ingress-controller.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fcoreos%2Falb-ingress-controller?ref=badge_large)\n" }, { "path": "docs/ingress/kube-vip.md", "content": "# kube-vip\n\nkube-vip provides Kubernetes clusters with a virtual IP and load balancer for both the control plane (for building a highly-available cluster) and Kubernetes Services of type LoadBalancer without relying on any external hardware or software.\n\n## Prerequisites\n\nYou have to configure `kube_proxy_strict_arp` when the kube_proxy_mode is `ipvs` and kube-vip ARP is enabled.\n\n```yaml\nkube_proxy_strict_arp: true\n```\n\n## Install\n\nYou have to explicitly enable the kube-vip extension:\n\n```yaml\nkube_vip_enabled: true\n```\n\nYou also need to enable\n[kube-vip as HA, Load Balancer, or both](https://kube-vip.io/docs/installation/static/#kube-vip-as-ha-load-balancer-or-both):\n\n```yaml\n# HA for control-plane, requires a VIP\nkube_vip_controlplane_enabled: true\nkube_vip_address: 10.42.42.42\nloadbalancer_apiserver:\n address: \"{{ kube_vip_address }}\"\n port: 6443\n# kube_vip_interface: ens160\n\n# LoadBalancer for services\nkube_vip_services_enabled: false\n# kube_vip_services_interface: ens320\n```\n\n> Note: When using `kube-vip` as LoadBalancer for services,\n[additional manual steps](https://kube-vip.io/docs/usage/cloud-provider/)\nare needed.\n\nIf using [local traffic policy](https://kube-vip.io/docs/usage/kubernetes-services/#external-traffic-policy-kube-vip-v050):\n\n```yaml\nkube_vip_enableServicesElection: true\n```\n\nIf using [ARP mode](https://kube-vip.io/docs/installation/static/#arp) :\n\n```yaml\nkube_vip_arp_enabled: true\n```\n\nIf using [BGP mode](https://kube-vip.io/docs/installation/static/#bgp) :\n\n```yaml\nkube_vip_bgp_enabled: true\nkube_vip_local_as: 65000\nkube_vip_bgp_routerid: 192.168.0.2\nkube_vip_bgppeers:\n- 192.168.0.10:65000::false\n- 192.168.0.11:65000::false\n# kube_vip_bgp_peeraddress:\n# kube_vip_bgp_peerpass:\n# kube_vip_bgp_peeras:\n# kube_vip_bgp_sourceip:\n# kube_vip_bgp_sourceif:\n```\n\nIf using [control plane load-balancing](https://kube-vip.io/docs/about/architecture/#control-plane-load-balancing):\n\n```yaml\nkube_vip_lb_enable: true\n```\n\nIn addition, [load-balancing method](https://kube-vip.io/docs/installation/flags/#environment-variables) could be changed:\n\n```yaml\nkube_vip_lb_fwdmethod: masquerade\n```\n\nIf you want to adjust the parameters of [kube-vip LeaderElection](https://kube-vip.io/docs/installation/flags/#environment-variables):\n\n```yaml\nkube_vip_leaseduration: 30\nkube_vip_renewdeadline: 20\nkube_vip_retryperiod: 4\n```\n" }, { "path": "docs/ingress/metallb.md", "content": "# MetalLB\n\nMetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation.\nIt allows you to create Kubernetes services of type \"LoadBalancer\" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers.\nThe default operating mode of MetalLB is in [\"Layer2\"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in [\"BGP\"](https://metallb.universe.tf/concepts/bgp/) mode.\n\n## Prerequisites\n\nYou have to configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface for MetalLB to work.\n\n```yaml\nkube_proxy_strict_arp: true\n```\n\n## Install\n\nYou have to explicitly enable the MetalLB extension.\n\n```yaml\nmetallb_enabled: true\nmetallb_speaker_enabled: true\n```\n\nBy default, MetalLB resources are deployed into the `metallb-system` namespace. You can override this namespace using a variable.\n\n```yaml\nmetallb_namespace: woodenlb-system\n```\n\nBy default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller:\n\n```yaml\nmetallb_config:\n controller:\n nodeselector:\n kubernetes.io/os: linux\n tolerations:\n - key: \"node-role.kubernetes.io/control-plane\"\n operator: \"Equal\"\n value: \"\"\n effect: \"NoSchedule\"\n```\n\nIf you'd like to set additional nodeSelector and tolerations values, you can do so in the following fashion:\n\n```yaml\nmetallb_config:\n controller:\n nodeselector:\n kubernetes.io/os: linux\n tolerations:\n - key: \"node-role.kubernetes.io/control-plane\"\n operator: \"Equal\"\n value: \"\"\n effect: \"NoSchedule\"\n speaker:\n nodeselector:\n kubernetes.io/os: linux\n tolerations:\n - key: \"node-role.kubernetes.io/control-plane\"\n operator: \"Equal\"\n value: \"\"\n effect: \"NoSchedule\"\n```\n\n## Pools\n\nFirst you need to specify all of the pools you are going to use:\n\n```yaml\nmetallb_config:\n\n address_pools:\n\n primary:\n ip_range:\n - 192.0.1.0-192.0.1.254\n\n pool1:\n ip_range:\n - 192.0.2.1-192.0.2.1\n auto_assign: false # When set to false, you need to explicitly set the loadBalancerIP in the service!\n\n pool2:\n ip_range:\n - 192.0.3.0/24\n avoid_buggy_ips: true # When set to true, .0 and .255 addresses will be avoided.\n```\n\n## Layer2 Mode\n\nPools that need to be configured in layer2 mode, need to be specified in a list:\n\n```yaml\nmetallb_config:\n\n layer2:\n - primary\n```\n\n## BGP Mode\n\nWhen operating in BGP Mode MetalLB needs to have defined upstream peers and link the pool(s) specified above to the correct peer:\n\n```yaml\nmetallb_config:\n\n layer3:\n defaults:\n\n peer_port: 179 # The TCP port to talk to. Defaults to 179, you shouldn't need to set this in production.\n hold_time: 120s # Requested BGP hold time, per RFC4271.\n\n communities:\n vpn-only: \"1234:1\"\n NO_ADVERTISE: \"65535:65282\"\n\n metallb_peers:\n\n peer1:\n peer_address: 192.0.2.1\n peer_asn: 64512\n my_asn: 4200000000\n communities:\n - vpn-only\n address_pool:\n - pool1\n\n # (optional) The source IP address to use when establishing the BGP session. In most cases the source-address field should only be used with per-node peers, i.e. peers with node selectors which select only one node. CURRENTLY NOT SUPPORTED\n source_address: 192.0.2.2\n\n # (optional) The router ID to use when connecting to this peer. Defaults to the node IP address.\n # Generally only useful when you need to peer with another BGP router running on the same machine as MetalLB.\n router_id: 1.2.3.4\n\n # (optional) Password for TCPMD5 authenticated BGP sessions offered by some peers.\n password: \"changeme\"\n\n peer2:\n peer_address: 192.0.2.2\n peer_asn: 64513\n my_asn: 4200000000\n communities:\n - NO_ADVERTISE\n address_pool:\n - pool2\n\n # (optional) The source IP address to use when establishing the BGP session. In most cases the source-address field should only be used with per-node peers, i.e. peers with node selectors which select only one node. CURRENTLY NOT SUPPORTED\n source_address: 192.0.2.1\n\n # (optional) The router ID to use when connecting to this peer. Defaults to the node IP address.\n # Generally only useful when you need to peer with another BGP router running on the same machine as MetalLB.\n router_id: 1.2.3.5\n\n # (optional) Password for TCPMD5 authenticated BGP sessions offered by some peers.\n password: \"changeme\"\n```\n\nWhen using calico >= 3.18 you can replace MetalLB speaker by calico Service LoadBalancer IP advertisement.\nSee [calico service IPs advertisement documentation](https://docs.projectcalico.org/archive/v3.18/networking/advertise-service-ips#advertise-service-load-balancer-ip-addresses).\nIn this scenario you should disable the MetalLB speaker and configure the `calico_advertise_service_loadbalancer_ips` to match your `ip_range`\n\n```yaml\nmetallb_speaker_enabled: false\nmetallb_config:\n address_pools:\n primary:\n ip_range:\n - 10.5.0.0/16\n auto_assign: true\n layer2:\n - primary\ncalico_advertise_service_loadbalancer_ips: \"{{ metallb_config.address_pools.primary.ip_range }}\"\n```\n\nIf you have additional loadbalancer IP pool in `metallb_config.address_pools` , ensure to add them to the list.\n\n```yaml\nmetallb_speaker_enabled: false\nmetallb_config:\n address_pools:\n primary:\n ip_range:\n - 10.5.0.0/16\n auto_assign: true\n pool1:\n ip_range:\n - 10.6.0.0/16\n auto_assign: true\n pool2:\n ip_range:\n - 10.10.0.0/16\n auto_assign: true\n layer2:\n - primary\n layer3:\n defaults:\n peer_port: 179\n hold_time: 120s\n communities:\n vpn-only: \"1234:1\"\n NO_ADVERTISE: \"65535:65282\"\n metallb_peers:\n peer1:\n peer_address: 10.6.0.1\n peer_asn: 64512\n my_asn: 4200000000\n communities:\n - vpn-only\n address_pool:\n - pool1\n peer2:\n peer_address: 10.10.0.1\n peer_asn: 64513\n my_asn: 4200000000\n communities:\n - NO_ADVERTISE\n address_pool:\n - pool2\ncalico_advertise_service_loadbalancer_ips:\n - 10.5.0.0/16\n - 10.6.0.0/16\n - 10.10.0.0/16\n```\n" }, { "path": "docs/operating_systems/amazonlinux.md", "content": "# Amazon Linux 2\n\nAmazon Linux is supported with docker,containerd and cri-o runtimes.\n\n**Note:** that Amazon Linux is not currently covered in kubespray CI and\nsupport for it is currently considered experimental.\n\nAmazon Linux 2, while derived from the Redhat OS family, does not keep in\nsync with RHEL upstream like CentOS/AlmaLinux/Oracle Linux. In order to use\nAmazon Linux as the ansible host for your kubespray deployments you need to\nmanually install `python3` and deploy ansible and kubespray dependencies in\na python virtual environment or use the official kubespray containers.\n\nThere are no special considerations for using Amazon Linux as the target OS\nfor Kubespray deployments.\n" }, { "path": "docs/operating_systems/bootstrap-os.md", "content": "# bootstrap_os\n\nBootstrap an Ansible host to be able to run Ansible modules.\n\nThis role will:\n\n* configure the package manager (if applicable) to be able to fetch packages\n* install Python\n* install the necessary packages to use Ansible's package manager modules\n* set the hostname of the host to `{{ inventory_hostname }}` when requested\n\n## Requirements\n\nA host running an operating system that is supported by Kubespray.\nSee [Supported Linux Distributions](https://github.com/kubernetes-sigs/kubespray#supported-linux-distributions) for a current list.\n\nSSH access to the host.\n\n## Role Variables\n\nVariables are listed with their default values, if applicable.\n\n### General variables\n\n* `http_proxy`/`https_proxy`\n The role will configure the package manager (if applicable) to download packages via a proxy.\n\n* `override_system_hostname: true`\n The role will set the hostname of the machine to the name it has according to Ansible's inventory (the variable `{{ inventory_hostname }}`).\n\n### Per distribution variables\n\n#### Flatcar Container Linux\n\n* `coreos_locksmithd_disable: false`\n Whether `locksmithd` (responsible for rolling restarts) should be disabled or be left alone.\n\n#### CentOS/RHEL/AlmaLinux/Rocky Linux\n\n* `centos_fastestmirror_enabled: false`\n Whether the [fastestmirror](https://wiki.centos.org/PackageManagement/Yum/FastestMirror) yum plugin should be enabled.\n\n## Example Playbook\n\nRemember to disable fact gathering since Python might not be present on hosts.\n\n```yaml\n- hosts: all\n gather_facts: false # not all hosts might be able to run modules yet\n roles:\n - kubespray_defaults\n - bootstrap_os\n```\n\n## License\n\nApache 2.0\n" }, { "path": "docs/operating_systems/fcos.md", "content": "# Fedora CoreOS\n\nTested with stable version 37.20230322.3.0\n\nBecause package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.\nRestart playbook again.\n\n## Containers\n\nTested with\n\n- containerd\n- crio\n\n## Network\n\n### calico\n\nTo use calico create sysctl file with ignition:\n\n```yaml\nfiles:\n - path: /etc/sysctl.d/reverse-path-filter.conf\n contents:\n inline: |\n net.ipv4.conf.all.rp_filter=1\n```\n\n## libvirt setup\n\n### Prepare\n\nPrepare ignition and serve via http (a.e. python -m http.server )\n\n```json\n{\n \"ignition\": {\n \"version\": \"3.0.0\"\n },\n\n \"passwd\": {\n \"users\": [\n {\n \"name\": \"ansibleUser\",\n \"sshAuthorizedKeys\": [\n \"ssh-rsa ..publickey..\"\n ],\n \"groups\": [ \"wheel\" ]\n }\n ]\n }\n}\n```\n\n### create guest\n\n```ShellSeasion\nmachine_name=myfcos1\nignition_url=http://mywebserver/fcos.ign\n\nfcos_version=34.20210611.3.0\nkernel=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-kernel-x86_64\ninitrd=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-initramfs.x86_64.img\nrootfs=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-rootfs.x86_64.img\nkernel_args=\"console=ttyS0 coreos.live.rootfs_url=${rootfs} coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}\"\nsudo virt-install --name ${machine_name} --ram 4048 --graphics=none --vcpus 2 --disk size=20 \\\n --network bridge=virbr0 \\\n --install kernel=${kernel},initrd=${initrd},kernel_args_overwrite=yes,kernel_args=\"${kernel_args}\"\n```\n" }, { "path": "docs/operating_systems/flatcar.md", "content": "Flatcar Container Linux bootstrap\n===============\n\nExample with Ansible:\n\nBefore running the cluster playbook you must satisfy the following requirements:\n\nGeneral Flatcar Pre-Installation Notes:\n\n- Ensure that the bin_dir is set to `/opt/bin`\n- ansible_python_interpreter should be `/opt/bin/python`. This will be laid down by the bootstrap task.\n- The resolvconf_mode setting of `docker_dns` **does not** work for Flatcar. This is because we do not edit the systemd service file for docker on Flatcar nodes. Instead, just use the default `host_resolvconf` mode. It should work out of the box.\n\nThen you can proceed to [cluster deployment](#run-deployment)\n" }, { "path": "docs/operating_systems/kylinlinux.md", "content": "# Kylin Linux\n\nKylin Linux is supported with docker and containerd runtimes.\n\n**Note:** that Kylin Linux is not currently covered in kubespray CI and\nsupport for it is currently considered experimental.\n\nAt present, only `Kylin Linux Advanced Server V10 (Sword)` has been adapted, which can support the deployment of aarch64 and x86_64 platforms.\n\nThere are no special considerations for using Kylin Linux as the target OS\nfor Kubespray deployments.\n" }, { "path": "docs/operating_systems/openeuler.md", "content": "# OpenEuler\n\n[OpenEuler](https://www.openeuler.org/en/) Linux is supported with docker and containerd runtimes.\n\n**Note:** that OpenEuler Linux is not currently covered in kubespray CI and\nsupport for it is currently considered experimental.\n\nAt present, only `openEuler 22.03 LTS` has been adapted, which can support the deployment of aarch64 and x86_64 platforms.\n\nThere are no special considerations for using OpenEuler Linux as the target OS\nfor Kubespray deployments.\n" }, { "path": "docs/operating_systems/opensuse.md", "content": "# openSUSE Leap 15.6 and Tumbleweed\n\nopenSUSE Leap installation Notes:\n\n- Install Ansible\n\n ```ShellSession\n sudo zypper ref\n sudo zypper -n install ansible\n\n ```\n\n- Install Jinja2 and Python-Netaddr\n\n ```sudo zypper -n install python-Jinja2 python-netaddr```\n\nNow you can continue with [Preparing your deployment](getting-started.md#starting-custom-deployment)\n" }, { "path": "docs/operating_systems/rhel.md", "content": "# Red Hat Enterprise Linux (RHEL)\n\nThe documentation also applies to Red Hat derivatives, including Alma Linux, Rocky Linux, Oracle Linux, and CentOS.\n\n## RHEL Support Subscription Registration\n\nThe content of this section does not apply to open-source derivatives.\n\nIn order to install packages via yum or dnf, RHEL hosts are required to be registered for a valid Red Hat support subscription.\n\nYou can apply for a 1-year Development support subscription by creating a [Red Hat Developers](https://developers.redhat.com/) account. Be aware though that as the Red Hat Developers subscription is limited to only 1 year, it should not be used to register RHEL hosts provisioned in Production environments.\n\nOnce you have a Red Hat support account, simply add the credentials to the Ansible inventory parameters `rh_subscription_username` and `rh_subscription_password` prior to deploying Kubespray. If your company has a Corporate Red Hat support account, then obtain an **Organization ID** and **Activation Key**, and add these to the Ansible inventory parameters `rh_subscription_org_id` and `rh_subscription_activation_key` instead of using your Red Hat support account credentials.\n\n```ini\nrh_subscription_username: \"\"\nrh_subscription_password: \"\"\n# rh_subscription_org_id: \"\"\n# rh_subscription_activation_key: \"\"\n```\n\nEither the Red Hat support account username/password, or Organization ID/Activation Key combination must be specified in the Ansible inventory in order for the Red Hat subscription registration to complete successfully during the deployment of Kubespray.\n\nUpdate the Ansible inventory parameters `rh_subscription_usage`, `rh_subscription_role` and `rh_subscription_sla` if necessary to suit your specific requirements.\n\n```ini\nrh_subscription_usage: \"Development\"\nrh_subscription_role: \"Red Hat Enterprise Server\"\nrh_subscription_sla: \"Self-Support\"\n```\n\nIf the RHEL hosts are already registered to a valid Red Hat support subscription via an alternative configuration management approach prior to the deployment of Kubespray, the successful RHEL `subscription-manager` status check will simply result in the RHEL subscription registration tasks being skipped.\n\n## Rocky Linux 10\n\n(Experimental in Kubespray CI)\n\nThe official Rocky Linux 10 cloud image does not include `kernel-module-extra`. Both Kube Proxy and CNI rely on this package, and since it relates to kernel version compatibility (which may require VM reboots, etc.), we haven't found an ideal solution.\n\nHowever, some users report that it doesn't affect them (minimal version). Therefore, the Kubespray CI Rocky Linux 10 image is built by Kubespray maintainers using `diskimage-builder`. For detailed methods, please refer to [the comments](https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093).\n" }, { "path": "docs/operating_systems/uoslinux.md", "content": "# UOS Linux\n\nUOS Linux(UnionTech OS Server 20) is supported with docker and containerd runtimes.\n\n**Note:** that UOS Linux is not currently covered in kubespray CI and\nsupport for it is currently considered experimental.\n\nThere are no special considerations for using UOS Linux as the target OS\nfor Kubespray deployments.\n" }, { "path": "docs/operations/cgroups.md", "content": "# cgroups\n\nTo avoid resource contention between containers and host daemons in Kubernetes, the kubelet components can use cgroups to limit resource usage.\n\n## Enforcing Node Allocatable\n\nYou can use `kubelet_enforce_node_allocatable` to set node allocatable enforcement.\n\n```yaml\n# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.\nkubelet_enforce_node_allocatable: \"pods\"\n# kubelet_enforce_node_allocatable: \"pods,kube-reserved\"\n# kubelet_enforce_node_allocatable: \"pods,kube-reserved,system-reserved\"\n```\n\nNote that to enforce kube-reserved or system-reserved, `kube_reserved_cgroups` or `system_reserved_cgroups` needs to be specified respectively.\n\nHere is an example:\n\n```yaml\nkubelet_enforce_node_allocatable: \"pods,kube-reserved,system-reserved\"\n\n# Set kube_reserved to true to run kubelet and container-engine daemons in a dedicated cgroup.\n# This is required if you want to enforce limits on the resource usage of these daemons.\n# It is not required if you just want to make resource reservations (kube_memory_reserved, kube_cpu_reserved, etc.)\nkube_reserved: true\nkube_reserved_cgroups_for_service_slice: kube.slice\nkube_reserved_cgroups: \"/{{ kube_reserved_cgroups_for_service_slice }}\"\nkube_memory_reserved: 256Mi\nkube_cpu_reserved: 100m\n# kube_ephemeral_storage_reserved: 2Gi\n# kube_pid_reserved: \"1000\"\n\n# Set to true to reserve resources for system daemons\nsystem_reserved: true\nsystem_reserved_cgroups_for_service_slice: system.slice\nsystem_reserved_cgroups: \"/{{ system_reserved_cgroups_for_service_slice }}\"\nsystem_memory_reserved: 512Mi\nsystem_cpu_reserved: 500m\n# system_ephemeral_storage_reserved: 2Gi\n# system_pid_reserved: \"1000\"\n```\n\nAfter the setup, the cgroups hierarchy is as follows:\n\n```bash\n/ (Cgroups Root)\n├── kubepods.slice\n│ ├── ...\n│ ├── kubepods-besteffort.slice\n│ ├── kubepods-burstable.slice\n│ └── ...\n├── kube.slice\n│ ├── ...\n│ ├── {{container_manager}}.service\n│ ├── kubelet.service\n│ └── ...\n├── system.slice\n│ └── ...\n└── ...\n```\n\nYou can learn more in the [official kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/).\n" }, { "path": "docs/operations/encrypting-secret-data-at-rest.md", "content": "# Encrypting Secret Data at Rest\n\nBefore enabling Encrypting Secret Data at Rest, please read the following documentation carefully.\n\n\n\nAs you can see from the documentation above, 5 encryption providers are supported as of today (22.02.2022).\n\nAs default value for the provider we have chosen `secretbox`.\n\nAlternatively you can use the values `identity`, `aesgcm`, `aescbc` or `kms`.\n\n| Provider | Why we have decided against the value as default |\n|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| identity | no encryption |\n| aesgcm | Must be rotated every 200k writes |\n| aescbc | Not recommended due to CBC's vulnerability to padding oracle attacks. |\n| kms | Is the official recommended way, but assumes that a key management service independent of Kubernetes exists, we cannot assume this in all environments, so not a suitable default value. |\n\n## Details about Secretbox\n\nSecretbox uses [Poly1305](https://cr.yp.to/mac.html) as message-authentication code and [XSalsa20](https://www.xsalsa20.com/) as secret-key authenticated encryption and secret-key encryption.\n" }, { "path": "docs/operations/etcd.md", "content": "# etcd\n\n## Deployment Types\n\nIt is possible to deploy etcd with three methods. To change the default deployment method (host), use the `etcd_deployment_type` variable. Possible values are `host`, `kubeadm`, and `docker`.\n\n### Host\n\nHost deployment is the default method. Using this method will result in etcd installed as a systemd service.\n\n### Docker\n\nInstalls docker in etcd group members and runs etcd on docker containers. Only usable when `container_manager` is set to `docker`.\n\n### Kubeadm\n\nThis deployment method is experimental and is only available for new deployments. This deploys etcd as a static pod on control plane hosts.\n\n## Metrics\n\nTo expose metrics on a separate HTTP port, define it in the inventory with:\n\n```yaml\netcd_metrics_port: 2381\n```\n\nTo create a service `etcd-metrics` and associated endpoints in the `kube-system` namespace,\ndefine its labels in the inventory with:\n\n```yaml\netcd_metrics_service_labels:\n k8s-app: etcd\n app.kubernetes.io/managed-by: Kubespray\n app: kube-prometheus-stack-kube-etcd\n release: kube-prometheus-stack\n```\n\nThe last two labels in the above example allows to scrape the metrics from the\n[kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack)\nchart when it is installed with the release name `kube-prometheus-stack` and the following Helm `values.yaml`:\n\n```yaml\nkubeEtcd:\n service:\n enabled: false\n```\n\nIf your Helm release name is different, adjust the `release` label accordingly.\n\nTo fully override metrics exposition URLs, define it in the inventory with:\n\n```yaml\netcd_listen_metrics_urls: \"http://0.0.0.0:2381\"\n```\n\nIf you choose to expose metrics on specific node IPs (for example `10.141.4.22`, `10.141.4.23`, `10.141.4.24`) in `etcd_listen_metrics_urls`,\nyou can configure kube-prometheus-stack to scrape those endpoints directly with:\n\n```yaml\nkubeEtcd:\n enabled: true\n endpoints:\n - 10.141.4.22\n - 10.141.4.23\n - 10.141.4.24\n```\n" }, { "path": "docs/operations/ha-mode.md", "content": "# HA endpoints for K8s\n\nThe following components require a highly available endpoints:\n\n* etcd cluster,\n* kube-apiserver service instances.\n\nThe latter relies on a 3rd side reverse proxy, like Nginx or HAProxy, to\nachieve the same goal.\n\n## Etcd\n\nThe etcd clients (kube-api-masters) are configured with the list of all etcd peers. If the etcd-cluster has multiple instances, it's configured in HA already.\n\n## Kube-apiserver\n\nK8s components require a loadbalancer to access the apiservers via a reverse\nproxy. Kubespray includes support for an nginx-based proxy that resides on each\nnon-master Kubernetes node. This is referred to as localhost loadbalancing. It\nis less efficient than a dedicated load balancer because it creates extra\nhealth checks on the Kubernetes apiserver, but is more practical for scenarios\nwhere an external LB or virtual IP management is inconvenient. This option is\nconfigured by the variable `loadbalancer_apiserver_localhost` (defaults to\n`True`. Or `False`, if there is an external `loadbalancer_apiserver` defined).\nYou may also define the port the local internal loadbalancer uses by changing,\n`loadbalancer_apiserver_port`. This defaults to the value of\n`kube_apiserver_port`. It is also important to note that Kubespray will only\nconfigure kubelet and kube-proxy on non-master nodes to use the local internal\nloadbalancer. If you wish to control the name of the loadbalancer container,\nyou can set the variable `loadbalancer_apiserver_pod_name`.\n\nIf you choose to NOT use the local internal loadbalancer, you will need to\nuse the [kube-vip](/docs/ingress/kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the\n`access_ip` or IP address of the first server node in the `kube_control_plane` group.\nIt can also configure clients to use endpoints for a given loadbalancer type.\nThe following diagram shows how traffic to the apiserver is directed.\n\n![Image](/docs/figures/loadbalancer_localhost.png?raw=true)\n\nA user may opt to use an external loadbalancer (LB) instead. An external LB\nprovides access for external clients, while the internal LB accepts client\nconnections only to the localhost.\nGiven a frontend `VIP` address and `IP1, IP2` addresses of backends, here is\nan example configuration for a HAProxy service acting as an external LB:\n\n```raw\nlisten kubernetes-apiserver-https\n bind :8383\n mode tcp\n option log-health-checks\n timeout client 3h\n timeout server 3h\n server master1 :6443 check check-ssl verify none inter 10000\n server master2 :6443 check check-ssl verify none inter 10000\n balance roundrobin\n```\n\n Note: That's an example config managed elsewhere outside Kubespray.\n\nAnd the corresponding example global vars for such a \"cluster-aware\"\nexternal LB with the cluster API access modes configured in Kubespray:\n\n```yml\napiserver_loadbalancer_domain_name: \"my-apiserver-lb.example.com\"\nloadbalancer_apiserver:\n address: \n port: 8383\n```\n\n Note: The default kubernetes apiserver configuration binds to all interfaces,\n so you will need to use a different port for the vip from that the API is\n listening on, or set the `kube_apiserver_bind_address` so that the API only\n listens on a specific interface (to avoid conflict with haproxy binding the\n port on the VIP address)\n\nThis domain name, or default \"lb-apiserver.kubernetes.local\", will be inserted\ninto the `/etc/hosts` file of all servers in the `k8s_cluster` group and wired\ninto the generated self-signed TLS/SSL certificates as well. Note that\nthe HAProxy service should as well be HA and requires a VIP management, which\nis out of scope of this doc.\n\nThere is a special case for an internal and an externally configured (not with\nKubespray) LB used simultaneously. Keep in mind that the cluster is not aware\nof such an external LB and you need no to specify any configuration variables\nfor it.\n\n Note: TLS/SSL termination for externally accessed API endpoints' will **not**\n be covered by Kubespray for that case. Make sure your external LB provides it.\n Alternatively you may specify an external load balanced VIPs in the\n `supplementary_addresses_in_ssl_keys` list. Then, kubespray will add them into\n the generated cluster certificates as well.\n\nAside of that specific case, the `loadbalancer_apiserver` considered mutually\nexclusive to `loadbalancer_apiserver_localhost`.\n\nAccess API endpoints are evaluated automatically, as the following:\n\n| Endpoint type | kube_control_plane | non-master | external |\n|------------------------------|------------------------------------------|-------------------------|-----------------------|\n| Local LB (default) | `https://dbip:sp` | `https://lc:nsp` | `https://m[0].aip:sp` |\n| Local LB (default) + cbip | `https://cbip:sp` and `https://lc:nsp` | `https://lc:nsp` | `https://m[0].aip:sp` |\n| Local LB + Unmanaged here LB | `https://dbip:sp` | `https://lc:nsp` | `https://ext` |\n| External LB, no internal | `https://dbip:sp` | `` | `https://lb:lp` |\n| No ext/int LB | `https://dbip:sp` | `` | `https://m[0].aip:sp` |\n\nWhere:\n\n* `m[0]` - the first node in the `kube_control_plane` group;\n* `lb` - LB FQDN, `apiserver_loadbalancer_domain_name`;\n* `ext` - Externally load balanced VIP:port and FQDN, not managed by Kubespray;\n* `lc` - localhost;\n* `cbip` - a custom bind IP, `kube_apiserver_bind_address`;\n* `dbip` - localhost for the default bind IP '0.0.0.0';\n* `nsp` - nginx secure port, `loadbalancer_apiserver_port`, defers to `sp`;\n* `sp` - secure port, `kube_apiserver_port`;\n* `lp` - LB port, `loadbalancer_apiserver.port`, defers to the secure port;\n* `ip` - the node IP, defers to the ansible IP;\n* `aip` - `access_ip`, defers to the ip.\n\nA second and a third column represent internal cluster access modes. The last\ncolumn illustrates an example URI to access the cluster APIs externally.\nKubespray has nothing to do with it, this is informational only.\n\nAs you can see, the masters' internal API endpoints are always\ncontacted via the local bind IP, which is `https://bip:sp`.\n\n## Optional configurations\n\n### ETCD with a LB\n\nIn order to use an external loadbalancing (L4/TCP or L7 w/ SSL Passthrough VIP), the following variables need to be overridden in group_vars\n\n* `etcd_access_addresses`\n* `etcd_client_url`\n* `etcd_cert_alt_names`\n* `etcd_cert_alt_ips`\n\n#### Example of a VIP w/ FQDN\n\n```yaml\netcd_access_addresses: https://etcd.example.com:2379\netcd_client_url: https://etcd.example.com:2379\netcd_cert_alt_names:\n - \"etcd.kube-system.svc.{{ dns_domain }}\"\n - \"etcd.kube-system.svc\"\n - \"etcd.kube-system\"\n - \"etcd\"\n - \"etcd.example.com\" # This one needs to be added to the default etcd_cert_alt_names\n```\n\n#### Example of a VIP w/o FQDN (IP only)\n\n```yaml\netcd_access_addresses: https://2.3.7.9:2379\netcd_client_url: https://2.3.7.9:2379\netcd_cert_alt_ips:\n - \"2.3.7.9\"\n```\n" }, { "path": "docs/operations/hardening.md", "content": "# Cluster Hardening\n\nIf you want to improve the security on your cluster and make it compliant with the [CIS Benchmarks](https://learn.cisecurity.org/benchmarks), here you can find a configuration to harden your **kubernetes** installation.\n\nTo apply the hardening configuration, create a file (eg. `hardening.yaml`) and paste the content of the following code snippet into that.\n\n## Minimum Requirements\n\nThe **kubernetes** version should be at least `v1.23.6` to have all the most recent security features (eg. the new `PodSecurity` admission plugin, etc).\n\n**N.B.** Some of these configurations have just been added to **kubespray**, so ensure that you have the latest version to make it works properly. Also, ensure that other configurations doesn't override these.\n\n`hardening.yaml`:\n\n```yaml\n# Hardening\n---\n\n## kube-apiserver\nauthorization_modes: ['Node', 'RBAC']\nkube_apiserver_request_timeout: 120s\nkube_apiserver_service_account_lookup: true\n\n# enable kubernetes audit\nkubernetes_audit: true\naudit_log_path: \"/var/log/kube-apiserver-log.json\"\naudit_log_maxage: 30\naudit_log_maxbackups: 10\naudit_log_maxsize: 100\n\ntls_min_version: VersionTLS12\ntls_cipher_suites:\n - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305\n\n# enable encryption at rest\nkube_encrypt_secret_data: true\nkube_encryption_resources: [secrets]\nkube_encryption_algorithm: \"secretbox\"\n\nkube_apiserver_enable_admission_plugins:\n - EventRateLimit\n - AlwaysPullImages\n - ServiceAccount\n - NamespaceLifecycle\n - NodeRestriction\n - LimitRanger\n - ResourceQuota\n - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - PodNodeSelector\n - PodSecurity\nkube_apiserver_admission_control_config_file: true\n# Creates config file for PodNodeSelector\n# kube_apiserver_admission_plugins_needs_configuration: [PodNodeSelector]\n# Define the default node selector, by default all the workloads will be scheduled on nodes\n# with label network=srv1\n# kube_apiserver_admission_plugins_podnodeselector_default_node_selector: \"network=srv1\"\n# EventRateLimit plugin configuration\nkube_apiserver_admission_event_rate_limits:\n limit_1:\n type: Namespace\n qps: 50\n burst: 100\n cache_size: 2000\n limit_2:\n type: User\n qps: 50\n burst: 100\nkube_profiling: false\n# Remove anonymous access to cluster\nremove_anonymous_access: true\n\n## kube-controller-manager\nkube_controller_manager_bind_address: 127.0.0.1\nkube_controller_terminated_pod_gc_threshold: 50\nkube_controller_feature_gates: [\"RotateKubeletServerCertificate=true\"]\n\n## kube-scheduler\nkube_scheduler_bind_address: 127.0.0.1\n\n## etcd\n# Running etcd (on dedicated hosts) outside the Kubernetes cluster is the most secure deployment option,\n# as it isolates etcd from the cluster's CNI network and removes direct pod-level attack vectors.\n# This approach prevents RBAC misconfigurations that potentially compromise etcd,\n# creating an additional security boundary that protects the cluster's critical state store.\netcd_deployment_type: host\n\n## kubelet\nkubelet_authorization_mode_webhook: true\nkubelet_authentication_token_webhook: true\nkube_read_only_port: 0\nkubelet_rotate_server_certificates: true\nkubelet_protect_kernel_defaults: true\nkubelet_event_record_qps: 1\nkubelet_rotate_certificates: true\nkubelet_streaming_connection_idle_timeout: \"5m\"\nkubelet_make_iptables_util_chains: true\nkubelet_feature_gates: [\"RotateKubeletServerCertificate=true\"]\nkubelet_seccomp_default: true\nkubelet_systemd_hardening: true\n# In case you have multiple interfaces in your\n# control plane nodes and you want to specify the right\n# IP addresses, kubelet_secure_addresses allows you\n# to specify the IP from which the kubelet\n# will receive the packets.\nkubelet_secure_addresses: \"localhost link-local {{ kube_pods_subnet }} 192.168.10.110 192.168.10.111 192.168.10.112\"\n\n# additional configurations\nkube_owner: root\nkube_cert_group: root\n\n# create a default Pod Security Configuration and deny running of insecure pods\n# kube_system namespace is exempted by default\nkube_pod_security_use_default: true\nkube_pod_security_default_enforce: restricted\n```\n\nLet's take a deep look to the resultant **kubernetes** configuration:\n\n* The `anonymous-auth` (on `kube-apiserver`) is set to `true` by default. This is fine, because it is considered safe if you enable `RBAC` for the `authorization-mode`.\n* The `enable-admission-plugins` includes `PodSecurity` (for more details, please take a look here: ). Then, we set the `EventRateLimit` plugin, providing additional configuration files (that are automatically created under the hood and mounted inside the `kube-apiserver` container) to make it work.\n* The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this).\n* The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`.\n See for more information on the subject.\n* The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image:\n ![kubelet hardening](../img/kubelet-hardening.png)\n\nOnce you have the file properly filled, you can run the **Ansible** command to start the installation:\n\n```bash\nansible-playbook -v cluster.yml \\\n -i inventory.ini \\\n -b --become-user=root \\\n --private-key ~/.ssh/id_ecdsa \\\n -e \"@vars.yaml\" \\\n -e \"@hardening.yaml\"\n```\n\n**N.B.** The `vars.yaml` contains our general cluster information (SANs, load balancer, dns, etc..) and `hardening.yaml` is the file described above.\n" }, { "path": "docs/operations/integration.md", "content": "# Kubespray (kubespray) in own ansible playbooks repo\n\n1. Fork [kubespray repo](https://github.com/kubernetes-sigs/kubespray) to your personal/organisation account on github.\n Note:\n * All forked public repos at github will be also public, so **never commit sensitive data to your public forks**.\n * List of all forked repos could be retrieved from github page of original project.\n\n2. Add **forked repo** as submodule to desired folder in your existent ansible repo (for example 3d/kubespray):\n\n ```ShellSession\n git submodule add https://github.com/YOUR_GITHUB/kubespray.git kubespray\n ```\n\n Git will create `.gitmodules` file in your existent ansible repo:\n\n ```ini\n [submodule \"3d/kubespray\"]\n path = 3d/kubespray\n url = https://github.com/YOUR_GITHUB/kubespray.git\n ```\n\n3. Configure git to show submodule status:\n\n ```ShellSession\n git config --global status.submoduleSummary true\n ```\n\n4. Add *original* kubespray repo as upstream:\n\n ```ShellSession\n cd kubespray && git remote add upstream https://github.com/kubernetes-sigs/kubespray.git\n ```\n\n5. Sync your master branch with upstream:\n\n ```ShellSession\n git checkout master\n git fetch upstream\n git merge upstream/master\n git push origin master\n ```\n\n6. Create a new branch which you will use in your working environment:\n\n ```ShellSession\n git checkout -b work\n ```\n\n ***Never*** use master branch of your repository for your commits.\n\n7. Modify path to library and roles in your ansible.cfg file (role naming should be unique, you may have to rename your existent roles if they have same names as kubespray project),\n if you had roles in your existing ansible project before, you can add the path to those separated with `:`:\n\n ```ini\n ...\n library = ./library/:3d/kubespray/library/\n roles_path = ./roles/:3d/kubespray/roles/\n ...\n ```\n\n8. Copy and modify configs from kubespray `group_vars` folder to corresponding `group_vars` folder in your existent project.\n\n You could rename *all.yml* config to something else, i.e. *kubespray.yml* and create corresponding group in your inventory file, which will include all hosts groups related to kubernetes setup.\n\n9. Modify your ansible inventory file by adding mapping of your existent groups (if any) to kubespray naming.\n For example:\n\n ```ini\n ...\n #Kubespray groups:\n [kube_node:children]\n kubenode\n\n [etcd:children]\n kubemaster\n kubemaster-ha\n\n [kube_control_plane:children]\n kubemaster\n kubemaster-ha\n ```\n\n* Last entry here needed to apply kubespray.yml config file, renamed from all.yml of kubespray project.\n\n10. Now you can include kubespray tasks in you existent playbooks by including cluster.yml file:\n\n ```yml\n - name: Import kubespray playbook\n ansible.builtin.import_playbook: 3d/kubespray/cluster.yml\n ```\n\n Or you could copy separate tasks from cluster.yml into your ansible repository.\n\n11. Commit changes to your ansible repo. Keep in mind, that submodule folder is just a link to the git commit hash of your forked repo.\n\n When you update your \"work\" branch you need to commit changes to ansible repo as well.\nOther members of your team should use ```git submodule sync```, ```git submodule update --init``` to get actual code from submodule.\n\n## Contributing\n\nIf you made useful changes or fixed a bug in existent kubespray repo, use this flow for PRs to original kubespray repo.\n\n1. Sign the [CNCF CLA](https://git.k8s.io/community/CLA.md).\n\n2. Change working directory to git submodule directory (3d/kubespray).\n\n3. Setup desired user.name and user.email for submodule.\n\n If kubespray is only one submodule in your repo you could use something like:\n\n ```ShellSession\n git submodule foreach --recursive 'git config user.name \"First Last\" && git config user.email \"your-email-address@used.for.cncf\"'\n ```\n\n4. Sync with upstream master:\n\n ```ShellSession\n git fetch upstream\n git merge upstream/master\n git push origin master\n ```\n\n5. Create new branch for the specific fixes that you want to contribute:\n\n ```ShellSession\n git checkout -b fixes-name-date-index\n ```\n\n Branch name should be self explaining to you, adding date and/or index will help you to track/delete your old PRs.\n\n6. Find git hash of your commit in \"work\" repo and apply it to newly created \"fix\" repo:\n\n ```ShellSession\n git cherry-pick \n ```\n\n7. If you have several temporary-stage commits - squash them using [git rebase -i](https://eli.thegreenplace.net/2014/02/19/squashing-github-pull-requests-into-a-single-commit)\n\n Also you could use interactive rebase\n\n ```ShellSession\n git rebase -i HEAD~10\n ```\n\n to delete commits which you don't want to contribute into original repo.\n\n8. When your changes is in place, you need to check upstream repo one more time because it could be changed during your work.\n\n Check that you're on correct branch:\n\n ```ShellSession\n git status\n ```\n\n And pull changes from upstream (if any):\n\n ```ShellSession\n git pull --rebase upstream master\n ```\n\n9. Now push your changes to your **fork** repo with\n\n ```ShellSession\n git push\n ```\n\n If your branch doesn't exist on github, git will propose you to use something like\n\n ```ShellSession\n git push --set-upstream origin fixes-name-date-index\n ```\n\n10. Open you forked repo in browser, on the main page you will see proposition to create pull request for your newly created branch. Check proposed diff of your PR. If something is wrong you could safely delete \"fix\" branch on github using\n\n ```ShellSession\n git push origin --delete fixes-name-date-index\n git branch -D fixes-name-date-index\n ```\n\n and start whole process from the beginning.\n\n If everything is fine - add description about your changes (what they do and why they're needed) and confirm pull request creation.\n" }, { "path": "docs/operations/kernel-requirements.md", "content": "# Kernel Requirements\n\nFor Kubernetes >=1.32.0, the recommended kernel LTS version from the 4.x series is 4.19. Any 5.x or 6.x versions are also supported. For cgroups v2 support, the minimum version is 4.15 and the recommended version is 5.8+. Refer to [this link](https://github.com/kubernetes/kubernetes/blob/v1.32.0/vendor/k8s.io/system-validators/validators/types_unix.go#L33). For more information, see [kernel version requirements](https://kubernetes.io/docs/reference/node/kernel-version-requirements).\n\nIf the OS kernel version is lower than required, add the following configuration to ignore the kubeadm preflight errors:\n\n```yaml\nkubeadm_ignore_preflight_errors:\n - SystemVerification\n```\n\nThe Kernel Version Matrixs:\n\n| OS Version | Kernel Version | Kernel >=4.19 |\n|--- | --- | --- |\n| RHEL 9 | 5.14 | :white_check_mark: |\n| RHEL 8 | 4.18 | :x: |\n| Alma Linux 9 | 5.14 | :white_check_mark: |\n| Alma Linux 8 | 4.18 | :x: |\n| Rocky Linux 9 | 5.14 | :white_check_mark: |\n| Rocky Linux 8 | 4.18 | :x: |\n| Oracle Linux 9 | 5.14 | :white_check_mark: |\n| Oracle Linux 8 | 4.18 | :x: |\n| Ubuntu 24.04 | 6.6 | :white_check_mark: |\n| Ubuntu 22.04 | 5.15 | :white_check_mark: |\n| Ubuntu 20.04 | 5.4 | :white_check_mark: |\n| Debian 12 | 6.1 | :white_check_mark: |\n| Debian 11 | 5.10 | :white_check_mark: |\n| Fedora 40 | 6.8 | :white_check_mark: |\n| Fedora 39 | 6.5 | :white_check_mark: |\n| openSUSE Leap 15.5 | 5.14 | :white_check_mark: |\n| Amazon Linux 2 | 4.14 | :x: |\n| openEuler 24.03 | 6.6 | :white_check_mark: |\n| openEuler 22.03 | 5.10 | :white_check_mark: |\n| openEuler 20.03 | 4.19 | :white_check_mark: |\n" }, { "path": "docs/operations/large-deployments.md", "content": "Large deployments of K8s\n========================\n\nFor a large scaled deployments, consider the following configuration changes:\n\n* Tune [ansible settings](https://docs.ansible.com/ansible/latest/intro_configuration.html)\n for `forks` and `timeout` vars to fit large numbers of nodes being deployed.\n\n* Override containers' `foo_image_repo` vars to point to intranet registry.\n\n* Override the ``download_run_once: true`` and/or ``download_localhost: true``.\n See [Downloading binaries and containers](/docs/advanced/downloads.md) for details.\n\n* Adjust the `retry_stagger` global var as appropriate. It should provide sane\n load on a delegate (the first K8s control plane node) then retrying failed\n push or download operations.\n\n* Tune parameters for DNS related applications\n Those are ``dns_replicas``, ``dns_cpu_limit``,\n ``dns_cpu_requests``, ``dns_memory_limit``, ``dns_memory_requests``.\n Please note that limits must always be greater than or equal to requests.\n\n* Tune CPU/memory limits and requests. Those are located in roles' defaults\n and named like ``foo_memory_limit``, ``foo_memory_requests`` and\n ``foo_cpu_limit``, ``foo_cpu_requests``. Note that 'Mi' memory units for K8s\n will be submitted as 'M', if applied for ``docker run``, and cpu K8s units\n will end up with the 'm' skipped for docker as well. This is required as\n docker does not understand k8s units well.\n\n* Tune ``kubelet_status_update_frequency`` to increase reliability of kubelet.\n ``kube_controller_node_monitor_grace_period``,\n ``kube_controller_node_monitor_period``,\n ``kube_apiserver_pod_eviction_not_ready_timeout_seconds`` &\n ``kube_apiserver_pod_eviction_unreachable_timeout_seconds`` for better Kubernetes reliability.\n Check out [Kubernetes Reliability](/docs/advanced/kubernetes-reliability.md)\n\n* Tune network prefix sizes. Those are ``kube_network_node_prefix``,\n ``kube_service_addresses`` and ``kube_pods_subnet``.\n\n* Add calico_rr nodes if you are deploying with Calico or Canal. Nodes recover\n from host/network interruption much quicker with calico_rr.\n\n* Check out the\n [Inventory](/docs/getting_started/getting-started.md#building-your-own-inventory)\n section of the Getting started guide for tips on creating a large scale\n Ansible inventory.\n\n* Override the ``etcd_events_cluster_setup: true`` store events in a separate\n dedicated etcd instance.\n\nFor example, when deploying 200 nodes, you may want to run ansible with\n``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``.\n" }, { "path": "docs/operations/mirror.md", "content": "# Public Download Mirror\n\nThe public mirror is useful to make the public resources download quickly in some areas of the world. (such as China).\n\n## Configuring Kubespray to use a mirror site\n\nYou can follow the [offline](offline-environment.md) to config the image/file download configuration to the public mirror site. If you want to download quickly in China, the configuration can be like:\n\n```shell\n# this should be in /group_vars/k8s_cluster.yml\ngcr_image_repo: \"gcr.m.daocloud.io\"\nkube_image_repo: \"k8s.m.daocloud.io\"\ndocker_image_repo: \"docker.m.daocloud.io\"\nquay_image_repo: \"quay.m.daocloud.io\"\ngithub_image_repo: \"ghcr.m.daocloud.io\"\n\nfiles_repo: \"https://files.m.daocloud.io\"\n```\n\nUse mirror sites only if you trust the provider. The Kubespray team cannot verify their reliability or security.\nYou can replace the `m.daocloud.io` with any site you want.\n\n## Community-run mirror sites\n\nDaoCloud(China)\n\n* [image-mirror](https://github.com/DaoCloud/public-image-mirror)\n* [files-mirror](https://github.com/DaoCloud/public-binary-files-mirror)\n" }, { "path": "docs/operations/nodes.md", "content": "# Adding/replacing a node\n\nModified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/issues/3471#issuecomment-530036084)\n\n## Adding/replacing a worker node\n\nThis should be the easiest.\n\n### 1) Add new node to the inventory\n\n### 2) Run `scale.yml`\n\nYou can use `--limit=NODE_NAME` to limit Kubespray to avoid disturbing other nodes in the cluster.\n\nBefore using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes.\n\n### 3) Remove an old node with remove-node.yml\n\nWith the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.\n\nIf the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars: `-e node=NODE_NAME -e reset_nodes=false -e allow_ungraceful_removal=true`.\nUse this flag even when you remove other types of nodes like a control plane or etcd nodes.\n\n### 4) Remove the node from the inventory\n\nThat's it.\n\n## Adding/replacing a control plane node\n\n### 1) Run `cluster.yml`\n\nAppend the new host to the inventory and run `cluster.yml`. You can NOT use `scale.yml` for that.\n\n**Note:** When adding new control plane nodes, always append them to the end of the `kube_control_plane` group in your inventory. Adding control plane nodes in the first position is not supported and will cause the playbook to fail.\n\n### 2) Restart kube-system/nginx-proxy\n\nIn all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserver. Kubespray will update its static config, but it needs to be restarted in order to reload.\n\n```sh\n# run in every host\ndocker ps | grep k8s_nginx-proxy_nginx-proxy | awk '{print $1}' | xargs docker restart\n\n# or with containerd\ncrictl ps | grep nginx-proxy | awk '{print $1}' | xargs crictl stop\n```\n\n### 3) Remove old control plane nodes\n\nWith the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.\nIf the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.\n\n## Adding/Removal of first `kube_control_plane` and etcd-master\n\nCurrently you can't remove the first node in your `kube_control_plane` and etcd-master list. If you still want to remove this node you have to:\n\n### 1) Change order of current control planes\n\nModify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:\n\n```yaml\nall:\n hosts:\n children:\n kube_control_plane:\n hosts:\n node-1:\n node-2:\n node-3:\n kube_node:\n hosts:\n node-1:\n node-2:\n node-3:\n etcd:\n hosts:\n node-1:\n node-2:\n node-3:\n```\n\nchange your inventory to:\n\n```yaml\nall:\n hosts:\n children:\n kube_control_plane:\n hosts:\n node-2:\n node-3:\n node-1:\n kube_node:\n hosts:\n node-2:\n node-3:\n node-1:\n etcd:\n hosts:\n node-2:\n node-3:\n node-1:\n```\n\n### 2) Upgrade the cluster\n\nrun `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.\n\n### 3) Remove old first control plane node from cluster\n\nWith the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.\nIf the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.\n\n### 4) Edit cluster-info configmap in kube-public namespace\n\n`kubectl edit cm -n kube-public cluster-info`\n\nChange ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.\n\n### 5) Add new control plane node\n\nUpdate inventory (if needed)\n\nRun `cluster.yml` with `--limit=kube_control_plane`\n\n## Adding an etcd node\n\nYou need to make sure there are always an odd number of etcd nodes in the cluster. In such a way, this is always a replacement or scale up operation. Either add two new nodes or remove an old one.\n\n### 1) Add the new node running cluster.yml\n\nUpdate the inventory and run `cluster.yml` passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`.\nIf the node you want to add as an etcd node is already a worker or control plane node in your cluster, you have to remove him first using `remove-node.yml`.\n\nRun `upgrade-cluster.yml` also passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`. This is necessary to update all etcd configuration in the cluster.\n\nAt this point, you will have an even number of nodes.\nEverything should still be working, and you should only have problems if the cluster decides to elect a new etcd leader before you remove a node.\nEven so, running applications should continue to be available.\n\nIf you add multiple etcd nodes with one run, you might want to append `-e etcd_retries=10` to increase the amount of retries between each etcd node join.\nOtherwise the etcd cluster might still be processing the first join and fail on subsequent nodes. `etcd_retries=10` might work to join 3 new nodes.\n\n### 2) Add the new node to apiserver config\n\nIn every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure the new etcd nodes are present in the apiserver command line parameter `--etcd-servers=...`.\n\n## Removing an etcd node\n\n### 1) Remove an old etcd node\n\nWith the node still in the inventory, run `remove-node.yml` passing `-e node=NODE_NAME` as the name of the node that should be removed.\nIf the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.\n\n### 2) Make sure only remaining nodes are in your inventory\n\nRemove `NODE_NAME` from your inventory file.\n\n### 3) Update kubernetes and network configuration files with the valid list of etcd members\n\nRun `cluster.yml` to regenerate the configuration files on all remaining nodes.\n\n### 4) Remove the old etcd node from apiserver config\n\nIn every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure only active etcd nodes are still present in the apiserver command line parameter `--etcd-servers=...`.\n\n### 5) Shutdown the old instance\n\nThat's it.\n" }, { "path": "docs/operations/offline-environment.md", "content": "# Offline environment\n\nIn case your servers don't have access to the internet directly (for example\nwhen deploying on premises with security constraints), you need to get the\nfollowing artifacts in advance from another environment where has access to the internet.\n\n* Some static files (zips and binaries)\n* OS packages (rpm/deb files)\n* Container images used by Kubespray. Exhaustive list depends on your setup\n* [Optional] Python packages used by Kubespray (only required if your OS doesn't provide all python packages/versions\n listed in `requirements.txt`)\n* [Optional] Helm chart files (only required if `helm_enabled=true`)\n\nThen you need to setup the following services on your offline environment:\n\n* an HTTP reverse proxy/cache/mirror to serve some static files (zips and binaries)\n* an internal Yum/Deb repository for OS packages\n* an internal container image registry that need to be populated with all container images used by Kubespray\n* [Optional] an internal PyPi server for python packages used by Kubespray\n* [Optional] an internal Helm registry for Helm chart files\n\nYou can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script.\nIn addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md).\n\n## Access Control\n\n### Note: access controlled files_repo\n\nTo specify a username and password for \"{{ files_repo }}\", used to download the binaries, you can use url-encoding. Be aware that the Boolean `unsafe_show_logs` will show these credentials when `roles/download/tasks/download_file.yml` runs the task \"Download_file | Show url of file to download\". You can disable that Boolean in a job-template when running AWX/AAP/Semaphore.\n\n```yaml\nfiles_repo_host: example.com\nfiles_repo_path: /repo\nfiles_repo_user: download\nfiles_repo_pass: !vault |\n $ANSIBLE_VAULT;1.1;AES256\n 61663232643236353864663038616361373739613338623338656434386662363539613462626661\n 6435333438313034346164313631303534346564316361370a306661393232626364376436386439\n 64653965663965356137333436616536643132336630313235333232336661373761643766356366\n 6232353233386534380a373262313634613833623537626132633033373064336261383166323230\n 3164\nfiles_repo: \"https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_repo_host ~ files_repo_path }}\"\n```\n\n### Note: access controlled registry\n\nTo specify a username and password for \"{{ registry_host }}\", used to download the container images, you can use url-encoding too.\n\n```yaml\nregistry_pass: !vault |\n $ANSIBLE_VAULT;1.1;AES256\n 61663232643236353864663038616361373739613338623338656434386662363539613462626661\n 6435333438313034346164313631303534346564316361370a306661393232626364376436386439\n 64653965663965356137333436616536643132336630313235333232336661373761643766356366\n 6232353233386534380a373262313634613833623537626132633033373064336261383166323230\n 3164\n\ncontainerd_registry_auth:\n - registry: \"{{ registry_host }}\"\n username: \"{{ registry_user }}\"\n password: \"{{ registry_pass }}\"\n```\n\n## Configure Inventory\n\nOnce all artifacts are accessible from your internal network, **adjust** the following variables\nin [your inventory](/inventory/sample/group_vars/all/offline.yml) to match your environment:\n\n```yaml\n# Registry overrides\nkube_image_repo: \"{{ registry_host }}\"\ngcr_image_repo: \"{{ registry_host }}\"\ndocker_image_repo: \"{{ registry_host }}\"\nquay_image_repo: \"{{ registry_host }}\"\ngithub_image_repo: \"{{ registry_host }}\"\n\nlocal_path_provisioner_helper_image_repo: \"{{ registry_host }}/busybox\"\nkubeadm_download_url: \"{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm\"\nkubectl_download_url: \"{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl\"\nkubelet_download_url: \"{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet\"\n# etcd is optional if you **DON'T** use etcd_deployment=host\netcd_download_url: \"{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz\"\ncni_download_url: \"{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz\"\ncrictl_download_url: \"{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz\"\n# If using Calico\ncalicoctl_download_url: \"{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}\"\n# If using Calico with kdd\ncalico_crds_download_url: \"{{ files_repo }}/github.com/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml\"\n# Containerd\ncontainerd_download_url: \"{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz\"\nrunc_download_url: \"{{ files_repo }}/runc.{{ image_arch }}\"\nnerdctl_download_url: \"{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz\"\nget_helm_url: \"{{ files_repo }}/get.helm.sh\"\n# Insecure registries for containerd\ncontainerd_registries_mirrors:\n - prefix: \"{{ registry_addr }}\"\n mirrors:\n - host: \"{{ registry_host }}\"\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: true\n\n# CentOS/Redhat/AlmaLinux/Rocky Linux\n## Docker / Containerd\ndocker_rh_repo_base_url: \"{{ yum_repo }}/docker-ce/$releasever/$basearch\"\ndocker_rh_repo_gpgkey: \"{{ yum_repo }}/docker-ce/gpg\"\n\n# Fedora\n## Docker\ndocker_fedora_repo_base_url: \"{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}\"\ndocker_fedora_repo_gpgkey: \"{{ yum_repo }}/docker-ce/gpg\"\n## Containerd\ncontainerd_fedora_repo_base_url: \"{{ yum_repo }}/containerd\"\ncontainerd_fedora_repo_gpgkey: \"{{ yum_repo }}/docker-ce/gpg\"\n\n# Debian\n## Docker\ndocker_debian_repo_base_url: \"{{ debian_repo }}/docker-ce\"\ndocker_debian_repo_gpgkey: \"{{ debian_repo }}/docker-ce/gpg\"\n## Containerd\ncontainerd_debian_repo_base_url: \"{{ ubuntu_repo }}/containerd\"\ncontainerd_debian_repo_gpgkey: \"{{ ubuntu_repo }}/containerd/gpg\"\ncontainerd_debian_repo_repokey: 'YOURREPOKEY'\n\n# Ubuntu\n## Docker\ndocker_ubuntu_repo_base_url: \"{{ ubuntu_repo }}/docker-ce\"\ndocker_ubuntu_repo_gpgkey: \"{{ ubuntu_repo }}/docker-ce/gpg\"\n## Containerd\ncontainerd_ubuntu_repo_base_url: \"{{ ubuntu_repo }}/containerd\"\ncontainerd_ubuntu_repo_gpgkey: \"{{ ubuntu_repo }}/containerd/gpg\"\ncontainerd_ubuntu_repo_repokey: 'YOURREPOKEY'\n```\n\nFor the OS specific settings, just define the one matching your OS.\nIf you use the settings like the one above, you'll need to define in your inventory the following variables:\n\n* `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that\n the ones defined\n in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray_defaults/defaults/main/download.yml)\n , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the\n same repository path, you won't have to override anything else.\n* `registry_addr`: Container image registry, but only have [domain or ip]:[port].\n* `files_repo`: HTTP webserver or reverse proxy that is able to serve the files listed above. Path is not important, you\n can store them anywhere as long as it's accessible by kubespray. It's recommended to use `*_version` in the path so\n that you don't need to modify this setting everytime kubespray upgrades one of these components.\n* `yum_repo`/`debian_repo`/`ubuntu_repo`: OS package repository depending on your OS, should point to your internal\n repository. Adjust the path accordingly. Used only for Docker/Containerd packages (if needed); other packages might\n be installed from other repositories. You might disable installing packages from other repositories by skipping\n the `system-packages` tag\n\n## Install Kubespray Python Packages\n\n### Recommended way: Kubespray Container Image\n\nThe easiest way is to use [kubespray container image](https://quay.io/kubespray/kubespray) as all the required packages\nare baked in the image.\nJust copy the container image in your private container image registry and you are all set!\n\n### Manual installation\n\nLook at the `requirements.txt` file and check if your OS provides all packages out-of-the-box (Using the OS package\nmanager). For those missing, you need to either use a proxy that has Internet access (typically from a DMZ) or setup a\nPyPi server in your network that will host these packages.\n\nIf you're using an HTTP(S) proxy to download your python packages:\n\n```bash\nsudo pip install --proxy=https://[username:password@]proxyserver:port -r requirements.txt\n```\n\nWhen using an internal PyPi server:\n\n```bash\n# If you host all required packages\npip install -i https://pypiserver/pypi -r requirements.txt\n\n# If you only need the ones missing from the OS package manager\npip install -i https://pypiserver/pypi package_you_miss\n```\n\n## Run Kubespray as usual\n\nOnce all artifacts are in place and your inventory properly set up, you can run kubespray with the\nregular `cluster.yaml` command:\n\n```bash\nansible-playbook -i inventory/my_airgap_cluster/hosts.yaml -b cluster.yml\n```\n\nIf you use [Kubespray Container Image](#recommended-way:-kubespray-container-image), you can mount your inventory inside\nthe container:\n\n```bash\ndocker run --rm -it -v path_to_inventory/my_airgap_cluster:inventory/my_airgap_cluster myprivateregisry.com/kubespray/kubespray:v2.14.0 ansible-playbook -i inventory/my_airgap_cluster/hosts.yaml -b cluster.yml\n```\n" }, { "path": "docs/operations/port-requirements.md", "content": "# Port Requirements\n\nTo operate properly, Kubespray requires some ports to be opened. If the network is configured with firewall rules, it is needed to ensure infrastructure components can communicate with each other through specific ports.\n\nEnsure the following ports required by Kubespray are open on the network and configured to allow access between hosts. Some ports are optional depending on the configuration and usage.\n\n## Kubernetes\n\n### Control plane\n\n| Protocol | Port | Description |\n|----------|--------| ------------ |\n| TCP | 22 | ssh for ansible |\n| TCP | 2379 | etcd client port|\n| TCP | 2380 | etcd peer port |\n| TCP | 6443 | kubernetes api |\n| TCP | 10250 | kubelet api |\n| TCP | 10257 | kube-scheduler |\n| TCP | 10259 | kube-controller-manager |\n\n### Worker node(s)\n\n| Protocol | Port | Description |\n|----------|-------- | ------------ |\n| TCP | 22 | ssh for ansible |\n| TCP | 10250 | kubelet api |\n| TCP | 30000-32767| kube nodePort range |\n\nrefers to: [Kubernetes Docs](https://kubernetes.io/docs/reference/networking/ports-and-protocols/)\n\n## Calico\n\nIf Calico is used, it requires:\n\n| Protocol | Port | Description |\n|----------|-------- | ------------ |\n| TCP | 179 | Calico networking (BGP) |\n| UDP | 4789 | Calico CNI with VXLAN enabled |\n| TCP | 5473 | Calico CNI with Typha enabled |\n| UDP | 51820 | Calico with IPv4 Wireguard enabled |\n| UDP | 51821 | Calico with IPv6 Wireguard enabled |\n| IPENCAP / IPIP | - | Calico CNI with IPIP enabled |\n\nrefers to: [Calico Docs](https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#network-requirements)\n\n## Cilium\n\nIf Cilium is used, it requires:\n\n| Protocol | Port | Description |\n|----------|-------- | ------------ |\n| TCP | 4240 | Cilium Health checks (``cilium-health``) |\n| TCP | 4244 | Hubble server |\n| TCP | 4245 | Hubble Relay |\n| UDP | 8472 | VXLAN overlay |\n| TCP | 9962 | Cilium-agent Prometheus metrics |\n| TCP | 9963 | Cilium-operator Prometheus metrics |\n| TCP | 9964 | Cilium-proxy Prometheus metrics |\n| UDP | 51871 | WireGuard encryption tunnel endpoint |\n| ICMP | - | health checks |\n\nrefers to: [Cilium Docs](https://docs.cilium.io/en/v1.13/operations/system_requirements/)\n\n## Addons\n\n| Protocol | Port | Description |\n|----------|-------- | ------------ |\n| TCP | 9100 | node exporter |\n| TCP/UDP | 7472 | metallb metrics ports |\n| TCP/UDP | 7946 | metallb L2 operating mode |\n" }, { "path": "docs/operations/recover-control-plane.md", "content": "# Recovering the control plane\n\nTo recover from broken nodes in the control plane use the \"recover\\-control\\-plane.yml\" playbook.\n\nExamples of what broken means in this context:\n\n* One or more bare metal node(s) suffer from unrecoverable hardware failure\n* One or more node(s) fail during patching or upgrading\n* Etcd database corruption\n* Other node related failures leaving your control plane degraded or nonfunctional\n\n__Note that you need at least one functional node to be able to recover using this method.__\n\n## Runbook\n\n* Backup what you can\n* Provision new nodes to replace the broken ones\n* Copy any broken etcd nodes into the \"broken\\_etcd\" group, make sure the \"etcd\\_member\\_name\" variable is set.\n* Copy any broken control plane nodes into the \"broken\\_kube\\_control\\_plane\" group.\n* Place the surviving nodes of the control plane first in the \"etcd\" and \"kube\\_control\\_plane\" groups\n* Add the new nodes below the surviving control plane nodes in the \"etcd\" and \"kube\\_control\\_plane\" groups\n\nThen run the playbook with ```--limit etcd,kube_control_plane``` and increase the number of ETCD retries by setting ```-e etcd_retries=10``` or something even larger. The amount of retries required is difficult to predict.\n\nWhen finished you should have a fully working control plane again.\n\n## Recover from lost quorum\n\nThe playbook attempts to figure out it the etcd quorum is intact. If quorum is lost it will attempt to take a snapshot from the first node in the \"etcd\" group and restore from that. If you would like to restore from an alternate snapshot set the path to that snapshot in the \"etcd\\_snapshot\" variable.\n\n```-e etcd_snapshot=/tmp/etcd_snapshot```\n\n## Caveats\n\n* The playbook has only been tested with fairly small etcd databases.\n* There may be disruptions while running the playbook.\n* There are absolutely no guarantees.\n\nIf possible try to break a cluster in the same way that your target cluster is broken and test to recover that before trying on the real target cluster.\n" }, { "path": "docs/operations/upgrades.md", "content": "# Upgrading Kubernetes in Kubespray\n\nKubespray handles upgrades the same way it handles initial deployment. That is to\nsay that each component is laid down in a fixed order.\n\nYou can also individually control versions of components by explicitly defining their\nversions. Here are all version vars for each component:\n\n* docker_version\n* docker_containerd_version (relevant when `container_manager` == `docker`)\n* containerd_version (relevant when `container_manager` == `containerd`)\n* kube_version\n* etcd_version\n* calico_version\n* calico_cni_version\n* flannel_version\n\n> **Warning**\n> [Attempting to upgrade from an older release straight to the latest release is unsupported and likely to break something](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515)\n\nSee [Multiple Upgrades](#multiple-upgrades) for how to upgrade from older Kubespray release to the latest release\n\n## Unsafe upgrade example\n\nIf you wanted to upgrade just kube_version from v1.18.10 to v1.19.7, you could\ndeploy the following way:\n\n```ShellSession\nansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=1.18.10 -e upgrade_cluster_setup=true\n```\n\nAnd then repeat with 1.19.7 as kube_version:\n\n```ShellSession\nansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=1.19.7 -e upgrade_cluster_setup=true\n```\n\nThe var ```-e upgrade_cluster_setup=true``` is needed to be set in order to migrate the deploys of e.g kube-apiserver inside the cluster immediately which is usually only done in the graceful upgrade. (Refer to [#4139](https://github.com/kubernetes-sigs/kubespray/issues/4139) and [#4736](https://github.com/kubernetes-sigs/kubespray/issues/4736))\n\n## Graceful upgrade\n\nKubespray also supports cordon, drain and uncordoning of nodes when performing\na cluster upgrade. There is a separate playbook used for this purpose. It is\nimportant to note that upgrade-cluster.yml can only be used for upgrading an\nexisting cluster. That means there must be at least 1 kube_control_plane already\ndeployed.\n\n```ShellSession\nansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.19.7\n```\n\nAfter a successful upgrade, the Server Version should be updated:\n\n```ShellSession\n$ kubectl version\nClient Version: version.Info{Major:\"1\", Minor:\"19\", GitVersion:\"v1.19.7\", GitCommit:\"1dd5338295409edcfff11505e7bb246f0d325d15\", GitTreeState:\"clean\", BuildDate:\"2021-01-13T13:23:52Z\", GoVersion:\"go1.15.5\", Compiler:\"gc\", Platform:\"linux/amd64\"}\nServer Version: version.Info{Major:\"1\", Minor:\"19\", GitVersion:\"v1.19.7\", GitCommit:\"1dd5338295409edcfff11505e7bb246f0d325d15\", GitTreeState:\"clean\", BuildDate:\"2021-01-13T13:15:20Z\", GoVersion:\"go1.15.5\", Compiler:\"gc\", Platform:\"linux/amd64\"}\n```\n\nYou can control how many nodes are upgraded at the same time by modifying the ansible variable named `serial`, as explained [here](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_strategies.html#setting-the-batch-size-with-serial). If you don't set this variable, it will upgrade the cluster nodes in batches of 20% of the available nodes. Setting `serial=1` would mean upgrade one node at a time.\n\n```ShellSession\nansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 -e \"serial=1\"\n```\n\n### Pausing the upgrade\n\nIf you want to manually control the upgrade procedure, you can set some variables to pause the upgrade playbook. Pausing *before* upgrading each upgrade may be useful for inspecting pods running on that node, or performing manual actions on the node:\n\n* `upgrade_node_confirm: true` - This will pause the playbook execution prior to upgrading each node. The play will resume when manually approved by typing \"yes\" at the terminal.\n* `upgrade_node_pause_seconds: 60` - This will pause the playbook execution for 60 seconds prior to upgrading each node. The play will resume automatically after 60 seconds.\n\nPausing *after* upgrading each node may be useful for rebooting the node to apply kernel updates, or testing the still-cordoned node:\n\n* `upgrade_node_post_upgrade_confirm: true` - This will pause the playbook execution after upgrading each node, but before the node is uncordoned. The play will resume when manually approved by typing \"yes\" at the terminal.\n* `upgrade_node_post_upgrade_pause_seconds: 60` - This will pause the playbook execution for 60 seconds after upgrading each node, but before the node is uncordoned. The play will resume automatically after 60 seconds.\n\n## Node-based upgrade\n\nIf you don't want to upgrade all nodes in one run, you can use `--limit` [patterns](https://docs.ansible.com/ansible/latest/user_guide/intro_patterns.html#patterns-and-ansible-playbook-flags).\n\nBefore using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:\n\n```ShellSession\nansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini\n```\n\nAfter this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):\n\n```ShellSession\nansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit \"kube_control_plane:etcd\"\n```\n\nNow you can upgrade other nodes in any order and quantity:\n\n```ShellSession\nansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit \"node4:node6:node7:node12\"\nansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit \"node5*\"\n```\n\n## Multiple upgrades\n\n> **Warning**\n> [Do not skip minor releases (patches releases are ok) when upgrading--upgrade by one tag at a\n> time.](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515)\n\nFor instances, given the tag list:\n\n```console\n$ git tag\nv2.20.0\nv2.21.0\nv2.22.0\nv2.22.1\nv2.23.0\nv2.23.1\nv2.23.2\nv2.24.0\n...\n```\n\nv2.22.0 -> v2.23.2 -> v2.24.0 : ✓\nv.22.0 -> v2.24.0 : ✕\n\nAssuming you don't explicitly define a kubernetes version in your k8s_cluster.yml, you simply check out the next tag and run the upgrade-cluster.yml playbook\n\n* If you do define kubernetes version in your inventory (e.g. group_vars/k8s_cluster.yml) then either make sure to update it before running upgrade-cluster, or specify the new version you're upgrading to: `ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml -e kube_version=1.11.3`\n\n Otherwise, the upgrade will leave your cluster at the same k8s version defined in your inventory vars.\n\nThe below example shows taking a cluster that was set up for v2.6.0 up to v2.10.0\n\n```ShellSession\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 1h v1.10.4\nboomer Ready master,node 42m v1.10.4\ncaprica Ready master,node 42m v1.10.4\n\n$ git describe --tags\nv2.6.0\n\n$ git tag\n...\nv2.6.0\nv2.7.0\nv2.8.0\nv2.8.1\nv2.8.2\n...\n\n$ git checkout v2.7.0\nPrevious HEAD position was 8b3ce6e4 bump upgrade tests to v2.5.0 commit (#3087)\nHEAD is now at 05dabb7e Fix Bionic networking restart error #3430 (#3431)\n\n# NOTE: May need to `pip3 install -r requirements.txt` when upgrading.\n\nansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 1h v1.11.3\nboomer Ready master,node 1h v1.11.3\ncaprica Ready master,node 1h v1.11.3\n\n$ git checkout v2.8.0\nPrevious HEAD position was 05dabb7e Fix Bionic networking restart error #3430 (#3431)\nHEAD is now at 9051aa52 Fix ubuntu-contiv test failed (#3808)\n```\n\n> **Note**\n> Review changes between the sample inventory and your inventory when upgrading versions.\n\nSome deprecations between versions that mean you can't just upgrade straight from 2.7.0 to 2.8.0 if you started with the sample inventory.\n\nIn this case, I set \"kubeadm_enabled\" to false, knowing that it is deprecated and removed by 2.9.0, to delay converting the cluster to kubeadm as long as I could.\n\n```ShellSession\n$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n...\n \"msg\": \"DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release.\"\n...\nAre you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden):\nyes\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 114m v1.12.3\nboomer Ready master,node 114m v1.12.3\ncaprica Ready master,node 114m v1.12.3\n\n$ git checkout v2.8.1\nPrevious HEAD position was 9051aa52 Fix ubuntu-contiv test failed (#3808)\nHEAD is now at 2ac1c756 More Feature/2.8 backports for 2.8.1 (#3911)\n\n$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n...\n \"msg\": \"DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release.\"\n...\nAre you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden):\nyes\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 2h36m v1.12.4\nboomer Ready master,node 2h36m v1.12.4\ncaprica Ready master,node 2h36m v1.12.4\n\n$ git checkout v2.8.2\nPrevious HEAD position was 2ac1c756 More Feature/2.8 backports for 2.8.1 (#3911)\nHEAD is now at 4167807f Upgrade to 1.12.5 (#4066)\n\n$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n...\n \"msg\": \"DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release.\"\n...\nAre you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden):\nyes\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 3h3m v1.12.5\nboomer Ready master,node 3h3m v1.12.5\ncaprica Ready master,node 3h3m v1.12.5\n\n$ git checkout v2.8.3\nPrevious HEAD position was 4167807f Upgrade to 1.12.5 (#4066)\nHEAD is now at ea41fc5e backport cve-2019-5736 to release-2.8 (#4234)\n\n$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n...\n \"msg\": \"DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release.\"\n...\nAre you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden):\nyes\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 5h18m v1.12.5\nboomer Ready master,node 5h18m v1.12.5\ncaprica Ready master,node 5h18m v1.12.5\n\n$ git checkout v2.8.4\nPrevious HEAD position was ea41fc5e backport cve-2019-5736 to release-2.8 (#4234)\nHEAD is now at 3901480b go to k8s 1.12.7 (#4400)\n\n$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n...\n \"msg\": \"DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release.\"\n...\nAre you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden):\nyes\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 5h37m v1.12.7\nboomer Ready master,node 5h37m v1.12.7\ncaprica Ready master,node 5h37m v1.12.7\n\n$ git checkout v2.8.5\nPrevious HEAD position was 3901480b go to k8s 1.12.7 (#4400)\nHEAD is now at 6f97687d Release 2.8 robust san handling (#4478)\n\n$ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n...\n \"msg\": \"DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release.\"\n...\nAre you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden):\nyes\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 5h45m v1.12.7\nboomer Ready master,node 5h45m v1.12.7\ncaprica Ready master,node 5h45m v1.12.7\n\n$ git checkout v2.9.0\nPrevious HEAD position was 6f97687d Release 2.8 robust san handling (#4478)\nHEAD is now at a4e65c7c Upgrade to Ansible >2.7.0 (#4471)\n```\n\n> **Warning**\n> IMPORTANT: Some variable formats changed in the k8s_cluster.yml between 2.8.5 and 2.9.0\n\nIf you do not keep your inventory copy up to date, **your upgrade will fail** and your first master will be left non-functional until fixed and re-run.\n\nIt is at this point the cluster was upgraded from non-kubeadm to kubeadm as per the deprecation warning.\n\n```ShellSession\nansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 6h54m v1.13.5\nboomer Ready master,node 6h55m v1.13.5\ncaprica Ready master,node 6h54m v1.13.5\n\n# Watch out: 2.10.0 is hiding between 2.1.2 and 2.2.0\n\n$ git tag\n...\nv2.1.0\nv2.1.1\nv2.1.2\nv2.10.0\nv2.2.0\n...\n\n$ git checkout v2.10.0\nPrevious HEAD position was a4e65c7c Upgrade to Ansible >2.7.0 (#4471)\nHEAD is now at dcd9c950 Add etcd role dependency on kube user to avoid etcd role failure when running scale.yml with a fresh node. (#3240) (#4479)\n\nansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml\n\n...\n\n$ kubectl get node\nNAME STATUS ROLES AGE VERSION\napollo Ready master,node 7h40m v1.14.1\nboomer Ready master,node 7h40m v1.14.1\ncaprica Ready master,node 7h40m v1.14.1\n\n\n```\n\n## Upgrading to v2.19\n\n`etcd_kubeadm_enabled` is being deprecated at v2.19. The same functionality is achievable by setting `etcd_deployment_type` to `kubeadm`.\nDeploying etcd using kubeadm is experimental and is only available for either new or deployments where `etcd_kubeadm_enabled` was set to `true` while deploying the cluster.\n\nFrom 2.19 and onward `etcd_deployment_type` variable will be placed in `group_vars/all/etcd.yml` instead of `group_vars/etcd.yml`, due to scope issues.\nThe placement of the variable is only important for `etcd_deployment_type: kubeadm` right now. However, since this might change in future updates, it is recommended to move the variable.\n\nUpgrading is straightforward; no changes are required if `etcd_kubeadm_enabled` was not set to `true` when deploying.\n\nIf you have a cluster where `etcd` was deployed using `kubeadm`, you will need to remove `etcd_kubeadm_enabled` the variable. Then move `etcd_deployment_type` variable from `group_vars/etcd.yml` to `group_vars/all/etcd.yml` due to scope issues and set `etcd_deployment_type` to `kubeadm`.\n\n## Upgrade order\n\nAs mentioned above, components are upgraded in the order in which they were\ninstalled in the Ansible playbook. The order of component installation is as\nfollows:\n\n* Docker\n* Containerd\n* etcd\n* kubelet and kube-proxy\n* network_plugin (such as Calico)\n* kube-apiserver, kube-scheduler, and kube-controller-manager\n* Add-ons (such as KubeDNS)\n\n### Component-based upgrades\n\nA deployer may want to upgrade specific components in order to minimize risk\nor save time. This strategy is not covered by CI as of this writing, so it is\nnot guaranteed to work.\n\nThese commands are useful only for upgrading fully-deployed, healthy, existing\nhosts. This will definitely not work for undeployed or partially deployed\nhosts.\n\nUpgrade docker:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=docker\n```\n\nUpgrade etcd:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd\n```\n\nUpgrade etcd without rotating etcd certs:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd --limit=etcd --skip-tags=etcd-secrets\n```\n\nUpgrade kubelet:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs\n```\n\nUpgrade Kubernetes master components:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=master\n```\n\nUpgrade network plugins:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=network\n```\n\nUpgrade all add-ons:\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=apps\n```\n\nUpgrade just helm (assuming `helm_enabled` is true):\n\n```ShellSession\nansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=helm\n```\n\n## Migrate from Docker to Containerd\n\nPlease note that **migrating container engines is not officially supported by Kubespray**. While this procedure can be used to migrate your cluster, it applies to one particular scenario and will likely evolve over time. At the moment, they are intended as an additional resource to provide insight into how these steps can be officially integrated into the Kubespray playbooks.\n\nAs of Kubespray 2.18.0, containerd is already the default container engine. If you have the chance, it is advisable and safer to reset and redeploy the entire cluster with a new container engine.\n\n* [Migrating from Docker to Containerd](/docs/upgrades/migrate_docker2containerd.md)\n\n## System upgrade\n\nIf you want to upgrade the APT or YUM packages while the nodes are cordoned, you can use:\n\n```ShellSession\nansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e system_upgrade=true\n```\n\nNodes will be rebooted when there are package upgrades (`system_upgrade_reboot: on-upgrade`).\nThis can be changed to `always` or `never`.\n\nNote: Downloads will happen twice unless `system_upgrade_reboot` is `never`.\n" }, { "path": "docs/roadmap/roadmap.md", "content": "# Kubespray's roadmap\n\nWe are tracking the evolution towards Kubespray 3.0 in [#6400](https://github.com/kubernetes-sigs/kubespray/issues/6400) as well as in other open issue in our [github issues](https://github.com/kubernetes-sigs/kubespray/issues/) section.\n" }, { "path": "docs/upgrades/migrate_docker2containerd.md", "content": "# Migrating from Docker to Containerd\n\n❗MAKE SURE YOU READ BEFORE PROCEEDING❗\n\n**Migrating container engines is not officially supported by Kubespray**. The following procedure covers one particular scenario and involves manual steps, along with multiple runs of `cluster.yml`. It provides no guarantees that it will actually work or that any further action is needed. Please, consider these instructions as experimental guidelines. While they can be used to migrate your cluster, they will likely evolve over time. At the moment, they are intended as an additional resource to provide insight into how these steps can be officially integrated into the Kubespray playbooks.\n\nAs of Kubespray 2.18.0, containerd is already the default container engine. If you have the chance, it is still advisable and safer to reset and redeploy the entire cluster with a new container engine.\n\nInput and feedback are always appreciated.\n\n## Tested environment\n\nNodes: Ubuntu 18.04 LTS\\\nCloud Provider: None (baremetal or VMs)\\\nKubernetes version: 1.21.5\\\nKubespray version: 2.18.0\n\n## Important considerations\n\nIf you require minimum downtime, nodes need to be cordoned and drained before being processed, one by one. If you wish to run `cluster.yml` only once and get it all done in one swoop, downtime will be significantly higher. Docker will need to be manually removed from all nodes before the playbook runs (see [#8431](https://github.com/kubernetes-sigs/kubespray/issues/8431)). For minimum downtime, the following steps will be executed multiple times, once for each node.\n\nProcessing nodes one by one also means you will not be able to update any other cluster configuration using Kubespray before this procedure is finished and the cluster is fully migrated.\n\nEverything done here requires full root access to every node.\n\n## Migration steps\n\nBefore you begin, adjust your inventory:\n\n```yaml\n# Filename: k8s_cluster/k8s-cluster.yml\nresolvconf_mode: host_resolvconf\ncontainer_manager: containerd\n\n# Filename: etcd.yml\netcd_deployment_type: host\n```\n\n### 1) Pick one or more nodes for processing\n\nIt is still unclear how the order might affect this procedure. So, to be sure, it might be best to start with the control plane and etcd nodes all together, followed by each worker node individually.\n\n### 2) Cordon and drain the node\n\n... because, downtime.\n\n### 3) Stop docker and kubelet daemons\n\n```commandline\nservice kubelet stop\nservice docker stop\n```\n\n### 4) Uninstall docker + dependencies\n\n```commandline\napt-get remove -y --allow-change-held-packages containerd.io docker-ce docker-ce-cli docker-ce-rootless-extras\n```\n\nIn some cases, there might a `pigz` missing dependency. Some image layers need this to be extracted.\n\n```shell\napt-get install pigz\n```\n\n### 5) Run `cluster.yml` playbook with `--limit`\n\n```commandline\nansible-playbook -i inventory/sample/hosts.ini cluster.yml --limit=NODENAME\n```\n\nThis effectively reinstalls containerd and seems to place all config files in the right place. When this completes, kubelet will immediately pick up the new container engine and start spinning up DaemonSets and kube-system Pods.\n\nOptionally, if you feel confident, you can remove `/var/lib/docker` anytime after this step.\n\n```commandline\nrm -fr /var/lib/docker\n```\n\nYou can watch new containers using `crictl`.\n\n```commandline\ncrictl ps -a\n```\n\n### 6) Replace the cri-socket node annotation\n\nNode annotations need to be adjusted. Kubespray will not do this, but a simple kubectl is enough.\n\n```commandline\nkubectl annotate node NODENAME --overwrite kubeadm.alpha.kubernetes.io/cri-socket=/var/run/containerd/containerd.sock\n```\n\nThe annotation is required by kubeadm to follow through future cluster upgrades.\n\n### 7) Reboot the node\n\nReboot, just to make sure everything restarts fresh before the node is uncordoned.\n\n## After thoughts\n\nIf your cluster runs a log aggregator, like fluentd+Graylog, you will likely need to adjust collection filters and parsers. While docker generates Json logs, containerd has its own space delimited format. Example:\n\n```text\n2020-01-10T18:10:40.01576219Z stdout F application log message...\n```\n" }, { "path": "extra_playbooks/files/get_cinder_pvs.sh", "content": "#!/bin/sh\nkubectl get pv -o go-template --template='{{ range .items }}{{ $metadata := .metadata }}{{ with $value := index .metadata.annotations \"pv.kubernetes.io/provisioned-by\" }}{{ if eq $value \"kubernetes.io/cinder\" }}{{printf \"%s\\n\" $metadata.name}}{{ end }}{{ end }}{{ end }}'\n" }, { "path": "extra_playbooks/migrate_openstack_provider.yml", "content": "---\n- name: Remove old cloud provider config\n hosts: kube_node:kube_control_plane\n tasks:\n - name: Remove old cloud provider config\n file:\n path: \"{{ item }}\"\n state: absent\n with_items:\n - /etc/kubernetes/cloud_config\n- name: Migrate intree Cinder PV\n hosts: kube_control_plane[0]\n tasks:\n - name: Include kubespray-default variables\n include_vars: ../roles/kubespray_defaults/defaults/main/main.yml\n - name: Copy get_cinder_pvs.sh to first control plane node\n copy:\n src: get_cinder_pvs.sh\n dest: /tmp\n mode: u+rwx\n - name: Get PVs provisioned by in-tree cloud provider\n command: /tmp/get_cinder_pvs.sh\n register: pvs\n - name: Remove get_cinder_pvs.sh\n file:\n path: /tmp/get_cinder_pvs.sh\n state: absent\n - name: Rewrite the \"pv.kubernetes.io/provisioned-by\" annotation\n command: \"{{ bin_dir }}/kubectl annotate --overwrite pv {{ item }} pv.kubernetes.io/provisioned-by=cinder.csi.openstack.org\"\n loop: \"{{ pvs.stdout_lines | list }}\"\n" }, { "path": "extra_playbooks/upgrade-only-k8s.yml", "content": "---\n### NOTE: This playbook cannot be used to deploy any new nodes to the cluster.\n### Additional information:\n### * Will not upgrade etcd\n### * Will not upgrade network plugins\n### * Will not upgrade Docker\n### * Will not pre-download containers or kubeadm\n### * Currently does not support Vault deployment.\n###\n### In most cases, you probably want to use upgrade-cluster.yml playbook and\n### not this one.\n\n- name: Setup ssh config to use the bastion\n hosts: localhost\n gather_facts: false\n roles:\n - { role: kubespray_defaults}\n - { role: bastion-ssh-config, tags: [\"localhost\", \"bastion\"]}\n\n- name: Bootstrap hosts OS for Ansible\n hosts: k8s_cluster:etcd:calico_rr\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n gather_facts: false\n vars:\n # Need to disable pipelining for bootstrap_os as some systems have requiretty in sudoers set, which makes pipelining\n # fail. bootstrap_os fixes this on these systems, so in later plays it can be enabled.\n ansible_ssh_pipelining: false\n roles:\n - { role: kubespray_defaults}\n - { role: bootstrap_os, tags: bootstrap_os}\n\n- name: Preinstall\n hosts: k8s_cluster:etcd:calico_rr\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n roles:\n - { role: kubespray_defaults}\n - { role: kubernetes/preinstall, tags: preinstall }\n\n- name: Handle upgrades to control plane components first to maintain backwards compat.\n hosts: kube_control_plane\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n serial: 1\n roles:\n - { role: kubespray_defaults}\n - { role: upgrade/pre-upgrade, tags: pre-upgrade }\n - { role: kubernetes/node, tags: node }\n - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }\n - { role: kubernetes/client, tags: client }\n - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }\n - { role: upgrade/post-upgrade, tags: post-upgrade }\n\n- name: Finally handle worker upgrades, based on given batch size\n hosts: kube_node:!kube_control_plane\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n serial: \"{{ serial | default('20%') }}\"\n roles:\n - { role: kubespray_defaults}\n - { role: upgrade/pre-upgrade, tags: pre-upgrade }\n - { role: kubernetes/node, tags: node }\n - { role: upgrade/post-upgrade, tags: post-upgrade }\n - { role: kubespray_defaults}\n" }, { "path": "extra_playbooks/wait-for-cloud-init.yml", "content": "---\n- name: Wait for cloud-init to finish\n hosts: all\n tasks:\n - name: Wait for cloud-init to finish\n command: cloud-init status --wait\n" }, { "path": "galaxy.yml", "content": "---\nnamespace: kubernetes_sigs\ndescription: Deploy a production ready Kubernetes cluster\nname: kubespray\nversion: 2.31.0\nreadme: README.md\nauthors:\n - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)\ntags:\n - infrastructure\nrepository: https://github.com/kubernetes-sigs/kubespray\nissues: https://github.com/kubernetes-sigs/kubespray/issues\ndocumentation: https://kubespray.io\nlicense_file: LICENSE\ndependencies:\n ansible.utils: '>=2.5.0'\n community.crypto: '>=2.22.3'\n community.general: '>=7.0.0'\n ansible.netcommon: '>=5.3.0'\n ansible.posix: '>=1.5.4'\n community.docker: '>=3.11.0'\n kubernetes.core: '>=2.4.2'\nmanifest:\n directives:\n - recursive-exclude tests **\n - recursive-include roles **/files/*\n" }, { "path": "index.html", "content": "\n\n\n \n Kubespray - Deploy a Production Ready Kubernetes Cluster\n \n \n \n \n \n\n\n
\n\n\n\n\n\n\n\n" }, { "path": "inventory/local/hosts.ini", "content": "node1 ansible_connection=local local_release_dir={{ansible_env.HOME}}/releases\n\n[kube_control_plane]\nnode1\n\n[etcd]\nnode1\n\n[kube_node]\nnode1\n" }, { "path": "inventory/sample/group_vars/all/all.yml", "content": "---\n## Directory where the binaries will be installed\nbin_dir: /usr/local/bin\n\n## The access_ip variable is used to define how other nodes should access\n## the node. This is used in flannel to allow other flannel nodes to see\n## this node for example. The access_ip is really useful AWS and Google\n## environments where the nodes are accessed remotely by the \"public\" ip,\n## but don't know about that address themselves.\n# access_ip: 1.1.1.1\n\n\n## External LB example config\n## apiserver_loadbalancer_domain_name: \"elb.some.domain\"\n# loadbalancer_apiserver:\n# address: 1.2.3.4\n# port: 1234\n\n## Internal loadbalancers for apiservers\n# loadbalancer_apiserver_localhost: true\n# valid options are \"nginx\" or \"haproxy\"\n# loadbalancer_apiserver_type: nginx # valid values \"nginx\" or \"haproxy\"\n\n## Local loadbalancer should use this port\n## And must be set port 6443\nloadbalancer_apiserver_port: 6443\n\n## If loadbalancer_apiserver_healthcheck_port variable defined, enables proxy liveness check for nginx.\nloadbalancer_apiserver_healthcheck_port: 8081\n\n### OTHER OPTIONAL VARIABLES\n\n## By default, Kubespray collects nameservers on the host. It then adds the previously collected nameservers in nameserverentries.\n## If true, Kubespray does not include host nameservers in nameserverentries in dns_late stage. However, It uses the nameserver to make sure cluster installed safely in dns_early stage.\n## Use this option with caution, you may need to define your dns servers. Otherwise, the outbound queries such as www.google.com may fail.\n# disable_host_nameservers: false\n\n## Upstream dns servers\n# upstream_dns_servers:\n# - 8.8.8.8\n# - 8.8.4.4\n\n## There are some changes specific to the cloud providers\n## for instance we need to encapsulate packets with some network plugins\n## If set the possible values only 'external' after K8s v1.31.\n# cloud_provider:\n\n# External Cloud Controller Manager (Formerly known as cloud provider)\n# cloud_provider must be \"external\", otherwise this setting is invalid.\n# Supported external cloud controllers are: 'openstack', 'vsphere', 'oci', 'huaweicloud', 'hcloud' and 'manual'\n# 'manual' does not install the cloud controller manager used by Kubespray.\n# If you fill in a value other than the above, the check will fail.\n# external_cloud_provider:\n\n## Set these proxy values in order to update package manager and docker daemon to use proxies and custom CA for https_proxy if needed\n# http_proxy: \"\"\n# https_proxy: \"\"\n# https_proxy_cert_file: \"\"\n\n## Refer to roles/kubespray_defaults/defaults/main/main.yml before modifying no_proxy\n# no_proxy: \"\"\n\n## Some problems may occur when downloading files over https proxy due to ansible bug\n## https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable\n## SSL validation of get_url module. Note that kubespray will still be performing checksum validation.\n# download_validate_certs: False\n\n## If you need exclude all cluster nodes from proxy and other resources, add other resources here.\n# additional_no_proxy: \"\"\n\n## If you need to disable proxying of os package repositories but are still behind an http_proxy set\n## skip_http_proxy_on_os_packages to true\n## This will cause kubespray not to set proxy environment in /etc/yum.conf for centos and in /etc/apt/apt.conf for debian/ubuntu\n## Special information for debian/ubuntu - you have to set the no_proxy variable, then apt package will install from your source of wish\n# skip_http_proxy_on_os_packages: false\n\n## Since workers are included in the no_proxy variable by default, docker engine will be restarted on all nodes (all\n## pods will restart) when adding or removing workers. To override this behaviour by only including control plane nodes\n## in the no_proxy variable, set below to true:\nno_proxy_exclude_workers: false\n\n## Certificate Management\n## This setting determines whether certs are generated via scripts.\n## Chose 'none' if you provide your own certificates.\n## Option is \"script\", \"none\"\n# cert_management: script\n\n## Set to true to allow pre-checks to fail and continue deployment\n# ignore_assert_errors: false\n\n## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable.\n# kube_read_only_port: 10255\n\n## Set true to download and cache container\n# download_container: true\n\n## Deploy container engine\n# Set false if you want to deploy container engine manually.\n# deploy_container_engine: true\n\n## Red Hat Enterprise Linux subscription registration\n## Add either RHEL subscription Username/Password or Organization ID/Activation Key combination\n## Update RHEL subscription purpose usage, role and SLA if necessary\n# rh_subscription_username: \"\"\n# rh_subscription_password: \"\"\n# rh_subscription_org_id: \"\"\n# rh_subscription_activation_key: \"\"\n# rh_subscription_usage: \"Development\"\n# rh_subscription_role: \"Red Hat Enterprise Server\"\n# rh_subscription_sla: \"Self-Support\"\n\n## Check if access_ip responds to ping. Set false if your firewall blocks ICMP.\n# ping_access_ip: true\n\n# sysctl_file_path to add sysctl conf to\n# sysctl_file_path: \"/etc/sysctl.d/99-sysctl.conf\"\n\n# ignore sysctl errors about unknown keys\n# sysctl_ignore_unknown_keys: false\n\n## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication\nkube_webhook_token_auth: false\nkube_webhook_token_auth_url_skip_tls_verify: false\n# kube_webhook_token_auth_url: https://...\n## base64-encoded string of the webhook's CA certificate\n# kube_webhook_token_auth_ca_data: \"LS0t...\"\n\n## NTP Settings\n# Start the ntpd or chrony service and enable it at system boot.\nntp_enabled: false\nntp_manage_config: false\nntp_servers:\n - \"0.pool.ntp.org iburst\"\n - \"1.pool.ntp.org iburst\"\n - \"2.pool.ntp.org iburst\"\n - \"3.pool.ntp.org iburst\"\n\n## Used to control no_log attribute\nunsafe_show_logs: false\n\n## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases.\nallow_unsupported_distribution_setup: false\n" }, { "path": "inventory/sample/group_vars/all/aws.yml", "content": "## To use AWS EBS CSI Driver to provision volumes, uncomment the first value\n## and configure the parameters below\n# aws_ebs_csi_enabled: true\n# aws_ebs_csi_enable_volume_scheduling: true\n# aws_ebs_csi_enable_volume_snapshot: false\n# aws_ebs_csi_enable_volume_resizing: false\n# aws_ebs_csi_controller_replicas: 1\n# aws_ebs_csi_plugin_image_tag: latest\n# aws_ebs_csi_extra_volume_tags: \"Owner=owner,Team=team,Environment=environment'\n" }, { "path": "inventory/sample/group_vars/all/azure.yml", "content": "## When azure is used, you need to also set the following variables.\n## see docs/azure.md for details on how to get these values\n\n# azure_cloud:\n# azure_tenant_id:\n# azure_subscription_id:\n# azure_aad_client_id:\n# azure_aad_client_secret:\n# azure_resource_group:\n# azure_location:\n# azure_subnet_name:\n# azure_security_group_name:\n# azure_security_group_resource_group:\n# azure_vnet_name:\n# azure_vnet_resource_group:\n# azure_route_table_name:\n# azure_route_table_resource_group:\n# supported values are 'standard' or 'vmss'\n# azure_vmtype: standard\n\n## Azure Disk CSI credentials and parameters\n## see docs/azure-csi.md for details on how to get these values\n\n# azure_csi_tenant_id:\n# azure_csi_subscription_id:\n# azure_csi_aad_client_id:\n# azure_csi_aad_client_secret:\n# azure_csi_location:\n# azure_csi_resource_group:\n# azure_csi_vnet_name:\n# azure_csi_vnet_resource_group:\n# azure_csi_subnet_name:\n# azure_csi_security_group_name:\n# azure_csi_use_instance_metadata:\n# azure_csi_tags: \"Owner=owner,Team=team,Environment=environment'\n\n## To enable Azure Disk CSI, uncomment below\n# azure_csi_enabled: true\n# azure_csi_controller_replicas: 1\n# azure_csi_plugin_image_tag: latest\n" }, { "path": "inventory/sample/group_vars/all/containerd.yml", "content": "---\n# Please see roles/container-engine/containerd/defaults/main.yml for more configuration options\n\n# containerd_storage_dir: \"/var/lib/containerd\"\n# containerd_state_dir: \"/run/containerd\"\n# containerd_oom_score: 0\n\n# containerd_default_runtime: \"runc\"\n# containerd_snapshotter: \"native\"\n\n# containerd_runc_runtime:\n# name: runc\n# type: \"io.containerd.runc.v2\"\n# options:\n# Root: \"\"\n\n# containerd_additional_runtimes:\n# Example for Kata Containers as additional runtime:\n# - name: kata\n# type: \"io.containerd.kata.v2\"\n# options:\n# Root: \"\"\n\n# containerd_grpc_max_recv_message_size: 16777216\n# containerd_grpc_max_send_message_size: 16777216\n\n# Containerd debug socket location: unix or tcp format\n# containerd_debug_address: \"\"\n\n# Containerd log level\n# containerd_debug_level: \"info\"\n\n# Containerd logs format, supported values: text, json\n# containerd_debug_format: \"\"\n\n# Containerd debug socket UID\n# containerd_debug_uid: 0\n\n# Containerd debug socket GID\n# containerd_debug_gid: 0\n\n# containerd_metrics_address: \"\"\n\n# containerd_metrics_grpc_histogram: false\n\n# Registries defined within containerd.\n# containerd_registries_mirrors:\n# - prefix: docker.io\n# mirrors:\n# - host: https://registry-1.docker.io\n# capabilities: [\"pull\", \"resolve\"]\n# skip_verify: false\n# header:\n# Authorization: \"Basic XXX\"\n\n# containerd_max_container_log_line_size: 16384\n\n# containerd_registry_auth:\n# - registry: 10.0.0.2:5000\n# username: user\n# password: pass\n" }, { "path": "inventory/sample/group_vars/all/coreos.yml", "content": "## Does coreos need auto upgrade, default is true\n# coreos_auto_upgrade: true\n" }, { "path": "inventory/sample/group_vars/all/cri-o.yml", "content": "# Registries defined within cri-o.\n# crio_insecure_registries:\n# - 10.0.0.2:5000\n\n# Auth config for the registries\n# crio_registry_auth:\n# - registry: 10.0.0.2:5000\n# username: user\n# password: pass\n" }, { "path": "inventory/sample/group_vars/all/docker.yml", "content": "---\n## Uncomment this if you want to force overlay/overlay2 as docker storage driver\n## Please note that overlay2 is only supported on newer kernels\n# docker_storage_options: -s overlay2\n\n## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7.\ndocker_container_storage_setup: false\n\n## It must be define a disk path for docker_container_storage_setup_devs.\n## Otherwise docker-storage-setup will be executed incorrectly.\n# docker_container_storage_setup_devs: /dev/vdb\n\n## Uncomment this if you want to change the Docker Cgroup driver (native.cgroupdriver)\n## Valid options are systemd or cgroupfs, default is systemd\n# docker_cgroup_driver: systemd\n\n## Only set this if you have more than 3 nameservers:\n## If true Kubespray will only use the first 3, otherwise it will fail\ndocker_dns_servers_strict: false\n\n# Path used to store Docker data\ndocker_daemon_graph: \"/var/lib/docker\"\n\n## Used to set docker daemon iptables options to true\ndocker_iptables_enabled: \"false\"\n\n# Docker log options\n# Rotate container stderr/stdout logs at 50m and keep last 5\ndocker_log_opts: \"--log-opt max-size=50m --log-opt max-file=5\"\n\n# define docker bin_dir\ndocker_bin_dir: \"/usr/bin\"\n\n# keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'\n# kubespray deletes the docker package on each run, so caching the package makes sense\ndocker_rpm_keepcache: 1\n\n## An obvious use case is allowing insecure-registry access to self hosted registries.\n## Can be ipaddress and domain_name.\n## example define 172.19.16.11 or mirror.registry.io\n# docker_insecure_registries:\n# - mirror.registry.io\n# - 172.19.16.11\n\n## Add other registry,example China registry mirror.\n# docker_registry_mirrors:\n# - https://registry.docker-cn.com\n# - https://mirror.aliyuncs.com\n\n## If non-empty will override default system MountFlags value.\n## This option takes a mount propagation flag: shared, slave\n## or private, which control whether mounts in the file system\n## namespace set up for docker will receive or propagate mounts\n## and unmounts. Leave empty for system default\n# docker_mount_flags:\n\n## A string of extra options to pass to the docker daemon.\n## This string should be exactly as you wish it to appear.\n# docker_options: \"\"\n" }, { "path": "inventory/sample/group_vars/all/etcd.yml", "content": "---\n## Directory where etcd data stored\netcd_data_dir: /var/lib/etcd\n\n## Container runtime\n## docker for docker, crio for cri-o and containerd for containerd.\n## Additionally you can set this to kubeadm if you want to install etcd using kubeadm\n## Kubeadm etcd deployment is experimental and only available for new deployments\n## If this is not set, container manager will be inherited from the Kubespray defaults\n## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want.\n## Also this makes possible to use different container manager for etcd nodes.\n# container_manager: containerd\n\n## Settings for etcd deployment type\n# Set this to docker if you are using container_manager: docker\netcd_deployment_type: host\n" }, { "path": "inventory/sample/group_vars/all/gcp.yml", "content": "## GCP compute Persistent Disk CSI Driver credentials and parameters\n## See docs/gcp-pd-csi.md for information about the implementation\n\n## Specify the path to the file containing the service account credentials\n# gcp_pd_csi_sa_cred_file: \"/my/safe/credentials/directory/cloud-sa.json\"\n\n## To enable GCP Persistent Disk CSI driver, uncomment below\n# gcp_pd_csi_enabled: true\n# gcp_pd_csi_controller_replicas: 1\n# gcp_pd_csi_driver_image_tag: \"v0.7.0-gke.0\"\n" }, { "path": "inventory/sample/group_vars/all/hcloud.yml", "content": "## Values for the external Hcloud Cloud Controller\n# external_hcloud_cloud:\n# hcloud_api_token: \"\"\n# token_secret_name: hcloud\n# with_networks: false # Use the hcloud controller-manager with networks support https://github.com/hetznercloud/hcloud-cloud-controller-manager#networks-support\n# network_name: # network name/ID: If you manage the network yourself it might still be required to let the CCM know about private networks\n# service_account_name: cloud-controller-manager\n#\n# controller_image_tag: \"latest\"\n# ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset\n# ## Format:\n# ## external_hcloud_cloud.controller_extra_args:\n# ## arg1: \"value1\"\n# ## arg2: \"value2\"\n# controller_extra_args: {}\n#\n# load_balancers_location: # mutually exclusive with load_balancers_network_zone\n# load_balancers_network_zone:\n# load_balancers_disable_private_ingress: # set to true if using IPVS based plugins https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/main/docs/load_balancers.md#sample-service-with-networks\n# load_balancers_use_private_ip: # set to true if using private networks\n# load_balancers_enabled:\n# network_routes_enabled:\n" }, { "path": "inventory/sample/group_vars/all/huaweicloud.yml", "content": "## Values for the external Huawei Cloud Controller\n# external_huaweicloud_lbaas_subnet_id: \"Neutron subnet ID to create LBaaS VIP\"\n# external_huaweicloud_lbaas_network_id: \"Neutron network ID to create LBaaS VIP\"\n\n## Credentials to authenticate against Keystone API\n## All of them are required Per default these values will be\n## read from the environment.\n# external_huaweicloud_auth_url: \"{{ lookup('env','OS_AUTH_URL') }}\"\n# external_huaweicloud_access_key: \"{{ lookup('env','OS_ACCESS_KEY') }}\"\n# external_huaweicloud_secret_key: \"{{ lookup('env','OS_SECRET_KEY') }}\"\n# external_huaweicloud_region: \"{{ lookup('env','OS_REGION_NAME') }}\"\n# external_huaweicloud_project_id: \"{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}\"\n# external_huaweicloud_cloud: \"{{ lookup('env','OS_CLOUD') }}\"\n\n## The repo and tag of the external Huawei Cloud Controller image\n# external_huawei_cloud_controller_image_repo: \"swr.ap-southeast-1.myhuaweicloud.com\"\n# external_huawei_cloud_controller_image_tag: \"v0.26.8\"\n" }, { "path": "inventory/sample/group_vars/all/oci.yml", "content": "## When External Oracle Cloud Infrastructure is used, set these variables\n## External OCI Cloud Controller Manager\n## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml\n# external_oracle_auth_region: \"\"\n# external_oracle_auth_tenancy: \"\"\n# external_oracle_auth_user: \"\"\n# external_oracle_auth_key: \"\"\n# external_oracle_auth_passphrase: \"\"\n# external_oracle_auth_fingerprint: \"\"\n# external_oracle_auth_use_instance_principals: false\n\n# external_oracle_compartment: \"\"\n# external_oracle_vcn: \"\"\n# external_oracle_load_balancer_subnet1: \"\"\n# external_oracle_load_balancer_subnet2: \"\"\n# external_oracle_load_balancer_security_list_management_mode: All\n# external_oracle_load_balancer_security_lists: {}\n\n# external_oracle_ratelimiter_qps_read: 20.0\n# external_oracle_ratelimiter_bucket_read: 5\n# external_oracle_ratelimiter_qps_write: 20.0\n# external_oracle_ratelimiter_bucket_write: 5\n\n# external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci\n# external_oracle_cloud_controller_image_tag: \"v1.29.0\"\n\n\n## When Oracle Cloud Infrastructure is used, set these variables\n# oci_private_key:\n# oci_region_id:\n# oci_tenancy_id:\n# oci_user_id:\n# oci_user_fingerprint:\n# oci_compartment_id:\n# oci_vnc_id:\n# oci_subnet1_id:\n# oci_subnet2_id:\n## Override these default/optional behaviors if you wish\n# oci_security_list_management: All\n## If you would like the controller to manage specific lists per subnet. This is a mapping of subnet ocids to security list ocids. Below are examples.\n# oci_security_lists:\n# ocid1.subnet.oc1.phx.aaaaaaaasa53hlkzk6nzksqfccegk2qnkxmphkblst3riclzs4rhwg7rg57q: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q\n# ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q\n## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint\n# oci_use_instance_principals: false\n## If you would like to control OCI query rate limits for the controller\n# oci_rate_limit:\n# rate_limit_qps_read:\n# rate_limit_qps_write:\n# rate_limit_bucket_read:\n# rate_limit_bucket_write:\n## Other optional variables\n# oci_cloud_controller_pull_source: (default iad.ocir.io/oracle/cloud-provider-oci)\n# oci_cloud_controller_pull_secret: (name of pull secret to use if you define your own mirror above)\n" }, { "path": "inventory/sample/group_vars/all/offline.yml", "content": "---\n## Global Offline settings\n### Private Container Image Registry\n# registry_host: \"myprivateregisry.com\"\n# files_repo: \"http://myprivatehttpd\"\n### If using CentOS, RedHat, AlmaLinux or Fedora\n# yum_repo: \"http://myinternalyumrepo\"\n### If using Debian\n# debian_repo: \"http://myinternaldebianrepo\"\n### If using Ubuntu\n# ubuntu_repo: \"http://myinternalubunturepo\"\n\n## Container Registry overrides\n# kube_image_repo: \"{{ registry_host }}\"\n# gcr_image_repo: \"{{ registry_host }}\"\n# github_image_repo: \"{{ registry_host }}\"\n# docker_image_repo: \"{{ registry_host }}\"\n# quay_image_repo: \"{{ registry_host }}\"\n\n## Kubernetes components\n# kubeadm_download_url: \"{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm\"\n# kubectl_download_url: \"{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl\"\n# kubelet_download_url: \"{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet\"\n\n\n## Two options - Override entire repository or override only a single binary.\n\n## [Optional] 1 - Override entire binary repository\n# github_url: \"https://my_github_proxy\"\n# dl_k8s_io_url: \"https://my_dl_k8s_io_proxy\"\n# storage_googleapis_url: \"https://my_storage_googleapi_proxy\"\n# get_helm_url: \"https://my_helm_sh_proxy\"\n\n## [Optional] 2 - Override a specific binary\n## CNI Plugins\n# cni_download_url: \"{{ files_repo }}/github.com/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz\"\n\n## cri-tools\n# crictl_download_url: \"{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz\"\n\n## [Optional] etcd: only if you use etcd_deployment=host\n# etcd_download_url: \"{{ files_repo }}/github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz\"\n\n# [Optional] Calico: If using Calico network plugin\n# calicoctl_download_url: \"{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}\"\n# [Optional] Calico with kdd: If using Calico network plugin with kdd datastore\n# calico_crds_download_url: \"{{ files_repo }}/github.com/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml\"\n\n# [Optional] Cilium: If using Cilium network plugin\n# ciliumcli_download_url: \"{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz\"\n\n# [Optional] helm: only if you set helm_enabled: true\n# helm_download_url: \"{{ files_repo }}/get.helm.sh/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz\"\n\n# [Optional] crun: only if you set crun_enabled: true\n# crun_download_url: \"{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}\"\n\n# [Optional] kata: only if you set kata_containers_enabled: true\n# kata_containers_download_url: \"{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz\"\n\n# [Optional] cri-dockerd: only if you set container_manager: docker\n# cri_dockerd_download_url: \"{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz\"\n\n# [Optional] runc: if you set container_manager to containerd or crio\n# runc_download_url: \"{{ files_repo }}/github.com/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}\"\n\n# [Optional] cri-o: only if you set container_manager: crio\n# crio_download_base: \"download.opensuse.org/repositories/devel:kubic:libcontainers:stable\"\n# crio_download_crio: \"http://{{ crio_download_base }}:/cri-o:/\"\n# crio_download_url: \"{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz\"\n# skopeo_download_url: \"{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}\"\n\n# [Optional] containerd: only if you set container_runtime: containerd\n# containerd_download_url: \"{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz\"\n# nerdctl_download_url: \"{{ files_repo }}/github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz\"\n\n# [Optional] runsc,containerd-shim-runsc: only if you set gvisor_enabled: true\n# gvisor_runsc_download_url: \"{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc\"\n# gvisor_containerd_shim_runsc_download_url: \"{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1\"\n\n\n## CentOS/Redhat/AlmaLinux\n### For EL8, baseos and appstream must be available,\n### By default we enable those repo automatically\n# rhel_enable_repos: false\n### Docker / Containerd\n# docker_rh_repo_base_url: \"{{ yum_repo }}/docker-ce/$releasever/$basearch\"\n# docker_rh_repo_gpgkey: \"{{ yum_repo }}/docker-ce/gpg\"\n\n## Fedora\n### Docker\n# docker_fedora_repo_base_url: \"{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}\"\n# docker_fedora_repo_gpgkey: \"{{ yum_repo }}/docker-ce/gpg\"\n### Containerd\n# containerd_fedora_repo_base_url: \"{{ yum_repo }}/containerd\"\n# containerd_fedora_repo_gpgkey: \"{{ yum_repo }}/docker-ce/gpg\"\n\n## Debian\n### Docker\n# docker_debian_repo_base_url: \"{{ debian_repo }}/docker-ce\"\n# docker_debian_repo_gpgkey: \"{{ debian_repo }}/docker-ce/gpg\"\n### Containerd\n# containerd_debian_repo_base_url: \"{{ debian_repo }}/containerd\"\n# containerd_debian_repo_gpgkey: \"{{ debian_repo }}/containerd/gpg\"\n# containerd_debian_repo_repokey: 'YOURREPOKEY'\n\n## Ubuntu\n### Docker\n# docker_ubuntu_repo_base_url: \"{{ ubuntu_repo }}/docker-ce\"\n# docker_ubuntu_repo_gpgkey: \"{{ ubuntu_repo }}/docker-ce/gpg\"\n### Containerd\n# containerd_ubuntu_repo_base_url: \"{{ ubuntu_repo }}/containerd\"\n# containerd_ubuntu_repo_gpgkey: \"{{ ubuntu_repo }}/containerd/gpg\"\n# containerd_ubuntu_repo_repokey: 'YOURREPOKEY'\n" }, { "path": "inventory/sample/group_vars/all/openstack.yml", "content": "## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)\n# openstack_blockstorage_ignore_volume_az: yes\n## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.\n# openstack_lbaas_enabled: True\n# openstack_lbaas_subnet_id: \"Neutron subnet ID (not network ID) to create LBaaS VIP\"\n## To enable automatic floating ip provisioning, specify a subnet.\n# openstack_lbaas_floating_network_id: \"Neutron network ID (not subnet ID) to get floating IP from, disabled by default\"\n## Override default LBaaS behavior\n# openstack_lbaas_use_octavia: False\n# openstack_lbaas_method: \"ROUND_ROBIN\"\n# openstack_lbaas_provider: \"haproxy\"\n# openstack_lbaas_create_monitor: \"yes\"\n# openstack_lbaas_monitor_delay: \"1m\"\n# openstack_lbaas_monitor_timeout: \"30s\"\n# openstack_lbaas_monitor_max_retries: \"3\"\n\n## Values for the external OpenStack Cloud Controller\n# external_openstack_lbaas_enabled: true\n# external_openstack_lbaas_floating_network_id: \"Neutron network ID to get floating IP from\"\n# external_openstack_lbaas_floating_subnet_id: \"Neutron subnet ID to get floating IP from\"\n# external_openstack_lbaas_method: ROUND_ROBIN\n# external_openstack_lbaas_provider: amphora\n# external_openstack_lbaas_subnet_id: \"Neutron subnet ID to create LBaaS VIP\"\n# external_openstack_lbaas_network_id: \"Neutron network ID to create LBaaS VIP\"\n# external_openstack_lbaas_manage_security_groups: false\n# external_openstack_lbaas_create_monitor: false\n# external_openstack_lbaas_monitor_delay: 5s\n# external_openstack_lbaas_monitor_max_retries: 1\n# external_openstack_lbaas_monitor_timeout: 3s\n# external_openstack_lbaas_internal_lb: false\n# external_openstack_network_ipv6_disabled: false\n# external_openstack_network_internal_networks: []\n# external_openstack_network_public_networks: []\n# external_openstack_metadata_search_order: \"configDrive,metadataService\"\n\n## Application credentials to authenticate against Keystone API\n## Those settings will take precedence over username and password that might be set your environment\n## All of them are required\n# external_openstack_application_credential_name:\n# external_openstack_application_credential_id:\n# external_openstack_application_credential_secret:\n\n## Tags for the Cinder CSI images\n## registry.k8s.io/sig-storage/csi-attacher\n# cinder_csi_attacher_image_tag: \"v4.4.2\"\n## registry.k8s.io/sig-storage/csi-provisioner\n# cinder_csi_provisioner_image_tag: \"v3.6.2\"\n## registry.k8s.io/sig-storage/csi-snapshotter\n# cinder_csi_snapshotter_image_tag: \"v6.3.2\"\n## registry.k8s.io/sig-storage/csi-resizer\n# cinder_csi_resizer_image_tag: \"v1.9.2\"\n## registry.k8s.io/sig-storage/livenessprobe\n# cinder_csi_livenessprobe_image_tag: \"v2.11.0\"\n\n## To use Cinder CSI plugin to provision volumes set this value to true\n## Make sure to source in the openstack credentials\n# cinder_csi_enabled: true\n# cinder_csi_controller_replicas: 1\n# storage_classes:\n# - name: \"cinder-csi\"\n# provisioner: \"kubernetes.io/cinder\"\n# mount_options:\n# - \"discard\"\n# parameters:\n# type: \"thin\"\n# availability: \"nova\"\n# reclaim_policy: \"Delete\"\n# volume_binding_mode: \"WaitForFirstConsumer\"\n" }, { "path": "inventory/sample/group_vars/all/upcloud.yml", "content": "## Repo for UpClouds csi-driver: https://github.com/UpCloudLtd/upcloud-csi\n## To use UpClouds CSI plugin to provision volumes set this value to true\n## Remember to set UPCLOUD_USERNAME and UPCLOUD_PASSWORD\n# upcloud_csi_enabled: true\n# upcloud_csi_controller_replicas: 1\n## Override used image tags\n# upcloud_csi_provisioner_image_tag: \"v3.1.0\"\n# upcloud_csi_attacher_image_tag: \"v3.4.0\"\n# upcloud_csi_resizer_image_tag: \"v1.4.0\"\n# upcloud_csi_plugin_image_tag: \"v0.3.3\"\n# upcloud_csi_node_image_tag: \"v2.5.0\"\n# upcloud_tolerations: []\n## Storage class options\n# storage_classes:\n# - name: standard\n# is_default: true\n# expand_persistent_volumes: true\n# parameters:\n# tier: maxiops\n# - name: hdd\n# is_default: false\n# expand_persistent_volumes: true\n# parameters:\n# tier: hdd\n" }, { "path": "inventory/sample/group_vars/all/vsphere.yml", "content": "## Values for the external vSphere Cloud Provider\n# external_vsphere_vcenter_ip: \"myvcenter.domain.com\"\n# external_vsphere_vcenter_port: \"443\"\n# external_vsphere_insecure: \"true\"\n# external_vsphere_user: \"administrator@vsphere.local\" # Can also be set via the `VSPHERE_USER` environment variable\n# external_vsphere_password: \"K8s_admin\" # Can also be set via the `VSPHERE_PASSWORD` environment variable\n# external_vsphere_datacenter: \"DATACENTER_name\"\n# external_vsphere_kubernetes_cluster_id: \"kubernetes-cluster-id\"\n\n## To use vSphere CSI plugin to provision volumes set this value to true\n# vsphere_csi_enabled: true\n# vsphere_csi_controller_replicas: 1\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/addons.yml", "content": "---\n# Helm deployment\nhelm_enabled: false\n\n# Registry deployment\nregistry_enabled: false\n# registry_namespace: kube-system\n# registry_storage_class: \"\"\n# registry_disk_size: \"10Gi\"\n\n# Metrics Server deployment\nmetrics_server_enabled: false\n# metrics_server_container_port: 10250\n# metrics_server_kubelet_insecure_tls: true\n# metrics_server_metric_resolution: 15s\n# metrics_server_kubelet_preferred_address_types: \"InternalIP,ExternalIP,Hostname\"\n# metrics_server_host_network: false\n# metrics_server_replicas: 1\n\n# Rancher Local Path Provisioner\nlocal_path_provisioner_enabled: false\n# local_path_provisioner_namespace: \"local-path-storage\"\n# local_path_provisioner_storage_class: \"local-path\"\n# local_path_provisioner_reclaim_policy: Delete\n# local_path_provisioner_claim_root: /opt/local-path-provisioner/\n# local_path_provisioner_debug: false\n# local_path_provisioner_image_repo: \"{{ docker_image_repo }}/rancher/local-path-provisioner\"\n# local_path_provisioner_helper_image_repo: \"busybox\"\n# local_path_provisioner_helper_image_tag: \"latest\"\n\n# Local volume provisioner deployment\nlocal_volume_provisioner_enabled: false\n# local_volume_provisioner_namespace: kube-system\n# local_volume_provisioner_nodelabels:\n# - kubernetes.io/hostname\n# - topology.kubernetes.io/region\n# - topology.kubernetes.io/zone\n# local_volume_provisioner_storage_classes:\n# local-storage:\n# host_dir: /mnt/disks\n# mount_dir: /mnt/disks\n# volume_mode: Filesystem\n# fs_type: ext4\n# fast-disks:\n# host_dir: /mnt/fast-disks\n# mount_dir: /mnt/fast-disks\n# block_cleaner_command:\n# - \"/scripts/shred.sh\"\n# - \"2\"\n# volume_mode: Filesystem\n# fs_type: ext4\n# local_volume_provisioner_tolerations:\n# - effect: NoSchedule\n# operator: Exists\n\n# CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots\n# currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller\n# Longhorn is an external CSI that would also require setting this to true but it is not included in kubespray\n# csi_snapshot_controller_enabled: false\n# csi snapshot namespace\n# snapshot_controller_namespace: kube-system\n\n# Gateway API CRDs\ngateway_api_enabled: false\n\n# ALB ingress controller deployment\ningress_alb_enabled: false\n# alb_ingress_aws_region: \"us-east-1\"\n# alb_ingress_restrict_scheme: \"false\"\n# Enables logging on all outbound requests sent to the AWS API.\n# If logging is desired, set to true.\n# alb_ingress_aws_debug: \"false\"\n\n# Cert manager deployment\ncert_manager_enabled: false\n# cert_manager_namespace: \"cert-manager\"\n# cert_manager_tolerations:\n# - key: node-role.kubernetes.io/control-plane\n# effect: NoSchedule\n# cert_manager_affinity:\n# nodeAffinity:\n# preferredDuringSchedulingIgnoredDuringExecution:\n# - weight: 100\n# preference:\n# matchExpressions:\n# - key: node-role.kubernetes.io/control-plane\n# operator: In\n# values:\n# - \"\"\n# cert_manager_nodeselector:\n# kubernetes.io/os: \"linux\"\n\n# cert_manager_trusted_internal_ca: |\n# -----BEGIN CERTIFICATE-----\n# [REPLACE with your CA certificate]\n# -----END CERTIFICATE-----\n# cert_manager_leader_election_namespace: kube-system\n\n# cert_manager_dns_policy: \"ClusterFirst\"\n# cert_manager_dns_config:\n# nameservers:\n# - \"1.1.1.1\"\n# - \"8.8.8.8\"\n\n# cert_manager_controller_extra_args:\n# - \"--dns01-recursive-nameservers-only=true\"\n# - \"--dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53\"\n\n# MetalLB deployment\nmetallb_enabled: false\nmetallb_speaker_enabled: \"{{ metallb_enabled }}\"\nmetallb_namespace: \"metallb-system\"\n# metallb_protocol: \"layer2\"\n# metallb_port: \"7472\"\n# metallb_memberlist_port: \"7946\"\n# metallb_config:\n# speaker:\n# nodeselector:\n# kubernetes.io/os: \"linux\"\n# tolerations:\n# - key: \"node-role.kubernetes.io/control-plane\"\n# operator: \"Equal\"\n# value: \"\"\n# effect: \"NoSchedule\"\n# controller:\n# nodeselector:\n# kubernetes.io/os: \"linux\"\n# tolerations:\n# - key: \"node-role.kubernetes.io/control-plane\"\n# operator: \"Equal\"\n# value: \"\"\n# effect: \"NoSchedule\"\n# address_pools:\n# primary:\n# ip_range:\n# - 10.5.0.0/16\n# auto_assign: true\n# pool1:\n# ip_range:\n# - 10.6.0.0/16\n# auto_assign: true\n# pool2:\n# ip_range:\n# - 10.10.0.0/16\n# auto_assign: true\n# layer2:\n# - primary\n# layer3:\n# defaults:\n# peer_port: 179\n# hold_time: 120s\n# communities:\n# vpn-only: \"1234:1\"\n# NO_ADVERTISE: \"65535:65282\"\n# metallb_peers:\n# peer1:\n# peer_address: 10.6.0.1\n# peer_asn: 64512\n# my_asn: 4200000000\n# communities:\n# - vpn-only\n# address_pool:\n# - pool1\n# peer2:\n# peer_address: 10.10.0.1\n# peer_asn: 64513\n# my_asn: 4200000000\n# communities:\n# - NO_ADVERTISE\n# address_pool:\n# - pool2\n\nargocd_enabled: false\n# argocd_namespace: argocd\n# Default password:\n# - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli\n# ---\n# The initial password is autogenerated and stored in `argocd-initial-admin-secret` in the argocd namespace defined above.\n# Using the argocd CLI the generated password can be automatically be fetched from the current kubectl context with the command:\n# argocd admin initial-password -n argocd\n# ---\n# Use the following var to set admin password\n# argocd_admin_password: \"password\"\n\n# The plugin manager for kubectl\n\n# Kube VIP\nkube_vip_enabled: false\n# kube_vip_arp_enabled: true\n# kube_vip_controlplane_enabled: true\n# kube_vip_address: 192.168.56.120\n# loadbalancer_apiserver:\n# address: \"{{ kube_vip_address }}\"\n# port: 6443\n# kube_vip_interface: eth0\n# kube_vip_services_enabled: false\n# kube_vip_dns_mode: first\n# kube_vip_cp_detect: false\n# kube_vip_leasename: plndr-cp-lock\n# kube_vip_enable_node_labeling: false\n# kube_vip_lb_fwdmethod: local\n# kube_vip_bgp_sourceip:\n# kube_vip_bgp_sourceif:\n\n# Node Feature Discovery\nnode_feature_discovery_enabled: false\n# node_feature_discovery_gc_sa_name: node-feature-discovery\n# node_feature_discovery_gc_sa_create: false\n# node_feature_discovery_worker_sa_name: node-feature-discovery\n# node_feature_discovery_worker_sa_create: false\n# node_feature_discovery_master_config:\n# extraLabelNs: [\"nvidia.com\"]\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml", "content": "---\n# Kubernetes configuration dirs and system namespace.\n# Those are where all the additional config stuff goes\n# the kubernetes normally puts in /srv/kubernetes.\n# This puts them in a sane location and namespace.\n# Editing those values will almost surely break something.\nkube_config_dir: /etc/kubernetes\nkube_script_dir: \"{{ bin_dir }}/kubernetes-scripts\"\nkube_manifest_dir: \"{{ kube_config_dir }}/manifests\"\n\n# This is where all the cert scripts and certs will be located\nkube_cert_dir: \"{{ kube_config_dir }}/ssl\"\n\n# This is where all of the bearer tokens will be stored\nkube_token_dir: \"{{ kube_config_dir }}/tokens\"\n\nkube_api_anonymous_auth: true\n\n# Where the binaries will be downloaded.\n# Note: ensure that you've enough disk space (about 1G)\nlocal_release_dir: \"/tmp/releases\"\n# Random shifts for retrying failed ops like pushing/downloading\nretry_stagger: 5\n\n# This is the user that owns the cluster installation.\n# Note: cilium needs to set kube_owner to root https://kubespray.io/#/docs/CNI/cilium?id=unprivileged-agent-configuration\nkube_owner: kube\n\n# This is the group that the cert creation scripts chgrp the\n# cert files to. Not really changeable...\nkube_cert_group: kube-cert\n\n# Cluster Loglevel configuration\nkube_log_level: 2\n\n# Directory where credentials will be stored\ncredentials_dir: \"{{ inventory_dir }}/credentials\"\n\n## It is possible to activate / deactivate selected authentication methods (oidc, static token auth)\n# kube_oidc_auth: false\n# kube_token_auth: false\n\n\n## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/\n## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)\n\n# kube_oidc_url: https:// ...\n# kube_oidc_client_id: kubernetes\n## Optional settings for OIDC\n# kube_oidc_ca_file: \"{{ kube_cert_dir }}/ca.pem\"\n# kube_oidc_username_claim: sub\n# kube_oidc_username_prefix: 'oidc:'\n# kube_oidc_groups_claim: groups\n# kube_oidc_groups_prefix: 'oidc:'\n\n## Variables to control webhook authn/authz\n# kube_webhook_token_auth: false\n# kube_webhook_token_auth_url: https://...\n# kube_webhook_token_auth_url_skip_tls_verify: false\n\n## For webhook authorization, authorization_modes must include Webhook or kube_apiserver_authorization_config_authorizers must configure a type: Webhook\n# kube_webhook_authorization: false\n# kube_webhook_authorization_url: https://...\n# kube_webhook_authorization_url_skip_tls_verify: false\n\n# Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin)\n# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing\nkube_network_plugin: calico\n\n# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni\nkube_network_plugin_multus: false\n\n# Kubernetes internal network for services, unused block of space.\nkube_service_addresses: 10.233.0.0/18\n\n# internal network. When used, it will assign IP\n# addresses from this range to individual pods.\n# This network must be unused in your network infrastructure!\nkube_pods_subnet: 10.233.64.0/18\n\n# internal network node size allocation (optional). This is the size allocated\n# to each node for pod IP address allocation. Note that the number of pods per node is\n# also limited by the kubelet_max_pods variable which defaults to 110.\n#\n# Example:\n# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node:\n# - kube_pods_subnet: 10.233.64.0/18\n# - kube_network_node_prefix: 24\n# - kubelet_max_pods: 110\n#\n# Example:\n# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node:\n# - kube_pods_subnet: 10.233.64.0/18\n# - kube_network_node_prefix: 25\n# - kubelet_max_pods: 110\nkube_network_node_prefix: 24\n\n# Kubernetes internal network for IPv6 services, unused block of space.\n# This is only used if ipv6_stack is set to true\n# This provides 4096 IPv6 IPs\nkube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116\n\n# Internal network. When used, it will assign IPv6 addresses from this range to individual pods.\n# This network must not already be in your network infrastructure!\n# This is only used if ipv6_stack is set to true.\n# This provides room for 256 nodes with 254 pods per node.\nkube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n\n# IPv6 subnet size allocated to each for pods.\n# This is only used if ipv6_stack is set to true\n# This provides room for 254 pods per node.\nkube_network_node_prefix_ipv6: 120\n\n# The port the API Server will be listening on.\nkube_apiserver_ip: \"{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}\"\nkube_apiserver_port: 6443 # (https)\n\n# Kube-proxy proxyMode configuration.\n# Can be ipvs, iptables, nftables\n# TODO: it needs to be changed to nftables when the upstream use nftables as default\nkube_proxy_mode: ipvs\n\n# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface\n# must be set to true for MetalLB, kube-vip(ARP enabled) to work\nkube_proxy_strict_arp: false\n\n# A string slice of values which specify the addresses to use for NodePorts.\n# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).\n# The default empty string slice ([]) means to use all local addresses.\n# kube_proxy_nodeport_addresses_cidr is retained for legacy config\nkube_proxy_nodeport_addresses: >-\n {%- if kube_proxy_nodeport_addresses_cidr is defined -%}\n [{{ kube_proxy_nodeport_addresses_cidr }}]\n {%- else -%}\n []\n {%- endif -%}\n\n# If non-empty, will use this string as identification instead of the actual hostname\n# kube_override_hostname: {{ inventory_hostname }}\n\n## Encrypting Secret Data at Rest\nkube_encrypt_secret_data: false\n\n# Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/\n# kubelet_shutdown_grace_period had to be greater than kubelet_shutdown_grace_period_critical_pods to allow\n# non-critical podsa to also terminate gracefully\n# kubelet_shutdown_grace_period: 60s\n# kubelet_shutdown_grace_period_critical_pods: 20s\n\n# DNS configuration.\n# Kubernetes cluster name, also will be used as DNS domain\ncluster_name: cluster.local\n# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods\nndots: 2\n# dns_timeout: 2\n# dns_attempts: 2\n# Custom search domains to be added in addition to the default cluster search domains\n# searchdomains:\n# - svc.{{ cluster_name }}\n# - default.svc.{{ cluster_name }}\n# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).\n# remove_default_searchdomains: false\n# Can be coredns, coredns_dual, manual or none\ndns_mode: coredns\n# Set manual server if using a custom cluster DNS server\n# manual_dns_server: 10.x.x.x\n# Enable nodelocal dns cache\nenable_nodelocaldns: true\nenable_nodelocaldns_secondary: false\nnodelocaldns_ip: 169.254.25.10\nnodelocaldns_health_port: 9254\nnodelocaldns_second_health_port: 9256\nnodelocaldns_bind_metrics_host_ip: false\nnodelocaldns_secondary_skew_seconds: 5\n# nodelocaldns_external_zones:\n# - zones:\n# - example.com\n# - example.io:1053\n# nameservers:\n# - 1.1.1.1\n# - 2.2.2.2\n# cache: 5\n# - zones:\n# - https://mycompany.local:4453\n# nameservers:\n# - 192.168.0.53\n# cache: 0\n# - zones:\n# - mydomain.tld\n# nameservers:\n# - 10.233.0.3\n# cache: 5\n# rewrite:\n# - name website.tld website.namespace.svc.cluster.local\n# Enable k8s_external plugin for CoreDNS\nenable_coredns_k8s_external: false\ncoredns_k8s_external_zone: k8s_external.local\n# Enable endpoint_pod_names option for kubernetes plugin\nenable_coredns_k8s_endpoint_pod_names: false\n# Set forward options for upstream DNS servers in coredns (and nodelocaldns) config\n# dns_upstream_forward_extra_opts:\n# policy: sequential\n# Apply extra options to coredns kubernetes plugin\n# coredns_kubernetes_extra_opts:\n# - 'fallthrough example.local'\n# Forward extra domains to the coredns kubernetes plugin\n# coredns_kubernetes_extra_domains: ''\n\n# Can be docker_dns, host_resolvconf or none\nresolvconf_mode: host_resolvconf\n# Deploy netchecker app to verify DNS resolve as an HTTP service\ndeploy_netchecker: false\n# Ip address of the kubernetes skydns service\nskydns_server: \"{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}\"\nskydns_server_secondary: \"{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}\"\ndns_domain: \"{{ cluster_name }}\"\n\n## Container runtime\n## docker for docker, crio for cri-o and containerd for containerd.\n## Default: containerd\ncontainer_manager: containerd\n\n# Additional container runtimes\nkata_containers_enabled: false\n\nkubeadm_certificate_key: \"{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}\"\n\n# K8s image pull policy (imagePullPolicy)\nk8s_image_pull_policy: IfNotPresent\n\n# audit log for kubernetes\nkubernetes_audit: false\n\n# define kubelet config dir for dynamic kubelet\n# kubelet_config_dir:\ndefault_kubelet_config_dir: \"{{ kube_config_dir }}/dynamic_kubelet_dir\"\n\n# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts\n# kubeconfig_localhost: false\n# Use ansible_host as external api ip when copying over kubeconfig.\n# kubeconfig_localhost_ansible_host: false\n# Download kubectl onto the host that runs Ansible in {{ bin_dir }}\n# kubectl_localhost: false\n\n# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.\n# Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is \"\".\n# kubelet_enforce_node_allocatable: pods\n\n## Set runtime and kubelet cgroups when using systemd as cgroup driver (default)\n# kubelet_runtime_cgroups: \"/{{ kube_service_cgroups }}/{{ container_manager }}.service\"\n# kubelet_kubelet_cgroups: \"/{{ kube_service_cgroups }}/kubelet.service\"\n\n## Set runtime and kubelet cgroups when using cgroupfs as cgroup driver\n# kubelet_runtime_cgroups_cgroupfs: \"/system.slice/{{ container_manager }}.service\"\n# kubelet_kubelet_cgroups_cgroupfs: \"/system.slice/kubelet.service\"\n\n# Whether to run kubelet and container-engine daemons in a dedicated cgroup.\n# kube_reserved: false\n## Uncomment to override default values\n## The following two items need to be set when kube_reserved is true\n# kube_reserved_cgroups_for_service_slice: kube.slice\n# kube_reserved_cgroups: \"/{{ kube_reserved_cgroups_for_service_slice }}\"\n# kube_memory_reserved: 256Mi\n# kube_cpu_reserved: 100m\n# kube_ephemeral_storage_reserved: 2Gi\n# kube_pid_reserved: \"1000\"\n\n## Optionally reserve resources for OS system daemons.\n# system_reserved: true\n## Uncomment to override default values\n## The following two items need to be set when system_reserved is true\n# system_reserved_cgroups_for_service_slice: system.slice\n# system_reserved_cgroups: \"/{{ system_reserved_cgroups_for_service_slice }}\"\n# system_memory_reserved: 512Mi\n# system_cpu_reserved: 500m\n# system_ephemeral_storage_reserved: 2Gi\n\n## Eviction Thresholds to avoid system OOMs\n# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds\n# eviction_hard: {}\n# eviction_hard_control_plane: {}\n\n# An alternative flexvolume plugin directory\n# kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec\n\n## Supplementary addresses that can be added in kubernetes ssl keys.\n## That can be useful for example to setup a keepalived virtual IP\n# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3]\n\n## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler.\n## See https://github.com/kubernetes-sigs/kubespray/issues/2141\n## Set this variable to true to get rid of this issue\nvolume_cross_zone_attachment: false\n## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI,\n## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI)\npersistent_volumes_enabled: false\n\n## Container Engine Acceleration\n## Enable container acceleration feature, for example use gpu acceleration in containers\n# nvidia_accelerator_enabled: true\n## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset.\n## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2'\n## Array with nvida_gpu_nodes, leave empty or comment if you don't want to install drivers.\n## Labels and taints won't be set to nodes if they are not in the array.\n# nvidia_gpu_nodes:\n# - kube-gpu-001\n# nvidia_driver_version: \"384.111\"\n## flavor can be tesla or gtx\n# nvidia_gpu_flavor: gtx\n## NVIDIA driver installer images. Change them if you have trouble accessing gcr.io.\n# nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2\n# nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63\n## NVIDIA GPU device plugin image.\n# nvidia_gpu_device_plugin_container: \"registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e\"\n\n## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.\n# tls_min_version: \"\"\n\n## Support tls cipher suites.\n# tls_cipher_suites: {}\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\n# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_RSA_WITH_RC4_128_SHA\n# - TLS_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_RSA_WITH_AES_256_CBC_SHA\n# - TLS_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_RSA_WITH_RC4_128_SHA\n\n## Amount of time to retain events. (default 1h0m0s)\nevent_ttl_duration: \"1h0m0s\"\n\n## Automatically renew K8S control plane certificates on first Monday of each month\nauto_renew_certificates: false\n# First Monday of each month\n# auto_renew_certificates_systemd_calendar: \"Mon *-*-1,2,3,4,5,6,7 03:00:00\"\n\nkubeadm_patches_dir: \"{{ kube_config_dir }}/patches\"\nkubeadm_patches: []\n# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches\n# Correspondance with this link\n# patchtype = type\n# target = target\n# suffix -> managed automatically\n# extension -> always \"yaml\"\n# kubeadm_patches:\n# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration\n# type: strategic(default)|json|merge\n# patch:\n# metadata:\n# annotations:\n# example.com/test: \"true\"\n# labels:\n# example.com/prod_level: \"{{ prod_level }}\"\n# - ...\n# Patches are applied in the order they are specified.\n\n# Set to true to remove the role binding to anonymous users created by kubeadm\nremove_anonymous_access: false\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml", "content": "---\n# see roles/network_plugin/calico/defaults/main.yml\n\n# the default value of name\ncalico_cni_name: k8s-pod-network\n\n## With calico it is possible to distributed routes with border routers of the datacenter.\n## Warning : enabling router peering will disable calico's default behavior ('node mesh').\n## The subnets of each nodes will be distributed by the datacenter router\n# peer_with_router: false\n\n# Enables Internet connectivity from containers\n# nat_outgoing: true\n# nat_outgoing_ipv6: true\n\n# Enables Calico CNI \"host-local\" IPAM plugin\n# calico_ipam_host_local: true\n\n# add default ippool name\n# calico_pool_name: \"default-pool\"\n\n# add default ippool blockSize\ncalico_pool_blocksize: 26\n\n# add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)\n# calico_pool_cidr: 1.2.3.4/5\n\n# Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set.\n# calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n\n# Global as_num (/calico/bgp/v1/global/as_num)\n# global_as_num: \"64512\"\n\n# If doing peering with node-assigned asn where the globas does not match your nodes, you want this\n# to be true. All other cases, false.\n# calico_no_global_as_num: false\n\n# You can set MTU value here. If left undefined or empty, it will\n# not be specified in calico CNI config, so Calico will use built-in\n# defaults. The value should be a number, not a string.\n# calico_mtu: 1500\n\n# Configure the MTU to use for workload interfaces and tunnels.\n# - If Wireguard is enabled, subtract 60 from your network MTU (i.e 1500-60=1440)\n# - Otherwise, if VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450)\n# - Otherwise, if IPIP is enabled, subtract 20 from your network MTU (i.e. 1500-20=1480)\n# - Otherwise, if not using any encapsulation, set to your network MTU (i.e. 1500)\n# calico_veth_mtu: 1440\n\n# Advertise Cluster IPs\n# calico_advertise_cluster_ips: true\n\n# Advertise Service External IPs\n# calico_advertise_service_external_ips:\n# - x.x.x.x/24\n# - y.y.y.y/32\n\n# Advertise Service LoadBalancer IPs\n# calico_advertise_service_loadbalancer_ips:\n# - x.x.x.x/24\n# - y.y.y.y/16\n\n# Choose data store type for calico: \"etcd\" or \"kdd\" (kubernetes datastore)\n# calico_datastore: \"kdd\"\n\n# Choose Calico iptables backend: \"Legacy\", \"Auto\" or \"NFT\"\n# calico_iptables_backend: \"Auto\"\n\n# Use typha (only with kdd)\n# typha_enabled: false\n\n# Generate TLS certs for secure typha<->calico-node communication\n# typha_secure: false\n\n# Scaling typha: 1 replica per 100 nodes is adequate\n# Number of typha replicas\n# typha_replicas: 1\n\n# Set max typha connections\n# typha_max_connections_lower_limit: 300\n\n# Set calico network backend: \"bird\", \"vxlan\" or \"none\"\n# bird enable BGP routing, required for ipip and no encapsulation modes\n# calico_network_backend: vxlan\n\n# IP in IP and VXLAN is mutually exclusive modes.\n# set IP in IP encapsulation mode: \"Always\", \"CrossSubnet\", \"Never\"\n# calico_ipip_mode: 'Never'\n\n# set VXLAN encapsulation mode: \"Always\", \"CrossSubnet\", \"Never\"\n# calico_vxlan_mode: 'Always'\n\n# set VXLAN port and VNI\n# calico_vxlan_vni: 4096\n# calico_vxlan_port: 4789\n\n# Enable eBPF mode\n# calico_bpf_enabled: false\n\n# If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of:\n# * can-reach=DESTINATION\n# * interface=INTERFACE-REGEX\n# see https://docs.projectcalico.org/reference/node/configuration\n# calico_ip_auto_method: \"interface=eth.*\"\n# calico_ip6_auto_method: \"interface=eth.*\"\n\n# Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection.\n# see https://projectcalico.docs.tigera.io/reference/felix/configuration\n# calico_felix_mtu_iface_pattern: \"^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)\"\n\n# Choose the iptables insert mode for Calico: \"Insert\" or \"Append\".\n# calico_felix_chaininsertmode: Insert\n\n# If you want use the default route interface when you use multiple interface with dynamique route (iproute2)\n# see https://docs.projectcalico.org/reference/node/configuration : FELIX_DEVICEROUTESOURCEADDRESS\n# calico_use_default_route_src_ipaddr: false\n\n# Enable calico traffic encryption with wireguard\n# calico_wireguard_enabled: false\n\n# Under certain situations liveness and readiness probes may need tunning\n# calico_node_livenessprobe_timeout: 10\n# calico_node_readinessprobe_timeout: 10\n\n# Calico apiserver (only with kdd)\n# calico_apiserver_enabled: false\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml", "content": "---\n# Log-level\n# cilium_debug: false\n\n# cilium_mtu: \"\"\n# cilium_enable_ipv4: true\n# cilium_enable_ipv6: false\n\n# Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/\ncilium_l2announcements: false\n\n# Cilium agent health port\n# cilium_agent_health_port: \"9879\"\n\n# Identity allocation mode selects how identities are shared between cilium\n# nodes by setting how they are stored. The options are \"crd\" or \"kvstore\".\n# - \"crd\" stores identities in kubernetes as CRDs (custom resource definition).\n# These can be queried with:\n# `kubectl get ciliumid`\n# - \"kvstore\" stores identities in an etcd kvstore.\n# - In order to support External Workloads, \"crd\" is required\n# - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta\n# - KVStore operations are only required when cilium-operator is running with any of the below options:\n# - --synchronize-k8s-services\n# - --synchronize-k8s-nodes\n# - --identity-allocation-mode=kvstore\n# - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations\n# cilium_identity_allocation_mode: kvstore\n\n# Etcd SSL dirs\n# cilium_cert_dir: /etc/cilium/certs\n# kube_etcd_cacert_file: ca.pem\n# kube_etcd_cert_file: cert.pem\n# kube_etcd_key_file: cert-key.pem\n\n# Limits for apps\n# cilium_memory_limit: 500M\n# cilium_cpu_limit: 500m\n# cilium_memory_requests: 64M\n# cilium_cpu_requests: 100m\n\n# Overlay Network Mode\n# cilium_tunnel_mode: vxlan\n\n# LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode\n# cilium_loadbalancer_mode: snat\n\n# Optional features\n# cilium_enable_prometheus: false\n# Enable if you want to make use of hostPort mappings\n# cilium_enable_portmap: false\n# Monitor aggregation level (none/low/medium/maximum)\n# cilium_monitor_aggregation: medium\n# The monitor aggregation flags determine which TCP flags which, upon the\n# first observation, cause monitor notifications to be generated.\n#\n# Only effective when monitor aggregation is set to \"medium\" or higher.\n# cilium_monitor_aggregation_flags: \"all\"\n# Kube Proxy Replacement mode (true/false)\n# cilium_kube_proxy_replacement: false\n\n# If upgrading from Cilium < 1.5, you may want to override some of these options\n# to prevent service disruptions. See also:\n# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action\n# cilium_preallocate_bpf_maps: false\n\n# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9\n# cilium_tofqdns_enable_poller: false\n\n# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9\n# cilium_enable_legacy_services: false\n\n# Unique ID of the cluster. Must be unique across all connected clusters and\n# in the range of 1 and 255. Only relevant when building a mesh of clusters.\n# This value is not defined by default\n# cilium_cluster_id:\n\n# Deploy cilium even if kube_network_plugin is not cilium.\n# This enables to deploy cilium alongside another CNI to replace kube-proxy.\n# cilium_deploy_additionally: false\n\n# Auto direct nodes routes can be used to advertise pods routes in your cluster\n# without any tunneling (with `cilium_tunnel_mode` sets to `disabled`).\n# This works only if you have a L2 connectivity between all your nodes.\n# You wil also have to specify the variable `cilium_native_routing_cidr` to\n# make this work. Please refer to the cilium documentation for more\n# information about this kind of setups.\n# cilium_auto_direct_node_routes: false\n\n# Allows to explicitly specify the IPv4 CIDR for native routing.\n# When specified, Cilium assumes networking for this CIDR is preconfigured and\n# hands traffic destined for that range to the Linux network stack without\n# applying any SNAT.\n# Generally speaking, specifying a native routing CIDR implies that Cilium can\n# depend on the underlying networking stack to route packets to their\n# destination. To offer a concrete example, if Cilium is configured to use\n# direct routing and the Kubernetes CIDR is included in the native routing CIDR,\n# the user must configure the routes to reach pods, either manually or by\n# setting the auto-direct-node-routes flag.\n# cilium_native_routing_cidr: \"\"\n\n# Allows to explicitly specify the IPv6 CIDR for native routing.\n# cilium_native_routing_cidr_ipv6: \"\"\n\n# Enable transparent network encryption.\n# cilium_encryption_enabled: false\n\n# Encryption method. Can be either ipsec or wireguard.\n# Only effective when `cilium_encryption_enabled` is set to true.\n# cilium_encryption_type: \"ipsec\"\n\n# Enable encryption for pure node to node traffic.\n# This option is only effective when `cilium_encryption_type` is set to `ipsec`.\n# cilium_ipsec_node_encryption: false\n\n# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.\n# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,\n# it will fallback on the wireguard-go user-space implementation of WireGuard.\n# This option is only effective when `cilium_encryption_type` is set to `wireguard`.\n# cilium_wireguard_userspace_fallback: false\n\n# IP Masquerade Agent\n# https://docs.cilium.io/en/stable/concepts/networking/masquerading/\n# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded\n# cilium_ip_masq_agent_enable: false\n\n### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded\n# cilium_non_masquerade_cidrs:\n# - 10.0.0.0/8\n# - 172.16.0.0/12\n# - 192.168.0.0/16\n# - 100.64.0.0/10\n# - 192.0.0.0/24\n# - 192.0.2.0/24\n# - 192.88.99.0/24\n# - 198.18.0.0/15\n# - 198.51.100.0/24\n# - 203.0.113.0/24\n# - 240.0.0.0/4\n### Indicates whether to masquerade traffic to the link local prefix.\n### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.\n# cilium_masq_link_local: false\n### A time interval at which the agent attempts to reload config from disk\n# cilium_ip_masq_resync_interval: 60s\n\n### Host Firewall and Policy Audit Mode\n# cilium_enable_host_firewall: false\n# cilium_policy_audit_mode: false\n\n# Hubble\n### Enable Hubble without install\n# cilium_enable_hubble: false\n### Enable Hubble-ui\n### Installed by default when hubble is enabled. To disable set to false\n# cilium_enable_hubble_ui: \"{{ cilium_enable_hubble }}\"\n### Enable Hubble Metrics\n# cilium_enable_hubble_metrics: false\n### if cilium_enable_hubble_metrics: true\n# cilium_hubble_metrics: {}\n# - dns\n# - drop\n# - tcp\n# - flow\n# - icmp\n# - http\n### Enable Hubble install\n# cilium_hubble_install: false\n### Enable auto generate certs if cilium_hubble_install: true\n# cilium_hubble_tls_generate: false\n\n### Tune cilium_hubble_event_buffer_capacity & cilium_hubble_event_queue_size values to avoid dropping events when hubble is under heavy load\n### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535\n### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095)\n# cilium_hubble_event_buffer_capacity: 4095\n### Buffer size of the channel to receive monitor events.\n# cilium_hubble_event_queue_size: 50\n\n# Override the DNS suffix that Hubble-Relay uses to resolve its peer service.\n# It defaults to the inventory's `dns_domain`.\n# cilium_hubble_peer_service_cluster_domain: \"{{ dns_domain }}\"\n\n# IP address management mode for v1.9+.\n# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/\n# cilium_ipam_mode: kubernetes\n\n# Extra arguments for the Cilium agent\n# cilium_agent_custom_args: []\n\n# For adding and mounting extra volumes to the cilium agent\n# cilium_agent_extra_volumes: []\n# cilium_agent_extra_volume_mounts: []\n\n# cilium_agent_extra_env_vars: []\n\n# cilium_operator_replicas: 2\n\n# The address at which the cillium operator bind health check api\n# cilium_operator_api_serve_addr: \"127.0.0.1:9234\"\n\n## A dictionary of extra config variables to add to cilium-config, formatted like:\n## cilium_config_extra_vars:\n## var1: \"value1\"\n## var2: \"value2\"\n# cilium_config_extra_vars: {}\n\n# For adding and mounting extra volumes to the cilium operator\n# cilium_operator_extra_volumes: []\n# cilium_operator_extra_volume_mounts: []\n\n# Extra arguments for the Cilium Operator\n# cilium_operator_custom_args: []\n\n# Name of the cluster. Only relevant when building a mesh of clusters.\n# cilium_cluster_name: default\n\n# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.\n# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.\n# Available for Cilium v1.10 and up.\n# cilium_cni_exclusive: true\n\n# Configure the log file for CNI logging with retention policy of 7 days.\n# Disable CNI file logging by setting this field to empty explicitly.\n# Available for Cilium v1.12 and up.\n# cilium_cni_log_file: \"/var/run/cilium/cilium-cni.log\"\n\n# -- Configure cgroup related configuration\n# -- Enable auto mount of cgroup2 filesystem.\n# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at\n# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.\n# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted\n# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the\n# volume will be mounted inside the cilium agent pod at the same path.\n# Available for Cilium v1.11 and up\n# cilium_cgroup_auto_mount: true\n# -- Configure cgroup root where cgroup2 filesystem is mounted on the host\n# cilium_cgroup_host_root: \"/run/cilium/cgroupv2\"\n\n# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic\n# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.\n# cilium_bpf_map_dynamic_size_ratio: \"0.0\"\n\n# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.\n# Available for Cilium v1.10 and up\n# cilium_enable_ipv4_masquerade: true\n# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.\n# Available for Cilium v1.10 and up\n# cilium_enable_ipv6_masquerade: true\n\n# -- Enable native IP masquerade support in eBPF\n# cilium_enable_bpf_masquerade: false\n\n# -- Enable BGP Control Plane\n# cilium_enable_bgp_control_plane: false\n\n# -- Configure Loadbalancer IP Pools\n# cilium_loadbalancer_ip_pools:\n# - name: \"blue-pool\"\n# cidrs:\n# - \"10.0.10.0/24\"\n# ranges:\n# - start: \"20.0.20.100\"\n# stop: \"20.0.20.200\"\n# - start: \"1.2.3.4\"\n\n# -- Configure BGP Instances (New bgpv2 API v1.16+)\n# cilium_bgp_cluster_configs:\n# - name: \"cilium-bgp\"\n# spec:\n# bgpInstances:\n# - name: \"instance-64512\"\n# localASN: 64512\n# peers:\n# - name: \"peer-64512-tor1\"\n# peerASN: 64512\n# peerAddress: '10.47.1.1'\n# peerConfigRef:\n# name: \"cilium-peer\"\n# nodeSelector:\n# matchExpressions:\n# - {key: somekey, operator: NotIn, values: ['never-used-value']}\n\n# -- Configure BGP Peers (New bgpv2 API v1.16+)\n# cilium_bgp_peer_configs:\n# - name: cilium-peer\n# spec:\n# # authSecretRef: bgp-auth-secret\n# gracefulRestart:\n# enabled: true\n# restartTimeSeconds: 15\n# families:\n# - afi: ipv4\n# safi: unicast\n# advertisements:\n# matchLabels:\n# advertise: \"bgp\"\n# - afi: ipv6\n# safi: unicast\n# advertisements:\n# matchLabels:\n# advertise: \"bgp\"\n\n# -- Configure BGP Advertisements (New bgpv2 API v1.16+)\n# cilium_bgp_advertisements:\n# - name: bgp-advertisements\n# labels:\n# advertise: bgp\n# spec:\n# advertisements:\n# # - advertisementType: \"PodCIDR\"\n# # attributes:\n# # communities:\n# # standard: [ \"64512:99\" ]\n# - advertisementType: \"Service\"\n# service:\n# addresses:\n# - ClusterIP\n# - ExternalIP\n# - LoadBalancerIP\n# selector:\n# matchExpressions:\n# - {key: somekey, operator: NotIn, values: ['never-used-value']}\n\n# -- Configure BGP Node Config Overrides (New bgpv2 API v1.16+)\n# cilium_bgp_node_config_overrides:\n# - name: bgp-node-config-override\n# spec:\n# bgpInstances:\n# - name: \"instance-65000\"\n# routerID: \"192.168.10.1\"\n# localPort: 1790\n# peers:\n# - name: \"peer-65000-tor1\"\n# localAddress: fd00:10:0:2::2\n# - name: \"peer-65000-tor2\"\n# localAddress: fd00:11:0:2::2\n\n# -- Configure BGP Peers (Legacy v1.16+)\n# cilium_bgp_peering_policies:\n# - name: \"01-bgp-peering-policy\"\n# spec:\n# virtualRouters:\n# - localASN: 64512\n# exportPodCIDR: false\n# neighbors:\n# - peerAddress: '10.47.1.1/24'\n# peerASN: 64512\n# serviceSelector:\n# matchExpressions:\n# - {key: somekey, operator: NotIn, values: ['never-used-value']}\n\n# -- Configure whether direct routing mode should route traffic via\n# host stack (true) or directly and more efficiently out of BPF (false) if\n# the kernel supports it. The latter has the implication that it will also\n# bypass netfilter in the host namespace.\n# cilium_enable_host_legacy_routing: true\n\n# -- Enable use of the remote node identity.\n# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity\n# cilium_enable_remote_node_identity: true\n\n# -- Enable the use of well-known identities.\n# cilium_enable_well_known_identities: false\n\n# -- Whether to enable CNP status updates.\n# cilium_disable_cnp_status_updates: true\n\n# A list of extra rules variables to add to clusterrole for cilium operator, formatted like:\n# cilium_clusterrole_rules_operator_extra_vars:\n# - apiGroups:\n# - '\"\"'\n# resources:\n# - pods\n# verbs:\n# - delete\n# - apiGroups:\n# - '\"\"'\n# resources:\n# - nodes\n# verbs:\n# - list\n# - watch\n# resourceNames:\n# - toto\n# cilium_clusterrole_rules_operator_extra_vars: []\n\n# Cilium extra values, use any values from cilium Helm Chart\n# ref: https://docs.cilium.io/en/stable/helm-reference/\n# cilium_extra_values: {}\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-custom-cni.yml", "content": "---\n# custom_cni network plugin configuration\n# There are two deployment options to choose from, select one\n\n## OPTION 1 - Static manifest files\n## With this option, referred manifest file will be deployed\n## as if the `kubectl apply -f` method was used with it.\n#\n## List of Kubernetes resource manifest files\n## See tests/files/custom_cni/README.md for example\n# custom_cni_manifests: []\n\n## OPTION 1 EXAMPLE - Cilium static manifests in Kubespray tree\n# custom_cni_manifests:\n# - \"{{ playbook_dir }}/../tests/files/custom_cni/cilium.yaml\"\n\n## OPTION 2 - Helm chart application\n## This allows the CNI backend to be deployed to Kubespray cluster\n## as common Helm application.\n#\n## Helm release name - how the local instance of deployed chart will be named\n# custom_cni_chart_release_name: \"\"\n#\n## Kubernetes namespace to deploy into\n# custom_cni_chart_namespace: \"kube-system\"\n#\n## Helm repository name - how the local record of Helm repository will be named\n# custom_cni_chart_repository_name: \"\"\n#\n## Helm repository URL\n# custom_cni_chart_repository_url: \"\"\n#\n## Helm chart reference - path to the chart in the repository\n# custom_cni_chart_ref: \"\"\n#\n## Helm chart version\n# custom_cni_chart_version: \"\"\n#\n## Custom Helm values to be used for deployment\n# custom_cni_chart_values: {}\n\n## OPTION 2 EXAMPLE - Cilium deployed from official public Helm chart\n# custom_cni_chart_namespace: kube-system\n# custom_cni_chart_release_name: cilium\n# custom_cni_chart_repository_name: cilium\n# custom_cni_chart_repository_url: https://helm.cilium.io\n# custom_cni_chart_ref: cilium/cilium\n# custom_cni_chart_version: (e.g.: 1.14.3)\n# custom_cni_chart_values:\n# cluster:\n# name: \"cilium-demo\"\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-flannel.yml", "content": "# see roles/network_plugin/flannel/defaults/main.yml\n\n## interface that should be used for flannel operations\n## This is actually an inventory cluster-level item\n# flannel_interface:\n\n## Select interface that should be used for flannel operations by regexp on Name or IP\n## This is actually an inventory cluster-level item\n## example: select interface with ip from net 10.0.0.0/23\n## single quote and escape backslashes\n# flannel_interface_regexp: '10\\\\.0\\\\.[0-2]\\\\.\\\\d{1,3}'\n\n# You can choose what type of flannel backend to use: 'vxlan', 'host-gw' or 'wireguard'\n# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md\n# flannel_backend_type: \"vxlan\"\n# flannel_vxlan_vni: 1\n# flannel_vxlan_port: 8472\n# flannel_vxlan_direct_routing: false\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-kube-ovn.yml", "content": "---\n\n# geneve or vlan\nkube_ovn_network_type: geneve\n\n# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module\nkube_ovn_tunnel_type: geneve\n\n## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use.\n# kube_ovn_iface: eth1\n## The MTU used by pod iface in overlay networks (default iface MTU - 100)\n# kube_ovn_mtu: 1333\n\n## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port.\nkube_ovn_hw_offload: false\n# traffic mirror\nkube_ovn_traffic_mirror: false\n\n# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n# kube_ovn_default_interface_name: eth0\n\nkube_ovn_external_address: 8.8.8.8\nkube_ovn_external_address_ipv6: 2400:3200::1\nkube_ovn_external_dns: alauda.cn\n\n# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0\nkube_ovn_default_gateway_check: true\nkube_ovn_default_logical_gateway: false\n# kube_ovn_default_exclude_ips: 10.16.0.1\nkube_ovn_node_switch_cidr: 100.64.0.0/16\nkube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64\n\n## vlan config, set default interface name and vlan id\n# kube_ovn_default_interface_name: eth0\nkube_ovn_default_vlan_id: 100\nkube_ovn_vlan_name: product\n\n## pod nic type, support: veth-pair or internal-port\nkube_ovn_pod_nic_type: veth_pair\n\n## Enable load balancer\nkube_ovn_enable_lb: true\n\n## Enable network policy support\nkube_ovn_enable_np: true\n\n## Enable external vpc support\nkube_ovn_enable_external_vpc: true\n\n## Enable checksum\nkube_ovn_encap_checksum: true\n\n## enable ssl\nkube_ovn_enable_ssl: false\n\n## dpdk\nkube_ovn_dpdk_enabled: false\n\n## enable interconnection to an existing IC database server.\nkube_ovn_ic_enable: false\nkube_ovn_ic_autoroute: true\nkube_ovn_ic_dbhost: \"127.0.0.1\"\nkube_ovn_ic_zone: \"kubernetes\"\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml", "content": "# See roles/network_plugin/kube-router/defaults/main.yml\n\n# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP\n# kube_router_run_router: true\n\n# Enables Network Policy -- sets up iptables to provide ingress firewall for pods\n# kube_router_run_firewall: true\n\n# Enables Service Proxy -- sets up IPVS for Kubernetes Services\n# see docs/kube-router.md \"Caveats\" section\n# kube_router_run_service_proxy: false\n\n# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.\n# kube_router_advertise_cluster_ip: false\n\n# Add External IP of service to the RIB so that it gets advertised to the BGP peers.\n# kube_router_advertise_external_ip: false\n\n# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.\n# kube_router_advertise_loadbalancer_ip: false\n\n# Enables BGP graceful restarts\n# kube_router_bgp_graceful_restart: true\n\n# Adjust manifest of kube-router daemonset template with DSR needed changes\n# kube_router_enable_dsr: false\n\n# Array of arbitrary extra arguments to kube-router, see\n# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md\n# kube_router_extra_args: []\n\n# ASN number of the cluster, used when communicating with external BGP routers\n# kube_router_cluster_asn: ~\n\n# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.\n# kube_router_peer_router_asns: ~\n\n# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.\n# kube_router_peer_router_ips: ~\n\n# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.\n# kube_router_peer_router_ports: ~\n\n# Setups node CNI to allow hairpin mode, requires node reboots, see\n# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode\n# kube_router_support_hairpin_mode: false\n\n# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc.\n# kube_router_dns_policy: ClusterFirstWithHostNet\n\n# Array of annotations for master\n# kube_router_annotations_master: []\n\n# Array of annotations for every node\n# kube_router_annotations_node: []\n\n# Array of common annotations for every node\n# kube_router_annotations_all: []\n\n# Enables scraping kube-router metrics with Prometheus\n# kube_router_enable_metrics: false\n\n# Path to serve Prometheus metrics on\n# kube_router_metrics_path: /metrics\n\n# Prometheus metrics port to use\n# kube_router_metrics_port: 9255\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/k8s-net-macvlan.yml", "content": "---\n# private interface, on a l2-network\nmacvlan_interface: \"eth1\"\n\n# Enable nat in default gateway network interface\nenable_nat_default_gateway: true\n" }, { "path": "inventory/sample/group_vars/k8s_cluster/kube_control_plane.yml", "content": "# Reservation for control plane kubernetes components\n# kube_memory_reserved: 512Mi\n# kube_cpu_reserved: 200m\n# kube_ephemeral_storage_reserved: 2Gi\n# kube_pid_reserved: \"1000\"\n\n# Reservation for control plane host system\n# system_memory_reserved: 256Mi\n# system_cpu_reserved: 250m\n# system_ephemeral_storage_reserved: 2Gi\n# system_pid_reserved: \"1000\"\n" }, { "path": "inventory/sample/inventory.ini", "content": "# This inventory describe a HA typology with stacked etcd (== same nodes as control plane)\n# and 3 worker nodes\n# See https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html\n# for tips on building your # inventory\n\n# Configure 'ip' variable to bind kubernetes services on a different ip than the default iface\n# We should set etcd_member_name for etcd cluster. The node that are not etcd members do not need to set the value,\n# or can set the empty string value.\n[kube_control_plane]\n# node1 ansible_host=95.54.0.12 # ip=10.3.0.1 etcd_member_name=etcd1\n# node2 ansible_host=95.54.0.13 # ip=10.3.0.2 etcd_member_name=etcd2\n# node3 ansible_host=95.54.0.14 # ip=10.3.0.3 etcd_member_name=etcd3\n\n[etcd:children]\nkube_control_plane\n\n[kube_node]\n# node4 ansible_host=95.54.0.15 # ip=10.3.0.4\n# node5 ansible_host=95.54.0.16 # ip=10.3.0.5\n# node6 ansible_host=95.54.0.17 # ip=10.3.0.6\n" }, { "path": "logo/LICENSE", "content": "# The Kubespray logo files are licensed under a choice of either Apache-2.0 or CC-BY-4.0 (Creative Commons Attribution 4.0 International).\n" }, { "path": "logo/usage_guidelines.md", "content": "# Kubernetes Branding Guidelines\n\nThese guidelines provide you with guidance for using the Kubespray logo.\nAll artwork is made available under the Linux Foundation trademark usage\n[guidelines](https://www.linuxfoundation.org/trademark-usage/). This text from\nthose guidelines, and the correct and incorrect usage examples, are particularly\nhelpful:\n>Certain marks of The Linux Foundation have been created to enable you to\n>communicate compatibility or interoperability of software or products. In\n>addition to the requirement that any use of a mark to make an assertion of\n>compatibility must, of course, be accurate, the use of these marks must\n>avoid confusion regarding The Linux Foundation’s association with the\n>product. The use of the mark cannot imply that The Linux Foundation or\n>its projects are sponsoring or endorsing the product.\n\nAdditionally, permission is granted to modify the Kubespray mark for non-commercial uses such as t-shirts and stickers.\n" }, { "path": "meta/runtime.yml", "content": "---\nrequires_ansible: \">=2.18.0,<2.19.0\"\n" }, { "path": "pipeline.Dockerfile", "content": "# Use immutable image tags rather than mutable tags (like ubuntu:24.04)\nFROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b\n# Some tools like yamllint need this\n# Pip needs this as well at the moment to install ansible\n# (and potentially other packages)\n# See: https://github.com/pypa/pip/issues/10219\nENV VAGRANT_VERSION=2.4.1 \\\n VAGRANT_DEFAULT_PROVIDER=libvirt \\\n VAGRANT_ANSIBLE_TAGS=facts \\\n LANG=C.UTF-8 \\\n DEBIAN_FRONTEND=noninteractive \\\n PYTHONDONTWRITEBYTECODE=1\n\nRUN apt update -q \\\n && apt install -yq \\\n libssl-dev \\\n python3-dev \\\n python3-pip \\\n sshpass \\\n apt-transport-https \\\n jq \\\n moreutils \\\n libvirt-dev \\\n openssh-client \\\n rsync \\\n git \\\n ca-certificates \\\n curl \\\n gnupg2 \\\n unzip \\\n libvirt-clients \\\n qemu-utils \\\n qemu-kvm \\\n dnsmasq \\\n && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \\\n && echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \\\n $(. /etc/os-release && echo \"${UBUNTU_CODENAME:-$VERSION_CODENAME}\") stable\" | tee /etc/apt/sources.list.d/docker.list \\\n && apt update -q \\\n && apt install --no-install-recommends -yq docker-ce \\\n && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*\n\nWORKDIR /kubespray\nADD ./requirements.txt /kubespray/requirements.txt\nADD ./tests/requirements.txt /kubespray/tests/requirements.txt\n\nRUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \\\n && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \\\n && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \\\n && curl -L https://dl.k8s.io/release/v1.35.1/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \\\n && echo $(curl -L https://dl.k8s.io/release/v1.35.1/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \\\n && chmod a+x /usr/local/bin/kubectl \\\n # Install Vagrant\n && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \\\n && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \\\n && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \\\n && vagrant plugin install vagrant-libvirt \\\n # Install Kubernetes collections\n && pip install --break-system-packages --no-compile --no-cache-dir kubernetes \\\n && ansible-galaxy collection install kubernetes.core\n" }, { "path": "playbooks/ansible_version.yml", "content": "---\n- name: Check Ansible version\n hosts: all\n gather_facts: false\n become: false\n run_once: true\n vars:\n minimal_ansible_version: 2.18.0\n maximal_ansible_version: 2.19.0\n tags: always\n tasks:\n - name: \"Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}\"\n assert:\n msg: \"Ansible must be between {{ minimal_ansible_version }} and {{ maximal_ansible_version }} exclusive - you have {{ ansible_version.string }}\"\n that:\n - ansible_version.string is version(minimal_ansible_version, \">=\")\n - ansible_version.string is version(maximal_ansible_version, \"<\")\n tags:\n - check\n\n - name: \"Check that python netaddr is installed\"\n assert:\n msg: \"Python netaddr is not present\"\n that: \"'127.0.0.1' | ansible.utils.ipaddr\"\n tags:\n - check\n\n - name: \"Check that jinja is not too old (install via pip)\"\n assert:\n msg: \"Your Jinja version is too old, install via pip\"\n that: \"{% set test %}It works{% endset %}{{ test == 'It works' }}\"\n tags:\n - check\n" }, { "path": "playbooks/boilerplate.yml", "content": "---\n- name: Check ansible version\n import_playbook: ansible_version.yml\n\n# These are inventory compatibility tasks with two purposes:\n# - to ensure we keep compatibility with old style group names\n# - to reduce inventory boilerplate (defining parent groups / empty groups)\n\n- name: Inventory setup and validation\n hosts: all\n gather_facts: false\n tags: always\n roles:\n - dynamic_groups\n - validate_inventory\n\n- name: Install bastion ssh config\n hosts: bastion[0]\n gather_facts: false\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: bastion-ssh-config, tags: [\"localhost\", \"bastion\"] }\n" }, { "path": "playbooks/cluster.yml", "content": "---\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Gather facts\n import_playbook: internal_facts.yml\n\n- name: Prepare for etcd install\n hosts: k8s_cluster:etcd\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/preinstall, tags: preinstall }\n - { role: \"container-engine\", tags: \"container-engine\", when: deploy_container_engine }\n - { role: download, tags: download, when: \"not skip_downloads\" }\n\n- name: Install etcd\n vars:\n etcd_cluster_setup: true\n etcd_events_cluster_setup: \"{{ etcd_events_cluster_enabled }}\"\n import_playbook: install_etcd.yml\n\n- name: Install Kubernetes nodes\n hosts: k8s_cluster\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/node, tags: node }\n\n- name: Install the control plane\n hosts: kube_control_plane\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/control-plane, tags: control-plane }\n - { role: kubernetes/client, tags: client }\n - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }\n\n- name: Invoke kubeadm and install a CNI\n hosts: k8s_cluster\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/kubeadm, tags: kubeadm}\n - { role: kubernetes/node-label, tags: node-label }\n - { role: kubernetes/node-taint, tags: node-taint }\n - { role: kubernetes-apps/common_crds }\n - { role: network_plugin, tags: network }\n\n- name: Install Calico Route Reflector\n hosts: calico_rr\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }\n\n- name: Patch Kubernetes for Windows\n hosts: kube_control_plane[0]\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: win_nodes/kubernetes_patch, tags: [\"control-plane\", \"win_nodes\"] }\n\n- name: Install Kubernetes apps\n hosts: kube_control_plane\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }\n - { role: kubernetes-apps/policy_controller, tags: policy-controller }\n - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }\n - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }\n - { role: kubernetes-apps, tags: apps }\n\n- name: Apply resolv.conf changes now that cluster DNS is up\n hosts: k8s_cluster\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/preinstall, when: \"dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'\", tags: resolvconf, dns_late: true }\n" }, { "path": "playbooks/facts.yml", "content": "---\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Gather facts\n import_playbook: internal_facts.yml\n" }, { "path": "playbooks/install_etcd.yml", "content": "---\n- name: Add worker nodes to the etcd play if needed\n hosts: kube_node\n roles:\n - { role: kubespray_defaults }\n tasks:\n - name: Check if nodes needs etcd client certs (depends on network_plugin)\n group_by:\n key: \"_kubespray_needs_etcd\"\n when:\n - kube_network_plugin in [\"flannel\", \"canal\", \"cilium\"] or\n (cilium_deploy_additionally | default(false)) or\n (kube_network_plugin == \"calico\" and calico_datastore == \"etcd\")\n - etcd_deployment_type != \"kubeadm\"\n tags: etcd\n\n- name: Install etcd\n hosts: etcd:kube_control_plane:_kubespray_needs_etcd\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - role: etcd\n tags: etcd\n when: etcd_deployment_type != \"kubeadm\"\n" }, { "path": "playbooks/internal_facts.yml", "content": "---\n- name: Bootstrap hosts for Ansible\n hosts: k8s_cluster:etcd:calico_rr\n strategy: linear\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n gather_facts: false\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: bootstrap_os, tags: bootstrap_os}\n\n- name: Gather facts\n hosts: k8s_cluster:etcd:calico_rr\n gather_facts: false\n tags: always\n tasks:\n - name: Gather and compute network facts\n import_role:\n name: network_facts\n tags:\n - always\n - name: Gather minimal facts\n setup:\n gather_subset: '!all'\n\n # filter match the following variables:\n # ansible_default_ipv4\n # ansible_default_ipv6\n # ansible_all_ipv4_addresses\n # ansible_all_ipv6_addresses\n - name: Gather necessary facts (network)\n setup:\n gather_subset: '!all,!min,network'\n filter: \"ansible_*_ipv[46]*\"\n\n # filter match the following variables:\n # ansible_memtotal_mb\n # ansible_swaptotal_mb\n - name: Gather necessary facts (hardware)\n setup:\n gather_subset: '!all,!min,hardware'\n filter: \"ansible_*total_mb\"\n" }, { "path": "playbooks/recover_control_plane.yml", "content": "---\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Recover etcd\n hosts: etcd[0]\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults}\n - role: recover_control_plane/etcd\n when: etcd_deployment_type != \"kubeadm\"\n\n- name: Recover control plane\n hosts: kube_control_plane[0]\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults}\n - { role: recover_control_plane/control-plane }\n\n- name: Apply whole cluster install\n import_playbook: cluster.yml\n\n- name: Perform post recover tasks\n hosts: kube_control_plane\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults}\n - { role: recover_control_plane/post-recover }\n" }, { "path": "playbooks/remove_node.yml", "content": "---\n- name: Validate nodes for removal\n hosts: localhost\n gather_facts: false\n become: false\n tasks:\n - name: Assert that nodes are specified for removal\n assert:\n that:\n - node is defined\n - node | length > 0\n msg: \"No nodes specified for removal. The `node` variable must be set explicitly.\"\n\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Confirm node removal\n hosts: \"{{ node | default('this_is_unreachable') }}\"\n gather_facts: false\n tasks:\n - name: Confirm Execution\n pause:\n prompt: \"Are you sure you want to delete nodes state? Type 'yes' to delete nodes.\"\n register: pause_result\n run_once: true\n when:\n - not (skip_confirmation | default(false) | bool)\n\n - name: Fail if user does not confirm deletion\n fail:\n msg: \"Delete nodes confirmation failed\"\n when: pause_result.user_input | default('yes') != 'yes'\n\n- name: Gather facts\n import_playbook: internal_facts.yml\n when: reset_nodes | default(True) | bool\n\n- name: Reset node\n hosts: \"{{ node | default('this_is_unreachable') }}\"\n gather_facts: false\n environment: \"{{ proxy_disable_env }}\"\n pre_tasks:\n - name: Gather information about installed services\n service_facts:\n when: reset_nodes | default(True) | bool\n roles:\n - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }\n - { role: remove_node/pre_remove, tags: pre-remove }\n - role: remove-node/remove-etcd-node\n when: \"'etcd' in group_names\"\n - { role: reset, tags: reset, when: reset_nodes | default(True) | bool }\n\n# Currently cannot remove first control plane node or first etcd node\n- name: Post node removal\n hosts: \"{{ node | default('this_is_unreachable') }}\"\n gather_facts: false\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }\n - { role: remove-node/post-remove, tags: post-remove }\n" }, { "path": "playbooks/reset.yml", "content": "---\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Gather facts\n import_playbook: internal_facts.yml\n\n- name: Reset cluster\n hosts: etcd:k8s_cluster:calico_rr\n gather_facts: false\n pre_tasks:\n - name: Reset Confirmation\n pause:\n prompt: \"Are you sure you want to reset cluster state? Type 'yes' to reset your cluster.\"\n register: reset_confirmation_prompt\n run_once: true\n when:\n - not (skip_confirmation | default(false) | bool)\n - reset_confirmation is not defined\n\n - name: Check confirmation\n fail:\n msg: \"Reset confirmation failed\"\n when:\n - not reset_confirmation | default(false) | bool\n - not reset_confirmation_prompt.user_input | default(\"\") == \"yes\"\n\n - name: Gather information about installed services\n service_facts:\n\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults}\n - { role: kubernetes/preinstall, when: \"dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'\", tags: resolvconf, dns_early: true }\n - { role: reset, tags: reset }\n" }, { "path": "playbooks/scale.yml", "content": "---\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Gather facts\n import_playbook: internal_facts.yml\n\n- name: Install etcd\n vars:\n etcd_cluster_setup: false\n etcd_events_cluster_setup: false\n import_playbook: install_etcd.yml\n\n- name: Download images to ansible host cache via first kube_control_plane node\n hosts: kube_control_plane[0]\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults, when: \"not skip_downloads and download_run_once and not download_localhost\" }\n - { role: kubernetes/preinstall, tags: preinstall, when: \"not skip_downloads and download_run_once and not download_localhost\" }\n - { role: download, tags: download, when: \"not skip_downloads and download_run_once and not download_localhost\" }\n\n- name: Target only workers to get kubelet installed and checking in on any new nodes(engine)\n hosts: kube_node\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/preinstall, tags: preinstall }\n - { role: container-engine, tags: \"container-engine\", when: deploy_container_engine }\n - { role: download, tags: download, when: \"not skip_downloads\" }\n - role: etcd\n tags: etcd\n vars:\n etcd_cluster_setup: false\n when:\n - etcd_deployment_type != \"kubeadm\"\n - kube_network_plugin in [\"calico\", \"flannel\", \"canal\", \"cilium\"] or cilium_deploy_additionally | default(false) | bool\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n\n- name: Target only workers to get kubelet installed and checking in on any new nodes(node)\n hosts: kube_node\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/node, tags: node }\n\n- name: Upload control plane certs and retrieve encryption key\n hosts: kube_control_plane | first\n environment: \"{{ proxy_disable_env }}\"\n gather_facts: false\n tags: kubeadm\n roles:\n - { role: kubespray_defaults }\n tasks:\n - name: Upload control plane certificates\n command: >-\n {{ bin_dir }}/kubeadm init phase\n --config {{ kube_config_dir }}/kubeadm-config.yaml\n upload-certs\n --upload-certs\n environment: \"{{ proxy_disable_env }}\"\n register: kubeadm_upload_cert\n changed_when: false\n - name: Set fact 'kubeadm_certificate_key' for later use\n set_fact:\n kubeadm_certificate_key: \"{{ kubeadm_upload_cert.stdout_lines[-1] | trim }}\"\n when: kubeadm_certificate_key is not defined\n\n- name: Target only workers to get kubelet installed and checking in on any new nodes(network)\n hosts: kube_node\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/kubeadm, tags: kubeadm }\n - { role: kubernetes/node-label, tags: node-label }\n - { role: kubernetes/node-taint, tags: node-taint }\n - { role: network_plugin, tags: network }\n\n- name: Apply resolv.conf changes now that cluster DNS is up\n hosts: k8s_cluster\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/preinstall, when: \"dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'\", tags: resolvconf, dns_late: true }\n" }, { "path": "playbooks/upgrade_cluster.yml", "content": "---\n- name: Common tasks for every playbooks\n import_playbook: boilerplate.yml\n\n- name: Gather facts\n import_playbook: internal_facts.yml\n\n- name: Download images to ansible host cache via first kube_control_plane node\n hosts: kube_control_plane[0]\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults, when: \"not skip_downloads and download_run_once and not download_localhost\"}\n - { role: kubernetes/preinstall, tags: preinstall, when: \"not skip_downloads and download_run_once and not download_localhost\" }\n - { role: download, tags: download, when: \"not skip_downloads and download_run_once and not download_localhost\" }\n\n- name: Prepare nodes for upgrade\n hosts: k8s_cluster:etcd:calico_rr\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/preinstall, tags: preinstall }\n - { role: download, tags: download, when: \"not skip_downloads\" }\n\n- name: Upgrade container engine on non-cluster nodes\n hosts: etcd:calico_rr:!k8s_cluster\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n serial: \"{{ serial | default('20%') }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: container-engine, tags: \"container-engine\", when: deploy_container_engine }\n\n- name: Install etcd\n vars:\n etcd_cluster_setup: true\n etcd_events_cluster_setup: \"{{ etcd_events_cluster_enabled }}\"\n import_playbook: install_etcd.yml\n\n- name: Handle upgrades to control plane components first to maintain backwards compat.\n gather_facts: false\n hosts: kube_control_plane\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n serial: 1\n roles:\n - { role: kubespray_defaults }\n - { role: upgrade/pre-upgrade, tags: pre-upgrade }\n - { role: upgrade/system-upgrade, tags: system-upgrade }\n - { role: download, tags: download, when: \"system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads\" }\n - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }\n - { role: container-engine, tags: \"container-engine\", when: deploy_container_engine }\n - { role: kubernetes/node, tags: node }\n - { role: kubernetes/control-plane, tags: control-plane, upgrade_cluster_setup: true }\n - { role: kubernetes/client, tags: client }\n - { role: kubernetes/node-label, tags: node-label }\n - { role: kubernetes/node-taint, tags: node-taint }\n - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }\n - { role: kubernetes-apps, tags: csi-driver }\n - { role: upgrade/post-upgrade, tags: post-upgrade }\n\n- name: Upgrade calico and external cloud provider on all control plane nodes, calico-rrs, and nodes\n hosts: kube_control_plane:calico_rr:kube_node\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n serial: \"{{ serial | default('20%') }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }\n - { role: network_plugin, tags: network }\n - { role: kubernetes-apps/policy_controller, tags: policy-controller }\n\n- name: Finally handle worker upgrades, based on given batch size\n hosts: kube_node:calico_rr:!kube_control_plane\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n serial: \"{{ serial | default('20%') }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: upgrade/pre-upgrade, tags: pre-upgrade }\n - { role: upgrade/system-upgrade, tags: system-upgrade }\n - { role: download, tags: download, when: \"system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads\" }\n - { role: container-engine, tags: \"container-engine\", when: deploy_container_engine }\n - { role: kubernetes/node, tags: node }\n - { role: kubernetes/kubeadm, tags: kubeadm }\n - { role: kubernetes/node-label, tags: node-label }\n - { role: kubernetes/node-taint, tags: node-taint }\n - { role: upgrade/post-upgrade, tags: post-upgrade }\n\n- name: Patch Kubernetes for Windows\n hosts: kube_control_plane[0]\n gather_facts: false\n any_errors_fatal: true\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: win_nodes/kubernetes_patch, tags: [\"control-plane\", \"win_nodes\"] }\n\n- name: Install Calico Route Reflector\n hosts: calico_rr\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: network_plugin/calico/rr, tags: network }\n\n- name: Install Kubernetes apps\n hosts: kube_control_plane\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }\n - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }\n - { role: kubernetes-apps, tags: apps }\n\n- name: Apply resolv.conf changes now that cluster DNS is up\n hosts: k8s_cluster\n gather_facts: false\n any_errors_fatal: \"{{ any_errors_fatal | default(true) }}\"\n environment: \"{{ proxy_disable_env }}\"\n roles:\n - { role: kubespray_defaults }\n - { role: kubernetes/preinstall, when: \"dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'\", tags: resolvconf, dns_late: true }\n" }, { "path": "plugins/modules/kube.py", "content": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\nDOCUMENTATION = \"\"\"\n---\nmodule: kube\nshort_description: Manage Kubernetes Cluster\ndescription:\n - Create, replace, remove, and stop resources within a Kubernetes Cluster\nversion_added: \"2.0\"\noptions:\n name:\n required: false\n default: null\n description:\n - The name associated with resource\n filename:\n required: false\n default: null\n description:\n - The path and filename of the resource(s) definition file(s).\n - To operate on several files this can accept a comma separated list of files or a list of files.\n aliases: [ 'files', 'file', 'filenames' ]\n kubectl:\n required: false\n default: null\n description:\n - The path to the kubectl bin\n namespace:\n required: false\n default: null\n description:\n - The namespace associated with the resource(s)\n resource:\n required: false\n default: null\n description:\n - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc)\n label:\n required: false\n default: null\n description:\n - The labels used to filter specific resources.\n server:\n required: false\n default: null\n description:\n - The url for the API server that commands are executed against.\n kubeconfig:\n required: false\n default: null\n description:\n - The path to the kubeconfig.\n force:\n required: false\n default: false\n description:\n - A flag to indicate to force delete, replace, or stop.\n wait:\n required: false\n default: false\n description:\n - A flag to indicate to wait for resources to be created before continuing to the next step\n all:\n required: false\n default: false\n description:\n - A flag to indicate delete all, stop all, or all namespaces when checking exists.\n log_level:\n required: false\n default: 0\n description:\n - Indicates the level of verbosity of logging by kubectl.\n state:\n required: false\n choices: ['present', 'absent', 'latest', 'reloaded', 'stopped']\n default: present\n description:\n - present handles checking existence or creating if definition file provided,\n absent handles deleting resource(s) based on other options,\n latest handles creating or updating based on existence,\n reloaded handles updating resource(s) definition using definition file,\n stopped handles stopping resource(s) based on other options.\n recursive:\n required: false\n default: false\n description:\n - Process the directory used in -f, --filename recursively.\n Useful when you want to manage related manifests organized\n within the same directory.\nrequirements:\n - kubectl\nauthor: \"Kenny Jones (@kenjones-cisco)\"\n\"\"\"\n\nEXAMPLES = \"\"\"\n- name: test nginx is present\n kube: name=nginx resource=rc state=present\n\n- name: test nginx is stopped\n kube: name=nginx resource=rc state=stopped\n\n- name: test nginx is absent\n kube: name=nginx resource=rc state=absent\n\n- name: test nginx is present\n kube: filename=/tmp/nginx.yml\n\n- name: test nginx and postgresql are present\n kube: files=/tmp/nginx.yml,/tmp/postgresql.yml\n\n- name: test nginx and postgresql are present\n kube:\n files:\n - /tmp/nginx.yml\n - /tmp/postgresql.yml\n\"\"\"\n\n\nclass KubeManager(object):\n\n def __init__(self, module):\n\n self.module = module\n\n self.kubectl = module.params.get('kubectl')\n if self.kubectl is None:\n self.kubectl = module.get_bin_path('kubectl', True)\n self.base_cmd = [self.kubectl]\n\n if module.params.get('server'):\n self.base_cmd.append('--server=' + module.params.get('server'))\n\n if module.params.get('kubeconfig'):\n self.base_cmd.append('--kubeconfig=' + module.params.get('kubeconfig'))\n\n if module.params.get('log_level'):\n self.base_cmd.append('--v=' + str(module.params.get('log_level')))\n\n if module.params.get('namespace'):\n self.base_cmd.append('--namespace=' + module.params.get('namespace'))\n\n\n self.all = module.params.get('all')\n self.force = module.params.get('force')\n self.wait = module.params.get('wait')\n self.name = module.params.get('name')\n self.filename = [f.strip() for f in module.params.get('filename') or []]\n self.resource = module.params.get('resource')\n self.label = module.params.get('label')\n self.recursive = module.params.get('recursive')\n\n def _execute(self, cmd):\n args = self.base_cmd + cmd\n try:\n rc, out, err = self.module.run_command(args)\n if rc != 0:\n self.module.fail_json(\n msg='error running kubectl (%s) command (rc=%d), out=\\'%s\\', err=\\'%s\\'' % (' '.join(args), rc, out, err))\n except Exception as exc:\n self.module.fail_json(\n msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc)))\n return out.splitlines()\n\n def _execute_nofail(self, cmd):\n args = self.base_cmd + cmd\n rc, out, err = self.module.run_command(args)\n if rc != 0:\n return None\n return out.splitlines()\n\n def create(self, check=True, force=True):\n if check and self.exists():\n return []\n\n cmd = ['apply']\n\n if force:\n cmd.append('--force')\n\n if self.wait:\n cmd.append('--wait')\n\n if self.recursive:\n cmd.append('--recursive={}'.format(self.recursive))\n\n if not self.filename:\n self.module.fail_json(msg='filename required to create')\n\n cmd.append('--filename=' + ','.join(self.filename))\n\n return self._execute(cmd)\n\n def replace(self, force=True):\n\n cmd = ['apply']\n\n if force:\n cmd.append('--force')\n\n if self.wait:\n cmd.append('--wait')\n\n if self.recursive:\n cmd.append('--recursive={}'.format(self.recursive))\n\n if not self.filename:\n self.module.fail_json(msg='filename required to reload')\n\n cmd.append('--filename=' + ','.join(self.filename))\n\n return self._execute(cmd)\n\n def delete(self):\n\n if not self.force and not self.exists():\n return []\n\n cmd = ['delete']\n\n if self.filename:\n cmd.append('--filename=' + ','.join(self.filename))\n if self.recursive:\n cmd.append('--recursive={}'.format(self.recursive))\n else:\n if not self.resource:\n self.module.fail_json(msg='resource required to delete without filename')\n\n cmd.append(self.resource)\n\n if self.name:\n cmd.append(self.name)\n\n if self.label:\n cmd.append('--selector=' + self.label)\n\n if self.all:\n cmd.append('--all')\n\n if self.force:\n cmd.append('--ignore-not-found')\n\n if self.recursive:\n cmd.append('--recursive={}'.format(self.recursive))\n\n return self._execute(cmd)\n\n def exists(self):\n cmd = ['get']\n\n if self.filename:\n cmd.append('--filename=' + ','.join(self.filename))\n if self.recursive:\n cmd.append('--recursive={}'.format(self.recursive))\n else:\n if not self.resource:\n self.module.fail_json(msg='resource required without filename')\n\n cmd.append(self.resource)\n\n if self.name:\n cmd.append(self.name)\n\n if self.label:\n cmd.append('--selector=' + self.label)\n\n if self.all:\n cmd.append('--all-namespaces')\n\n cmd.append('--no-headers')\n\n result = self._execute_nofail(cmd)\n if not result:\n return False\n return True\n\n # TODO: This is currently unused, perhaps convert to 'scale' with a replicas param?\n def stop(self):\n\n if not self.force and not self.exists():\n return []\n\n cmd = ['stop']\n\n if self.filename:\n cmd.append('--filename=' + ','.join(self.filename))\n if self.recursive:\n cmd.append('--recursive={}'.format(self.recursive))\n else:\n if not self.resource:\n self.module.fail_json(msg='resource required to stop without filename')\n\n cmd.append(self.resource)\n\n if self.name:\n cmd.append(self.name)\n\n if self.label:\n cmd.append('--selector=' + self.label)\n\n if self.all:\n cmd.append('--all')\n\n if self.force:\n cmd.append('--ignore-not-found')\n\n return self._execute(cmd)\n\n\ndef main():\n\n module = AnsibleModule(\n argument_spec=dict(\n name=dict(),\n filename=dict(type='list', aliases=['files', 'file', 'filenames']),\n namespace=dict(),\n resource=dict(),\n label=dict(),\n server=dict(),\n kubeconfig=dict(),\n kubectl=dict(),\n force=dict(default=False, type='bool'),\n wait=dict(default=False, type='bool'),\n all=dict(default=False, type='bool'),\n log_level=dict(default=0, type='int'),\n state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped', 'exists']),\n recursive=dict(default=False, type='bool'),\n ),\n mutually_exclusive=[['filename', 'list']]\n )\n\n changed = False\n\n manager = KubeManager(module)\n state = module.params.get('state')\n if state == 'present':\n result = manager.create(check=False)\n\n elif state == 'absent':\n result = manager.delete()\n\n elif state == 'reloaded':\n result = manager.replace()\n\n elif state == 'stopped':\n result = manager.stop()\n\n elif state == 'latest':\n result = manager.replace()\n\n elif state == 'exists':\n result = manager.exists()\n module.exit_json(changed=changed,\n msg='%s' % result)\n\n else:\n module.fail_json(msg='Unrecognized state %s.' % state)\n\n module.exit_json(changed=changed,\n msg='success: %s' % (' '.join(result))\n )\n\n\nfrom ansible.module_utils.basic import * # noqa\nif __name__ == '__main__':\n main()\n" }, { "path": "recover-control-plane.yml", "content": "---\n- name: Recover control plane\n ansible.builtin.import_playbook: playbooks/recover_control_plane.yml\n" }, { "path": "remove-node.yml", "content": "---\n- name: Remove node\n ansible.builtin.import_playbook: playbooks/remove_node.yml\n" }, { "path": "remove_node.yml", "content": "---\n- name: Remove node\n ansible.builtin.import_playbook: playbooks/remove_node.yml\n" }, { "path": "requirements.txt", "content": "ansible==11.13.0\n# Needed for community.crypto module\ncryptography==46.0.5\n# Needed for jinja2 json_query templating\njmespath==1.1.0\n# Needed for ansible.utils.ipaddr\nnetaddr==1.3.0\n" }, { "path": "reset.yml", "content": "---\n- name: Reset the cluster\n ansible.builtin.import_playbook: playbooks/reset.yml\n" }, { "path": "roles/adduser/defaults/main.yml", "content": "---\nkube_owner: kube\nkube_cert_group: kube-cert\netcd_data_dir: \"/var/lib/etcd\"\n\naddusers:\n etcd:\n name: etcd\n comment: \"Etcd user\"\n create_home: false\n system: true\n shell: /sbin/nologin\n kube:\n name: kube\n comment: \"Kubernetes user\"\n create_home: false\n system: true\n shell: /sbin/nologin\n group: \"{{ kube_cert_group }}\"\n\nadduser:\n name: \"{{ user.name }}\"\n group: \"{{ user.name | default(None) }}\"\n comment: \"{{ user.comment | default(None) }}\"\n shell: \"{{ user.shell | default(None) }}\"\n system: \"{{ user.system | default(None) }}\"\n create_home: \"{{ user.create_home | default(None) }}\"\n" }, { "path": "roles/adduser/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n gather_facts: false\n roles:\n - role: adduser\n vars:\n user:\n name: foo\n" }, { "path": "roles/adduser/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\ndependency:\n name: galaxy\nplatforms:\n - name: ubuntu22\n cloud_image: ubuntu-2204\n vm_cpu_cores: 1\n vm_memory: 512\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n playbooks:\n create: ../../../../tests/cloud_playbooks/create-kubevirt.yml\nverifier:\n name: testinfra\n" }, { "path": "roles/adduser/tasks/main.yml", "content": "---\n- name: User | Create User Group\n group:\n name: \"{{ user.group | default(user.name) }}\"\n system: \"{{ user.system | default(omit) }}\"\n\n- name: User | Create User\n user:\n comment: \"{{ user.comment | default(omit) }}\"\n create_home: \"{{ user.create_home | default(omit) }}\"\n group: \"{{ user.group | default(user.name) }}\"\n home: \"{{ user.home | default(omit) }}\"\n shell: \"{{ user.shell | default(omit) }}\"\n name: \"{{ user.name }}\"\n system: \"{{ user.system | default(omit) }}\"\n when: user.name != \"root\"\n" }, { "path": "roles/adduser/vars/coreos.yml", "content": "---\naddusers:\n - name: kube\n comment: \"Kubernetes user\"\n shell: /sbin/nologin\n system: true\n group: \"{{ kube_cert_group }}\"\n create_home: false\n" }, { "path": "roles/adduser/vars/debian.yml", "content": "---\naddusers:\n - name: etcd\n comment: \"Etcd user\"\n create_home: true\n home: \"{{ etcd_data_dir }}\"\n system: true\n shell: /sbin/nologin\n\n - name: kube\n comment: \"Kubernetes user\"\n create_home: false\n system: true\n shell: /sbin/nologin\n group: \"{{ kube_cert_group }}\"\n" }, { "path": "roles/adduser/vars/redhat.yml", "content": "---\naddusers:\n - name: etcd\n comment: \"Etcd user\"\n create_home: true\n home: \"{{ etcd_data_dir }}\"\n system: true\n shell: /sbin/nologin\n\n - name: kube\n comment: \"Kubernetes user\"\n create_home: false\n system: true\n shell: /sbin/nologin\n group: \"{{ kube_cert_group }}\"\n" }, { "path": "roles/bastion-ssh-config/defaults/main.yml", "content": "---\nssh_bastion_config_name: ssh-bastion.conf\n" }, { "path": "roles/bastion-ssh-config/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n gather_facts: false\n roles:\n - role: bastion-ssh-config\n tasks:\n - name: Copy config to remote host\n copy:\n src: \"{{ playbook_dir }}/{{ ssh_bastion_config_name }}\"\n dest: \"{{ ssh_bastion_config_name }}\"\n owner: \"{{ ansible_user }}\"\n group: \"{{ ansible_user }}\"\n mode: \"0644\"\n" }, { "path": "roles/bastion-ssh-config/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\ndependency:\n name: galaxy\nplatforms:\n - name: bastion-01\n cloud_image: ubuntu-2204\n vm_cpu_cores: 1\n vm_memory: 512\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n inventory:\n hosts:\n all:\n hosts:\n children:\n bastion:\n hosts:\n bastion-01:\n playbooks:\n create: ../../../../tests/cloud_playbooks/create-kubevirt.yml\nverifier:\n name: testinfra\n" }, { "path": "roles/bastion-ssh-config/tasks/main.yml", "content": "---\n- name: Set bastion host IP and port\n set_fact:\n bastion_ip: \"{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}\"\n bastion_port: \"{{ hostvars[groups['bastion'][0]]['ansible_port'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_port']) | d(22) }}\"\n delegate_to: localhost\n connection: local\n\n# As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly\n# To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user\n- name: Store the current ansible_user in the real_user fact\n set_fact:\n real_user: \"{{ ansible_user }}\"\n\n- name: Create ssh bastion conf\n become: false\n delegate_to: localhost\n connection: local\n template:\n src: \"{{ ssh_bastion_config_name }}.j2\"\n dest: \"{{ playbook_dir }}/{{ ssh_bastion_config_name }}\"\n mode: \"0640\"\n" }, { "path": "roles/bastion-ssh-config/templates/ssh-bastion.conf.j2", "content": "{% set vars={'hosts': ''} %}\n{% set user='' %}\n\n{% for h in groups['all'] %}\n{% if h not in groups['bastion'] %}\n{% if vars.update({'hosts': vars['hosts'] + ' ' + (hostvars[h].get('ansible_ssh_host') or hostvars[h]['ansible_host'])}) %}{% endif %}\n{% endif %}\n{% endfor %}\n\nHost {{ bastion_ip }}\n Hostname {{ bastion_ip }}\n StrictHostKeyChecking no\n ControlMaster auto\n ControlPath ~/.ssh/ansible-%r@%h:%p\n ControlPersist 5m\n\nHost {{ vars['hosts'] }}\n ProxyCommand ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -p {{ bastion_port }} {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}\n" }, { "path": "roles/bootstrap-os/tasks/main.yml", "content": "---\n- name: Warn for usage of deprecated role\n fail:\n msg: bootstrap-os is deprecated, switch to bootstrap_os\n ignore_errors: true # noqa ignore-errors\n run_once: true\n\n- name: Compat for direct role import\n import_role:\n name: bootstrap_os\n" }, { "path": "roles/bootstrap_os/defaults/main.yml", "content": "---\n## CentOS/RHEL/AlmaLinux specific variables\n# Use the fastestmirror yum plugin\ncentos_fastestmirror_enabled: false\n# Timeout (in seconds) for checking RHEL subscription status\nrh_subscription_check_timeout: 180\n\n## Flatcar Container Linux specific variables\n# Disable locksmithd or leave it in its current state\ncoreos_locksmithd_disable: false\n\n# Install epel repo on Centos/RHEL\nepel_enabled: false\n\n## openEuler specific variables\n# Enable metalink for openEuler repos (auto-selects fastest mirror by location)\nopeneuler_metalink_enabled: false\n\n## Oracle Linux specific variables\n# Install public repo on Oracle Linux\nuse_oracle_public_repo: true\n\n## Ubuntu specific variables\n# Disable unattended-upgrades for Linux kernel and all packages start with linux- on Ubuntu\nubuntu_kernel_unattended_upgrades_disabled: false\n# Stop unattended-upgrades if it is currently running on Ubuntu\nubuntu_stop_unattended_upgrades: false\n\nfedora_coreos_packages:\n - python\n - python3-libselinux\n - ethtool # required in kubeadm preflight phase for verifying the environment\n - ipset # required in kubeadm preflight phase for verifying the environment\n - conntrack-tools # required by kube-proxy\n - containernetworking-plugins # required by crio\n\n## General\n# Set the hostname to inventory_hostname\noverride_system_hostname: true\n\nis_fedora_coreos: false\n\nskip_http_proxy_on_os_packages: false\n" }, { "path": "roles/bootstrap_os/files/bootstrap.sh", "content": "#!/bin/bash\nset -e\n\nBINDIR=\"/opt/bin\"\nif [[ -e $BINDIR/.bootstrapped ]]; then\n exit 0\nfi\n\nARCH=$(uname -m)\ncase $ARCH in\n \"x86_64\")\n PYPY_ARCH=linux64\n PYPI_HASH=46818cb3d74b96b34787548343d266e2562b531ddbaf330383ba930ff1930ed5\n ;;\n \"aarch64\")\n PYPY_ARCH=aarch64\n PYPI_HASH=2e1ae193d98bc51439642a7618d521ea019f45b8fb226940f7e334c548d2b4b9\n ;;\n *)\n echo \"Unsupported Architecture: ${ARCH}\"\n exit 1\nesac\n\nPYTHON_VERSION=3.9\nPYPY_VERSION=7.3.9\nPYPY_FILENAME=\"pypy${PYTHON_VERSION}-v${PYPY_VERSION}-${PYPY_ARCH}\"\nPYPI_URL=\"https://downloads.python.org/pypy/${PYPY_FILENAME}.tar.bz2\"\n\nmkdir -p $BINDIR\n\ncd $BINDIR\n\nTAR_FILE=pyp.tar.bz2\nwget -O \"${TAR_FILE}\" \"${PYPI_URL}\"\necho \"${PYPI_HASH} ${TAR_FILE}\" | sha256sum -c -\ntar -xjf \"${TAR_FILE}\" && rm \"${TAR_FILE}\"\nmv -n \"${PYPY_FILENAME}\" pypy3\n\nln -s ./pypy3/bin/pypy3 python\n$BINDIR/python --version\n\n# install PyYAML\n./python -m ensurepip\n./python -m pip install pyyaml\n\ntouch $BINDIR/.bootstrapped\n" }, { "path": "roles/bootstrap_os/handlers/main.yml", "content": "---\n- name: RHEL auto-attach subscription\n command: /sbin/subscription-manager attach --auto\n become: true\n" }, { "path": "roles/bootstrap_os/meta/main.yml", "content": "---\ndependencies:\n - role: kubespray_defaults\n" }, { "path": "roles/bootstrap_os/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n gather_facts: false\n become: true\n roles:\n - role: bootstrap_os\n" }, { "path": "roles/bootstrap_os/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\ndependency:\n name: galaxy\nplatforms:\n - name: ubuntu22\n cloud_image: ubuntu-2204\n vm_cpu_cores: 1\n vm_memory: 512\n - name: ubuntu24\n cloud_image: ubuntu-2404\n vm_cpu_cores: 1\n vm_memory: 512\n - name: almalinux9\n cloud_image: almalinux-9\n vm_cpu_cores: 1\n vm_memory: 512\n - name: debian12\n cloud_image: debian-12\n vm_cpu_cores: 1\n vm_memory: 512\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n inventory:\n group_vars:\n all:\n user:\n name: foo\n comment: My test comment\n playbooks:\n create: ../../../../tests/cloud_playbooks/create-kubevirt.yml\nverifier:\n name: testinfra\n" }, { "path": "roles/bootstrap_os/molecule/default/tests/test_default.py", "content": "import os\n\nimport testinfra.utils.ansible_runner\n\ntestinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(\n os.environ['MOLECULE_INVENTORY_FILE']\n).get_hosts('all')\n\n\ndef test_python(host):\n assert host.exists('python3') or host.exists('python')\n" }, { "path": "roles/bootstrap_os/tasks/almalinux.yml", "content": "---\n- name: Import Centos boostrap for Alma Linux\n import_tasks: centos.yml\n" }, { "path": "roles/bootstrap_os/tasks/amzn.yml", "content": "---\n- name: Enable selinux-ng repo for Amazon Linux for container-selinux\n command: amazon-linux-extras enable selinux-ng\n\n- name: Enable EPEL repo for Amazon Linux\n yum_repository:\n name: epel\n file: epel\n description: Extra Packages for Enterprise Linux 7 - $basearch\n baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch\n gpgcheck: true\n gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7\n skip_if_unavailable: true\n enabled: true\n repo_gpgcheck: false\n when: epel_enabled\n" }, { "path": "roles/bootstrap_os/tasks/centos.yml", "content": "---\n- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version\n setup:\n gather_subset: '!all'\n filter: ansible_distribution_*version\n\n- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined\n community.general.ini_file:\n path: \"{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}\"\n section: main\n option: proxy\n value: \"{{ http_proxy | default(omit) }}\"\n state: \"{{ http_proxy | default(False) | ternary('present', 'absent') }}\"\n no_extra_spaces: true\n mode: \"0644\"\n become: true\n when: not skip_http_proxy_on_os_packages\n\n# For Oracle Linux install public repo\n\n- name: Install EPEL for Oracle Linux repo package\n package:\n name: \"oracle-epel-release-el{{ ansible_distribution_major_version }}\"\n state: present\n when:\n - use_oracle_public_repo | default(true)\n - '''ID=\"ol\"'' in os_release.stdout_lines'\n - (ansible_distribution_version | float) >= 7.6\n\n- name: Enable Oracle Linux repo\n community.general.ini_file:\n dest: \"/etc/yum.repos.d/oracle-linux-ol{{ ansible_distribution_major_version }}.repo\"\n section: \"ol{{ ansible_distribution_major_version }}_addons\"\n option: \"{{ item.option }}\"\n value: \"{{ item.value }}\"\n mode: \"0644\"\n with_items:\n - { option: \"name\", value: \"ol{{ ansible_distribution_major_version }}_addons\" }\n - { option: \"enabled\", value: \"1\" }\n - { option: \"baseurl\", value: \"http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/\" }\n when:\n - use_oracle_public_repo | default(true)\n - '''ID=\"ol\"'' in os_release.stdout_lines'\n - (ansible_distribution_version | float) >= 7.6\n\n- name: Enable Centos extra repo for Oracle Linux\n community.general.ini_file:\n dest: \"/etc/yum.repos.d/centos-extras.repo\"\n section: \"extras\"\n option: \"{{ item.option }}\"\n value: \"{{ item.value }}\"\n mode: \"0644\"\n with_items:\n - { option: \"name\", value: \"CentOS-{{ ansible_distribution_major_version }} - Extras\" }\n - { option: \"enabled\", value: \"1\" }\n - { option: \"gpgcheck\", value: \"0\" }\n - { option: \"baseurl\", value: \"http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/extras/$basearch/os/\" }\n when:\n - use_oracle_public_repo | default(true)\n - '''ID=\"ol\"'' in os_release.stdout_lines'\n - (ansible_distribution_version | float) >= 7.6\n - (ansible_distribution_version | float) < 9\n\n# CentOS ships with python installed\n\n- name: Check presence of fastestmirror.conf\n stat:\n path: /etc/yum/pluginconf.d/fastestmirror.conf\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: fastestmirror\n\n# the fastestmirror plugin can actually slow down Ansible deployments\n- name: Disable fastestmirror plugin if requested\n lineinfile:\n dest: /etc/yum/pluginconf.d/fastestmirror.conf\n regexp: \"^enabled=.*\"\n line: \"enabled=0\"\n state: present\n become: true\n when:\n - fastestmirror.stat.exists\n - not centos_fastestmirror_enabled\n" }, { "path": "roles/bootstrap_os/tasks/debian.yml", "content": "---\n# Some Debian based distros ship without Python installed\n\n- name: Check if bootstrap is needed\n raw: which python3\n register: need_bootstrap\n failed_when: false\n changed_when: false\n # This command should always run, even in check mode\n check_mode: false\n tags:\n - facts\n\n- name: Check http::proxy in apt configuration files\n raw: apt-config dump | grep -qsi 'Acquire::http::proxy'\n register: need_http_proxy\n failed_when: false\n changed_when: false\n # This command should always run, even in check mode\n check_mode: false\n\n- name: Add http_proxy to /etc/apt/apt.conf if http_proxy is defined\n raw: echo 'Acquire::http::proxy \"{{ http_proxy }}\";' >> /etc/apt/apt.conf\n become: true\n when:\n - http_proxy is defined\n - need_http_proxy.rc != 0\n - not skip_http_proxy_on_os_packages\n\n- name: Check https::proxy in apt configuration files\n raw: apt-config dump | grep -qsi 'Acquire::https::proxy'\n register: need_https_proxy\n failed_when: false\n changed_when: false\n # This command should always run, even in check mode\n check_mode: false\n\n- name: Add https_proxy to /etc/apt/apt.conf if https_proxy is defined\n raw: echo 'Acquire::https::proxy \"{{ https_proxy }}\";' >> /etc/apt/apt.conf\n become: true\n when:\n - https_proxy is defined\n - need_https_proxy.rc != 0\n - not skip_http_proxy_on_os_packages\n\n- name: Install python3\n raw:\n apt-get update && \\\n DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal\n become: true\n when:\n - need_bootstrap.rc != 0\n" }, { "path": "roles/bootstrap_os/tasks/fedora-coreos.yml", "content": "---\n\n- name: Check if bootstrap is needed\n raw: which python\n register: need_bootstrap\n failed_when: false\n changed_when: false\n tags:\n - facts\n\n- name: Remove podman network cni\n raw: \"podman network rm podman\"\n become: true\n ignore_errors: true # noqa ignore-errors\n when: need_bootstrap.rc != 0\n\n- name: Clean up possible pending packages on fedora coreos\n raw: \"export http_proxy={{ http_proxy | default('') }};rpm-ostree cleanup -p }}\"\n become: true\n when: need_bootstrap.rc != 0\n\n- name: Install required packages on fedora coreos\n raw: \"export http_proxy={{ http_proxy | default('') }};rpm-ostree install --allow-inactive {{ fedora_coreos_packages | join(' ') }}\"\n become: true\n when: need_bootstrap.rc != 0\n\n- name: Reboot immediately for updated ostree\n raw: \"nohup bash -c 'sleep 5s && shutdown -r now'\"\n become: true\n ignore_errors: true # noqa ignore-errors\n ignore_unreachable: true\n when: need_bootstrap.rc != 0\n\n- name: Wait for the reboot to complete\n wait_for_connection:\n timeout: 240\n connect_timeout: 20\n delay: 5\n sleep: 5\n when: need_bootstrap.rc != 0\n" }, { "path": "roles/bootstrap_os/tasks/fedora.yml", "content": "---\n# Some Fedora based distros ship without Python installed\n\n- name: Check if bootstrap is needed\n raw: which python\n register: need_bootstrap\n failed_when: false\n changed_when: false\n tags:\n - facts\n\n- name: Add proxy to dnf.conf if http_proxy is defined\n community.general.ini_file:\n path: \"/etc/dnf/dnf.conf\"\n section: main\n option: proxy\n value: \"{{ http_proxy | default(omit) }}\"\n state: \"{{ http_proxy | default(False) | ternary('present', 'absent') }}\"\n no_extra_spaces: true\n mode: \"0644\"\n become: true\n when: not skip_http_proxy_on_os_packages\n\n# libselinux-python3 is required on SELinux enabled hosts\n# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements\n- name: Install ansible requirements\n raw: \"dnf install --assumeyes python3 python3-dnf libselinux-python3\"\n become: true\n when:\n - need_bootstrap.rc != 0\n" }, { "path": "roles/bootstrap_os/tasks/flatcar.yml", "content": "---\n# Flatcar Container Linux ships without Python installed\n\n- name: Check if bootstrap is needed\n raw: stat /opt/bin/.bootstrapped\n register: need_bootstrap\n failed_when: false\n changed_when: false\n tags:\n - facts\n\n- name: Run bootstrap.sh\n script: bootstrap.sh\n become: true\n environment: \"{{ proxy_env }}\"\n when:\n - need_bootstrap.rc != 0\n\n# Workaround ansible https://github.com/ansible/ansible/pull/82821\n# We set the interpreter rather than ansible_python_interpreter to allow\n# - using virtual env with task level ansible_python_interpreter later\n# - let users specify an ansible_python_interpreter in group_vars\n\n- name: Make interpreter discovery works on Flatcar\n set_fact:\n ansible_interpreter_python_fallback: \"{{ (ansible_interpreter_python_fallback | default([])) + ['/opt/bin/python'] }}\"\n\n- name: Disable auto-upgrade\n systemd_service:\n name: locksmithd.service\n masked: true\n state: stopped\n when:\n - coreos_locksmithd_disable\n" }, { "path": "roles/bootstrap_os/tasks/main.yml", "content": "---\n- name: Fetch /etc/os-release\n raw: cat /etc/os-release\n register: os_release\n changed_when: false\n # This command should always run, even in check mode\n check_mode: false\n\n- name: Include distro specifics vars and tasks\n vars:\n os_release_dict: \"{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\\\"', '') |\n map('split', '=') | community.general.dict }}\"\n block:\n - name: Include vars\n include_vars: \"{{ item }}\"\n tags:\n - facts\n with_first_found:\n - files: &search_files\n - \"{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml\"\n - \"{{ os_release_dict['ID'] }}.yml\"\n paths:\n - vars/\n skip: true\n - name: Include tasks\n include_tasks: \"{{ included_tasks_file }}\"\n with_first_found:\n - files: *search_files\n skip: true\n loop_control:\n loop_var: included_tasks_file\n\n- name: Install system packages\n import_role:\n name: system_packages\n tags:\n - system-packages\n\n- name: Create remote_tmp for it is used by another module\n file:\n path: \"{{ ansible_remote_tmp | default('~/.ansible/tmp') }}\"\n state: directory\n mode: \"0700\"\n\n- name: Gather facts\n setup:\n gather_subset: '!all'\n filter: ansible_*\n\n- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)\n hostname:\n name: \"{{ inventory_hostname }}\"\n when: override_system_hostname\n\n- name: Ensure bash_completion.d folder exists\n file:\n name: /etc/bash_completion.d/\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n" }, { "path": "roles/bootstrap_os/tasks/openEuler.yml", "content": "---\n- name: Import CentOS bootstrap for openEuler\n ansible.builtin.import_tasks: centos.yml\n\n- name: Get existing openEuler repo sections\n ansible.builtin.shell:\n cmd: \"set -o pipefail && grep '^\\\\[' /etc/yum.repos.d/openEuler.repo | tr -d '[]'\"\n executable: /bin/bash\n register: _openeuler_repo_sections\n changed_when: false\n failed_when: false\n check_mode: false\n become: true\n when: openeuler_metalink_enabled\n\n- name: Enable metalink for openEuler repos\n community.general.ini_file:\n path: /etc/yum.repos.d/openEuler.repo\n section: \"{{ item.key }}\"\n option: metalink\n value: \"{{ item.value }}\"\n no_extra_spaces: true\n mode: \"0644\"\n loop: \"{{ _openeuler_metalink_repos | dict2items | selectattr('key', 'in', _openeuler_repo_sections.stdout_lines | default([])) }}\"\n become: true\n when: openeuler_metalink_enabled\n register: _openeuler_metalink_result\n vars:\n _openeuler_metalink_repos:\n OS: \"https://mirrors.openeuler.org/metalink?repo=$releasever/OS&arch=$basearch\"\n everything: \"https://mirrors.openeuler.org/metalink?repo=$releasever/everything&arch=$basearch\"\n EPOL: \"https://mirrors.openeuler.org/metalink?repo=$releasever/EPOL/main&arch=$basearch\"\n debuginfo: \"https://mirrors.openeuler.org/metalink?repo=$releasever/debuginfo&arch=$basearch\"\n source: \"https://mirrors.openeuler.org/metalink?repo=$releasever&arch=source\"\n update: \"https://mirrors.openeuler.org/metalink?repo=$releasever/update&arch=$basearch\"\n update-source: \"https://mirrors.openeuler.org/metalink?repo=$releasever/update&arch=source\"\n\n- name: Clean dnf cache to apply metalink mirror selection\n ansible.builtin.command: dnf clean all\n become: true\n when:\n - openeuler_metalink_enabled\n - _openeuler_metalink_result.changed\n" }, { "path": "roles/bootstrap_os/tasks/opensuse-leap.yml", "content": "---\n- name: Import Opensuse bootstrap\n import_tasks: opensuse.yml\n" }, { "path": "roles/bootstrap_os/tasks/opensuse-tumbleweed.yml", "content": "---\n- name: Import Opensuse bootstrap\n import_tasks: opensuse.yml\n" }, { "path": "roles/bootstrap_os/tasks/opensuse.yml", "content": "---\n# OpenSUSE ships with Python installed\n- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version\n setup:\n gather_subset: '!all'\n filter: ansible_distribution_*version\n\n- name: Check that /etc/sysconfig/proxy file exists\n stat:\n path: /etc/sysconfig/proxy\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: stat_result\n\n- name: Create the /etc/sysconfig/proxy empty file\n file: # noqa risky-file-permissions\n path: /etc/sysconfig/proxy\n state: touch\n when:\n - http_proxy is defined or https_proxy is defined\n - not stat_result.stat.exists\n\n- name: Set the http_proxy in /etc/sysconfig/proxy\n lineinfile:\n path: /etc/sysconfig/proxy\n regexp: '^HTTP_PROXY='\n line: 'HTTP_PROXY=\"{{ http_proxy }}\"'\n become: true\n when:\n - http_proxy is defined\n\n- name: Set the https_proxy in /etc/sysconfig/proxy\n lineinfile:\n path: /etc/sysconfig/proxy\n regexp: '^HTTPS_PROXY='\n line: 'HTTPS_PROXY=\"{{ https_proxy }}\"'\n become: true\n when:\n - https_proxy is defined\n\n- name: Enable proxies\n lineinfile:\n path: /etc/sysconfig/proxy\n regexp: '^PROXY_ENABLED='\n line: 'PROXY_ENABLED=\"yes\"'\n become: true\n when:\n - http_proxy is defined or https_proxy is defined\n\n# Required for zypper module\n- name: Install python-xml\n shell: zypper refresh && zypper --non-interactive install python-xml\n changed_when: false\n become: true\n tags:\n - facts\n" }, { "path": "roles/bootstrap_os/tasks/rhel.yml", "content": "---\n- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version\n setup:\n gather_subset: '!all'\n filter: ansible_distribution_*version\n\n- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined\n community.general.ini_file:\n path: \"{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}\"\n section: main\n option: proxy\n value: \"{{ http_proxy | default(omit) }}\"\n state: \"{{ http_proxy | default(False) | ternary('present', 'absent') }}\"\n no_extra_spaces: true\n mode: \"0644\"\n become: true\n when: not skip_http_proxy_on_os_packages\n\n- name: Add proxy to RHEL subscription-manager if http_proxy is defined\n command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\\d+$') | regex_replace('^.*://') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }}\n become: true\n when:\n - not skip_http_proxy_on_os_packages\n - http_proxy is defined\n\n- name: Check RHEL subscription-manager status\n command: /sbin/subscription-manager status\n register: rh_subscription_status\n changed_when: \"rh_subscription_status.rc != 0\"\n ignore_errors: true # noqa ignore-errors\n timeout: \"{{ rh_subscription_check_timeout }}\"\n become: true\n\n- name: RHEL subscription Organization ID/Activation Key registration\n community.general.redhat_subscription:\n state: present\n org_id: \"{{ rh_subscription_org_id }}\"\n activationkey: \"{{ rh_subscription_activation_key }}\"\n force_register: true\n notify: RHEL auto-attach subscription\n become: true\n when:\n - rh_subscription_org_id is defined\n - rh_subscription_status.changed\n\n# this task has no_log set to prevent logging security sensitive information such as subscription passwords\n- name: RHEL subscription Username/Password registration\n community.general.redhat_subscription:\n state: present\n username: \"{{ rh_subscription_username }}\"\n password: \"{{ rh_subscription_password }}\"\n auto_attach: true\n force_register: true\n syspurpose:\n usage: \"{{ rh_subscription_usage }}\"\n role: \"{{ rh_subscription_role }}\"\n service_level_agreement: \"{{ rh_subscription_sla }}\"\n sync: true\n notify: RHEL auto-attach subscription\n become: true\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n when:\n - rh_subscription_username is defined\n - rh_subscription_status.changed\n\n# container-selinux is in appstream repo\n- name: Enable RHEL 8 repos\n community.general.rhsm_repository:\n name:\n - \"rhel-8-for-*-baseos-rpms\"\n - \"rhel-8-for-*-appstream-rpms\"\n state: \"{{ 'enabled' if (rhel_enable_repos | default(True) | bool) else 'disabled' }}\"\n when:\n - ansible_distribution_major_version == \"8\"\n - (not rh_subscription_status.changed) or (rh_subscription_username is defined) or (rh_subscription_org_id is defined)\n\n- name: Check presence of fastestmirror.conf\n stat:\n path: /etc/yum/pluginconf.d/fastestmirror.conf\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: fastestmirror\n\n# the fastestmirror plugin can actually slow down Ansible deployments\n- name: Disable fastestmirror plugin if requested\n lineinfile:\n dest: /etc/yum/pluginconf.d/fastestmirror.conf\n regexp: \"^enabled=.*\"\n line: \"enabled=0\"\n state: present\n become: true\n when:\n - fastestmirror.stat.exists\n - not centos_fastestmirror_enabled\n" }, { "path": "roles/bootstrap_os/tasks/rocky.yml", "content": "---\n- name: Import Centos boostrap for Rocky Linux\n import_tasks: centos.yml\n" }, { "path": "roles/bootstrap_os/tasks/ubuntu.yml", "content": "---\n- name: Import Debian bootstrap\n import_tasks: debian.yml\n\n- name: Check unattended-upgrades file exist\n stat:\n path: /etc/apt/apt.conf.d/50unattended-upgrades\n register: unattended_upgrades_file_stat\n when:\n - ubuntu_kernel_unattended_upgrades_disabled\n\n- name: Disable kernel unattended-upgrades\n lineinfile:\n path: \"{{ unattended_upgrades_file_stat.stat.path }}\"\n insertafter: \"Unattended-Upgrade::Package-Blacklist\"\n line: '\"linux-\";'\n state: present\n become: true\n when:\n - ubuntu_kernel_unattended_upgrades_disabled\n - unattended_upgrades_file_stat.stat.exists\n\n- name: Stop unattended-upgrades service\n service:\n name: unattended-upgrades\n state: stopped\n enabled: false\n become: true\n when: ubuntu_stop_unattended_upgrades\n" }, { "path": "roles/bootstrap_os/vars/fedora-coreos.yml", "content": "---\nis_fedora_coreos: true\n" }, { "path": "roles/bootstrap_os/vars/flatcar.yml", "content": "---\nbin_dir: \"/opt/bin\"\n" }, { "path": "roles/container-engine/containerd/defaults/main.yml", "content": "---\ncontainerd_storage_dir: \"/var/lib/containerd\"\ncontainerd_state_dir: \"/run/containerd\"\ncontainerd_systemd_dir: \"/etc/systemd/system/containerd.service.d\"\n# The default value is not -999 here because containerd's oom_score_adj has been\n# set to the -999 even if containerd_oom_score is 0.\n# Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242\ncontainerd_oom_score: 0\n\ncontainerd_default_runtime: \"runc\"\ncontainerd_snapshotter: \"overlayfs\"\n\ncontainerd_runc_runtime:\n name: runc\n type: \"io.containerd.runc.v2\"\n base_runtime_spec: cri-base.json\n options:\n Root: \"\"\n SystemdCgroup: \"{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}\"\n BinaryName: \"{{ bin_dir }}/runc\"\n\ncontainerd_additional_runtimes: []\n# Example for Kata Containers as additional runtime:\n# - name: kata\n# type: \"io.containerd.kata.v2\"\n# options:\n# Root: \"\"\n\ncontainerd_base_runtime_spec_rlimit_nofile: 65535\n\ncontainerd_default_base_runtime_spec_patch:\n process:\n rlimits:\n - type: RLIMIT_NOFILE\n hard: \"{{ containerd_base_runtime_spec_rlimit_nofile }}\"\n soft: \"{{ containerd_base_runtime_spec_rlimit_nofile }}\"\n\n# Only for containerd < 2.1; discard unpacked layers to save disk space\n# https://github.com/containerd/containerd/blob/release/2.1/docs/cri/config.md#image-pull-configuration-since-containerd-v21\ncontainerd_discard_unpacked_layers: true\n\ncontainerd_base_runtime_specs:\n cri-base.json: \"{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch, recursive=1) }}\"\n\ncontainerd_grpc_max_recv_message_size: 16777216\ncontainerd_grpc_max_send_message_size: 16777216\n\ncontainerd_debug_address: \"\"\ncontainerd_debug_level: \"info\"\ncontainerd_debug_format: \"\"\ncontainerd_debug_uid: 0\ncontainerd_debug_gid: 0\n\ncontainerd_metrics_address: \"\"\n\ncontainerd_metrics_grpc_histogram: false\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://registry-1.docker.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n# ca: [\"/etc/certs/mirror.pem\"]\n# client: [[\"/etc/certs/client.pem\", \"\"],[\"/etc/certs/client.cert\", \"/etc/certs/client.key\"]]\n# header:\n# Authorization: \"Basic XXX\"\ncontainerd_max_container_log_line_size: 16384\n\n# If enabled it will allow non root users to use port numbers <1024\ncontainerd_enable_unprivileged_ports: false\n# If enabled it will allow non root users to use icmp sockets\ncontainerd_enable_unprivileged_icmp: false\n\ncontainerd_enable_selinux: false\ncontainerd_disable_apparmor: false\ncontainerd_tolerate_missing_hugetlb_controller: true\ncontainerd_disable_hugetlb_controller: true\ncontainerd_image_pull_progress_timeout: 5m\n\ncontainerd_cfg_dir: /etc/containerd\n\n# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally\ncontainerd_extra_args: ''\n\n# Extra runtime configuration options to be injected into the containerd CRI runtime plugin section\n# [plugins.\"io.containerd.cri.v1.runtime\"]. This is useful for adding containerd runtime\n# configuration options that aren't explicitly supported by Kubespray's default variables.\n# Example:\n# containerd_extra_runtime_args:\n# device_ownership_from_security_context: true\n# another_option: \"value\"\ncontainerd_extra_runtime_args: {}\n\n# Configure registry auth (if applicable to secure/insecure registries)\ncontainerd_registry_auth: []\n# - registry: 10.0.0.2:5000\n# username: user\n# password: pass\n\n# Configure containerd service\ncontainerd_limit_proc_num: \"infinity\"\ncontainerd_limit_core: \"infinity\"\ncontainerd_limit_open_file_num: 1048576\ncontainerd_limit_mem_lock: \"infinity\"\n\n# OS distributions that already support containerd\ncontainerd_supported_distributions:\n - \"CentOS\"\n - \"OracleLinux\"\n - \"RedHat\"\n - \"Ubuntu\"\n - \"Debian\"\n - \"Fedora\"\n - \"AlmaLinux\"\n - \"Rocky\"\n - \"Amazon\"\n - \"Flatcar\"\n - \"Flatcar Container Linux by Kinvolk\"\n - \"Suse\"\n - \"openSUSE Leap\"\n - \"openSUSE Tumbleweed\"\n - \"Kylin Linux Advanced Server\"\n - \"UnionTech\"\n - \"UniontechOS\"\n - \"openEuler\"\n\n# Enable container device interface\nenable_cdi: false\n\n# For containerd tracing configuration please check out the official documentation:\n# https://github.com/containerd/containerd/blob/main/docs/tracing.md\ncontainerd_tracing_enabled: false\ncontainerd_tracing_endpoint: \"[::]:4317\"\ncontainerd_tracing_protocol: \"grpc\"\ncontainerd_tracing_sampling_ratio: 1.0\ncontainerd_tracing_service_name: \"containerd\"\n" }, { "path": "roles/container-engine/containerd/handlers/main.yml", "content": "---\n- name: Containerd | restart containerd\n systemd_service:\n name: containerd\n state: restarted\n enabled: true\n daemon-reload: true\n masked: false\n listen: Restart containerd\n\n- name: Containerd | wait for containerd\n command: \"{{ containerd_bin_dir }}/ctr images ls -q\"\n register: containerd_ready\n retries: 8\n delay: 4\n until: containerd_ready.rc == 0\n listen: Restart containerd\n" }, { "path": "roles/container-engine/containerd/handlers/reset.yml", "content": "---\n" }, { "path": "roles/container-engine/containerd/meta/main.yml", "content": "---\ndependencies:\n - role: container-engine/containerd-common\n - role: container-engine/runc\n - role: container-engine/crictl\n - role: container-engine/nerdctl\n" }, { "path": "roles/container-engine/containerd/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n vars:\n container_manager: containerd\n roles:\n - role: kubespray_defaults\n - role: container-engine/containerd\n" }, { "path": "roles/container-engine/containerd/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\nplatforms:\n - cloud_image: ubuntu-2404\n name: ubuntu24\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\n - cloud_image: debian-12\n name: debian12\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\n - cloud_image: almalinux-9\n name: almalinux9\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n playbooks:\n create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml\n prepare: ../../../molecule/prepare.yml\nverifier:\n name: ansible\n" }, { "path": "roles/container-engine/containerd/molecule/default/verify.yml", "content": "---\n- name: Test containerd CRI\n import_playbook: ../../../molecule/test_cri.yml\n vars:\n container_manager: containerd\n cri_socket: unix:///var/run/containerd/containerd.sock\n cri_name: containerd\n\n- name: Test nerdctl\n hosts: all\n gather_facts: false\n become: true\n tasks:\n - name: Get kubespray defaults\n import_role:\n name: ../../../../../kubespray_defaults\n - name: Test nerdctl commands\n command: \"{{ bin_dir }}/nerdctl {{ item | join(' ') }}\"\n vars:\n image: quay.io/kubespray/hello-world:latest\n loop:\n - - pull\n - \"{{ image }}\"\n - - save\n - -o\n - /tmp/hello-world.tar\n - \"{{ image }}\"\n - - load\n - -i\n - /tmp/hello-world.tar\n - - -n\n - k8s.io\n - run\n - \"{{ image }}\"\n register: nerdctl\n - name: Check log from running a container\n assert:\n that:\n - ('Hello from Docker' in nerdctl.results[3].stdout)\n" }, { "path": "roles/container-engine/containerd/tasks/main.yml", "content": "---\n- name: Containerd | Download containerd\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.containerd) }}\"\n\n- name: Containerd | Unpack containerd archive\n unarchive:\n src: \"{{ downloads.containerd.dest }}\"\n dest: \"{{ containerd_bin_dir }}\"\n mode: \"0755\"\n remote_src: true\n extra_opts:\n - --strip-components=1\n notify: Restart containerd\n\n- name: Containerd | Generate systemd service for containerd\n template:\n src: containerd.service.j2\n dest: /etc/systemd/system/containerd.service\n mode: \"0644\"\n validate: \"sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:containerd.service'\"\n # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)\n # Remove once we drop support for systemd < 250\n notify: Restart containerd\n\n- name: Containerd | Ensure containerd directories exist\n file:\n dest: \"{{ item }}\"\n state: directory\n mode: \"0755\"\n owner: root\n group: root\n with_items:\n - \"{{ containerd_systemd_dir }}\"\n - \"{{ containerd_cfg_dir }}\"\n\n- name: Containerd | Write containerd proxy drop-in\n template:\n src: http-proxy.conf.j2\n dest: \"{{ containerd_systemd_dir }}/http-proxy.conf\"\n mode: \"0644\"\n notify: Restart containerd\n when: http_proxy is defined or https_proxy is defined\n\n- name: Containerd | Generate default base_runtime_spec\n register: ctr_oci_spec\n command: \"{{ containerd_bin_dir }}/ctr oci spec\"\n check_mode: false\n changed_when: false\n\n- name: Containerd | Store generated default base_runtime_spec\n set_fact:\n containerd_default_base_runtime_spec: \"{{ ctr_oci_spec.stdout | from_json }}\"\n\n- name: Containerd | Write base_runtime_specs\n copy:\n content: \"{{ item.value }}\"\n dest: \"{{ containerd_cfg_dir }}/{{ item.key }}\"\n owner: \"root\"\n mode: \"0644\"\n with_dict: \"{{ containerd_base_runtime_specs | default({}) }}\"\n notify: Restart containerd\n\n- name: Containerd | Copy containerd config file\n template:\n src: \"{{ 'config.toml.j2' if containerd_version is version('2.0.0', '>=') else 'config-v1.toml.j2' }}\"\n dest: \"{{ containerd_cfg_dir }}/config.toml\"\n owner: \"root\"\n mode: \"0640\"\n notify: Restart containerd\n\n- name: Containerd | Configure containerd registries\n # mirror configuration can contain sensitive information on headers configuration\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n block:\n - name: Containerd | Create registry directories\n file:\n path: \"{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}\"\n state: directory\n mode: \"0755\"\n loop: \"{{ containerd_registries_mirrors }}\"\n - name: Containerd | Write hosts.toml file\n template:\n src: hosts.toml.j2\n dest: \"{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml\"\n mode: \"0640\"\n loop: \"{{ containerd_registries_mirrors }}\"\n\n# you can sometimes end up in a state where everything is installed\n# but containerd was not started / enabled\n- name: Containerd | Flush handlers\n meta: flush_handlers\n\n- name: Containerd | Ensure containerd is started and enabled\n systemd_service:\n name: containerd\n daemon_reload: true\n enabled: true\n state: started\n" }, { "path": "roles/container-engine/containerd/tasks/reset.yml", "content": "---\n- name: Containerd | Stop containerd service\n service:\n name: containerd\n daemon_reload: true\n enabled: false\n state: stopped\n tags:\n - reset_containerd\n\n- name: Containerd | Remove configuration files\n file:\n path: \"{{ item }}\"\n state: absent\n loop:\n - /etc/systemd/system/containerd.service\n - \"{{ containerd_systemd_dir }}\"\n - \"{{ containerd_cfg_dir }}\"\n - \"{{ containerd_storage_dir }}\"\n - \"{{ containerd_state_dir }}\"\n tags:\n - reset_containerd\n" }, { "path": "roles/container-engine/containerd/templates/config-v1.toml.j2", "content": "# This is for containerd v1 for compatibility\nversion = 2\n\nroot = \"{{ containerd_storage_dir }}\"\nstate = \"{{ containerd_state_dir }}\"\noom_score = {{ containerd_oom_score }}\n\n{% if containerd_extra_args is defined %}\n{{ containerd_extra_args }}\n{% endif %}\n\n[grpc]\n max_recv_message_size = {{ containerd_grpc_max_recv_message_size }}\n max_send_message_size = {{ containerd_grpc_max_send_message_size }}\n\n[debug]\n address = \"{{ containerd_debug_address }}\"\n level = \"{{ containerd_debug_level }}\"\n format = \"{{ containerd_debug_format }}\"\n uid = {{ containerd_debug_uid }}\n gid = {{ containerd_debug_gid }}\n\n[metrics]\n address = \"{{ containerd_metrics_address }}\"\n grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }}\n\n[plugins]\n [plugins.\"io.containerd.grpc.v1.cri\"]\n sandbox_image = \"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}\"\n max_container_log_line_size = {{ containerd_max_container_log_line_size }}\n enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }}\n enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }}\n enable_selinux = {{ containerd_enable_selinux | lower }}\n disable_apparmor = {{ containerd_disable_apparmor | lower }}\n tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }}\n disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }}\n image_pull_progress_timeout = \"{{ containerd_image_pull_progress_timeout }}\"\n{% if enable_cdi %}\n enable_cdi = true\n cdi_spec_dirs = [\"/etc/cdi\", \"/var/run/cdi\"]\n{% endif %}\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n default_runtime_name = \"{{ containerd_default_runtime }}\"\n snapshotter = \"{{ containerd_snapshotter }}\"\n discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.{{ runtime.name }}]\n runtime_type = \"{{ runtime.type }}\"\n runtime_engine = \"{{ runtime.engine }}\"\n runtime_root = \"{{ runtime.root }}\"\n{% if runtime.base_runtime_spec is defined %}\n base_runtime_spec = \"{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}\"\n{% endif %}\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.{{ runtime.name }}.options]\n{% for key, value in runtime.options.items() %}\n{% if value | string != \"true\" and value | string != \"false\" %}\n {{ key }} = \"{{ value }}\"\n{% else %}\n {{ key }} = {{ value }}\n{% endif %}\n{% endfor %}\n{% endfor %}\n{% if kata_containers_enabled %}\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.kata-qemu]\n runtime_type = \"io.containerd.kata-qemu.v2\"\n{% endif %}\n{% if gvisor_enabled %}\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runsc]\n runtime_type = \"io.containerd.runsc.v1\"\n{% endif %}\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n config_path = \"{{ containerd_cfg_dir }}/certs.d\"\n{% for registry in containerd_registry_auth if registry['registry'] is defined %}\n{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}\n [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{ registry['registry'] }}\".auth]\n{% if registry['username'] is defined and registry['password'] is defined %}\n password = \"{{ registry['password'] }}\"\n username = \"{{ registry['username'] }}\"\n{% else %}\n auth = \"{{ registry['auth'] }}\"\n{% endif %}\n{% endif %}\n{% endfor %}\n\n{% if nri_enabled and containerd_version is version('1.7.0', '>=') %}\n [plugins.\"io.containerd.nri.v1.nri\"]\n disable = false\n{% endif %}\n\n{% if containerd_tracing_enabled %}\n [plugins.\"io.containerd.tracing.processor.v1.otlp\"]\n endpoint = \"{{ containerd_tracing_endpoint }}\"\n protocol = \"{{ containerd_tracing_protocol }}\"\n{% if containerd_tracing_protocol == \"grpc\" %}\n insecure = false\n{% endif %}\n [plugins.\"io.containerd.internal.v1.tracing\"]\n sampling_ratio = {{ containerd_tracing_sampling_ratio }}\n service_name = \"{{ containerd_tracing_service_name }}\"\n{% endif %}\n" }, { "path": "roles/container-engine/containerd/templates/config.toml.j2", "content": "version = 3\n\nroot = \"{{ containerd_storage_dir }}\"\nstate = \"{{ containerd_state_dir }}\"\noom_score = {{ containerd_oom_score }}\n\n{% if containerd_extra_args is defined %}\n{{ containerd_extra_args }}\n{% endif %}\n\n[grpc]\n max_recv_message_size = {{ containerd_grpc_max_recv_message_size }}\n max_send_message_size = {{ containerd_grpc_max_send_message_size }}\n\n[debug]\n address = \"{{ containerd_debug_address }}\"\n level = \"{{ containerd_debug_level }}\"\n format = \"{{ containerd_debug_format }}\"\n uid = {{ containerd_debug_uid }}\n gid = {{ containerd_debug_gid }}\n\n[metrics]\n address = \"{{ containerd_metrics_address }}\"\n grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }}\n\n[plugins]\n [plugins.\"io.containerd.cri.v1.runtime\"]\n max_container_log_line_size = {{ containerd_max_container_log_line_size }}\n enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }}\n enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }}\n enable_selinux = {{ containerd_enable_selinux | lower }}\n disable_apparmor = {{ containerd_disable_apparmor | lower }}\n tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }}\n disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }}\n{% if enable_cdi %}\n enable_cdi = true\n cdi_spec_dirs = [\"/etc/cdi\", \"/var/run/cdi\"]\n{% endif %}\n{% for key, value in containerd_extra_runtime_args.items() %}\n{% if value is string %}\n {{ key }} = \"{{ value }}\"\n{% elif value is boolean %}\n {{ key }} = {{ value | lower }}\n{% else %}\n {{ key }} = {{ value }}\n{% endif %}\n{% endfor %}\n\n [plugins.\"io.containerd.cri.v1.runtime\".containerd]\n default_runtime_name = \"{{ containerd_default_runtime }}\"\n [plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes]\n{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}\n [plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.{{ runtime.name }}]\n runtime_type = \"{{ runtime.type }}\"\n{% if runtime.base_runtime_spec is defined %}\n base_runtime_spec = \"{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}\"\n{% endif %}\n\n [plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.{{ runtime.name }}.options]\n{% for key, value in runtime.options.items() %}\n{% if value | string != \"true\" and value | string != \"false\" %}\n {{ key }} = \"{{ value }}\"\n{% else %}\n {{ key }} = {{ value }}\n{% endif %}\n{% endfor %}\n{% endfor %}\n{% if kata_containers_enabled %}\n [plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.kata-qemu]\n runtime_type = \"io.containerd.kata-qemu.v2\"\n{% endif %}\n{% if gvisor_enabled %}\n [plugins.\"io.containerd.cri.v1.runtime\".containerd.runtimes.runsc]\n runtime_type = \"io.containerd.runsc.v1\"\n{% endif %}\n\n [plugins.\"io.containerd.cri.v1.images\"]\n snapshotter = \"{{ containerd_snapshotter }}\"\n{% if containerd_discard_unpacked_layers and containerd_version is version('2.1.0', '<') %}\n discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}\n{% endif %}\n image_pull_progress_timeout = \"{{ containerd_image_pull_progress_timeout }}\"\n [plugins.\"io.containerd.cri.v1.images\".pinned_images]\n sandbox = \"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}\"\n [plugins.\"io.containerd.cri.v1.images\".registry]\n config_path = \"{{ containerd_cfg_dir }}/certs.d\"\n\n [plugins.\"io.containerd.nri.v1.nri\"]\n disable = {{ 'false' if nri_enabled else 'true' }}\n\n{% if containerd_tracing_enabled %}\n [plugins.\"io.containerd.tracing.processor.v1.otlp\"]\n endpoint = \"{{ containerd_tracing_endpoint }}\"\n protocol = \"{{ containerd_tracing_protocol }}\"\n{% if containerd_tracing_protocol == \"grpc\" %}\n insecure = false\n{% endif %}\n [plugins.\"io.containerd.internal.v1.tracing\"]\n sampling_ratio = {{ containerd_tracing_sampling_ratio }}\n service_name = \"{{ containerd_tracing_service_name }}\"\n{% endif %}\n" }, { "path": "roles/container-engine/containerd/templates/containerd.service.j2", "content": "# Copyright The containerd Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n[Unit]\nDescription=containerd container runtime\nDocumentation=https://containerd.io\nAfter=network.target local-fs.target dbus.service\n\n[Service]\nExecStartPre=-/sbin/modprobe overlay\nExecStart={{ containerd_bin_dir }}/containerd\n\nType=notify\nDelegate=yes\nKillMode=process\nRestart=always\nRestartSec=5\n# Having non-zero Limit*s causes performance problems due to accounting overhead\n# in the kernel. We recommend using cgroups to do container-local accounting.\nLimitNPROC={{ containerd_limit_proc_num }}\nLimitCORE={{ containerd_limit_core }}\nLimitNOFILE={{ containerd_limit_open_file_num }}\nLimitMEMLOCK={{ containerd_limit_mem_lock }}\n# Comment TasksMax if your systemd version does not supports it.\n# Only systemd 226 and above support this version.\nTasksMax=infinity\nOOMScoreAdjust=-999\n# Set the cgroup slice of the service so that kube reserved takes effect\n{% if kube_reserved is defined and kube_reserved|bool %}\nSlice={{ kube_reserved_cgroups_for_service_slice }}\n{% endif %}\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/container-engine/containerd/templates/hosts.toml.j2", "content": "server = \"{{ item.server | default(\"https://\" + item.prefix) }}\"\n{% for mirror in item.mirrors %}\n[host.\"{{ mirror.host }}\"]\n capabilities = [\"{{ ([ mirror.capabilities ] | flatten ) | join('\",\"') }}\"]\n skip_verify = {{ mirror.skip_verify | default('false') | string | lower }}\n override_path = {{ mirror.override_path | default('false') | string | lower }}\n{% if mirror.ca is defined %}\n ca = [\"{{ ([ mirror.ca ] | flatten ) | join('\",\"') }}\"]\n{% endif %}\n{% if mirror.client is defined %}\n client = [{% for pair in mirror.client %}[\"{{ pair[0] }}\", \"{{ pair[1] }}\"]{% if not loop.last %},{% endif %}{% endfor %}]\n{% endif %}\n{% if mirror.header is defined %}\n [host.\"{{ mirror.host }}\".header]\n{% for key, value in mirror.header.items() %}\n {{ key }} = [\"{{ ([ value ] | flatten ) | join('\",\"') }}\"]\n{% endfor %}\n{% endif %}\n{% endfor %}\n" }, { "path": "roles/container-engine/containerd/templates/http-proxy.conf.j2", "content": "[Service]\nEnvironment={% if http_proxy is defined %}\"HTTP_PROXY={{ http_proxy }}\"{% endif %} {% if https_proxy is defined %}\"HTTPS_PROXY={{ https_proxy }}\"{% endif %} {% if no_proxy is defined %}\"NO_PROXY={{ no_proxy }}\"{% endif %}\n" }, { "path": "roles/container-engine/containerd-common/defaults/main.yml", "content": "---\n# We keep these variables around to allow migration from package\n# manager controlled installs to direct download ones.\ncontainerd_package: 'containerd.io'\nyum_repo_dir: /etc/yum.repos.d\n" }, { "path": "roles/container-engine/containerd-common/meta/main.yml", "content": "---\nallow_duplicates: true\n" }, { "path": "roles/container-engine/containerd-common/tasks/main.yml", "content": "---\n- name: Containerd-common | check if fedora coreos\n stat:\n path: /run/ostree-booted\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: ostree\n\n- name: Containerd-common | set is_ostree\n set_fact:\n is_ostree: \"{{ ostree.stat.exists }}\"\n\n- name: Containerd-common | gather os specific variables\n include_vars: \"{{ item }}\"\n with_first_found:\n - files:\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ host_architecture }}.yml\"\n - \"{{ ansible_distribution | lower }}.yml\"\n - \"{{ ansible_os_family | lower }}-{{ host_architecture }}.yml\"\n - \"{{ ansible_os_family | lower }}.yml\"\n - defaults.yml\n paths:\n - ../vars\n skip: true\n tags:\n - facts\n" }, { "path": "roles/container-engine/containerd-common/vars/amazon.yml", "content": "---\ncontainerd_package: containerd\n" }, { "path": "roles/container-engine/containerd-common/vars/suse.yml", "content": "---\ncontainerd_package: containerd\n" }, { "path": "roles/container-engine/cri-dockerd/defaults/main.yml", "content": "---\n# Default is \"info\" (like if not provided). Possible values are any log level string parseable by logrus\ncri_dockerd_log_level: \"info\"\n" }, { "path": "roles/container-engine/cri-dockerd/handlers/main.yml", "content": "---\n- name: Cri-dockerd | reload systemd\n systemd_service:\n name: cri-dockerd\n daemon_reload: true\n masked: false\n listen: Restart and enable cri-dockerd\n\n- name: Cri-dockerd | reload cri-dockerd.socket\n service:\n name: cri-dockerd.socket\n state: restarted\n listen: Restart and enable cri-dockerd\n\n- name: Cri-dockerd | reload cri-dockerd.service\n service:\n name: cri-dockerd.service\n state: restarted\n listen: Restart and enable cri-dockerd\n\n- name: Cri-dockerd | enable cri-dockerd service\n service:\n name: cri-dockerd.service\n enabled: true\n listen: Restart and enable cri-dockerd\n" }, { "path": "roles/container-engine/cri-dockerd/meta/main.yml", "content": "---\ndependencies:\n - role: container-engine/docker\n - role: container-engine/crictl\n" }, { "path": "roles/container-engine/cri-dockerd/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n vars:\n container_manager: docker\n roles:\n - role: kubespray_defaults\n - role: container-engine/cri-dockerd\n" }, { "path": "roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf", "content": "{\n \"cniVersion\": \"0.2.0\",\n \"name\": \"mynet\",\n \"type\": \"bridge\",\n \"bridge\": \"cni0\",\n \"isGateway\": true,\n \"ipMasq\": true,\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"172.19.0.0/24\",\n \"routes\": [\n {\n \"dst\": \"0.0.0.0/0\"\n }\n ]\n }\n}\n" }, { "path": "roles/container-engine/cri-dockerd/molecule/default/files/container.json", "content": "{\n \"metadata\": {\n \"name\": \"cri-dockerd1\"\n },\n \"image\": {\n \"image\": \"quay.io/kubespray/hello-world:latest\"\n },\n \"log_path\": \"cri-dockerd1.0.log\",\n \"linux\": {}\n}\n" }, { "path": "roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json", "content": "{\n \"metadata\": {\n \"name\": \"cri-dockerd1\",\n \"namespace\": \"default\",\n \"attempt\": 1,\n \"uid\": \"hdishd83djaidwnduwk28bcsb\"\n },\n \"linux\": {},\n \"log_directory\": \"/tmp\"\n}\n" }, { "path": "roles/container-engine/cri-dockerd/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\nplatforms:\n - name: almalinux9\n cloud_image: almalinux-9\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - name: ubuntu22\n cloud_image: ubuntu-2204\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n inventory:\n group_vars:\n all:\n become: true\n k8s_cluster:\n container_manager: docker\n playbooks:\n create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml\n prepare: ../../../molecule/prepare.yml\nverifier:\n name: ansible\n" }, { "path": "roles/container-engine/cri-dockerd/molecule/default/verify.yml", "content": "---\n- name: Test cri-dockerd\n import_playbook: ../../../molecule/test_cri.yml\n vars:\n container_manager: cri-dockerd\n cri_socket: unix:///var/run/cri-dockerd.sock\n cri_name: docker\n\n- name: Test running a container with docker\n import_playbook: ../../../molecule/test_runtime.yml\n vars:\n container_runtime: docker\n # cri-dockerd does not support multiple runtime handler before 0.4.0\n # https://github.com/Mirantis/cri-dockerd/pull/350\n # TODO: check this when we upgrade cri-dockerd\n" }, { "path": "roles/container-engine/cri-dockerd/tasks/main.yml", "content": "---\n- name: Runc | Download cri-dockerd binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.cri_dockerd) }}\"\n\n- name: Copy cri-dockerd binary from download dir\n copy:\n src: \"{{ local_release_dir }}/cri-dockerd\"\n dest: \"{{ bin_dir }}/cri-dockerd\"\n mode: \"0755\"\n remote_src: true\n notify:\n - Restart and enable cri-dockerd\n\n- name: Generate cri-dockerd systemd unit files\n template:\n src: \"{{ item }}.j2\"\n dest: \"/etc/systemd/system/{{ item }}\"\n mode: \"0644\"\n validate: \"sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{ item }}'\"\n # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)\n # Remove once we drop support for systemd < 250\n with_items:\n - cri-dockerd.service\n - cri-dockerd.socket\n notify:\n - Restart and enable cri-dockerd\n\n- name: Flush handlers\n meta: flush_handlers\n" }, { "path": "roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2", "content": "[Unit]\nDescription=CRI Interface for Docker Application Container Engine\nDocumentation=https://docs.mirantis.com\nAfter=network-online.target firewalld.service docker.service\nWants=network-online.target docker.service\nRequires=cri-dockerd.socket\n\n[Service]\nType=notify\nExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnets }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if ipv6_stack %}--ipv6-dual-stack=True{% endif %}\n\nExecReload=/bin/kill -s HUP $MAINPID\nTimeoutSec=0\nRestartSec=2\nRestart=always\n\n# Note that StartLimit* options were moved from \"Service\" to \"Unit\" in systemd 229.\n# Both the old, and new location are accepted by systemd 229 and up, so using the old location\n# to make them work for either version of systemd.\nStartLimitBurst=3\n\n# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.\n# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make\n# this option work for either version of systemd.\nStartLimitInterval=60s\n\n# Having non-zero Limit*s causes performance problems due to accounting overhead\n# in the kernel. We recommend using cgroups to do container-local accounting.\nLimitNOFILE=infinity\nLimitNPROC=infinity\nLimitCORE=infinity\n\n# Comment TasksMax if your systemd version does not support it.\n# Only systemd 226 and above support this option.\nTasksMax=infinity\nDelegate=yes\nKillMode=process\n# Set the cgroup slice of the service so that kube reserved takes effect\n{% if kube_reserved is defined and kube_reserved|bool %}\nSlice={{ kube_reserved_cgroups_for_service_slice }}\n{% endif %}\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2", "content": "[Unit]\nDescription=CRI Docker Socket for the API\nPartOf=cri-dockerd.service\n\n[Socket]\nListenStream=%t/cri-dockerd.sock\nSocketMode=0660\nSocketUser=root\nSocketGroup=docker\n\n[Install]\nWantedBy=sockets.target\n" }, { "path": "roles/container-engine/cri-o/defaults/main.yml", "content": "---\n\ncrio_cgroup_manager: \"{{ kubelet_cgroup_driver | default('systemd') }}\"\ncrio_conmon: \"{{ bin_dir }}/conmon\"\ncrio_default_runtime: \"crun\"\ncrio_libexec_dir: \"/usr/libexec/crio\"\ncrio_runtime_switch: false\ncrio_enable_metrics: false\ncrio_log_level: \"info\"\ncrio_metrics_port: \"9090\"\ncrio_pause_image: \"{{ pod_infra_image_repo }}:{{ pod_infra_version }}\"\n\n# Registries defined within cri-o.\n# By default unqualified images are not allowed for security reasons\ncrio_registries: []\n# - prefix: docker.io\n# insecure: false\n# blocked: false\n# location: registry-1.docker.io ## REQUIRED\n# unqualified: false\n# mirrors:\n# - location: 172.20.100.52:5000\n# insecure: true\n# - location: mirror.gcr.io\n# insecure: false\n\ncrio_registry_auth: []\n# - registry: 10.0.0.2:5000\n# username: user\n# password: pass\n\ncrio_seccomp_profile: \"\"\ncrio_selinux: \"{{ (preinstall_selinux_state == 'enforcing') | lower }}\"\ncrio_signature_policy: \"{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}\"\n# Set the pull progress timeout\ncrio_pull_progress_timeout: \"10s\"\n\n# Override system default for storage driver\n# crio_storage_driver: \"overlay\"\n\ncrio_stream_port: \"10010\"\n\ncrio_required_version: \"{{ kube_version | regex_replace('^(?P\\\\d+).(?P\\\\d+).(?P\\\\d+)$', '\\\\g.\\\\g') }}\"\n\ncrio_root: \"/var/lib/containers/storage\"\n\n# The crio_runtimes variable defines a list of OCI compatible runtimes.\ncrio_runtimes:\n - name: crun\n path: \"{{ crio_runtime_bin_dir }}/crun\" # Use crun in cri-o distributions, don't use 'crun' role\n type: oci\n root: /run/crun\n\n# Kata Containers is an OCI runtime, where containers are run inside lightweight\n# VMs. Kata provides additional isolation towards the host, minimizing the host attack\n# surface and mitigating the consequences of containers breakout.\nkata_runtimes:\n # Kata Containers with the default configured VMM\n - name: kata-qemu\n path: /usr/local/bin/containerd-shim-kata-qemu-v2\n type: vm\n root: /run/kata-containers\n privileged_without_host_devices: true\n\nrunc_runtime:\n name: runc\n path: \"{{ crio_runtime_bin_dir }}/runc\"\n type: oci\n root: /run/runc\n\n# crun is a fast and low-memory footprint OCI Container Runtime fully written in C.\ncrun_runtime:\n name: crun\n path: \"{{ crio_runtime_bin_dir }}/crun\"\n type: oci\n root: /run/crun\n\n# youki is an implementation of the OCI runtime-spec in Rust, similar to runc.\nyouki_runtime:\n name: youki\n path: \"{{ youki_bin_dir }}/youki\"\n type: oci\n root: /run/youki\n\n# Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids)\n# at the end of the uid/gid space\ncrio_remap_enable: false\ncrio_remap_user: containers\ncrio_subuid_start: 2130706432\ncrio_subuid_length: 16777216\ncrio_subgid_start: 2130706432\ncrio_subgid_length: 16777216\n\n# cri-o manual files\ncrio_man_files:\n 5:\n - crio.conf\n - crio.conf.d\n 8:\n - crio\n - crio-status\n\n# If set to true, it will enable the CRIU support in cri-o\ncrio_criu_support_enabled: false\n\n# Configure default_capabilities in crio.conf\ncrio_default_capabilities:\n - CHOWN\n - DAC_OVERRIDE\n - FSETID\n - FOWNER\n - SETGID\n - SETUID\n - SETPCAP\n - NET_BIND_SERVICE\n - KILL\ncrio_additional_mounts: []\n" }, { "path": "roles/container-engine/cri-o/handlers/main.yml", "content": "---\n- name: CRI-O | reload systemd\n systemd_service:\n daemon_reload: true\n listen: Restart crio\n\n- name: CRI-O | reload crio\n service:\n name: crio\n state: restarted\n enabled: true\n listen: Restart crio\n" }, { "path": "roles/container-engine/cri-o/meta/main.yml", "content": "---\ndependencies:\n - role: container-engine/crictl\n - role: container-engine/skopeo\n" }, { "path": "roles/container-engine/cri-o/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n roles:\n - role: kubespray_defaults\n - role: container-engine/cri-o\n" }, { "path": "roles/container-engine/cri-o/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\nplatforms:\n - name: ubuntu22\n cloud_image: ubuntu-2204\n vm_cpu_cores: 2\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\n - name: almalinux9\n cloud_image: almalinux-9\n vm_cpu_cores: 2\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\n - name: fedora\n cloud_image: fedora-39\n vm_cpu_cores: 2\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\n - name: debian12\n cloud_image: debian-12\n vm_cpu_cores: 2\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - kube_node\n - k8s_cluster\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n inventory:\n group_vars:\n k8s_cluster:\n container_manager: crio\n playbooks:\n create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml\n prepare: ../../../molecule/prepare.yml\nverifier:\n name: ansible\n" }, { "path": "roles/container-engine/cri-o/molecule/default/verify.yml", "content": "---\n- name: Test CRI-O cri\n import_playbook: ../../../molecule/test_cri.yml\n vars:\n cri_socket: unix:///var/run/crio/crio.sock\n cri_name: cri-o\n- name: Test running a container with crun\n import_playbook: ../../../molecule/test_runtime.yml\n vars:\n container_runtime: crun\n" }, { "path": "roles/container-engine/cri-o/tasks/load_vars.yml", "content": "---\n- name: Cri-o | include vars/v1.29.yml\n include_vars: v1.29.yml\n when: crio_version is version(\"1.29.0\", operator=\">=\")\n\n- name: Cri-o | include vars/v1.31.yml\n include_vars: v1.31.yml\n when: crio_version is version(\"1.31.0\", operator=\">=\")\n" }, { "path": "roles/container-engine/cri-o/tasks/main.yaml", "content": "---\n- name: Cri-o | load vars\n import_tasks: load_vars.yml\n\n- name: Cri-o | check if fedora coreos\n stat:\n path: /run/ostree-booted\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: ostree\n\n- name: Cri-o | set is_ostree\n set_fact:\n is_ostree: \"{{ ostree.stat.exists }}\"\n\n- name: Cri-o | get ostree version\n shell: \"set -o pipefail && rpm-ostree --version | awk -F\\\\' '/Version/{print $2}'\"\n args:\n executable: /bin/bash\n register: ostree_version\n when: is_ostree\n\n- name: Cri-o | Download cri-o\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.crio) }}\"\n\n- name: Cri-o | special handling for amazon linux\n import_tasks: \"setup-amazon.yaml\"\n when: ansible_distribution in [\"Amazon\"]\n\n- name: Cri-o | build a list of crio runtimes with Katacontainers runtimes\n set_fact:\n crio_runtimes: \"{{ crio_runtimes + kata_runtimes }}\"\n when:\n - kata_containers_enabled\n\n## After CRI-O v1.31, crun is default runtime.\n# - name: Cri-o | build a list of crio runtimes with crun runtime\n# set_fact:\n# crio_runtimes: \"{{ crio_runtimes + [crun_runtime] }}\"\n# when:\n# - crun_enabled\n\n- name: Cri-o | build a list of crio runtimes with runc runtime\n set_fact:\n crio_runtimes: \"{{ crio_runtimes + [runc_runtime] }}\"\n when:\n - runc_enabled\n\n- name: Cri-o | build a list of crio runtimes with youki runtime\n set_fact:\n crio_runtimes: \"{{ crio_runtimes + [youki_runtime] }}\"\n when:\n - youki_enabled\n\n- name: Cri-o | Stop kubelet service if running\n service:\n name: kubelet\n state: stopped\n when:\n - crio_runtime_switch\n - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service'].state == 'running'\n\n- name: Cri-o | Get all pods\n ansible.builtin.command: \"{{ bin_dir }}/crictl pods -o json\"\n changed_when: false\n register: crio_pods\n when:\n - crio_runtime_switch\n - ansible_facts.services['crio.service'] is defined\n\n- name: Cri-o | Stop and remove pods not on host network\n ansible.builtin.command: \"{{ bin_dir }}/crictl rmp -f {{ item.id }}\"\n loop: \"{{ (crio_pods.stdout | from_json).items | default([]) | selectattr('metadata.namespace', 'ne', 'NODE') }}\"\n changed_when: true\n when:\n - crio_runtime_switch\n - ansible_facts.services['crio.service'] is defined\n - crio_pods.stdout is defined\n\n- name: Cri-o | Stop and remove all remaining pods\n ansible.builtin.command: \"{{ bin_dir }}/crictl rmp -fa\"\n changed_when: true\n when:\n - crio_runtime_switch\n - ansible_facts.services['crio.service'] is defined\n\n- name: Cri-o | stop crio service if running\n service:\n name: crio\n state: stopped\n when:\n - crio_runtime_switch\n - ansible_facts.services['crio.service'] is defined and ansible_facts.services['crio.service'].state == 'running'\n\n- name: Cri-o | make sure needed folders exist in the system\n with_items:\n - /etc/crio\n - /etc/containers\n - /etc/systemd/system/crio.service.d\n file:\n path: \"{{ item }}\"\n state: directory\n mode: \"0755\"\n\n- name: Cri-o | install cri-o config\n template:\n src: crio.conf.j2\n dest: /etc/crio/crio.conf\n mode: \"0644\"\n register: config_install\n\n- name: Cri-o | install config.json\n template:\n src: config.json.j2\n dest: /etc/crio/config.json\n mode: \"0644\"\n register: reg_auth_install\n\n- name: Cri-o | copy binaries\n copy:\n src: \"{{ local_release_dir }}/cri-o/bin/{{ item }}\"\n dest: \"{{ bin_dir }}/{{ item }}\"\n mode: \"0755\"\n remote_src: true\n with_items:\n - \"{{ crio_bin_files }}\"\n notify: Restart crio\n\n- name: Cri-o | create directory for libexec\n file:\n path: \"{{ crio_libexec_dir }}\"\n state: directory\n owner: root\n mode: \"0755\"\n\n- name: Cri-o | copy libexec\n copy:\n src: \"{{ local_release_dir }}/cri-o/bin/{{ item }}\"\n dest: \"{{ crio_libexec_dir }}/{{ item }}\"\n mode: \"0755\"\n remote_src: true\n with_items:\n - \"{{ crio_libexec_files }}\"\n notify: Restart crio\n\n- name: Cri-o | copy service file\n copy:\n src: \"{{ local_release_dir }}/cri-o/contrib/crio.service\"\n dest: /etc/systemd/system/crio.service\n mode: \"0755\"\n remote_src: true\n notify: Restart crio\n\n- name: Cri-o | configure crio to use kube reserved cgroups\n ansible.builtin.copy:\n dest: /etc/systemd/system/crio.service.d/00-slice.conf\n owner: root\n group: root\n mode: '0644'\n content: |\n [Service]\n Slice={{ kube_reserved_cgroups_for_service_slice }}\n notify: Restart crio\n when:\n - kube_reserved is defined and kube_reserved is true\n - kube_reserved_cgroups_for_service_slice is defined\n\n- name: Cri-o | update the bin dir for crio.service file\n replace:\n dest: /etc/systemd/system/crio.service\n regexp: \"/usr/local/bin/crio\"\n replace: \"{{ bin_dir }}/crio\"\n notify: Restart crio\n\n- name: Cri-o | copy default policy\n copy:\n src: \"{{ local_release_dir }}/cri-o/contrib/policy.json\"\n dest: /etc/containers/policy.json\n mode: \"0755\"\n remote_src: true\n notify: Restart crio\n\n- name: Cri-o | copy mounts.conf\n template:\n src: mounts.conf.j2\n dest: /etc/containers/mounts.conf\n mode: \"0644\"\n when:\n - ansible_os_family == 'RedHat'\n notify: Restart crio\n\n- name: Cri-o | create directory for oci hooks\n file:\n path: /etc/containers/oci/hooks.d\n state: directory\n owner: root\n mode: \"0755\"\n\n- name: Cri-o | set overlay driver\n community.general.ini_file:\n dest: /etc/containers/storage.conf\n section: storage\n option: \"{{ item.option }}\"\n value: \"{{ item.value }}\"\n mode: \"0644\"\n with_items:\n - option: driver\n value: '\"overlay\"'\n - option: graphroot\n value: '\"/var/lib/containers/storage\"'\n - option: runroot\n value: '\"/var/run/containers/storage\"'\n\n# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel\n- name: Cri-o | set metacopy mount options correctly\n community.general.ini_file:\n dest: /etc/containers/storage.conf\n section: storage.options.overlay\n option: mountopt\n value: '{{ ''\"nodev\"'' if ansible_kernel is version((\"4.18\" if ansible_os_family == \"RedHat\" else \"4.19\"), \"<\") else ''\"nodev,metacopy=on\"'' }}'\n mode: \"0644\"\n\n- name: Cri-o | create directory registries configs\n file:\n path: /etc/containers/registries.conf.d\n state: directory\n owner: root\n mode: \"0755\"\n\n- name: Cri-o | write registries configs\n template:\n src: registry.conf.j2\n dest: \"/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':|/', '_') }}.conf\"\n mode: \"0644\"\n loop: \"{{ crio_registries }}\"\n notify: Restart crio\n\n- name: Cri-o | configure unqualified registry settings\n template:\n src: unqualified.conf.j2\n dest: \"/etc/containers/registries.conf.d/01-unqualified.conf\"\n mode: \"0644\"\n notify: Restart crio\n\n- name: Cri-o | write cri-o proxy drop-in\n template:\n src: http-proxy.conf.j2\n dest: /etc/systemd/system/crio.service.d/http-proxy.conf\n mode: \"0644\"\n notify: Restart crio\n when: http_proxy is defined or https_proxy is defined\n\n- name: Cri-o | configure the uid/gid space for user namespaces\n lineinfile:\n path: '{{ item.path }}'\n line: '{{ item.entry }}'\n regex: '^\\s*{{ crio_remap_user }}:'\n state: '{{ \"present\" if crio_remap_enable | bool else \"absent\" }}'\n loop:\n - path: /etc/subuid\n entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}'\n - path: /etc/subgid\n entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}'\n loop_control:\n label: '{{ item.path }}'\n\n- name: Cri-o | ensure crio service is started and enabled\n service:\n name: crio\n daemon_reload: true\n enabled: true\n state: started\n register: service_start\n\n- name: Cri-o | trigger service restart only when needed\n service:\n name: crio\n state: restarted\n when:\n - config_install.changed or reg_auth_install.changed\n - not service_start.changed\n\n- name: Cri-o | verify that crio is running\n command: \"{{ bin_dir }}/{{ crio_status_command }} info\"\n register: get_crio_info\n until: get_crio_info is succeeded\n changed_when: false\n retries: 5\n delay: \"{{ retry_stagger | random + 3 }}\"\n\n# The kubelet service status can be 'not-found' if something depends on it.\n# This check prevents failures when the service is in this indeterminate state,\n# which can occur when adding new nodes to a cluster.\n# See: https://superuser.com/questions/1755211/cleaning-debugging-services/1755215#1755215\n\n- name: Cri-o | ensure kubelet service is started if present and stopped\n service:\n name: kubelet\n state: started\n when:\n - crio_runtime_switch\n - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service']['status'] != 'not-found'\n" }, { "path": "roles/container-engine/cri-o/tasks/reset.yml", "content": "---\n- name: Cri-o | load vars\n import_tasks: load_vars.yml\n\n- name: CRI-O | Kubic repo name for debian os family\n set_fact:\n crio_kubic_debian_repo_name: \"{{ ((ansible_distribution == 'Ubuntu') | ternary('x', '')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}\"\n when: ansible_os_family == \"Debian\"\n tags:\n - reset_crio\n\n- name: CRI-O | Remove kubic apt repo\n apt_repository:\n repo: \"deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /\"\n state: absent\n when: crio_kubic_debian_repo_name is defined\n tags:\n - reset_crio\n\n- name: CRI-O | Remove cri-o apt repo\n apt_repository:\n repo: \"deb {{ crio_download_crio }}v{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /\"\n state: absent\n filename: devel-kubic-libcontainers-stable-cri-o\n when: crio_kubic_debian_repo_name is defined\n tags:\n - reset_crio\n\n- name: CRI-O | Remove CRI-O kubic yum repo\n yum_repository:\n name: devel_kubic_libcontainers_stable\n state: absent\n when: ansible_distribution in [\"Amazon\"]\n tags:\n - reset_crio\n\n- name: CRI-O | Remove CRI-O kubic yum repo\n yum_repository:\n name: \"devel_kubic_libcontainers_stable_cri-o_v{{ crio_version }}\"\n state: absent\n when:\n - ansible_os_family == \"RedHat\"\n - ansible_distribution not in [\"Amazon\", \"Fedora\"]\n tags:\n - reset_crio\n\n- name: CRI-O | Run yum-clean-metadata\n command: yum clean metadata\n when:\n - ansible_os_family == \"RedHat\"\n tags:\n - reset_crio\n\n- name: CRI-O | Remove crictl\n file:\n name: \"{{ item }}\"\n state: absent\n loop:\n - /etc/crictl.yaml\n - \"{{ bin_dir }}/crictl\"\n tags:\n - reset_crio\n\n- name: CRI-O | Stop crio service\n service:\n name: crio\n daemon_reload: true\n enabled: false\n state: stopped\n tags:\n - reset_crio\n\n- name: CRI-O | Remove CRI-O configuration files\n file:\n name: \"{{ item }}\"\n state: absent\n loop:\n - /etc/crio\n - /etc/containers\n - /etc/systemd/system/crio.service.d\n tags:\n - reset_crio\n\n- name: CRI-O | Remove CRI-O binaries\n file:\n name: \"{{ item }}\"\n state: absent\n with_items: \"{{ crio_bin_files }}\"\n tags:\n - reset_crio\n\n- name: CRI-O | Remove CRI-O libexec\n file:\n name: \"{{ item }}\"\n state: absent\n with_items: \"{{ crio_libexec_files }}\"\n tags:\n - reset_crio\n" }, { "path": "roles/container-engine/cri-o/tasks/setup-amazon.yaml", "content": "---\n- name: Check that amzn2-extras.repo exists\n stat:\n path: /etc/yum.repos.d/amzn2-extras.repo\n register: amzn2_extras_file_stat\n\n- name: Find docker repo in amzn2-extras.repo file\n lineinfile:\n dest: /etc/yum.repos.d/amzn2-extras.repo\n line: \"[amzn2extra-docker]\"\n check_mode: true\n register: amzn2_extras_docker_repo\n when:\n - amzn2_extras_file_stat.stat.exists\n\n- name: Remove docker repository\n community.general.ini_file:\n dest: /etc/yum.repos.d/amzn2-extras.repo\n section: amzn2extra-docker\n option: enabled\n value: \"0\"\n backup: true\n mode: \"0644\"\n when:\n - amzn2_extras_file_stat.stat.exists\n - not amzn2_extras_docker_repo.changed\n" }, { "path": "roles/container-engine/cri-o/templates/config.json.j2", "content": "{% if crio_registry_auth is defined and crio_registry_auth|length %}\n{\n \"auths\": {\n{% for reg in crio_registry_auth %}\n \"{{ reg.registry }}\": {\n \"auth\": \"{{ (reg.username + ':' + reg.password) | string | b64encode }}\"\n{% if not loop.last %}\n },\n{% else %}\n }\n{% endif %}\n{% endfor %}\n }\n}\n{% else %}\n{}\n{% endif %}\n" }, { "path": "roles/container-engine/cri-o/templates/crio.conf.j2", "content": "# The CRI-O configuration file specifies all of the available configuration\n# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime\n# daemon, but in a TOML format that can be more easily modified and versioned.\n#\n# Please refer to crio.conf(5) for details of all configuration options.\n\n# CRI-O supports partial configuration reload during runtime, which can be\n# done by sending SIGHUP to the running process. Currently supported options\n# are explicitly mentioned with: 'This option supports live configuration\n# reload'.\n\n# CRI-O reads its storage defaults from the containers-storage.conf(5) file\n# located at /etc/containers/storage.conf. Modify this storage configuration if\n# you want to change the system's defaults. If you want to modify storage just\n# for CRI-O, you can change the storage configuration options here.\n[crio]\n\n# Path to the \"root directory\". CRI-O stores all of its data, including\n# containers images, in this directory.\nroot = \"{{ crio_root }}\"\n\n# Path to the \"run directory\". CRI-O stores all of its state in this directory.\n# Read from /etc/containers/storage.conf first so unnecessary here\n# runroot = \"/var/run/containers/storage\"\n\n# Storage driver used to manage the storage of images and containers. Please\n# refer to containers-storage.conf(5) to see all available storage drivers.\n{% if crio_storage_driver is defined %}\nstorage_driver = \"{{ crio_storage_driver }}\"\n{% endif %}\n\n# List to pass options to the storage driver. Please refer to\n# containers-storage.conf(5) to see all available storage options.\n#storage_option = [\n#]\n\n# The default log directory where all logs will go unless directly specified by\n# the kubelet. The log directory specified must be an absolute directory.\nlog_dir = \"/var/log/crio/pods\"\n\n# Location for CRI-O to lay down the temporary version file.\n# It is used to check if crio wipe should wipe containers, which should\n# always happen on a node reboot\nversion_file = \"/var/run/crio/version\"\n\n# Location for CRI-O to lay down the persistent version file.\n# It is used to check if crio wipe should wipe images, which should\n# only happen when CRI-O has been upgraded\nversion_file_persist = \"/var/lib/crio/version\"\n\n# The crio.api table contains settings for the kubelet/gRPC interface.\n[crio.api]\n\n# Path to AF_LOCAL socket on which CRI-O will listen.\nlisten = \"/var/run/crio/crio.sock\"\n\n# IP address on which the stream server will listen.\nstream_address = \"127.0.0.1\"\n\n# The port on which the stream server will listen. If the port is set to \"0\", then\n# CRI-O will allocate a random free port number.\nstream_port = \"{{ crio_stream_port }}\"\n\n# Enable encrypted TLS transport of the stream server.\nstream_enable_tls = false\n\n# Path to the x509 certificate file used to serve the encrypted stream. This\n# file can change, and CRI-O will automatically pick up the changes within 5\n# minutes.\nstream_tls_cert = \"\"\n\n# Path to the key file used to serve the encrypted stream. This file can\n# change and CRI-O will automatically pick up the changes within 5 minutes.\nstream_tls_key = \"\"\n\n# Path to the x509 CA(s) file used to verify and authenticate client\n# communication with the encrypted stream. This file can change and CRI-O will\n# automatically pick up the changes within 5 minutes.\nstream_tls_ca = \"\"\n\n# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.\ngrpc_max_send_msg_size = 16777216\n\n# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.\ngrpc_max_recv_msg_size = 16777216\n\n# The crio.runtime table contains settings pertaining to the OCI runtime used\n# and options for how to set up and manage the OCI runtime.\n[crio.runtime]\n\n# A list of ulimits to be set in containers by default, specified as\n# \"=:\", for example:\n# \"nofile=1024:2048\"\n# If nothing is set here, settings will be inherited from the CRI-O daemon\n#default_ulimits = [\n#]\n\n# default_runtime is the _name_ of the OCI runtime to be used as the default.\n# The name is matched against the runtimes map below.\ndefault_runtime = \"{{ crio_default_runtime }}\"\n\n# If true, the runtime will not use pivot_root, but instead use MS_MOVE.\nno_pivot = false\n\n# decryption_keys_path is the path where the keys required for\n# image decryption are stored. This option supports live configuration reload.\ndecryption_keys_path = \"/etc/crio/keys/\"\n\n# Path to the conmon binary, used for monitoring the OCI runtime.\n# Will be searched for using $PATH if empty.\nconmon = \"{{ crio_conmon }}\"\n\n# Cgroup setting for conmon\n{% if crio_cgroup_manager == \"cgroupfs\" %}\nconmon_cgroup = \"pod\"\n{% else %}\n{% if kube_reserved is defined and kube_reserved|bool %}\nconmon_cgroup = \"{{ kube_reserved_cgroups_for_service_slice }}\"\n{% else %}\nconmon_cgroup = \"system.slice\"\n{% endif %}\n{% endif %}\n\n# Environment variable list for the conmon process, used for passing necessary\n# environment variables to conmon or the runtime.\nconmon_env = [\n\t\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n]\n\n# Additional environment variables to set for all the\n# containers. These are overridden if set in the\n# container image spec or in the container runtime configuration.\ndefault_env = [\n]\n\n# If true, SELinux will be used for pod separation on the host.\nselinux = {{ crio_selinux }}\n\n# Path to the seccomp.json profile which is used as the default seccomp profile\n# for the runtime. If not specified, then the internal default seccomp profile\n# will be used. This option supports live configuration reload.\nseccomp_profile = \"{{ crio_seccomp_profile }}\"\n\n# Used to change the name of the default AppArmor profile of CRI-O. The default\n# profile name is \"crio-default\". This profile only takes effect if the user\n# does not specify a profile via the Kubernetes Pod's metadata annotation. If\n# the profile is set to \"unconfined\", then this equals to disabling AppArmor.\n# This option supports live configuration reload.\n# apparmor_profile = \"crio-default\"\n\n# Cgroup management implementation used for the runtime.\ncgroup_manager = \"{{ crio_cgroup_manager }}\"\n\n# List of default capabilities for containers. If it is empty or commented out,\n# only the capabilities defined in the containers json file by the user/kube\n# will be added.\ndefault_capabilities = [\n{%- for item in crio_default_capabilities %}\n \"{{ item }}\",\n{%- endfor %}\n]\n\n# List of default sysctls. If it is empty or commented out, only the sysctls\n# defined in the container json file by the user/kube will be added.\ndefault_sysctls = [\n]\n\n# List of additional devices. specified as\n# \"::\", for example: \"--device=/dev/sdc:/dev/xvdc:rwm\".\n#If it is empty or commented out, only the devices\n# defined in the container json file by the user/kube will be added.\nadditional_devices = [\n]\n\n# Path to OCI hooks directories for automatically executed hooks. If one of the\n# directories does not exist, then CRI-O will automatically skip them.\nhooks_dir = [\n\t\"/usr/share/containers/oci/hooks.d\",\n]\n\n# List of default mounts for each container. **Deprecated:** this option will\n# be removed in future versions in favor of default_mounts_file.\ndefault_mounts = [\n]\n\n# Path to the file specifying the defaults mounts for each container. The\n# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads\n# its default mounts from the following two files:\n#\n# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the\n# override file, where users can either add in their own default mounts, or\n# override the default mounts shipped with the package.\n#\n# 2) /usr/share/containers/mounts.conf: This is the default file read for\n# mounts. If you want CRI-O to read from a different, specific mounts file,\n# you can change the default_mounts_file. Note, if this is done, CRI-O will\n# only add mounts it finds in this file.\n#\n#default_mounts_file = \"\"\n\n# Maximum sized allowed for the container log file. Negative numbers indicate\n# that no size limit is imposed. If it is positive, it must be >= 8192 to\n# match/exceed conmon's read buffer. The file is truncated and re-opened so the\n# limit is never exceeded.\nlog_size_max = -1\n\n# Whether container output should be logged to journald in addition to the kubernetes log file\nlog_to_journald = false\n\n# Path to directory in which container exit files are written to by conmon.\ncontainer_exits_dir = \"/var/run/crio/exits\"\n\n# Path to directory for container attach sockets.\ncontainer_attach_socket_dir = \"/var/run/crio\"\n\n# The prefix to use for the source of the bind mounts.\nbind_mount_prefix = \"\"\n\n# If set to true, all containers will run in read-only mode.\nread_only = false\n\n# Changes the verbosity of the logs based on the level it is set to. Options\n# are fatal, panic, error, warn, info, debug and trace. This option supports\n# live configuration reload.\nlog_level = \"{{ crio_log_level }}\"\n\n# Filter the log messages by the provided regular expression.\n# This option supports live configuration reload.\nlog_filter = \"\"\n\n# The UID mappings for the user namespace of each container. A range is\n# specified in the form containerUID:HostUID:Size. Multiple ranges must be\n# separated by comma.\nuid_mappings = \"\"\n\n# The GID mappings for the user namespace of each container. A range is\n# specified in the form containerGID:HostGID:Size. Multiple ranges must be\n# separated by comma.\ngid_mappings = \"\"\n\n# The minimal amount of time in seconds to wait before issuing a timeout\n# regarding the proper termination of the container. The lowest possible\n# value is 30s, whereas lower values are not considered by CRI-O.\nctr_stop_timeout = 30\n\n# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below.\n# manage_network_ns_lifecycle = false\n\n# manage_ns_lifecycle determines whether we pin and remove namespaces\n# and manage their lifecycle\n{% if kata_containers_enabled %}\nmanage_ns_lifecycle = true\n{% else %}\nmanage_ns_lifecycle = false\n{% endif %}\n\n# The directory where the state of the managed namespaces gets tracked.\n# Only used when manage_ns_lifecycle is true.\nnamespaces_dir = \"/var/run\"\n\n# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle\n{% if bin_dir == \"/usr/local/bin\" %}\npinns_path = \"\"\n{% else %}\npinns_path = \"{{ bin_dir }}/pinns\"\n{% endif %}\n\n{% if crio_criu_support_enabled %}\n# Enable CRIU integration, requires that the criu binary is available in $PATH.\nenable_criu_support = true\n{% endif %}\n\n# The \"crio.runtime.runtimes\" table defines a list of OCI compatible runtimes.\n# The runtime to use is picked based on the runtime_handler provided by the CRI.\n# If no runtime_handler is provided, the runtime will be picked based on the level\n# of trust of the workload. Each entry in the table should follow the format:\n#\n#[crio.runtime.runtimes.runtime-handler]\n# runtime_path = \"/path/to/the/executable\"\n# runtime_type = \"oci\"\n# runtime_root = \"/path/to/the/root\"\n#\n# Where:\n# - runtime-handler: name used to identify the runtime\n# - runtime_path (optional, string): absolute path to the runtime executable in\n# the host filesystem. If omitted, the runtime-handler identifier should match\n# the runtime executable name, and the runtime executable should be placed\n# in $PATH.\n# - runtime_type (optional, string): type of runtime, one of: \"oci\", \"vm\". If\n# omitted, an \"oci\" runtime is assumed.\n# - runtime_root (optional, string): root directory for storage of containers\n# state.\n\n{% for runtime in crio_runtimes %}\n[crio.runtime.runtimes.{{ runtime.name }}]\nruntime_path = \"{{ runtime.path }}\"\nruntime_type = \"{{ runtime.type }}\"\nruntime_root = \"{{ runtime.root }}\"\nprivileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }}\nallowed_annotations = {{ runtime.allowed_annotations|default([])|to_json }}\n{% endfor %}\n\n# Kata Containers with the Firecracker VMM\n#[crio.runtime.runtimes.kata-fc]\n\n# The crio.image table contains settings pertaining to the management of OCI images.\n#\n# CRI-O reads its configured registries defaults from the system wide\n# containers-registries.conf(5) located in /etc/containers/registries.conf. If\n# you want to modify just CRI-O, you can change the registries configuration in\n# this file. Otherwise, leave insecure_registries and registries commented out to\n# use the system's defaults from /etc/containers/registries.conf.\n[crio.image]\n{% if crio_insecure_registries is defined and crio_insecure_registries|length>0 %}\ninsecure_registries = {{ crio_insecure_registries }}\n{% endif %}\n\n# Default transport for pulling images from a remote container storage.\ndefault_transport = \"docker://\"\n\n# The path to a file containing credentials necessary for pulling images from\n# secure registries. The file is similar to that of /var/lib/kubelet/config.json\nglobal_auth_file = \"/etc/crio/config.json\"\n\n# The image used to instantiate infra containers.\n# This option supports live configuration reload.\npause_image = \"{{ crio_pause_image }}\"\n\n# The path to a file containing credentials specific for pulling the pause_image from\n# above. The file is similar to that of /var/lib/kubelet/config.json\n# This option supports live configuration reload.\npause_image_auth_file = \"\"\n\n# The command to run to have a container stay in the paused state.\n# When explicitly set to \"\", it will fallback to the entrypoint and command\n# specified in the pause image. When commented out, it will fallback to the\n# default: \"/pause\". This option supports live configuration reload.\npause_command = \"/pause\"\n\n# Path to the file which decides what sort of policy we use when deciding\n# whether or not to trust an image that we've pulled. It is not recommended that\n# this option be used, as the default behavior of using the system-wide default\n# policy (i.e., /etc/containers/policy.json) is most often preferred. Please\n# refer to containers-policy.json(5) for more details.\nsignature_policy = \"{{ crio_signature_policy }}\"\n\n# Controls how image volumes are handled. The valid values are mkdir, bind and\n# ignore; the latter will ignore volumes entirely.\nimage_volumes = \"mkdir\"\n\n# The timeout for an image pull to make progress until the pull operation gets\n# canceled. This value will be also used for calculating the pull progress interval\n# to pull_progress_timeout / 10. Can be set to 0 to disable the timeout as well as\n# the progress output.\npull_progress_timeout = \"{{ crio_pull_progress_timeout }}\"\n\n# The crio.network table containers settings pertaining to the management of\n# CNI plugins.\n[crio.network]\n\n# The default CNI network name to be selected. If not set or \"\", then\n# CRI-O will pick-up the first one found in network_dir.\n# cni_default_network = \"\"\n\n# Path to the directory where CNI configuration files are located.\nnetwork_dir = \"/etc/cni/net.d/\"\n\n# Paths to directories where CNI plugin binaries are located.\nplugin_dirs = [\n\t\"/opt/cni/bin\",\n\t\"/usr/libexec/cni\",\n]\n\n# A necessary configuration for Prometheus based metrics retrieval\n[crio.metrics]\n\n# Globally enable or disable metrics support.\nenable_metrics = {{ crio_enable_metrics | bool | lower }}\n\n# The port on which the metrics server will listen.\nmetrics_port = {{ crio_metrics_port }}\n\n{% if nri_enabled and crio_version is version('1.26.0', operator='>=') %}\n[crio.nri]\n\nenable_nri=true\n{% endif %}\n" }, { "path": "roles/container-engine/cri-o/templates/http-proxy.conf.j2", "content": "[Service]\nEnvironment={% if http_proxy is defined %}\"HTTP_PROXY={{ http_proxy }}\"{% endif %} {% if https_proxy is defined %}\"HTTPS_PROXY={{ https_proxy }}\"{% endif %} {% if no_proxy is defined %}\"NO_PROXY={{ no_proxy }}\"{% endif %}\n" }, { "path": "roles/container-engine/cri-o/templates/mounts.conf.j2", "content": "/usr/share/rhel/secrets:/run/secrets\n{% for mount in crio_additional_mounts %}\n{{ mount }}\n{% endfor %}\n" }, { "path": "roles/container-engine/cri-o/templates/registry.conf.j2", "content": "[[registry]]\nprefix = \"{{ item.prefix | default(item.location) }}\"\ninsecure = {{ item.insecure | default('false') | string | lower }}\nblocked = {{ item.blocked | default('false') | string | lower }}\nlocation = \"{{ item.location }}\"\n{% if item.mirrors is defined %}\n{% for mirror in item.mirrors %}\n\n[[registry.mirror]]\nlocation = \"{{ mirror.location }}\"\ninsecure = {{ mirror.insecure | default('false') | string | lower }}\n{% endfor %}\n{% endif %}\n" }, { "path": "roles/container-engine/cri-o/templates/unqualified.conf.j2", "content": "{%- set _unqualified_registries = [] -%}\n{% for _registry in crio_registries if _registry.unqualified -%}\n{% if _registry.prefix is defined -%}\n{{ _unqualified_registries.append(_registry.prefix) }}\n{% else %}\n{{ _unqualified_registries.append(_registry.location) }}\n{%- endif %}\n{%- endfor %}\n\nunqualified-search-registries = {{ _unqualified_registries | string }}\n" }, { "path": "roles/container-engine/cri-o/vars/v1.29.yml", "content": "---\ncrio_conmon: \"{{ bin_dir }}/crio-conmon\"\ncrio_runtime_bin_dir: \"{{ bin_dir }}\"\n\n# cri-o binary files\ncrio_bin_files:\n - crio-conmon\n - crio-conmonrs\n - crio-crun\n - crio-runc\n - crio\n - pinns\n\ncrio_status_command: crio status\n" }, { "path": "roles/container-engine/cri-o/vars/v1.31.yml", "content": "---\ncrio_conmon: \"{{ crio_libexec_dir }}/conmon\"\ncrio_runtime_bin_dir: \"{{ crio_libexec_dir }}\"\n\n# cri-o binary files\ncrio_bin_files:\n - crio\n - pinns\n\ncrio_libexec_files:\n - conmon\n - conmonrs\n - crun\n - runc\n\ncrio_status_command: crio status\n" }, { "path": "roles/container-engine/crictl/handlers/main.yml", "content": "---\n- name: Get crictl completion\n command: \"{{ bin_dir }}/crictl completion\"\n changed_when: false\n register: cri_completion\n check_mode: false\n\n- name: Install crictl completion\n copy:\n dest: /etc/bash_completion.d/crictl\n content: \"{{ cri_completion.stdout }}\"\n mode: \"0644\"\n" }, { "path": "roles/container-engine/crictl/tasks/main.yml", "content": "---\n- name: Crictl | Download crictl\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.crictl) }}\"\n\n- name: Install crictl config\n template:\n src: crictl.yaml.j2\n dest: /etc/crictl.yaml\n owner: root\n mode: \"0644\"\n\n- name: Copy crictl binary from download dir\n copy:\n src: \"{{ local_release_dir }}/crictl\"\n dest: \"{{ bin_dir }}/crictl\"\n mode: \"0755\"\n remote_src: true\n notify:\n - Get crictl completion\n - Install crictl completion\n" }, { "path": "roles/container-engine/crictl/templates/crictl.yaml.j2", "content": "runtime-endpoint: {{ cri_socket }}\nimage-endpoint: {{ cri_socket }}\ntimeout: 30\ndebug: false\n" }, { "path": "roles/container-engine/crun/tasks/main.yml", "content": "---\n- name: Crun | Download crun binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.crun) }}\"\n\n- name: Copy crun binary from download dir\n copy:\n src: \"{{ downloads.crun.dest }}\"\n dest: \"{{ bin_dir }}/crun\"\n mode: \"0755\"\n remote_src: true\n" }, { "path": "roles/container-engine/docker/defaults/main.yml", "content": "---\ndocker_version: '28.3'\ndocker_cli_version: \"{{ docker_version }}\"\n\ndocker_package_info:\n pkgs:\n\n# Path where to store repo key\n# docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg\n\ndocker_repo_key_info:\n repo_keys:\n\ndocker_repo_info:\n repos:\n\ndocker_cgroup_driver: systemd\n\ndocker_bin_dir: \"/usr/bin\"\n\n# flag to enable/disable docker cleanup\ndocker_orphan_clean_up: false\n\n# old docker package names to be removed\ndocker_remove_packages_yum:\n - docker\n - docker-common\n - docker-engine\n - docker-selinux.noarch\n - docker-client\n - docker-client-latest\n - docker-latest\n - docker-latest-logrotate\n - docker-logrotate\n - docker-engine-selinux.noarch\n\n# remove podman to avoid containerd.io confliction\npodman_remove_packages_yum:\n - podman\n\ndocker_remove_packages_apt:\n - docker\n - docker-engine\n - docker.io\n\n# Docker specific repos should be part of the docker role not containerd-common anymore\n# Optional values for containerd apt repo\ncontainerd_package_info:\n pkgs:\n\n# Fedora docker-ce repo\ndocker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'\ndocker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'\n\n# CentOS/RedHat docker-ce repo\ndocker_rh_repo_base_url: 'https://download.docker.com/linux/rhel/{{ ansible_distribution_major_version }}/$basearch/stable'\ndocker_rh_repo_gpgkey: 'https://download.docker.com/linux/rhel/gpg'\n\n# Ubuntu docker-ce repo\ndocker_ubuntu_repo_base_url: \"https://download.docker.com/linux/ubuntu\"\ndocker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'\ndocker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'\n\n# Debian docker-ce repo\ndocker_debian_repo_base_url: \"https://download.docker.com/linux/debian\"\ndocker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'\ndocker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'\n" }, { "path": "roles/container-engine/docker/files/cleanup-docker-orphans.sh", "content": "#!/bin/bash\nlist_descendants ()\n{\n local children=$(ps -o pid= --ppid \"$1\")\n for pid in $children\n do\n list_descendants \"$pid\"\n done\n [[ -n \"$children\" ]] && echo \"$children\"\n}\n\nshim_search=\"^docker-containerd-shim|^containerd-shim\"\ncount_shim_processes=$(pgrep -f $shim_search | wc -l)\n\nif [ ${count_shim_processes} -gt 0 ]; then\n # Find all container pids from shims\n orphans=$(pgrep -P $(pgrep -d ',' -f $shim_search) |\\\n # Filter out valid docker pids, leaving the orphans\n egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf \"%s%s\",sep,$1; sep=\"|\"}'))\n\n if [[ -n \"$orphans\" && -n \"$(ps -o ppid= $orphans)\" ]]\n then\n # Get shim pids of orphans\n orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans))\n\n # Find all orphaned container PIDs\n orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done)\n\n # Recursively kill all child PIDs of orphan shims\n echo -e \"Killing orphan container PIDs and descendants: \\n$(ps -O ppid= $orphan_container_pids)\"\n kill -9 $orphan_container_pids || true\n\n else\n echo \"No orphaned containers found\"\n fi\nelse\n echo \"The node doesn't have any shim processes.\"\nfi\n" }, { "path": "roles/container-engine/docker/handlers/main.yml", "content": "---\n- name: Docker | reload systemd\n systemd_service:\n name: docker\n daemon_reload: true\n masked: false\n listen: Restart docker\n\n- name: Docker | reload docker.socket\n service:\n name: docker.socket\n state: restarted\n when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos\n listen: Restart docker\n\n\n- name: Docker | reload docker\n service:\n name: docker\n state: restarted\n listen: Restart docker\n\n\n- name: Docker | wait for docker\n command: \"{{ docker_bin_dir }}/docker images\"\n register: docker_ready\n retries: 20\n delay: 1\n until: docker_ready.rc == 0\n listen: Restart docker\n" }, { "path": "roles/container-engine/docker/meta/main.yml", "content": "---\ndependencies:\n - role: container-engine/containerd-common\n" }, { "path": "roles/container-engine/docker/tasks/docker_plugin.yml", "content": "---\n- name: Install Docker plugin\n command: docker plugin install --grant-all-permissions {{ docker_plugin | quote }}\n when: docker_plugin is defined\n register: docker_plugin_status\n failed_when:\n - docker_plugin_status.failed\n - '\"already exists\" not in docker_plugin_status.stderr'\n" }, { "path": "roles/container-engine/docker/tasks/main.yml", "content": "---\n- name: Check if fedora coreos\n stat:\n path: /run/ostree-booted\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: ostree\n\n- name: Set is_ostree\n set_fact:\n is_ostree: \"{{ ostree.stat.exists }}\"\n\n- name: Gather os specific variables\n include_vars: \"{{ item }}\"\n with_first_found:\n - files:\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ host_architecture }}.yml\"\n - \"{{ ansible_distribution | lower }}.yml\"\n - \"{{ ansible_distribution.split(' ')[0] | lower }}.yml\"\n - \"{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_os_family | lower }}-{{ host_architecture }}.yml\"\n - \"{{ ansible_os_family | lower }}.yml\"\n - defaults.yml\n paths:\n - ../vars\n skip: true\n tags:\n - facts\n\n- name: Warn about Docker version on SUSE\n debug:\n msg: \"SUSE distributions always install Docker from the distro repos\"\n when: ansible_pkg_mgr == 'zypper'\n\n- name: Gather DNS facts\n include_tasks: set_facts_dns.yml\n when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'\n tags:\n - facts\n\n- name: Pre-upgrade docker\n import_tasks: pre-upgrade.yml\n\n- name: Ensure docker-ce repository public key is installed\n apt_key:\n id: \"{{ item }}\"\n url: \"{{ docker_repo_key_info.url }}\"\n keyring: \"{{ docker_repo_key_keyring | default(omit) }}\"\n state: present\n register: keyserver_task_result\n until: keyserver_task_result is succeeded\n retries: 4\n delay: \"{{ retry_stagger }}\"\n with_items: \"{{ docker_repo_key_info.repo_keys }}\"\n environment: \"{{ proxy_env }}\"\n when: ansible_pkg_mgr == 'apt'\n\n# ref to https://github.com/kubernetes-sigs/kubespray/issues/11086 & 12424\n- name: Convert -backports sources to archive.debian.org for bullseye and older\n replace:\n path: \"{{ item }}\"\n regexp: '^(deb(?:-src)?\\s+)(?:https?://)?(?:[^ ]+debian\\.org)?([^ ]*/debian)(\\s+{{ ansible_distribution_release }}-backports\\b.*)'\n replace: '\\1http://archive.debian.org/debian\\3'\n backup: true\n loop: \"{{ query('fileglob', '/etc/apt/sources.list') }}\"\n when:\n - ansible_os_family == 'Debian'\n - ansible_distribution_release in ['bullseye', 'buster']\n\n- name: Ensure docker-ce repository is enabled\n apt_repository:\n repo: \"{{ item }}\"\n state: present\n with_items: \"{{ docker_repo_info.repos }}\"\n when: ansible_pkg_mgr == 'apt'\n\n- name: Configure docker repository on Fedora\n template:\n src: \"fedora_docker.repo.j2\"\n dest: \"{{ yum_repo_dir }}/docker.repo\"\n mode: \"0644\"\n when: ansible_distribution == \"Fedora\" and not is_ostree\n\n- name: Configure docker repository on RedHat/CentOS/OracleLinux/AlmaLinux/KylinLinux\n template:\n src: \"rh_docker.repo.j2\"\n dest: \"{{ yum_repo_dir }}/docker-ce.repo\"\n mode: \"0644\"\n when:\n - ansible_os_family == \"RedHat\"\n - ansible_distribution != \"Fedora\"\n - not is_ostree\n\n- name: Remove dpkg hold\n dpkg_selections:\n name: \"{{ item }}\"\n selection: install\n when: ansible_pkg_mgr == 'apt'\n register: ret\n changed_when: false\n failed_when:\n - ret is failed\n - ret.msg != ( \"Failed to find package '\" + item + \"' to perform selection 'install'.\" )\n with_items:\n - \"{{ containerd_package }}\"\n - docker-ce\n - docker-ce-cli\n\n- name: Ensure docker packages are installed\n package:\n name: \"{{ docker_package_info.pkgs }}\"\n state: \"{{ docker_package_info.state | default('present') }}\"\n module_defaults:\n apt:\n update_cache: true\n dnf:\n enablerepo: \"{{ docker_package_info.enablerepo | default(omit) }}\"\n disablerepo: \"{{ docker_package_info.disablerepo | default(omit) }}\"\n yum:\n enablerepo: \"{{ docker_package_info.enablerepo | default(omit) }}\"\n zypper:\n update_cache: true\n register: docker_task_result\n until: docker_task_result is succeeded\n retries: 4\n delay: \"{{ retry_stagger }}\"\n notify: Restart docker\n when:\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n - not is_ostree\n - docker_package_info.pkgs | length > 0\n\n# This is required to ensure any apt upgrade will not break kubernetes\n- name: Tell Debian hosts not to change the docker version with apt upgrade\n dpkg_selections:\n name: \"{{ item }}\"\n selection: hold\n when: ansible_pkg_mgr == 'apt'\n changed_when: false\n with_items:\n - \"{{ containerd_package }}\"\n - docker-ce\n - docker-ce-cli\n\n- name: Ensure docker started, remove our config if docker start failed and try again\n block:\n - name: Ensure service is started if docker packages are already present\n service:\n name: docker\n state: started\n when: docker_task_result is not changed\n rescue:\n - debug: # noqa name[missing]\n msg: \"Docker start failed. Try to remove our config\"\n - name: Remove kubespray generated config\n file:\n path: \"{{ item }}\"\n state: absent\n with_items:\n - /etc/systemd/system/docker.service.d/http-proxy.conf\n - /etc/systemd/system/docker.service.d/docker-options.conf\n - /etc/systemd/system/docker.service.d/docker-dns.conf\n - /etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf\n notify: Restart docker\n\n- name: Flush handlers so we can wait for docker to come up\n meta: flush_handlers\n\n# Install each plugin using a looped include to make error handling in the included task simpler.\n- name: Install docker plugin\n include_tasks: docker_plugin.yml\n loop: \"{{ docker_plugins }}\"\n loop_control:\n loop_var: docker_plugin\n\n- name: Set docker systemd config\n import_tasks: systemd.yml\n\n- name: Ensure docker service is started and enabled\n service:\n name: \"{{ item }}\"\n enabled: true\n state: started\n with_items:\n - docker\n" }, { "path": "roles/container-engine/docker/tasks/pre-upgrade.yml", "content": "---\n- name: Remove legacy docker repo file\n file:\n path: \"{{ yum_repo_dir }}/docker.repo\"\n state: absent\n when:\n - ansible_os_family == 'RedHat'\n - not is_ostree\n\n- name: Ensure old versions of Docker are not installed. | Debian\n apt:\n name: '{{ docker_remove_packages_apt }}'\n state: absent\n when:\n - ansible_os_family == 'Debian'\n - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))\n\n\n- name: Ensure podman not installed. | RedHat\n package:\n name: '{{ podman_remove_packages_yum }}'\n state: absent\n when:\n - ansible_os_family == 'RedHat'\n - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))\n - not is_ostree\n\n\n- name: Ensure old versions of Docker are not installed. | RedHat\n package:\n name: '{{ docker_remove_packages_yum }}'\n state: absent\n when:\n - ansible_os_family == 'RedHat'\n - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))\n - not is_ostree\n" }, { "path": "roles/container-engine/docker/tasks/reset.yml", "content": "---\n\n- name: Docker | Get package facts\n package_facts:\n manager: auto\n\n- name: Docker | Find docker packages\n set_fact:\n docker_packages_list: \"{{ ansible_facts.packages.keys() | select('search', '^docker+') }}\"\n containerd_package: \"{{ ansible_facts.packages.keys() | select('search', '^containerd+') }}\"\n\n- name: Docker | Stop all running container\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -q | xargs -r {{ docker_bin_dir }}/docker kill\"\n args:\n executable: /bin/bash\n register: stop_all_containers\n retries: 5\n until: stop_all_containers.rc == 0\n changed_when: true\n delay: 5\n ignore_errors: true # noqa ignore-errors\n when: docker_packages_list | length>0\n\n- name: Reset | remove all containers\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv\"\n args:\n executable: /bin/bash\n register: remove_all_containers\n retries: 4\n until: remove_all_containers.rc == 0\n delay: 5\n when: docker_packages_list | length>0\n\n- name: Docker | Stop docker service\n service:\n name: \"{{ item }}\"\n enabled: false\n state: stopped\n loop:\n - docker\n - docker.socket\n - containerd\n when: docker_packages_list | length>0\n\n- name: Docker | Remove dpkg hold\n dpkg_selections:\n name: \"{{ item }}\"\n selection: install\n when: ansible_pkg_mgr == 'apt'\n changed_when: false\n with_items:\n - \"{{ docker_packages_list }}\"\n - \"{{ containerd_package }}\"\n\n- name: Docker | Remove docker package\n package:\n name: \"{{ item }}\"\n state: absent\n changed_when: false\n with_items:\n - \"{{ docker_packages_list }}\"\n - \"{{ containerd_package }}\"\n when:\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n - not is_ostree\n - docker_packages_list | length > 0\n\n- name: Docker | ensure docker-ce repository is removed\n apt_repository:\n repo: \"{{ item }}\"\n state: absent\n with_items: \"{{ docker_repo_info.repos }}\"\n when: ansible_pkg_mgr == 'apt'\n\n- name: Docker | Remove docker repository on Fedora\n file:\n name: \"{{ yum_repo_dir }}/docker.repo\"\n state: absent\n when: ansible_distribution == \"Fedora\" and not is_ostree\n\n- name: Docker | Remove docker repository on RedHat/CentOS/Oracle/AlmaLinux Linux\n file:\n name: \"{{ yum_repo_dir }}/docker-ce.repo\"\n state: absent\n when:\n - ansible_os_family == \"RedHat\"\n - ansible_distribution != \"Fedora\"\n - not is_ostree\n\n- name: Docker | Remove docker configuration files\n file:\n name: \"{{ item }}\"\n state: absent\n loop:\n - /etc/systemd/system/docker.service.d/\n - /etc/systemd/system/docker.socket\n - /etc/systemd/system/docker.service\n - /etc/systemd/system/containerd.service\n - /etc/systemd/system/containerd.service.d\n - /var/lib/docker\n - /etc/docker\n ignore_errors: true # noqa ignore-errors\n\n- name: Docker | systemctl daemon-reload # noqa no-handler\n systemd_service:\n daemon_reload: true\n" }, { "path": "roles/container-engine/docker/tasks/set_facts_dns.yml", "content": "---\n\n- name: Set dns server for docker\n set_fact:\n docker_dns_servers: \"{{ dns_servers }}\"\n\n- name: Show docker_dns_servers\n debug:\n msg: \"{{ docker_dns_servers }}\"\n\n- name: Add upstream dns servers\n set_fact:\n docker_dns_servers: \"{{ docker_dns_servers + upstream_dns_servers }}\"\n when: dns_mode in ['coredns', 'coredns_dual']\n\n- name: Add global searchdomains\n set_fact:\n docker_dns_search_domains: \"{{ docker_dns_search_domains + searchdomains }}\"\n\n- name: Check system nameservers\n shell: set -o pipefail && grep \"^nameserver\" /etc/resolv.conf | sed -r 's/^nameserver\\s*([^#\\s]+)\\s*(#.*)?/\\1/'\n args:\n executable: /bin/bash\n changed_when: false\n register: system_nameservers\n check_mode: false\n\n- name: Check system search domains\n # noqa risky-shell-pipe - if resolf.conf has no search domain, grep will exit 1 which would force us to add failed_when: false\n # Therefore -o pipefail is not applicable in this specific instance\n shell: grep \"^search\" /etc/resolv.conf | sed -r 's/^search\\s*([^#]+)\\s*(#.*)?/\\1/'\n args:\n executable: /bin/bash\n changed_when: false\n register: system_search_domains\n check_mode: false\n\n- name: Add system nameservers to docker options\n set_fact:\n docker_dns_servers: \"{{ docker_dns_servers | union(system_nameservers.stdout_lines) | unique }}\"\n when: system_nameservers.stdout\n\n- name: Add system search domains to docker options\n set_fact:\n docker_dns_search_domains: \"{{ docker_dns_search_domains | union(system_search_domains.stdout.split() | default([])) | unique }}\"\n when: system_search_domains.stdout\n\n- name: Check number of nameservers\n fail:\n msg: \"Too many nameservers. You can relax this check by set docker_dns_servers_strict=false in docker.yml and we will only use the first 3.\"\n when: docker_dns_servers | length > 3 and docker_dns_servers_strict | bool\n\n- name: Rtrim number of nameservers to 3\n set_fact:\n docker_dns_servers: \"{{ docker_dns_servers[0:3] }}\"\n when: docker_dns_servers | length > 3 and not docker_dns_servers_strict | bool\n\n- name: Check number of search domains\n fail:\n msg: \"Too many search domains\"\n when: docker_dns_search_domains | length > 6\n\n- name: Check length of search domains\n fail:\n msg: \"Search domains exceeded limit of 256 characters\"\n when: docker_dns_search_domains | join(' ') | length > 256\n" }, { "path": "roles/container-engine/docker/tasks/systemd.yml", "content": "---\n- name: Create docker service systemd directory if it doesn't exist\n file:\n path: /etc/systemd/system/docker.service.d\n state: directory\n mode: \"0755\"\n\n- name: Write docker proxy drop-in\n template:\n src: http-proxy.conf.j2\n dest: /etc/systemd/system/docker.service.d/http-proxy.conf\n mode: \"0644\"\n notify: Restart docker\n when: http_proxy is defined or https_proxy is defined\n\n- name: Write docker.service systemd file\n template:\n src: docker.service.j2\n dest: /etc/systemd/system/docker.service\n mode: \"0644\"\n register: docker_service_file\n notify: Restart docker\n when:\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n - not is_fedora_coreos\n\n- name: Write docker options systemd drop-in\n template:\n src: docker-options.conf.j2\n dest: \"/etc/systemd/system/docker.service.d/docker-options.conf\"\n mode: \"0644\"\n notify: Restart docker\n\n- name: Write docker dns systemd drop-in\n template:\n src: docker-dns.conf.j2\n dest: \"/etc/systemd/system/docker.service.d/docker-dns.conf\"\n mode: \"0644\"\n notify: Restart docker\n when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'\n\n- name: Copy docker orphan clean up script to the node\n copy:\n src: cleanup-docker-orphans.sh\n dest: \"{{ bin_dir }}/cleanup-docker-orphans.sh\"\n mode: \"0755\"\n when: docker_orphan_clean_up | bool\n\n- name: Write docker orphan clean up systemd drop-in\n template:\n src: docker-orphan-cleanup.conf.j2\n dest: \"/etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf\"\n mode: \"0644\"\n notify: Restart docker\n when: docker_orphan_clean_up | bool\n\n- name: Flush handlers\n meta: flush_handlers\n" }, { "path": "roles/container-engine/docker/templates/docker-dns.conf.j2", "content": "[Service]\nEnvironment=\"DOCKER_DNS_OPTIONS=\\\n {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \\\n {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \\\n {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \\\n\"\n" }, { "path": "roles/container-engine/docker/templates/docker-options.conf.j2", "content": "[Service]\nEnvironment=\"DOCKER_OPTS={{ docker_options|default('') }} --iptables={{ docker_iptables_enabled | default('false') }} \\\n--exec-opt native.cgroupdriver={{ docker_cgroup_driver }} \\\n{% for i in docker_insecure_registries %}--insecure-registry={{ i }} {% endfor %} \\\n{% for i in docker_registry_mirrors %}--registry-mirror={{ i }} {% endfor %} \\\n--data-root={{ docker_daemon_graph }} \\\n{% if ansible_os_family not in [\"openSUSE Leap\", \"openSUSE Tumbleweed\", \"Suse\"] %}{{ docker_log_opts }}{% endif %}\"\n\n{% if docker_mount_flags is defined and docker_mount_flags != \"\" %}\nMountFlags={{ docker_mount_flags }}\n{% endif %}\n" }, { "path": "roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2", "content": "[Service]\nExecStartPost=-{{ bin_dir }}/cleanup-docker-orphans.sh\n" }, { "path": "roles/container-engine/docker/templates/docker.service.j2", "content": "[Unit]\nDescription=Docker Application Container Engine\nDocumentation=http://docs.docker.com\nAfter=network.target docker.socket containerd.service lvm2-monitor.service SuSEfirewall2.service\n{% if ansible_os_family != \"Suse\" %}\nBindsTo=containerd.service\n{% endif %}\nWants=docker.socket\n\n[Service]\nType=notify\n{% if docker_storage_options is defined %}\nEnvironment=\"DOCKER_STORAGE_OPTIONS={{ docker_storage_options }}\"\n{% endif %}\nEnvironment=GOTRACEBACK=crash\nExecReload=/bin/kill -s HUP $MAINPID\nDelegate=yes\nKillMode=process\nExecStart={{ docker_bin_dir }}/dockerd \\\n{% if ansible_os_family == \"Suse\" %}\n --add-runtime oci=/usr/sbin/docker-runc \\\n{% endif %}\n $DOCKER_OPTS \\\n $DOCKER_STORAGE_OPTIONS \\\n $DOCKER_DNS_OPTIONS\nTasksMax=infinity\nLimitNOFILE=1048576\nLimitNPROC=1048576\nLimitCORE=infinity\nTimeoutStartSec=1min\n# restart the docker process if it exits prematurely\nRestart=on-failure\nStartLimitBurst=10\nStartLimitInterval=60s\n# Set the cgroup slice of the service so that kube reserved takes effect\n{% if kube_reserved is defined and kube_reserved|bool %}\nSlice={{ kube_reserved_cgroups_for_service_slice }}\n{% endif %}\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/container-engine/docker/templates/fedora_docker.repo.j2", "content": "[docker-ce]\nname=Docker-CE Repository\nbaseurl={{ docker_fedora_repo_base_url }}\nenabled=1\ngpgcheck={{ '1' if docker_fedora_repo_gpgkey else '0' }}\ngpgkey={{ docker_fedora_repo_gpgkey }}\n{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}\n" }, { "path": "roles/container-engine/docker/templates/http-proxy.conf.j2", "content": "[Service]\nEnvironment={% if http_proxy is defined %}\"HTTP_PROXY={{ http_proxy }}\"{% endif %} {% if https_proxy is defined %}\"HTTPS_PROXY={{ https_proxy }}\"{% endif %} {% if no_proxy is defined %}\"NO_PROXY={{ no_proxy }}\"{% endif %}\n" }, { "path": "roles/container-engine/docker/templates/rh_docker.repo.j2", "content": "[docker-ce]\nname=Docker-CE Repository\nbaseurl={{ docker_rh_repo_base_url }}\nenabled=0\ngpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }}\nkeepcache={{ docker_rpm_keepcache | default('1') }}\ngpgkey={{ docker_rh_repo_gpgkey }}\n{% if http_proxy is defined %}\nproxy={{ http_proxy }}\n{% endif %}\n" }, { "path": "roles/container-engine/docker/vars/amazon.yml", "content": "---\n# https://docs.aws.amazon.com/en_us/AmazonECS/latest/developerguide/docker-basics.html\n\ndocker_versioned_pkg:\n 'latest': docker\n '18.09': docker-18.09.9ce-2.amzn2\n '19.03': docker-19.03.13ce-1.amzn2\n '20.10': docker-20.10.7-5.amzn2\n '24.0': docker-24.0.5-1.amzn2\n '25.0': docker-25.0.3-1.amzn2\n\ndocker_version: \"latest\"\n\ndocker_package_info:\n pkgs:\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n enablerepo: amzn2extra-docker\n" }, { "path": "roles/container-engine/docker/vars/clearlinux.yml", "content": "---\ndocker_package_info:\n pkgs:\n - \"containers-basic\"\n" }, { "path": "roles/container-engine/docker/vars/debian.yml", "content": "---\n# containerd package info is only relevant for docker\ncontainerd_versioned_pkg:\n 'latest': \"{{ containerd_package }}\"\n '1.3.7': \"{{ containerd_package }}=1.3.7-1\"\n '1.3.9': \"{{ containerd_package }}=1.3.9-1\"\n '1.4.3': \"{{ containerd_package }}=1.4.3-2\"\n '1.4.4': \"{{ containerd_package }}=1.4.4-1\"\n '1.4.6': \"{{ containerd_package }}=1.4.6-1\"\n '1.4.9': \"{{ containerd_package }}=1.4.9-1\"\n '1.4.12': \"{{ containerd_package }}=1.4.12-1\"\n '1.6.4': \"{{ containerd_package }}=1.6.4-1\"\n '1.6.6': \"{{ containerd_package }}=1.6.6-1\"\n '1.6.7': \"{{ containerd_package }}=1.6.7-1\"\n '1.6.8': \"{{ containerd_package }}=1.6.8-1\"\n '1.6.9': \"{{ containerd_package }}=1.6.9-1\"\n '1.6.10': \"{{ containerd_package }}=1.6.10-1\"\n '1.6.11': \"{{ containerd_package }}=1.6.11-1\"\n '1.6.12': \"{{ containerd_package }}=1.6.12-1\"\n '1.6.13': \"{{ containerd_package }}=1.6.13-1\"\n '1.6.14': \"{{ containerd_package }}=1.6.14-1\"\n '1.6.15': \"{{ containerd_package }}=1.6.15-1\"\n '1.6.16': \"{{ containerd_package }}=1.6.16-1\"\n '1.6.18': \"{{ containerd_package }}=1.6.18-1\"\n '1.6.28': \"{{ containerd_package }}=1.6.28-2\"\n '1.6.31': \"{{ containerd_package }}=1.6.31-1\"\n '1.6.32': \"{{ containerd_package }}=1.6.32-1\"\n '1.6.33': \"{{ containerd_package }}=1.6.33-1\"\n '1.7.18': \"{{ containerd_package }}=1.7.18-1\"\n '1.7.19': \"{{ containerd_package }}=1.7.19-1\"\n '1.7.20': \"{{ containerd_package }}=1.7.20-1\"\n '1.7.21': \"{{ containerd_package }}=1.7.21-1\"\n '1.7.22': \"{{ containerd_package }}=1.7.22-1\"\n '1.7.23': \"{{ containerd_package }}=1.7.23-1\"\n '1.7.24': \"{{ containerd_package }}=1.7.24-1\"\n '1.7.25': \"{{ containerd_package }}=1.7.25-1\"\n '1.7.26': \"{{ containerd_package }}=1.7.26-1\"\n '1.7.27': \"{{ containerd_package }}=1.7.27-1\"\n 'stable': \"{{ containerd_package }}=1.7.27-1\"\n 'edge': \"{{ containerd_package }}=1.7.27-1\"\n\n# https://download.docker.com/linux/debian/\ndocker_versioned_pkg:\n 'latest': docker-ce\n '18.09': docker-ce=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }}\n '19.03': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }}\n '20.10': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }}\n '23.0': docker-ce=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '24.0': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '25.0': docker-ce=5:25.0.5-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '26.0': docker-ce=5:26.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '26.1': docker-ce=5:26.1.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.0': docker-ce=5:27.0.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.1': docker-ce=5:27.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.2': docker-ce=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.3': docker-ce=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.4': docker-ce=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.5': docker-ce=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.0': docker-ce=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.1': docker-ce=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.2': docker-ce=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.3': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n 'stable': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n 'edge': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n\ndocker_cli_versioned_pkg:\n 'latest': docker-ce-cli\n '18.09': docker-ce-cli=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }}\n '19.03': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }}\n '20.10': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }}\n '23.0': docker-ce-cli=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '24.0': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '25.0': docker-ce-cli=5:25.0.5-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '26.0': docker-ce-cli=5:26.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '26.1': docker-ce-cli=5:26.1.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.0': docker-ce-cli=5:27.0.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.1': docker-ce-cli=5:27.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.2': docker-ce-cli=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.3': docker-ce-cli=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.4': docker-ce-cli=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '27.5': docker-ce-cli=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.0': docker-ce-cli=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.1': docker-ce-cli=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.2': docker-ce-cli=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n '28.3': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n 'stable': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n 'edge': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}\n\ndocker_package_info:\n pkgs:\n - \"{{ containerd_versioned_pkg[docker_containerd_version | string] }}\"\n - \"{{ docker_cli_versioned_pkg[docker_cli_version | string] }}\"\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n\ndocker_repo_key_info:\n url: '{{ docker_debian_repo_gpgkey }}'\n repo_keys:\n - '{{ docker_debian_repo_repokey }}'\n\ndocker_repo_info:\n repos:\n - >\n deb {{ docker_debian_repo_base_url }}\n {{ ansible_distribution_release | lower }}\n stable\n" }, { "path": "roles/container-engine/docker/vars/fedora.yml", "content": "---\n# containerd versions are only relevant for docker\ncontainerd_versioned_pkg:\n 'latest': \"{{ containerd_package }}\"\n '1.3.7': \"{{ containerd_package }}-1.3.7-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.3.9': \"{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.4.3': \"{{ containerd_package }}-1.4.3-3.2.fc{{ ansible_distribution_major_version }}\"\n '1.4.4': \"{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.4.6': \"{{ containerd_package }}-1.4.6-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.4.9': \"{{ containerd_package }}-1.4.9-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.4.12': \"{{ containerd_package }}-1.4.12-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.4': \"{{ containerd_package }}-1.6.4-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.6': \"{{ containerd_package }}-1.6.6-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.7': \"{{ containerd_package }}-1.6.7-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.8': \"{{ containerd_package }}-1.6.8-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.9': \"{{ containerd_package }}-1.6.9-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.10': \"{{ containerd_package }}-1.6.10-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.11': \"{{ containerd_package }}-1.6.11-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.12': \"{{ containerd_package }}-1.6.12-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.13': \"{{ containerd_package }}-1.6.13-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.14': \"{{ containerd_package }}-1.6.14-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.15': \"{{ containerd_package }}-1.6.15-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.16': \"{{ containerd_package }}-1.6.16-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.18': \"{{ containerd_package }}-1.6.18-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.28': \"{{ containerd_package }}-1.6.28-3.2.fc{{ ansible_distribution_major_version }}\"\n '1.6.31': \"{{ containerd_package }}-1.6.31-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.32': \"{{ containerd_package }}-1.6.32-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.6.33': \"{{ containerd_package }}-1.6.33-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.18': \"{{ containerd_package }}-1.7.18-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.19': \"{{ containerd_package }}-1.7.19-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.20': \"{{ containerd_package }}-1.7.20-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.21': \"{{ containerd_package }}-1.7.21-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.22': \"{{ containerd_package }}-1.7.22-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.23': \"{{ containerd_package }}-1.7.23-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.24': \"{{ containerd_package }}-1.7.24-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.25': \"{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.26': \"{{ containerd_package }}-1.7.26-3.1.fc{{ ansible_distribution_major_version }}\"\n '1.7.27': \"{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}\"\n 'stable': \"{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}\"\n 'edge': \"{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}\"\n\n# https://docs.docker.com/install/linux/docker-ce/fedora/\n# https://download.docker.com/linux/fedora//x86_64/stable/Packages/\ndocker_versioned_pkg:\n 'latest': docker-ce\n '19.03': docker-ce-19.03.15-3.fc{{ ansible_distribution_major_version }}\n '20.10': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }}\n '23.0': docker-ce-3:23.0.6-1.fc{{ ansible_distribution_major_version }}\n '24.0': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }}\n '26.0': docker-ce-3:26.0.2-1.fc{{ ansible_distribution_major_version }}\n '26.1': docker-ce-3:26.1.4-1.fc{{ ansible_distribution_major_version }}\n '27.0': docker-ce-3:27.0.3-1.fc{{ ansible_distribution_major_version }}\n '27.1': docker-ce-3:27.1.2-1.fc{{ ansible_distribution_major_version }}\n '27.2': docker-ce-3:27.2.1-1.fc{{ ansible_distribution_major_version }}\n '27.3': docker-ce-3:27.3.1-1.fc{{ ansible_distribution_major_version }}\n '27.4': docker-ce-3:27.4.1-1.fc{{ ansible_distribution_major_version }}\n '27.5': docker-ce-3:27.5.1-1.fc{{ ansible_distribution_major_version }}\n '28.0': docker-ce-3:28.0.4-1.fc{{ ansible_distribution_major_version }}\n '28.1': docker-ce-3:28.1.1-1.fc{{ ansible_distribution_major_version }}\n '28.2': docker-ce-3:28.2.2-1.fc{{ ansible_distribution_major_version }}\n '28.3': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }}\n 'stable': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }}\n 'edge': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }}\n\n\ndocker_cli_versioned_pkg:\n 'latest': docker-ce-cli\n '19.03': docker-ce-cli-19.03.15-3.fc{{ ansible_distribution_major_version }}\n '20.10': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }}\n '23.0': docker-ce-cli-1:23.0.6-1.fc{{ ansible_distribution_major_version }}\n '24.0': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }}\n '26.0': docker-ce-cli-1:26.0.2-1.fc{{ ansible_distribution_major_version }}\n '26.1': docker-ce-cli-1:26.1.4-1.fc{{ ansible_distribution_major_version }}\n '27.0': docker-ce-cli-1:27.0.3-1.fc{{ ansible_distribution_major_version }}\n '27.1': docker-ce-cli-1:27.1.2-1.fc{{ ansible_distribution_major_version }}\n '27.2': docker-ce-cli-1:27.2.1-1.fc{{ ansible_distribution_major_version }}\n '27.3': docker-ce-cli-1:27.3.1-1.fc{{ ansible_distribution_major_version }}\n '27.4': docker-ce-cli-1:27.4.1-1.fc{{ ansible_distribution_major_version }}\n '27.5': docker-ce-cli-1:27.5.1-1.fc{{ ansible_distribution_major_version }}\n '28.0': docker-ce-cli-1:28.0.4-1.fc{{ ansible_distribution_major_version }}\n '28.1': docker-ce-cli-1:28.1.1-1.fc{{ ansible_distribution_major_version }}\n '28.2': docker-ce-cli-1:28.2.2-1.fc{{ ansible_distribution_major_version }}\n '28.3': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }}\n 'stable': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }}\n 'edge': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }}\n\ndocker_package_info:\n enablerepo: \"docker-ce\"\n pkgs:\n - \"{{ containerd_versioned_pkg[docker_containerd_version | string] }}\"\n - \"{{ docker_cli_versioned_pkg[docker_cli_version | string] }}\"\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n" }, { "path": "roles/container-engine/docker/vars/kylin.yml", "content": "---\n\ndocker_version: 26.1\ndocker_cli_version: \"{{ docker_version }}\"\ndocker_rh_repo_base_url: 'https://download.docker.com/linux/centos/8/$basearch/stable'\n\n# containerd versions are only relevant for docker\ncontainerd_versioned_pkg:\n 'latest': \"{{ containerd_package }}\"\n '1.3.7': \"{{ containerd_package }}-1.3.7-3.1.el8\"\n '1.3.9': \"{{ containerd_package }}-1.3.9-3.1.el8\"\n '1.4.3': \"{{ containerd_package }}-1.4.3-3.2.el8\"\n '1.4.4': \"{{ containerd_package }}-1.4.4-3.1.el8\"\n '1.4.6': \"{{ containerd_package }}-1.4.6-3.1.el8\"\n '1.4.9': \"{{ containerd_package }}-1.4.9-3.1.el8\"\n '1.4.12': \"{{ containerd_package }}-1.4.12-3.1.el8\"\n '1.6.4': \"{{ containerd_package }}-1.6.4-3.1.el8\"\n '1.6.6': \"{{ containerd_package }}-1.6.6-3.1.el8\"\n '1.6.7': \"{{ containerd_package }}-1.6.7-3.1.el8\"\n '1.6.8': \"{{ containerd_package }}-1.6.8-3.1.el8\"\n '1.6.9': \"{{ containerd_package }}-1.6.9-3.1.el8\"\n '1.6.10': \"{{ containerd_package }}-1.6.10-3.1.el8\"\n '1.6.11': \"{{ containerd_package }}-1.6.11-3.1.el8\"\n '1.6.12': \"{{ containerd_package }}-1.6.12-3.1.el8\"\n '1.6.13': \"{{ containerd_package }}-1.6.13-3.1.el8\"\n '1.6.14': \"{{ containerd_package }}-1.6.14-3.1.el8\"\n '1.6.15': \"{{ containerd_package }}-1.6.15-3.1.el8\"\n '1.6.16': \"{{ containerd_package }}-1.6.16-3.1.el8\"\n '1.6.18': \"{{ containerd_package }}-1.6.18-3.1.el8\"\n '1.6.28': \"{{ containerd_package }}-1.6.28-3.1.el8\"\n '1.6.31': \"{{ containerd_package }}-1.6.31-3.1.el8\"\n '1.6.32': \"{{ containerd_package }}-1.6.32-3.1.el8\"\n 'stable': \"{{ containerd_package }}-1.6.32-3.1.el8\"\n 'edge': \"{{ containerd_package }}-1.6.32-3.1.el8\"\n\n# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package\n# https://download.docker.com/linux/centos/8/x86_64/stable/Packages/\n# or do 'yum --showduplicates list docker-engine'\ndocker_versioned_pkg:\n 'latest': docker-ce\n '18.09': docker-ce-3:18.09.9-3.el8\n '19.03': docker-ce-3:19.03.15-3.el8\n '23.0': docker-ce-3:23.0.6-1.el8\n '24.0': docker-ce-3:24.0.9-1.el8\n '26.0': docker-ce-26.0.2-1.el8\n '26.1': docker-ce-26.1.2-1.el8\n 'stable': docker-ce-26.1.2-1.el8\n 'edge': docker-ce-26.1.2-1.el8\n\ndocker_cli_versioned_pkg:\n 'latest': docker-ce-cli\n '18.09': docker-ce-cli-1:18.09.9-3.el8\n '19.03': docker-ce-cli-1:19.03.15-3.el8\n '23.0': docker-ce-cli-1:23.0.6-1.el8\n '24.0': docker-ce-cli-1:24.0.9-1.el8\n '26.0': docker-ce-cli-26.0.2-1.el8\n '26.1': docker-ce-cli-26.1.2-1.el8\n 'stable': docker-ce-cli-26.1.2-1.el8\n 'edge': docker-ce-cli-26.1.2-1.el8\n\ndocker_package_info:\n enablerepo: \"docker-ce\"\n pkgs:\n - \"{{ containerd_versioned_pkg[docker_containerd_version | string] }}\"\n - \"{{ docker_cli_versioned_pkg[docker_cli_version | string] }}\"\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n" }, { "path": "roles/container-engine/docker/vars/redhat.yml", "content": "---\n# containerd versions are only relevant for docker\ncontainerd_versioned_pkg:\n 'latest': \"{{ containerd_package }}\"\n '1.3.7': \"{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}\"\n '1.3.9': \"{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.3': \"{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}\"\n '1.4.4': \"{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.6': \"{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.9': \"{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.12': \"{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.4': \"{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.6': \"{{ containerd_package }}-1.6.6-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.7': \"{{ containerd_package }}-1.6.7-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.8': \"{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.9': \"{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.10': \"{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.11': \"{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.12': \"{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.13': \"{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.14': \"{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.15': \"{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.16': \"{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.18': \"{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.28': \"{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.31': \"{{ containerd_package }}-1.6.31-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.32': \"{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.33': \"{{ containerd_package }}-1.6.33-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.18': \"{{ containerd_package }}-1.7.18-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.19': \"{{ containerd_package }}-1.7.19-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.20': \"{{ containerd_package }}-1.7.20-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.21': \"{{ containerd_package }}-1.7.21-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.22': \"{{ containerd_package }}-1.7.22-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.23': \"{{ containerd_package }}-1.7.23-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.24': \"{{ containerd_package }}-1.7.24-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.25': \"{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.26': \"{{ containerd_package }}-1.7.26-3.1.el{{ ansible_distribution_major_version }}\"\n '1.7.27': \"{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}\"\n 'stable': \"{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}\"\n 'edge': \"{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}\"\n\n# https://docs.docker.com/engine/installation/linux/rhel/#install-from-a-package\n# https://download.docker.com/linux/rhel/>/x86_64/stable/Packages/\n# or do 'yum --showduplicates list docker-engine'\ndocker_versioned_pkg:\n 'latest': docker-ce\n '18.09': docker-ce-3:18.09.9-3.el7\n '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}\n '20.10': docker-ce-3:20.10.24-3.el{{ ansible_distribution_major_version }}\n '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }}\n '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }}\n '26.0': docker-ce-3:26.0.2-1.el{{ ansible_distribution_major_version }}\n '26.1': docker-ce-3:26.1.4-1.el{{ ansible_distribution_major_version }}\n '27.0': docker-ce-3:27.0.3-1.el{{ ansible_distribution_major_version }}\n '27.1': docker-ce-3:27.1.2-1.el{{ ansible_distribution_major_version }}\n '27.2': docker-ce-3:27.2.1-1.el{{ ansible_distribution_major_version }}\n '27.3': docker-ce-3:27.3.1-1.el{{ ansible_distribution_major_version }}\n '27.4': docker-ce-3:27.4.1-1.el{{ ansible_distribution_major_version }}\n '27.5': docker-ce-3:27.5.1-1.el{{ ansible_distribution_major_version }}\n '28.0': docker-ce-3:28.0.4-1.el{{ ansible_distribution_major_version }}\n '28.1': docker-ce-3:28.1.1-1.el{{ ansible_distribution_major_version }}\n '28.2': docker-ce-3:28.2.2-1.el{{ ansible_distribution_major_version }}\n '28.3': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }}\n 'stable': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }}\n 'edge': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }}\n\ndocker_cli_versioned_pkg:\n 'latest': docker-ce-cli\n '18.09': docker-ce-cli-1:18.09.9-3.el7\n '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}\n '20.10': docker-ce-cli-1:20.10.24-3.el{{ ansible_distribution_major_version }}\n '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }}\n '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }}\n '26.0': docker-ce-cli-1:26.0.2-1.el{{ ansible_distribution_major_version }}\n '26.1': docker-ce-cli-1:26.1.4-1.el{{ ansible_distribution_major_version }}\n '27.0': docker-ce-cli-1:27.0.3-1.el{{ ansible_distribution_major_version }}\n '27.1': docker-ce-cli-1:27.1.2-1.el{{ ansible_distribution_major_version }}\n '27.2': docker-ce-cli-1:27.2.1-1.el{{ ansible_distribution_major_version }}\n '27.3': docker-ce-cli-1:27.3.1-1.el{{ ansible_distribution_major_version }}\n '27.4': docker-ce-cli-1:27.4.1-1.el{{ ansible_distribution_major_version }}\n '27.5': docker-ce-cli-1:27.5.1-1.el{{ ansible_distribution_major_version }}\n '28.0': docker-ce-cli-1:28.0.4-1.el{{ ansible_distribution_major_version }}\n '28.1': docker-ce-cli-1:28.1.1-1.el{{ ansible_distribution_major_version }}\n '28.2': docker-ce-cli-1:28.2.2-1.el{{ ansible_distribution_major_version }}\n '28.3': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }}\n 'stable': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }}\n 'edge': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }}\n\ndocker_package_info:\n enablerepo: \"docker-ce\"\n pkgs:\n - \"{{ containerd_versioned_pkg[docker_containerd_version | string] }}\"\n - \"{{ docker_cli_versioned_pkg[docker_cli_version | string] }}\"\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n" }, { "path": "roles/container-engine/docker/vars/suse.yml", "content": "---\ndocker_package_info:\n state: latest\n pkgs:\n - docker\n - containerd\n" }, { "path": "roles/container-engine/docker/vars/ubuntu.yml", "content": "---\n# containerd versions are only relevant for docker\ncontainerd_versioned_pkg:\n 'latest': \"{{ containerd_package }}\"\n '1.6.4': \"{{ containerd_package }}=1.6.4-1\"\n '1.6.6': \"{{ containerd_package }}=1.6.6-1\"\n '1.6.7': \"{{ containerd_package }}=1.6.7-1\"\n '1.6.8': \"{{ containerd_package }}=1.6.8-1\"\n '1.6.9': \"{{ containerd_package }}=1.6.9-1\"\n '1.6.10': \"{{ containerd_package }}=1.6.10-1\"\n '1.6.11': \"{{ containerd_package }}=1.6.11-1\"\n '1.6.12': \"{{ containerd_package }}=1.6.12-1\"\n '1.6.13': \"{{ containerd_package }}=1.6.13-1\"\n '1.6.14': \"{{ containerd_package }}=1.6.14-1\"\n '1.6.15': \"{{ containerd_package }}=1.6.15-1\"\n '1.6.16': \"{{ containerd_package }}=1.6.16-1\"\n '1.6.18': \"{{ containerd_package }}=1.6.18-1\"\n '1.6.28': \"{{ containerd_package }}=1.6.28-2\"\n '1.6.31': \"{{ containerd_package }}=1.6.31-1\"\n '1.6.32': \"{{ containerd_package }}=1.6.32-1\"\n '1.6.33': \"{{ containerd_package }}=1.6.33-1\"\n '1.7.18': \"{{ containerd_package }}=1.7.18-1\"\n '1.7.19': \"{{ containerd_package }}=1.7.19-1\"\n '1.7.20': \"{{ containerd_package }}=1.7.20-1\"\n '1.7.21': \"{{ containerd_package }}=1.7.21-1\"\n '1.7.22': \"{{ containerd_package }}=1.7.22-1\"\n '1.7.23': \"{{ containerd_package }}=1.7.23-1\"\n '1.7.24': \"{{ containerd_package }}=1.7.24-1\"\n '1.7.25': \"{{ containerd_package }}=1.7.25-1\"\n '1.7.26': \"{{ containerd_package }}=1.7.26-1\"\n '1.7.27': \"{{ containerd_package }}=1.7.27-1\"\n 'stable': \"{{ containerd_package }}=1.7.27-1\"\n 'edge': \"{{ containerd_package }}=1.7.27-1\"\n\n# https://download.docker.com/linux/ubuntu/\ndocker_versioned_pkg:\n 'latest': docker-ce\n '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }}\n '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }}\n '20.10': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }}\n '23.0': docker-ce=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '24.0': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '26.0': docker-ce=5:26.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '26.1': docker-ce=5:26.1.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.0': docker-ce=5:27.0.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.1': docker-ce=5:27.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.2': docker-ce=5:27.2.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.3': docker-ce=5:27.3.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.4': docker-ce=5:27.4.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.5': docker-ce=5:27.5.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.0': docker-ce=5:28.0.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.1': docker-ce=5:28.1.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.2': docker-ce=5:28.2.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.3': docker-ce=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n\ndocker_cli_versioned_pkg:\n 'latest': docker-ce-cli\n '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }}\n '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }}\n '20.10': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }}\n '23.0': docker-ce-cli=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '24.0': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '26.0': docker-ce-cli=5:26.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '26.1': docker-ce-cli=5:26.1.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.0': docker-ce-cli=5:27.0.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.1': docker-ce-cli=5:27.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.2': docker-ce-cli=5:27.2.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.3': docker-ce-cli=5:27.3.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.4': docker-ce-cli=5:27.4.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '27.5': docker-ce-cli=5:27.5.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.0': docker-ce-cli=5:28.0.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.1': docker-ce-cli=5:28.1.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.2': docker-ce-cli=5:28.2.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n '28.3': docker-ce-cli=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n 'stable': docker-ce-cli=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n 'edge': docker-ce-cli=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }}\n\ndocker_package_info:\n pkgs:\n - \"{{ containerd_versioned_pkg[docker_containerd_version | string] }}\"\n - \"{{ docker_cli_versioned_pkg[docker_cli_version | string] }}\"\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n\ndocker_repo_key_info:\n url: '{{ docker_ubuntu_repo_gpgkey }}'\n repo_keys:\n - '{{ docker_ubuntu_repo_repokey }}'\n\ndocker_repo_info:\n repos:\n - >\n deb [arch={{ host_architecture }}] {{ docker_ubuntu_repo_base_url }}\n {{ ansible_distribution_release | lower }}\n stable\n" }, { "path": "roles/container-engine/docker/vars/uniontech.yml", "content": "---\n# containerd versions are only relevant for docker\ncontainerd_versioned_pkg:\n 'latest': \"{{ containerd_package }}\"\n '1.3.7': \"{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}\"\n '1.3.9': \"{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.3': \"{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}\"\n '1.4.4': \"{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.6': \"{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.9': \"{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}\"\n '1.4.12': \"{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.4': \"{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.8': \"{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.9': \"{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.10': \"{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.11': \"{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.12': \"{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.13': \"{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.14': \"{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.15': \"{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.16': \"{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.18': \"{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.28': \"{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}\"\n '1.6.32': \"{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}\"\n 'stable': \"{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}\"\n 'edge': \"{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}\"\n\ndocker_version: 19.03\ndocker_cli_version: 19.03\n\n# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package\n# https://download.docker.com/linux/centos/>/x86_64/stable/Packages/\n# or do 'yum --showduplicates list docker-engine'\ndocker_versioned_pkg:\n 'latest': docker-ce\n '18.09': docker-ce-3:18.09.9-3.el7\n '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}\n '20.10': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }}\n '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }}\n '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }}\n 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }}\n 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }}\n\ndocker_cli_versioned_pkg:\n 'latest': docker-ce-cli\n '18.09': docker-ce-cli-1:18.09.9-3.el7\n '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}\n '20.10': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }}\n '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }}\n '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }}\n 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }}\n 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }}\n\ndocker_package_info:\n enablerepo: \"docker-ce\"\n disablerepo: \"UniontechOS-20-AppStream\"\n pkgs:\n - \"{{ containerd_versioned_pkg[docker_containerd_version | string] }}\"\n - \"{{ docker_cli_versioned_pkg[docker_cli_version | string] }}\"\n - \"{{ docker_versioned_pkg[docker_version | string] }}\"\n" }, { "path": "roles/container-engine/gvisor/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n vars:\n gvisor_enabled: true\n container_manager: containerd\n roles:\n - role: kubespray_defaults\n - role: container-engine/containerd\n - role: container-engine/gvisor\n" }, { "path": "roles/container-engine/gvisor/molecule/default/files/10-mynet.conf", "content": "{\n \"cniVersion\": \"0.2.0\",\n \"name\": \"mynet\",\n \"type\": \"bridge\",\n \"bridge\": \"cni0\",\n \"isGateway\": true,\n \"ipMasq\": true,\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"172.19.0.0/24\",\n \"routes\": [\n {\n \"dst\": \"0.0.0.0/0\"\n }\n ]\n }\n}\n" }, { "path": "roles/container-engine/gvisor/molecule/default/files/container.json", "content": "{\n \"metadata\": {\n \"name\": \"gvisor1\"\n },\n \"image\": {\n \"image\": \"quay.io/kubespray/hello-world:latest\"\n },\n \"log_path\": \"gvisor1.0.log\",\n \"linux\": {}\n}\n" }, { "path": "roles/container-engine/gvisor/molecule/default/files/sandbox.json", "content": "{\n \"metadata\": {\n \"name\": \"gvisor1\",\n \"namespace\": \"default\",\n \"attempt\": 1,\n \"uid\": \"hdishd83djaidwnduwk28bcsb\"\n },\n \"linux\": {},\n \"log_directory\": \"/tmp\"\n}\n" }, { "path": "roles/container-engine/gvisor/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\nplatforms:\n - cloud_image: ubuntu-2404\n name: ubuntu24\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - name: almalinux9\n cloud_image: almalinux-9\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n inventory:\n group_vars:\n k8s_cluster:\n gvisor_enabled: true\n container_manager: containerd\n playbooks:\n create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml\n prepare: ../../../molecule/prepare.yml\nverifier:\n name: ansible\n" }, { "path": "roles/container-engine/gvisor/molecule/default/verify.yml", "content": "---\n- name: Test gvisor\n hosts: all\n gather_facts: false\n tasks:\n - name: Get kubespray defaults\n import_role:\n name: ../../../../../kubespray_defaults\n - name: Test version\n command: \"{{ bin_dir }}/runsc --version\"\n register: runsc_version\n failed_when: >\n runsc_version is failed or\n 'runsc version' not in runsc_version.stdout\n\n- name: Test run container\n import_playbook: ../../../molecule/test_runtime.yml\n vars:\n container_runtime: runsc\n" }, { "path": "roles/container-engine/gvisor/tasks/main.yml", "content": "---\n- name: GVisor | Download runsc binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.gvisor_runsc) }}\"\n\n- name: GVisor | Download containerd-shim-runsc-v1 binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}\"\n\n- name: GVisor | Copy binaries\n copy:\n src: \"{{ item.src }}\"\n dest: \"{{ bin_dir }}/{{ item.dest }}\"\n mode: \"0755\"\n remote_src: true\n with_items:\n - { src: \"{{ downloads.gvisor_runsc.dest }}\", dest: \"runsc\" }\n - { src: \"{{ downloads.gvisor_containerd_shim.dest }}\", dest: \"containerd-shim-runsc-v1\" }\n" }, { "path": "roles/container-engine/kata-containers/defaults/main.yml", "content": "---\nkata_containers_dir: /opt/kata\nkata_containers_config_dir: /etc/kata-containers\nkata_containers_containerd_bin_dir: /usr/local/bin\n\nkata_containers_qemu_default_memory: \"{{ ansible_memtotal_mb }}\"\nkata_containers_qemu_debug: 'false'\nkata_containers_qemu_sandbox_cgroup_only: 'true'\nkata_containers_qemu_enable_mem_prealloc: 'false'\nkata_containers_virtio_fs_cache: 'always'\n" }, { "path": "roles/container-engine/kata-containers/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n vars:\n kata_containers_enabled: true\n container_manager: containerd\n roles:\n - role: kubespray_defaults\n - role: container-engine/containerd\n - role: container-engine/kata-containers\n" }, { "path": "roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf", "content": "{\n \"cniVersion\": \"0.2.0\",\n \"name\": \"mynet\",\n \"type\": \"bridge\",\n \"bridge\": \"cni0\",\n \"isGateway\": true,\n \"ipMasq\": true,\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"172.19.0.0/24\",\n \"routes\": [\n {\n \"dst\": \"0.0.0.0/0\"\n }\n ]\n }\n}\n" }, { "path": "roles/container-engine/kata-containers/molecule/default/files/container.json", "content": "{\n \"metadata\": {\n \"name\": \"kata1\"\n },\n \"image\": {\n \"image\": \"quay.io/kubespray/hello-world:latest\"\n },\n \"log_path\": \"kata1.0.log\",\n \"linux\": {}\n}\n" }, { "path": "roles/container-engine/kata-containers/molecule/default/files/sandbox.json", "content": "{\n \"metadata\": {\n \"name\": \"kata1\",\n \"namespace\": \"default\",\n \"attempt\": 1,\n \"uid\": \"hdishd83djaidwnduwk28bcsb\"\n },\n \"linux\": {},\n \"log_directory\": \"/tmp\"\n}\n" }, { "path": "roles/container-engine/kata-containers/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\nplatforms:\n - name: ubuntu22\n cloud_image: ubuntu-2204\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - name: ubuntu24\n cloud_image: ubuntu-2404\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n playbooks:\n create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml\n prepare: ../../../molecule/prepare.yml\nverifier:\n name: ansible\n" }, { "path": "roles/container-engine/kata-containers/molecule/default/verify.yml", "content": "---\n- name: Test kata-containers\n hosts: all\n gather_facts: false\n tasks:\n - name: Test version\n command: \"/opt/kata/bin/kata-runtime version\"\n register: version\n failed_when: >\n version is failed or\n 'kata-runtime' not in version.stdout\n - name: Test version\n command: \"/opt/kata/bin/kata-runtime check\"\n register: check\n failed_when: >\n check is failed or\n 'System is capable of running' not in check.stdout\n\n- name: Test run container\n import_playbook: ../../../molecule/test_runtime.yml\n vars:\n container_runtime: kata-qemu\n container_manager: containerd\n" }, { "path": "roles/container-engine/kata-containers/tasks/main.yml", "content": "---\n- name: Kata-containers | Download kata binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.kata_containers) }}\"\n\n- name: Kata-containers | Copy kata-containers binary\n unarchive:\n src: \"{{ downloads.kata_containers.dest }}\"\n dest: \"/\"\n mode: \"0755\"\n owner: root\n group: root\n remote_src: true\n\n- name: Kata-containers | Create config directory\n file:\n path: \"{{ kata_containers_config_dir }}\"\n state: directory\n mode: \"0755\"\n\n- name: Kata-containers | Set configuration\n template:\n src: \"{{ item }}.j2\"\n dest: \"{{ kata_containers_config_dir }}/{{ item }}\"\n mode: \"0644\"\n with_items:\n - configuration-qemu.toml\n\n- name: Kata-containers | Set containerd bin\n vars:\n shim: \"{{ item }}\"\n template:\n dest: \"{{ kata_containers_containerd_bin_dir }}/containerd-shim-kata-{{ item }}-v2\"\n src: containerd-shim-kata-v2.j2\n mode: \"0755\"\n with_items:\n - qemu\n\n- name: Kata-containers | Load vhost kernel modules\n community.general.modprobe:\n state: present\n name: \"{{ item }}\"\n with_items:\n - vhost_vsock\n - vhost_net\n\n- name: Kata-containers | Persist vhost kernel modules\n copy:\n dest: /etc/modules-load.d/kubespray-kata-containers.conf\n mode: \"0644\"\n content: |\n vhost_vsock\n vhost_net\n" }, { "path": "roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2", "content": "# Copyright (c) 2017-2019 Intel Corporation\n# Copyright (c) 2021 Adobe Inc.\n#\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# XXX: WARNING: this file is auto-generated.\n# XXX:\n# XXX: Source file: \"config/configuration-qemu.toml.in\"\n# XXX: Project:\n# XXX: Name: Kata Containers\n# XXX: Type: kata\n\n[hypervisor.qemu]\npath = \"/opt/kata/bin/qemu-system-x86_64\"\n{% if kata_containers_version is version('2.2.0', '>=') %}\nkernel = \"/opt/kata/share/kata-containers/vmlinux.container\"\n{% else %}\nkernel = \"/opt/kata/share/kata-containers/vmlinuz.container\"\n{% endif %}\nimage = \"/opt/kata/share/kata-containers/kata-containers.img\"\n# initrd = \"/opt/kata/share/kata-containers/kata-containers-initrd.img\"\nmachine_type = \"q35\"\n\n# rootfs filesystem type:\n# - ext4 (default)\n# - xfs\n# - erofs\nrootfs_type=\"ext4\"\n\n# Enable confidential guest support.\n# Toggling that setting may trigger different hardware features, ranging\n# from memory encryption to both memory and CPU-state encryption and integrity.\n# The Kata Containers runtime dynamically detects the available feature set and\n# aims at enabling the largest possible one, returning an error if none is\n# available, or none is supported by the hypervisor.\n#\n# Known limitations:\n# * Does not work by design:\n# - CPU Hotplug\n# - Memory Hotplug\n# - NVDIMM devices\n#\n# Default false\n# confidential_guest = true\n\n# Choose AMD SEV-SNP confidential guests\n# In case of using confidential guests on AMD hardware that supports both SEV\n# and SEV-SNP, the following enables SEV-SNP guests. SEV guests are default.\n# Default false\n# sev_snp_guest = true\n\n# Enable running QEMU VMM as a non-root user.\n# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as\n# a non-root random user. See documentation for the limitations of this mode.\n# rootless = true\n\n# List of valid annotation names for the hypervisor\n# Each member of the list is a regular expression, which is the base name\n# of the annotation, e.g. \"path\" for io.katacontainers.config.hypervisor.path\"\nenable_annotations = [\"enable_iommu\"]\n\n# List of valid annotations values for the hypervisor\n# Each member of the list is a path pattern as described by glob(3).\n# The default if not set is empty (all annotations rejected.)\n# Your distribution recommends: [\"/opt/kata/bin/qemu-system-x86_64\"]\nvalid_hypervisor_paths = [\"/opt/kata/bin/qemu-system-x86_64\"]\n\n# Optional space-separated list of options to pass to the guest kernel.\n# For example, use `kernel_params = \"vsyscall=emulate\"` if you are having\n# trouble running pre-2.15 glibc.\n#\n# WARNING: - any parameter specified here will take priority over the default\n# parameter value of the same name used to start the virtual machine.\n# Do not set values here unless you understand the impact of doing so as you\n# may stop the virtual machine from booting.\n# To see the list of default parameters, enable hypervisor debug, create a\n# container and look for 'default-kernel-parameters' log entries.\nkernel_params = \"\"\n\n# Path to the firmware.\n# If you want that qemu uses the default firmware leave this option empty\nfirmware = \"\"\n\n# Path to the firmware volume.\n# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables\n# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables\n# can be customized per each user while UEFI code is kept same.\nfirmware_volume = \"\"\n\n# Machine accelerators\n# comma-separated list of machine accelerators to pass to the hypervisor.\n# For example, `machine_accelerators = \"nosmm,nosmbus,nosata,nopit,static-prt,nofw\"`\nmachine_accelerators=\"\"\n\n# Qemu seccomp sandbox feature\n# comma-separated list of seccomp sandbox features to control the syscall access.\n# For example, `seccompsandbox= \"on,obsolete=deny,spawn=deny,resourcecontrol=deny\"`\n# Note: \"elevateprivileges=deny\" doesn't work with daemonize option, so it's removed from the seccomp sandbox\n# Another note: enabling this feature may reduce performance, you may enable\n# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html\n#seccompsandbox=\"on,obsolete=deny,spawn=deny,resourcecontrol=deny\"\n\n# CPU features\n# comma-separated list of cpu features to pass to the cpu\n# For example, `cpu_features = \"pmu=off,vmx=off\"\ncpu_features=\"pmu=off\"\n\n# Default number of vCPUs per SB/VM:\n# unspecified or 0 --> will be set to 1\n# < 0 --> will be set to the actual number of physical cores\n# > 0 <= number of physical cores --> will be set to the specified number\n# > number of physical cores --> will be set to the actual number of physical cores\ndefault_vcpus = 1\n\n# Default maximum number of vCPUs per SB/VM:\n# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number\n# of vCPUs supported by KVM if that number is exceeded\n# > 0 <= number of physical cores --> will be set to the specified number\n# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number\n# of vCPUs supported by KVM if that number is exceeded\n# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when\n# the actual number of physical cores is greater than it.\n# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU\n# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs\n# can be added to a SB/VM, but the memory footprint will be big. Another example, with\n# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of\n# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,\n# unless you know what are you doing.\n# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.\ndefault_maxvcpus = 0\n\n# Bridges can be used to hot plug devices.\n# Limitations:\n# * Currently only pci bridges are supported\n# * Until 30 devices per bridge can be hot plugged.\n# * Until 5 PCI bridges can be cold plugged per VM.\n# This limitation could be a bug in qemu or in the kernel\n# Default number of bridges per SB/VM:\n# unspecified or 0 --> will be set to 1\n# > 1 <= 5 --> will be set to the specified number\n# > 5 --> will be set to 5\ndefault_bridges = 1\n\n# Default memory size in MiB for SB/VM.\n# If unspecified then it will be set 2048 MiB.\ndefault_memory = {{ kata_containers_qemu_default_memory }}\n#\n# Default memory slots per SB/VM.\n# If unspecified then it will be set 10.\n# This is will determine the times that memory will be hotadded to sandbox/VM.\n#memory_slots = 10\n\n# Default maximum memory in MiB per SB / VM\n# unspecified or == 0 --> will be set to the actual amount of physical RAM\n# > 0 <= amount of physical RAM --> will be set to the specified number\n# > amount of physical RAM --> will be set to the actual amount of physical RAM\ndefault_maxmemory = 0\n\n# The size in MiB will be plused to max memory of hypervisor.\n# It is the memory address space for the NVDIMM devie.\n# If set block storage driver (block_device_driver) to \"nvdimm\",\n# should set memory_offset to the size of block device.\n# Default 0\n#memory_offset = 0\n\n# Specifies virtio-mem will be enabled or not.\n# Please note that this option should be used with the command\n# \"echo 1 > /proc/sys/vm/overcommit_memory\".\n# Default false\n#enable_virtio_mem = true\n\n# Disable block device from being used for a container's rootfs.\n# In case of a storage driver like devicemapper where a container's\n# root file system is backed by a block device, the block device is passed\n# directly to the hypervisor for performance reasons.\n# This flag prevents the block device from being passed to the hypervisor,\n# virtio-fs is used instead to pass the rootfs.\ndisable_block_device_use = false\n\n# Shared file system type:\n# - virtio-fs (default)\n# - virtio-9p\n# - virtio-fs-nydus\n{% if kata_containers_version is version('2.2.0', '>=') %}\nshared_fs = \"virtio-fs\"\n{% else %}\nshared_fs = \"virtio-9p\"\n{% endif %}\n\n# Path to vhost-user-fs daemon.\n{% if kata_containers_version is version('2.5.0', '>=') %}\nvirtio_fs_daemon = \"/opt/kata/libexec/virtiofsd\"\n{% else %}\nvirtio_fs_daemon = \"/opt/kata/libexec/kata-qemu/virtiofsd\"\n{% endif %}\n\n# List of valid annotations values for the virtiofs daemon\n# The default if not set is empty (all annotations rejected.)\n# Your distribution recommends: [\"/opt/kata/libexec/virtiofsd\"]\nvalid_virtio_fs_daemon_paths = [\n \"/opt/kata/libexec/virtiofsd\",\n \"/opt/kata/libexec/kata-qemu/virtiofsd\",\n]\n\n# Default size of DAX cache in MiB\nvirtio_fs_cache_size = 0\n\n# Default size of virtqueues\nvirtio_fs_queue_size = 1024\n\n# Extra args for virtiofsd daemon\n#\n# Format example:\n# [\"--arg1=xxx\", \"--arg2=yyy\"]\n# Examples:\n# Set virtiofsd log level to debug : [\"--log-level=debug\"]\n#\n# see `virtiofsd -h` for possible options.\nvirtio_fs_extra_args = [\"--thread-pool-size=1\", \"--announce-submounts\"]\n\n# Cache mode:\n#\n# - never\n# Metadata, data, and pathname lookup are not cached in guest. They are\n# always fetched from host and any changes are immediately pushed to host.\n#\n# - auto\n# Metadata and pathname lookup cache expires after a configured amount of\n# time (default is 1 second). Data is cached while the file is open (close\n# to open consistency).\n#\n# - always\n# Metadata, data, and pathname lookup are cached in guest and never expire.\nvirtio_fs_cache = \"{{ kata_containers_virtio_fs_cache }}\"\n\n# Block storage driver to be used for the hypervisor in case the container\n# rootfs is backed by a block device. This is virtio-scsi, virtio-blk\n# or nvdimm.\nblock_device_driver = \"virtio-scsi\"\n\n# aio is the I/O mechanism used by qemu\n# Options:\n#\n# - threads\n# Pthread based disk I/O.\n#\n# - native\n# Native Linux I/O.\n#\n# - io_uring\n# Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and\n# qemu >=5.0.\nblock_device_aio = \"io_uring\"\n\n# Specifies cache-related options will be set to block devices or not.\n# Default false\n#block_device_cache_set = true\n\n# Specifies cache-related options for block devices.\n# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.\n# Default false\n#block_device_cache_direct = true\n\n# Specifies cache-related options for block devices.\n# Denotes whether flush requests for the device are ignored.\n# Default false\n#block_device_cache_noflush = true\n\n# Enable iothreads (data-plane) to be used. This causes IO to be\n# handled in a separate IO thread. This is currently only implemented\n# for SCSI.\n#\nenable_iothreads = false\n\n# Enable pre allocation of VM RAM, default false\n# Enabling this will result in lower container density\n# as all of the memory will be allocated and locked\n# This is useful when you want to reserve all the memory\n# upfront or in the cases where you want memory latencies\n# to be very predictable\n# Default false\nenable_mem_prealloc = {{ kata_containers_qemu_enable_mem_prealloc }}\n\n# Enable huge pages for VM RAM, default false\n# Enabling this will result in the VM memory\n# being allocated using huge pages.\n# This is useful when you want to use vhost-user network\n# stacks within the container. This will automatically\n# result in memory pre allocation\n#enable_hugepages = true\n\n# Enable vhost-user storage device, default false\n# Enabling this will result in some Linux reserved block type\n# major range 240-254 being chosen to represent vhost-user devices.\nenable_vhost_user_store = false\n\n# The base directory specifically used for vhost-user devices.\n# Its sub-path \"block\" is used for block devices; \"block/sockets\" is\n# where we expect vhost-user sockets to live; \"block/devices\" is where\n# simulated block device nodes for vhost-user devices to live.\nvhost_user_store_path = \"/var/run/kata-containers/vhost-user\"\n\n# Enable vIOMMU, default false\n# Enabling this will result in the VM having a vIOMMU device\n# This will also add the following options to the kernel's\n# command line: intel_iommu=on,iommu=pt\n#enable_iommu = true\n\n# Enable IOMMU_PLATFORM, default false\n# Enabling this will result in the VM device having iommu_platform=on set\n#enable_iommu_platform = true\n\n# List of valid annotations values for the vhost user store path\n# The default if not set is empty (all annotations rejected.)\n# Your distribution recommends: [\"/var/run/kata-containers/vhost-user\"]\nvalid_vhost_user_store_paths = [\"/var/run/kata-containers/vhost-user\"]\n\n# The timeout for reconnecting on non-server spdk sockets when the remote end goes away.\n# qemu will delay this many seconds and then attempt to reconnect.\n# Zero disables reconnecting, and the default is zero.\nvhost_user_reconnect_timeout_sec = 0\n\n# Enable file based guest memory support. The default is an empty string which\n# will disable this feature. In the case of virtio-fs, this is enabled\n# automatically and '/dev/shm' is used as the backing folder.\n# This option will be ignored if VM templating is enabled.\n#file_mem_backend = \"\"\n\n# List of valid annotations values for the file_mem_backend annotation\n# The default if not set is empty (all annotations rejected.)\n# Your distribution recommends: [\"\"]\nvalid_file_mem_backends = [\"\"]\n\n# -pflash can add image file to VM. The arguments of it should be in format\n# of [\"/path/to/flash0.img\", \"/path/to/flash1.img\"]\npflashes = []\n\n# This option changes the default hypervisor and kernel parameters\n# to enable debug output where available. And Debug also enables the hmp socket.\n#\n# Default false\nenable_debug = {{ kata_containers_qemu_debug }}\n\n# Disable the customizations done in the runtime when it detects\n# that it is running on top a VMM. This will result in the runtime\n# behaving as it would when running on bare metal.\n#\n#disable_nesting_checks = true\n\n# This is the msize used for 9p shares. It is the number of bytes\n# used for 9p packet payload.\n#msize_9p = 8192\n\n# If false and nvdimm is supported, use nvdimm device to plug guest image.\n# Otherwise virtio-block device is used.\n#\n# nvdimm is not supported when `confidential_guest = true`.\n#\n# Default is false\n#disable_image_nvdimm = true\n\n# VFIO devices are hotplugged on a bridge by default.\n# Enable hotplugging on root bus. This may be required for devices with\n# a large PCI bar, as this is a current limitation with hotplugging on\n# a bridge.\n# Default false\n#hotplug_vfio_on_root_bus = true\n\n# Before hot plugging a PCIe device, you need to add a pcie_root_port device.\n# Use this parameter when using some large PCI bar devices, such as Nvidia GPU\n# The value means the number of pcie_root_port\n# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is \"q35\"\n# Default 0\n#pcie_root_port = 2\n\n# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off\n# security (vhost-net runs ring0) for network I/O performance.\n#disable_vhost_net = true\n\n#\n# Default entropy source.\n# The path to a host source of entropy (including a real hardware RNG)\n# /dev/urandom and /dev/random are two main options.\n# Be aware that /dev/random is a blocking source of entropy. If the host\n# runs out of entropy, the VMs boot time will increase leading to get startup\n# timeouts.\n# The source of entropy /dev/urandom is non-blocking and provides a\n# generally acceptable source of entropy. It should work well for pretty much\n# all practical purposes.\n#entropy_source= \"/dev/urandom\"\n\n# List of valid annotations values for entropy_source\n# The default if not set is empty (all annotations rejected.)\n# Your distribution recommends: [\"/dev/urandom\",\"/dev/random\",\"\"]\nvalid_entropy_sources = [\"/dev/urandom\",\"/dev/random\",\"\"]\n\n# Path to OCI hook binaries in the *guest rootfs*.\n# This does not affect host-side hooks which must instead be added to\n# the OCI spec passed to the runtime.\n#\n# You can create a rootfs with hooks by customizing the osbuilder scripts:\n# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder\n#\n# Hooks must be stored in a subdirectory of guest_hook_path according to their\n# hook type, i.e. \"guest_hook_path/{prestart,poststart,poststop}\".\n# The agent will scan these directories for executable files and add them, in\n# lexicographical order, to the lifecycle of the guest container.\n# Hooks are executed in the runtime namespace of the guest. See the official documentation:\n# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks\n# Warnings will be logged if any error is encountered while scanning for hooks,\n# but it will not abort container execution.\n#guest_hook_path = \"/usr/share/oci/hooks\"\n#\n# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM).\n# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic.\n# Default 0-sized value means unlimited rate.\n#rx_rate_limiter_max_rate = 0\n# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM).\n# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block)\n# to discipline traffic.\n# Default 0-sized value means unlimited rate.\n#tx_rate_limiter_max_rate = 0\n\n# Set where to save the guest memory dump file.\n# If set, when GUEST_PANICKED event occurred,\n# guest memeory will be dumped to host filesystem under guest_memory_dump_path,\n# This directory will be created automatically if it does not exist.\n#\n# The dumped file(also called vmcore) can be processed with crash or gdb.\n#\n# WARNING:\n# Dump guest’s memory can take very long depending on the amount of guest memory\n# and use much disk space.\n#guest_memory_dump_path=\"/var/crash/kata\"\n\n# If enable paging.\n# Basically, if you want to use \"gdb\" rather than \"crash\",\n# or need the guest-virtual addresses in the ELF vmcore,\n# then you should enable paging.\n#\n# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details\n#guest_memory_dump_paging=false\n\n# Enable swap in the guest. Default false.\n# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device\n# if the swappiness of a container (set by annotation \"io.katacontainers.container.resource.swappiness\")\n# is bigger than 0.\n# The size of the swap device should be\n# swap_in_bytes (set by annotation \"io.katacontainers.container.resource.swap_in_bytes\") - memory_limit_in_bytes.\n# If swap_in_bytes is not set, the size should be memory_limit_in_bytes.\n# If swap_in_bytes and memory_limit_in_bytes is not set, the size should\n# be default_memory.\n#enable_guest_swap = true\n\n# use legacy serial for guest console if available and implemented for architecture. Default false\n#use_legacy_serial = true\n\n# disable applying SELinux on the VMM process (default false)\ndisable_selinux=false\n\n# disable applying SELinux on the container process\n# If set to false, the type `container_t` is applied to the container process by default.\n# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built\n# with `SELINUX=yes`.\n# (default: true)\ndisable_guest_selinux=true\n\n[factory]\n# VM templating support. Once enabled, new VMs are created from template\n# using vm cloning. They will share the same initial kernel, initramfs and\n# agent memory by mapping it readonly. It helps speeding up new container\n# creation and saves a lot of memory if there are many kata containers running\n# on the same host.\n#\n# When disabled, new VMs are created from scratch.\n#\n# Note: Requires \"initrd=\" to be set (\"image=\" is not supported).\n#\n# Default false\n#enable_template = true\n\n# Specifies the path of template.\n#\n# Default \"/run/vc/vm/template\"\n#template_path = \"/run/vc/vm/template\"\n\n# The number of caches of VMCache:\n# unspecified or == 0 --> VMCache is disabled\n# > 0 --> will be set to the specified number\n#\n# VMCache is a function that creates VMs as caches before using it.\n# It helps speed up new container creation.\n# The function consists of a server and some clients communicating\n# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto.\n# The VMCache server will create some VMs and cache them by factory cache.\n# It will convert the VM to gRPC format and transport it when gets\n# requestion from clients.\n# Factory grpccache is the VMCache client. It will request gRPC format\n# VM and convert it back to a VM. If VMCache function is enabled,\n# kata-runtime will request VM from factory grpccache when it creates\n# a new sandbox.\n#\n# Default 0\n#vm_cache_number = 0\n\n# Specify the address of the Unix socket that is used by VMCache.\n#\n# Default /var/run/kata-containers/cache.sock\n#vm_cache_endpoint = \"/var/run/kata-containers/cache.sock\"\n\n[agent.kata]\n# If enabled, make the agent display debug-level messages.\n# (default: disabled)\nenable_debug = {{ kata_containers_qemu_debug }}\n\n# Enable agent tracing.\n#\n# If enabled, the agent will generate OpenTelemetry trace spans.\n#\n# Notes:\n#\n# - If the runtime also has tracing enabled, the agent spans will be\n# associated with the appropriate runtime parent span.\n# - If enabled, the runtime will wait for the container to shutdown,\n# increasing the container shutdown time slightly.\n#\n# (default: disabled)\n#enable_tracing = true\n\n# Comma separated list of kernel modules and their parameters.\n# These modules will be loaded in the guest kernel using modprobe(8).\n# The following example can be used to load two kernel modules with parameters\n# - kernel_modules=[\"e1000e InterruptThrottleRate=3000,3000,3000 EEE=1\", \"i915 enable_ppgtt=0\"]\n# The first word is considered as the module name and the rest as its parameters.\n# Container will not be started when:\n# * A kernel module is specified and the modprobe command is not installed in the guest\n# or it fails loading the module.\n# * The module is not available in the guest or it doesn't met the guest kernel\n# requirements, like architecture and version.\n#\nkernel_modules=[]\n\n# Enable debug console.\n\n# If enabled, user can connect guest OS running inside hypervisor\n# through \"kata-runtime exec \" command\n\n#debug_console_enabled = true\n\n# Agent connection dialing timeout value in seconds\n# (default: 30)\n#dial_timeout = 30\n\n[runtime]\n# If enabled, the runtime will log additional debug messages to the\n# system log\n# (default: disabled)\nenable_debug = {{ kata_containers_qemu_debug }}\n#\n# Internetworking model\n# Determines how the VM should be connected to the\n# the container network interface\n# Options:\n#\n# - macvtap\n# Used when the Container network interface can be bridged using\n# macvtap.\n#\n# - none\n# Used when customize network. Only creates a tap device. No veth pair.\n#\n# - tcfilter\n# Uses tc filter rules to redirect traffic from the network interface\n# provided by plugin to a tap interface connected to the VM.\n#\ninternetworking_model=\"tcfilter\"\n\n# disable guest seccomp\n# Determines whether container seccomp profiles are passed to the virtual\n# machine and applied by the kata agent. If set to true, seccomp is not applied\n# within the guest\n# (default: true)\ndisable_guest_seccomp=true\n\n# vCPUs pinning settings\n# if enabled, each vCPU thread will be scheduled to a fixed CPU\n# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet)\n# enable_vcpus_pinning = false\n\n# Apply a custom SELinux security policy to the container process inside the VM.\n# This is used when you want to apply a type other than the default `container_t`,\n# so general users should not uncomment and apply it.\n# (format: \"user:role:type\")\n# Note: You cannot specify MCS policy with the label because the sensitivity levels and\n# categories are determined automatically by high-level container runtimes such as containerd.\n#guest_selinux_label=\"system_u:system_r:container_t\"\n\n# If enabled, the runtime will create opentracing.io traces and spans.\n# (See https://www.jaegertracing.io/docs/getting-started).\n# (default: disabled)\n#enable_tracing = true\n\n# Set the full url to the Jaeger HTTP Thrift collector.\n# The default if not set will be \"http://localhost:14268/api/traces\"\n#jaeger_endpoint = \"\"\n\n# Sets the username to be used if basic auth is required for Jaeger.\n#jaeger_user = \"\"\n\n# Sets the password to be used if basic auth is required for Jaeger.\n#jaeger_password = \"\"\n\n# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.\n# This option may have some potential impacts to your host. It should only be used when you know what you're doing.\n# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only\n# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge\n# (like OVS) directly.\n# (default: false)\n#disable_new_netns = true\n\n# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.\n# The container cgroups in the host are not created, just one single cgroup per sandbox.\n# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.\n# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.\n# The sandbox cgroup is constrained if there is no container type annotation.\n# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType\nsandbox_cgroup_only={{ kata_containers_qemu_sandbox_cgroup_only }}\n\n# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In\n# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful\n# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug.\n# Compatibility for determining appropriate sandbox (VM) size:\n# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O\n# does not yet support sandbox sizing annotations.\n# - When running single containers using a tool like ctr, container sizing information will be available.\nstatic_sandbox_resource_mgmt=false\n\n# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.\n# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.\n# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`\n# These will not be exposed to the container workloads, and are only provided for potential guest services.\nsandbox_bind_mounts=[]\n\n# VFIO Mode\n# Determines how VFIO devices should be be presented to the container.\n# Options:\n#\n# - vfio\n# Matches behaviour of OCI runtimes (e.g. runc) as much as\n# possible. VFIO devices will appear in the container as VFIO\n# character devices under /dev/vfio. The exact names may differ\n# from the host (they need to match the VM's IOMMU group numbers\n# rather than the host's)\n#\n# - guest-kernel\n# This is a Kata-specific behaviour that's useful in certain cases.\n# The VFIO device is managed by whatever driver in the VM kernel\n# claims it. This means it will appear as one or more device nodes\n# or network interfaces depending on the nature of the device.\n# Using this mode requires specially built workloads that know how\n# to locate the relevant device interfaces within the VM.\n#\nvfio_mode=\"guest-kernel\"\n\n# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will\n# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.\ndisable_guest_empty_dir=false\n\n# Enabled experimental feature list, format: [\"a\", \"b\"].\n# Experimental features are features not stable enough for production,\n# they may break compatibility, and are prepared for a big version bump.\n# Supported experimental features:\n# (default: [])\nexperimental=[]\n\n# If enabled, user can run pprof tools with shim v2 process through kata-monitor.\n# (default: false)\n# enable_pprof = true\n\n# WARNING: All the options in the following section have not been implemented yet.\n# This section was added as a placeholder. DO NOT USE IT!\n[image]\n# Container image service.\n#\n# Offload the CRI image management service to the Kata agent.\n# (default: false)\n#service_offload = true\n\n# Container image decryption keys provisioning.\n# Applies only if service_offload is true.\n# Keys can be provisioned locally (e.g. through a special command or\n# a local file) or remotely (usually after the guest is remotely attested).\n# The provision setting is a complete URL that lets the Kata agent decide\n# which method to use in order to fetch the keys.\n#\n# Keys can be stored in a local file, in a measured and attested initrd:\n#provision=data:///local/key/file\n#\n# Keys could be fetched through a special command or binary from the\n# initrd (guest) image, e.g. a firmware call:\n#provision=file:///path/to/bin/fetcher/in/guest\n#\n# Keys can be remotely provisioned. The Kata agent fetches them from e.g.\n# a HTTPS URL:\n#provision=https://my-key-broker.foo/tenant/\n" }, { "path": "roles/container-engine/kata-containers/templates/containerd-shim-kata-v2.j2", "content": "#!/bin/bash\nKATA_CONF_FILE={{ kata_containers_config_dir }}/configuration-{{ shim }}.toml {{ kata_containers_dir }}/bin/containerd-shim-kata-v2 $@\n" }, { "path": "roles/container-engine/molecule/files/10-mynet.conf", "content": "{\n \"cniVersion\": \"0.2.0\",\n \"name\": \"mynet\",\n \"type\": \"bridge\",\n \"bridge\": \"cni0\",\n \"isGateway\": true,\n \"ipMasq\": true,\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"172.19.0.0/24\",\n \"routes\": [\n {\n \"dst\": \"0.0.0.0/0\"\n }\n ]\n }\n}\n" }, { "path": "roles/container-engine/molecule/prepare.yml", "content": "---\n- name: Prepare\n hosts: all\n gather_facts: false\n become: true\n vars:\n ignore_assert_errors: true\n roles:\n - role: dynamic_groups\n - role: bootstrap_os\n - role: network_facts\n - role: kubernetes/preinstall\n - role: adduser\n user: \"{{ addusers.kube }}\"\n tasks:\n - name: Download CNI\n include_tasks: \"../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.cni) }}\"\n\n- name: Prepare CNI\n hosts: all\n gather_facts: false\n become: true\n vars:\n ignore_assert_errors: true\n kube_network_plugin: cni\n roles:\n - role: kubespray_defaults\n - role: network_plugin/cni\n tasks:\n - name: Create /etc/cni/net.d directory\n file:\n path: /etc/cni/net.d\n state: directory\n owner: root\n mode: \"0755\"\n - name: Config bridge host-local CNI\n copy:\n src: \"10-mynet.conf\"\n dest: \"/etc/cni/net.d/\"\n owner: root\n mode: \"0644\"\n" }, { "path": "roles/container-engine/molecule/templates/container.json.j2", "content": "{\n \"metadata\": {\n \"name\": \"{{ container_runtime }}1\"\n },\n \"image\": {\n \"image\": \"quay.io/kubespray/hello-world:latest\"\n },\n \"log_path\": \"{{ container_runtime }}1.0.log\",\n \"linux\": {}\n}\n" }, { "path": "roles/container-engine/molecule/templates/sandbox.json.j2", "content": "{\n \"metadata\": {\n \"name\": \"{{ container_runtime }}1\",\n \"namespace\": \"default\",\n \"attempt\": 1,\n \"uid\": \"hdishd83djaidwnduwk28bcsb\"\n },\n \"linux\": {},\n \"log_directory\": \"/tmp\"\n}\n" }, { "path": "roles/container-engine/molecule/test_cri.yml", "content": "---\n- name: Test container manager\n hosts: all\n gather_facts: false\n become: true\n tasks:\n - name: Get kubespray defaults\n import_role:\n name: ../../kubespray_defaults\n - name: Collect services facts\n ansible.builtin.service_facts:\n\n - name: Check container manager service is running\n assert:\n that:\n - ansible_facts.services[container_manager + '.service'].state == 'running'\n - ansible_facts.services[container_manager + '.service'].status == 'enabled'\n\n - name: Check runtime version\n command: \"{{ bin_dir }}/crictl --runtime-endpoint {{ cri_socket }} version\"\n register: cri_version\n failed_when: >\n cri_version is failed or\n (\"RuntimeName: \" + cri_name) not in cri_version.stdout\n" }, { "path": "roles/container-engine/molecule/test_runtime.yml", "content": "---\n- name: Test container runtime\n hosts: all\n gather_facts: false\n become: true\n roles:\n - role: ../../kubespray_defaults\n tasks:\n - name: Copy test container files\n template:\n src: \"{{ item }}.j2\"\n dest: \"/tmp/{{ item }}\"\n owner: root\n mode: \"0644\"\n loop:\n - container.json\n - sandbox.json\n - name: Check running a container with runtime {{ container_runtime }}\n block:\n - name: Run container\n command:\n argv:\n - \"{{ bin_dir }}/crictl\"\n - run\n - --with-pull\n - --runtime\n - \"{{ container_runtime }}\"\n - /tmp/container.json\n - /tmp/sandbox.json\n - name: Check log file\n slurp:\n src: \"/tmp/{{ container_runtime }}1.0.log\"\n register: log_file\n failed_when: >\n log_file is failed or\n 'Hello from Docker' not in (log_file.content | b64decode)\n rescue:\n - name: Display container manager config on error\n command: \"{{ bin_dir }}/crictl info\"\n - name: Check container manager logs\n command: journalctl -u {{ container_manager }}\n failed_when: true\n" }, { "path": "roles/container-engine/nerdctl/handlers/main.yml", "content": "---\n- name: Get nerdctl completion\n command: \"{{ bin_dir }}/nerdctl completion bash\"\n changed_when: false\n register: nerdctl_completion\n check_mode: false\n\n- name: Install nerdctl completion\n copy:\n dest: /etc/bash_completion.d/nerdctl\n content: \"{{ nerdctl_completion.stdout }}\"\n mode: \"0644\"\n" }, { "path": "roles/container-engine/nerdctl/tasks/main.yml", "content": "---\n- name: Nerdctl | Download nerdctl\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.nerdctl) }}\"\n\n- name: Nerdctl | Copy nerdctl binary from download dir\n copy:\n src: \"{{ local_release_dir }}/nerdctl\"\n dest: \"{{ bin_dir }}/nerdctl\"\n mode: \"0755\"\n remote_src: true\n owner: root\n group: root\n become: true\n notify:\n - Get nerdctl completion\n - Install nerdctl completion\n\n- name: Nerdctl | Create configuration dir\n file:\n path: /etc/nerdctl\n state: directory\n mode: \"0755\"\n owner: root\n group: root\n become: true\n\n- name: Nerdctl | Install nerdctl configuration\n template:\n src: nerdctl.toml.j2\n dest: /etc/nerdctl/nerdctl.toml\n mode: \"0644\"\n owner: root\n group: root\n become: true\n" }, { "path": "roles/container-engine/nerdctl/templates/nerdctl.toml.j2", "content": "debug = false\ndebug_full = false\naddress = \"{{ cri_socket }}\"\nnamespace = \"k8s.io\"\nsnapshotter = \"{{ nerdctl_snapshotter | default('overlayfs') }}\"\ncni_path = \"/opt/cni/bin\"\ncni_netconfpath = \"/etc/cni/net.d\"\ncgroup_manager = \"{{ kubelet_cgroup_driver | default('systemd') }}\"\nhosts_dir = [\"{{ containerd_cfg_dir }}/certs.d\"]\n" }, { "path": "roles/container-engine/runc/defaults/main.yml", "content": "---\n\nrunc_bin_dir: \"{{ bin_dir }}\"\n\nrunc_package_name: runc\n" }, { "path": "roles/container-engine/runc/tasks/main.yml", "content": "---\n- name: Runc | check if fedora coreos\n stat:\n path: /run/ostree-booted\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: ostree\n\n- name: Runc | set is_ostree\n set_fact:\n is_ostree: \"{{ ostree.stat.exists }}\"\n\n- name: Runc | Uninstall runc package managed by package manager\n when:\n - not is_ostree\n - ansible_distribution != \"Flatcar Container Linux by Kinvolk\"\n - ansible_distribution != \"Flatcar\"\n block:\n - name: Runc | Remove package\n package:\n name: \"{{ runc_package_name }}\"\n state: absent\n - name: Runc | Remove orphaned binary\n file:\n path: /usr/bin/runc\n state: absent\n when: runc_bin_dir != \"/usr/bin\"\n\n- name: Runc | Download runc binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.runc) }}\"\n\n- name: Copy runc binary from download dir\n copy:\n src: \"{{ downloads.runc.dest }}\"\n dest: \"{{ runc_bin_dir }}/runc\"\n mode: \"0755\"\n remote_src: true\n" }, { "path": "roles/container-engine/skopeo/tasks/main.yml", "content": "---\n- name: Skopeo | check if fedora coreos\n stat:\n path: /run/ostree-booted\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: ostree\n\n- name: Skopeo | set is_ostree\n set_fact:\n is_ostree: \"{{ ostree.stat.exists }}\"\n\n- name: Skopeo | Uninstall skopeo package managed by package manager\n package:\n name: skopeo\n state: absent\n when:\n - not (is_ostree or (ansible_distribution == \"Flatcar Container Linux by Kinvolk\") or (ansible_distribution == \"Flatcar\"))\n ignore_errors: true # noqa ignore-errors\n\n- name: Skopeo | Download skopeo binary\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.skopeo) }}\"\n\n- name: Copy skopeo binary from download dir\n copy:\n src: \"{{ downloads.skopeo.dest }}\"\n dest: \"{{ bin_dir }}/skopeo\"\n mode: \"0755\"\n remote_src: true\n" }, { "path": "roles/container-engine/tasks/main.yml", "content": "---\n- name: Validate container engine\n import_role:\n name: container-engine/validate-container-engine\n tags:\n - container-engine\n - validate-container-engine\n\n- name: Container runtimes\n include_role:\n name: \"container-engine/{{ item.role }}\"\n apply:\n tags:\n - container-engine\n - \"{{ item.role }}\"\n loop:\n - { role: 'kata-containers', enabled: \"{{ kata_containers_enabled }}\" }\n - { role: 'gvisor', enabled: \"{{ gvisor_enabled and container_manager in ['docker', 'containerd'] }}\" }\n - { role: 'crun', enabled: \"{{ crun_enabled }}\" }\n - { role: 'youki', enabled: \"{{ youki_enabled and container_manager == 'crio' }}\" }\n # TODO: Technically, this is more container-runtime than engine\n when: item.enabled\n tags:\n - container-engine\n - kata-containers\n - gvisor\n - crun\n - youki\n\n- name: Container Manager\n vars:\n container_manager_role:\n crio: cri-o\n docker: cri-dockerd\n containerd: containerd\n include_role:\n name: \"container-engine/{{ container_manager_role[container_manager] }}\"\n apply:\n tags:\n - container-engine\n - crio\n - docker\n - containerd\n tags:\n - container-engine\n - crio\n - docker\n - containerd\n" }, { "path": "roles/container-engine/validate-container-engine/tasks/main.yml", "content": "---\n- name: Validate-container-engine | check if fedora coreos\n stat:\n path: /run/ostree-booted\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: ostree\n tags:\n - facts\n\n- name: Validate-container-engine | set is_ostree\n set_fact:\n is_ostree: \"{{ ostree.stat.exists }}\"\n tags:\n - facts\n\n- name: Ensure kubelet systemd unit exists\n stat:\n path: \"/etc/systemd/system/kubelet.service\"\n register: kubelet_systemd_unit_exists\n tags:\n - facts\n\n- name: Populate service facts\n service_facts:\n tags:\n - facts\n\n- name: Check if containerd is installed\n find:\n file_type: file\n recurse: true\n use_regex: true\n patterns:\n - containerd.service$\n paths:\n - /lib/systemd\n - /etc/systemd\n - /run/systemd\n register: containerd_installed\n tags:\n - facts\n\n- name: Check if docker is installed\n find:\n file_type: file\n recurse: true\n use_regex: true\n patterns:\n - docker.service$\n paths:\n - /lib/systemd\n - /etc/systemd\n - /run/systemd\n register: docker_installed\n tags:\n - facts\n\n- name: Check if crio is installed\n find:\n file_type: file\n recurse: true\n use_regex: true\n patterns:\n - crio.service$\n paths:\n - /lib/systemd\n - /etc/systemd\n - /run/systemd\n register: crio_installed\n tags:\n - facts\n\n- name: Uninstall containerd\n vars:\n service_name: containerd.service\n when:\n - not (is_ostree or (ansible_distribution == \"Flatcar Container Linux by Kinvolk\") or (ansible_distribution == \"Flatcar\"))\n - container_manager != \"containerd\"\n - docker_installed.matched == 0\n - containerd_installed.matched > 0\n - ansible_facts.services[service_name]['state'] == 'running'\n block:\n - name: Drain node\n include_role:\n name: remove_node/pre_remove\n apply:\n tags:\n - pre-remove\n when: kubelet_systemd_unit_exists.stat.exists\n - name: Stop kubelet\n service:\n name: kubelet\n state: stopped\n when: kubelet_systemd_unit_exists.stat.exists\n - name: Remove Containerd\n import_role:\n name: container-engine/containerd\n tasks_from: reset\n handlers_from: reset\n\n- name: Uninstall docker\n vars:\n service_name: docker.service\n when:\n - not (is_ostree or (ansible_distribution == \"Flatcar Container Linux by Kinvolk\") or (ansible_distribution == \"Flatcar\"))\n - container_manager != \"docker\"\n - docker_installed.matched > 0\n - ansible_facts.services[service_name]['state'] == 'running'\n block:\n - name: Drain node\n include_role:\n name: remove_node/pre_remove\n apply:\n tags:\n - pre-remove\n when: kubelet_systemd_unit_exists.stat.exists\n - name: Stop kubelet\n service:\n name: kubelet\n state: stopped\n when: kubelet_systemd_unit_exists.stat.exists\n - name: Remove Docker\n import_role:\n name: container-engine/docker\n tasks_from: reset\n\n- name: Uninstall crio\n vars:\n service_name: crio.service\n when:\n - not (is_ostree or (ansible_distribution == \"Flatcar Container Linux by Kinvolk\") or (ansible_distribution == \"Flatcar\"))\n - container_manager != \"crio\"\n - crio_installed.matched > 0\n - ansible_facts.services[service_name]['state'] == 'running'\n block:\n - name: Drain node\n include_role:\n name: remove_node/pre_remove\n apply:\n tags:\n - pre-remove\n when: kubelet_systemd_unit_exists.stat.exists\n - name: Stop kubelet\n service:\n name: kubelet\n state: stopped\n when: kubelet_systemd_unit_exists.stat.exists\n - name: Remove CRI-O\n import_role:\n name: container-engine/cri-o\n tasks_from: reset\n" }, { "path": "roles/container-engine/youki/defaults/main.yml", "content": "---\n\nyouki_bin_dir: \"{{ bin_dir }}\"\n" }, { "path": "roles/container-engine/youki/molecule/default/converge.yml", "content": "---\n- name: Converge\n hosts: all\n become: true\n vars:\n youki_enabled: true\n container_manager: crio\n roles:\n - role: kubespray_defaults\n - role: container-engine/cri-o\n - role: container-engine/youki\n" }, { "path": "roles/container-engine/youki/molecule/default/files/10-mynet.conf", "content": "{\n \"cniVersion\": \"0.4.0\",\n \"name\": \"mynet\",\n \"type\": \"bridge\",\n \"bridge\": \"cni0\",\n \"isGateway\": true,\n \"ipMasq\": true,\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"172.19.0.0/24\",\n \"routes\": [\n {\n \"dst\": \"0.0.0.0/0\"\n }\n ]\n }\n}\n" }, { "path": "roles/container-engine/youki/molecule/default/files/container.json", "content": "{\n \"metadata\": {\n \"name\": \"youki1\"\n },\n \"image\": {\n \"image\": \"quay.io/kubespray/hello-world:latest\"\n },\n \"log_path\": \"youki1.0.log\",\n \"linux\": {}\n}\n" }, { "path": "roles/container-engine/youki/molecule/default/files/sandbox.json", "content": "{\n \"metadata\": {\n \"name\": \"youki1\",\n \"namespace\": \"default\",\n \"attempt\": 1,\n \"uid\": \"hdishd83djaidwnduwk28bcsb\"\n },\n \"linux\": {},\n \"log_directory\": \"/tmp\"\n}\n" }, { "path": "roles/container-engine/youki/molecule/default/molecule.yml", "content": "---\nrole_name_check: 1\nplatforms:\n - cloud_image: ubuntu-2404\n name: ubuntu24\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\n - name: almalinux9\n cloud_image: almalinux-9\n vm_cpu_cores: 1\n vm_memory: 1024\n node_groups:\n - kube_control_plane\nprovisioner:\n name: ansible\n env:\n ANSIBLE_ROLES_PATH: ../../../../\n config_options:\n defaults:\n callbacks_enabled: profile_tasks\n timeout: 120\n inventory:\n group_vars:\n k8s_cluster:\n youki_enabled: true\n container_manager: crio\n playbooks:\n create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml\n prepare: ../../../molecule/prepare.yml\nverifier:\n name: ansible\n" }, { "path": "roles/container-engine/youki/molecule/default/verify.yml", "content": "---\n- name: Test youki\n hosts: all\n gather_facts: false\n tasks:\n - name: Get kubespray defaults\n import_role:\n name: ../../../../../kubespray_defaults\n - name: Test version\n command: \"{{ bin_dir }}/youki --version\"\n register: youki_version\n failed_when: >\n youki_version is failed or\n 'youki' not in youki_version.stdout\n\n- name: Test run container\n import_playbook: ../../../molecule/test_runtime.yml\n vars:\n container_runtime: youki\n" }, { "path": "roles/container-engine/youki/tasks/main.yml", "content": "---\n- name: Youki | Download youki\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.youki) }}\"\n\n- name: Youki | Copy youki binary from download dir\n copy:\n src: \"{{ local_release_dir }}/youki\"\n dest: \"{{ youki_bin_dir }}/youki\"\n mode: \"0755\"\n remote_src: true\n" }, { "path": "roles/download/meta/main.yml", "content": "---\nallow_duplicates: true\n" }, { "path": "roles/download/tasks/check_pull_required.yml", "content": "---\n# The image_info_command depends on the Container Runtime and will output something like the following:\n# nginx:1.15,gcr.io/google-containers/kube-proxy:v1.14.1,gcr.io/google-containers/kube-proxy@sha256:44af2833c6cbd9a7fc2e9d2f5244a39dfd2e31ad91bf9d4b7d810678db738ee9,gcr.io/google-containers/kube-apiserver:v1.14.1,etc...\n- name: Check_pull_required | Generate a list of information about the images on a node # noqa command-instead-of-shell - image_info_command contains a pipe, therefore requiring shell\n shell: \"{{ image_info_command }}\"\n register: docker_images\n changed_when: false\n check_mode: false\n when: not download_always_pull\n\n- name: Check_pull_required | Set pull_required if the desired image is not yet loaded\n set_fact:\n pull_required: >-\n {%- if image_reponame | regex_replace('^docker\\.io/(library/)?', '') in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}\n when: not download_always_pull\n\n- name: Check_pull_required | Check that the local digest sha256 corresponds to the given image tag\n assert:\n that: \"{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')\"\n when:\n - not download_always_pull\n - not pull_required\n - pull_by_digest\n tags:\n - asserts\n" }, { "path": "roles/download/tasks/download_container.yml", "content": "---\n- tags:\n - download\n block:\n - name: Set default values for flag variables\n set_fact:\n image_is_cached: false\n image_changed: false\n pull_required: \"{{ download_always_pull }}\"\n tags:\n - facts\n\n - name: Download_container | Set a few facts\n import_tasks: set_container_facts.yml\n tags:\n - facts\n\n - name: Download_container | Prepare container download\n include_tasks: check_pull_required.yml\n when:\n - not download_always_pull\n\n - debug: # noqa name[missing]\n msg: \"Pull {{ image_reponame }} required is: {{ pull_required }}\"\n\n - name: Download_container | Determine if image is in cache\n stat:\n path: \"{{ image_path_cached }}\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n delegate_to: localhost\n connection: local\n delegate_facts: false\n register: cache_image\n changed_when: false\n become: false\n when:\n - download_force_cache\n\n - name: Download_container | Set fact indicating if image is in cache\n set_fact:\n image_is_cached: \"{{ cache_image.stat.exists }}\"\n tags:\n - facts\n when:\n - download_force_cache\n\n - name: Stop if image not in cache on ansible host when download_force_cache=true\n assert:\n that: image_is_cached\n msg: \"Image cache file {{ image_path_cached }} not found for {{ image_reponame }} on localhost\"\n when:\n - download_force_cache\n - not download_run_once\n\n - name: Download_container | Download image if required\n command: \"{{ image_pull_command_on_localhost if download_localhost else image_pull_command }} {{ image_reponame }}\"\n delegate_to: \"{{ download_delegate if download_run_once else inventory_hostname }}\"\n delegate_facts: true\n run_once: \"{{ download_run_once }}\"\n register: pull_task_result\n until: pull_task_result is succeeded\n delay: \"{{ retry_stagger | random + 3 }}\"\n retries: \"{{ download_retries }}\"\n become: \"{{ user_can_become_root | default(false) or not download_localhost }}\"\n environment: \"{{ proxy_env if container_manager == 'containerd' else omit }}\"\n when:\n - pull_required or download_run_once\n - not image_is_cached\n\n - name: Download_container | Save and compress image\n shell: \"{{ image_save_command_on_localhost if download_localhost else image_save_command }}\" # noqa command-instead-of-shell - image_save_command_on_localhost contains a pipe, therefore requires shell\n delegate_to: \"{{ download_delegate }}\"\n delegate_facts: false\n register: container_save_status\n failed_when: container_save_status.stderr\n run_once: true\n become: \"{{ user_can_become_root | default(false) or not download_localhost }}\"\n when:\n - not image_is_cached\n - download_run_once\n\n - name: Download_container | Copy image to ansible host cache\n ansible.posix.synchronize:\n src: \"{{ image_path_final }}\"\n dest: \"{{ image_path_cached }}\"\n use_ssh_args: true\n mode: pull\n when:\n - not image_is_cached\n - download_run_once\n - not download_localhost\n - download_delegate == inventory_hostname\n\n - name: Download_container | Upload image to node if it is cached\n ansible.posix.synchronize:\n src: \"{{ image_path_cached }}\"\n dest: \"{{ image_path_final }}\"\n use_ssh_args: true\n mode: push\n delegate_facts: false\n register: upload_image\n failed_when: not upload_image\n until: upload_image is succeeded\n retries: \"{{ download_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n when:\n - pull_required\n - download_force_cache\n\n - name: Download_container | Load image into the local container registry\n shell: \"{{ image_load_command }}\" # noqa command-instead-of-shell - image_load_command uses pipes, therefore requires shell\n register: container_load_status\n failed_when: container_load_status is failed\n when:\n - pull_required\n - download_force_cache\n\n - name: Download_container | Remove container image from cache\n file:\n state: absent\n path: \"{{ image_path_final }}\"\n when:\n - not download_keep_remote_cache\n" }, { "path": "roles/download/tasks/download_file.yml", "content": "---\n- name: \"Download_file | download {{ download.dest }}\"\n tags:\n - download\n block:\n - name: Prep_download | Set a few facts\n set_fact:\n download_force_cache: \"{{ true if download_run_once else download_force_cache }}\"\n\n - name: Download_file | Show url of file to download\n when: unsafe_show_logs | bool\n debug:\n msg: \"{{ download.url }}\"\n run_once: \"{{ download_run_once }}\"\n\n - name: Download_file | Set pathname of cached file\n set_fact:\n file_path_cached: \"{{ download_cache_dir }}/{{ download.dest | basename }}\"\n tags:\n - facts\n\n - name: Download_file | Create dest directory on node\n file:\n path: \"{{ download.dest | dirname }}\"\n owner: \"{{ download.owner | default(omit) }}\"\n mode: \"0755\"\n state: directory\n recurse: true\n\n - name: Download_file | Create local cache directory\n file:\n path: \"{{ file_path_cached | dirname }}\"\n state: directory\n recurse: true\n delegate_to: localhost\n connection: local\n delegate_facts: false\n run_once: true\n become: false\n when:\n - download_force_cache\n tags:\n - localhost\n\n - name: Download_file | Create cache directory on download_delegate host\n file:\n path: \"{{ file_path_cached | dirname }}\"\n state: directory\n recurse: true\n delegate_to: \"{{ download_delegate }}\"\n delegate_facts: false\n run_once: true\n when:\n - download_force_cache\n - not download_localhost\n\n # This must always be called, to check if the checksum matches. On no-match the file is re-downloaded.\n # This task will avoid logging it's parameters to not leak environment passwords in the log\n - name: Download_file | Download item\n get_url:\n url: \"{{ download.url }}\"\n dest: \"{{ file_path_cached if download_force_cache else download.dest }}\"\n owner: \"{{ omit if download_localhost else (download.owner | default(omit)) }}\"\n mode: \"{{ omit if download_localhost else (download.mode | default(omit)) }}\"\n checksum: \"{{ download.checksum }}\"\n validate_certs: \"{{ download_validate_certs }}\"\n url_username: \"{{ download.username | default(omit) }}\"\n url_password: \"{{ download.password | default(omit) }}\"\n force_basic_auth: \"{{ download.force_basic_auth | default(omit) }}\"\n timeout: \"{{ download.timeout | default(omit) }}\"\n delegate_to: \"{{ download_delegate if download_force_cache else inventory_hostname }}\"\n run_once: \"{{ download_force_cache }}\"\n register: get_url_result\n become: \"{{ not download_localhost }}\"\n until: \"'OK' in get_url_result.msg or\n 'file already exists' in get_url_result.msg or\n get_url_result.status_code | default() == 304\"\n retries: \"{{ download_retries }}\"\n delay: \"{{ retry_stagger | default(5) }}\"\n environment: \"{{ proxy_env }}\"\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n\n - name: Download_file | Copy file back to ansible host file cache\n ansible.posix.synchronize:\n src: \"{{ file_path_cached }}\"\n dest: \"{{ file_path_cached }}\"\n use_ssh_args: true\n mode: pull\n when:\n - download_force_cache\n - not download_localhost\n - download_delegate == inventory_hostname\n\n - name: Download_file | Copy file from cache to nodes, if it is available\n ansible.posix.synchronize:\n src: \"{{ file_path_cached }}\"\n dest: \"{{ download.dest }}\"\n use_ssh_args: true\n mode: push\n register: get_task\n until: get_task is succeeded\n delay: \"{{ retry_stagger | random + 3 }}\"\n retries: \"{{ download_retries }}\"\n when:\n - download_force_cache\n\n - name: Download_file | Set mode and owner\n file:\n path: \"{{ download.dest }}\"\n mode: \"{{ download.mode | default(omit) }}\"\n owner: \"{{ download.owner | default(omit) }}\"\n when:\n - download_force_cache\n\n - name: \"Download_file | Extract file archives\"\n include_tasks: \"extract_file.yml\"\n" }, { "path": "roles/download/tasks/extract_file.yml", "content": "---\n- name: Extract_file | Unpacking archive\n unarchive:\n src: \"{{ download.dest }}\"\n dest: \"{{ download.dest | dirname }}\"\n owner: \"{{ download.owner | default(omit) }}\"\n mode: \"{{ download.mode | default(omit) }}\"\n remote_src: true\n extra_opts: \"{{ download.unarchive_extra_opts | default(omit) }}\"\n when:\n - download.unarchive | default(false)\n" }, { "path": "roles/download/tasks/main.yml", "content": "---\n- name: Download | Prepare working directories and variables\n import_tasks: prep_download.yml\n when:\n - not skip_downloads | default(false)\n tags:\n - download\n - upload\n\n- name: Download | Get kubeadm binary and list of required images\n include_tasks: prep_kubeadm_images.yml\n when:\n - not skip_downloads | default(false)\n - ('kube_control_plane' in group_names)\n tags:\n - download\n - upload\n\n- name: Download | Download files / images\n include_tasks: \"{{ include_file }}\"\n loop: \"{{ downloads | combine(kubeadm_images) | dict2items }}\"\n vars:\n download: \"{{ download_defaults | combine(item.value) }}\"\n include_file: \"download_{% if download.container %}container{% else %}file{% endif %}.yml\"\n when:\n - not skip_downloads | default(false)\n - download.enabled\n - item.value.enabled\n - (not (item.value.container | default(false))) or (item.value.container and download_container)\n - (download_run_once and inventory_hostname == download_delegate) or (group_names | intersect(download.groups) | length)\n" }, { "path": "roles/download/tasks/prep_download.yml", "content": "---\n- name: Prep_download | Set a few facts\n set_fact:\n download_force_cache: \"{{ true if download_run_once else download_force_cache }}\"\n tags:\n - facts\n\n- name: Prep_download | On localhost, check if passwordless root is possible\n command: \"true\"\n delegate_to: localhost\n connection: local\n run_once: true\n register: test_become\n changed_when: false\n ignore_errors: true # noqa ignore-errors\n become: true\n when:\n - download_localhost\n tags:\n - localhost\n - asserts\n\n- name: Prep_download | On localhost, check if user has access to the container runtime without using sudo\n shell: \"{{ image_info_command_on_localhost }}\" # noqa command-instead-of-shell - image_info_command_on_localhost contains pipe, therefore requires shell\n delegate_to: localhost\n connection: local\n run_once: true\n register: test_docker\n changed_when: false\n ignore_errors: true # noqa ignore-errors\n become: false\n when:\n - download_localhost\n tags:\n - localhost\n - asserts\n\n- name: Prep_download | Parse the outputs of the previous commands\n set_fact:\n user_in_docker_group: \"{{ not test_docker.failed }}\"\n user_can_become_root: \"{{ not test_become.failed }}\"\n when:\n - download_localhost\n tags:\n - localhost\n - asserts\n\n- name: Prep_download | Check that local user is in group or can become root\n assert:\n that: \"user_in_docker_group or user_can_become_root\"\n msg: >-\n Error: User is not in docker group and cannot become root. When download_localhost is true, at least one of these two conditions must be met.\n when:\n - download_localhost\n tags:\n - localhost\n - asserts\n\n- name: Prep_download | Register docker images info\n shell: \"{{ image_info_command }}\" # noqa command-instead-of-shell - image_info_command contains pipe therefore requires shell\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n register: docker_images\n failed_when: false\n changed_when: false\n check_mode: false\n when: download_container\n\n- name: Prep_download | Create staging directory on remote node\n file:\n path: \"{{ local_release_dir }}/images\"\n state: directory\n mode: \"0755\"\n owner: \"{{ ansible_ssh_user | default(ansible_user_id) }}\"\n when:\n - ansible_os_family not in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n\n- name: Prep_download | Create local cache for files and images on control node\n file:\n path: \"{{ download_cache_dir }}/images\"\n state: directory\n mode: \"0755\"\n delegate_to: localhost\n connection: local\n delegate_facts: false\n run_once: true\n become: false\n when: download_force_cache or download_run_once\n tags:\n - localhost\n" }, { "path": "roles/download/tasks/prep_kubeadm_images.yml", "content": "---\n- name: Prep_kubeadm_images | Download kubeadm binary\n include_tasks: \"download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.kubeadm) }}\"\n when:\n - not skip_downloads | default(false)\n - downloads.kubeadm.enabled\n\n- name: Prep_kubeadm_images | Copy kubeadm binary from download dir to system path\n copy:\n src: \"{{ downloads.kubeadm.dest }}\"\n dest: \"{{ bin_dir }}/kubeadm\"\n mode: \"0755\"\n remote_src: true\n\n- name: Prep_kubeadm_images | Create kubeadm config\n template:\n src: \"kubeadm-images.yaml.j2\"\n dest: \"{{ kube_config_dir }}/kubeadm-images.yaml\"\n mode: \"0644\"\n validate: \"{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}\"\n when:\n - not skip_kubeadm_images | default(false)\n\n- name: Prep_kubeadm_images | Generate list of required images\n shell: \"set -o pipefail && {{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml | grep -Ev 'coredns|pause'\"\n args:\n executable: /bin/bash\n register: kubeadm_images_raw\n run_once: true\n changed_when: false\n when:\n - not skip_kubeadm_images | default(false)\n\n- name: Prep_kubeadm_images | Parse list of images\n vars:\n kubeadm_images_list: \"{{ kubeadm_images_raw.stdout_lines }}\"\n set_fact:\n kubeadm_image:\n key: \"kubeadm_{{ (item | regex_replace('^(?:.*\\\\/)*', '')).split(':')[0] }}\"\n value:\n enabled: true\n container: true\n repo: \"{{ item | regex_replace('^(.*):.*$', '\\\\1') }}\"\n tag: \"{{ item | regex_replace('^.*:(.*)$', '\\\\1') }}\"\n groups:\n - k8s_cluster\n loop: \"{{ kubeadm_images_list | flatten(levels=1) }}\"\n register: kubeadm_images_cooked\n run_once: true\n when:\n - not skip_kubeadm_images | default(false)\n\n- name: Prep_kubeadm_images | Convert list of images to dict for later use\n set_fact:\n kubeadm_images: \"{{ kubeadm_images_cooked.results | map(attribute='ansible_facts.kubeadm_image') | list | items2dict }}\"\n run_once: true\n when:\n - not skip_kubeadm_images | default(false)\n" }, { "path": "roles/download/tasks/set_container_facts.yml", "content": "---\n- name: Set_container_facts | Display the name of the image being processed\n debug:\n msg: \"{{ download.repo }}\"\n\n- name: Set_container_facts | Set if containers should be pulled by digest\n set_fact:\n pull_by_digest: \"{{ download.sha256 is defined and download.sha256 }}\"\n\n- name: Set_container_facts | Define by what name to pull the image\n set_fact:\n image_reponame: >-\n {%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%}\n\n- name: Set_container_facts | Define file name of image\n set_fact:\n image_filename: \"{{ image_reponame | regex_replace('/|\\0|:', '_') }}.tar\"\n\n- name: Set_container_facts | Define path of image\n set_fact:\n image_path_cached: \"{{ download_cache_dir }}/images/{{ image_filename }}\"\n image_path_final: \"{{ local_release_dir }}/images/{{ image_filename }}\"\n\n- name: Set image save/load command for docker\n set_fact:\n image_save_command: \"{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_final }}\"\n image_load_command: \"{{ docker_bin_dir }}/docker load < {{ image_path_final }}\"\n when: container_manager == 'docker'\n\n- name: Set image save/load command for containerd\n set_fact:\n image_save_command: \"{{ bin_dir }}/nerdctl -n k8s.io image save -o {{ image_path_final }} {{ image_reponame }}\"\n image_load_command: \"{{ bin_dir }}/nerdctl -n k8s.io image load < {{ image_path_final }}\"\n when: container_manager == 'containerd'\n\n- name: Set image save/load command for crio\n set_fact:\n image_save_command: \"{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null\"\n image_load_command: \"{{ bin_dir }}/skopeo copy docker-archive:{{ image_path_final }} containers-storage:{{ image_reponame }} 2>/dev/null\"\n when: container_manager == 'crio'\n\n- name: Set image save/load command for docker on localhost\n set_fact:\n image_save_command_on_localhost: \"{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached }}\"\n when: container_manager_on_localhost == 'docker'\n\n- name: Set image save/load command for containerd on localhost\n set_fact:\n image_save_command_on_localhost: \"{{ containerd_bin_dir }}/ctr -n k8s.io image export --platform linux/{{ image_arch }} {{ image_path_cached }} {{ image_reponame }}\"\n when: container_manager_on_localhost == 'containerd'\n\n- name: Set image save/load command for crio on localhost\n set_fact:\n image_save_command_on_localhost: \"{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null\"\n when: container_manager_on_localhost == 'crio'\n" }, { "path": "roles/download/templates/kubeadm-images.yaml.j2", "content": "apiVersion: kubeadm.k8s.io/v1beta4\nkind: InitConfiguration\nnodeRegistration:\n criSocket: {{ cri_socket }}\n---\napiVersion: kubeadm.k8s.io/v1beta4\nkind: ClusterConfiguration\nimageRepository: {{ kubeadm_image_repo }}\nkubernetesVersion: v{{ kube_version }}\netcd:\n{% if etcd_deployment_type == \"kubeadm\" %}\n local:\n imageRepository: \"{{ etcd_image_repo | regex_replace(\"/etcd$\",\"\") }}\"\n imageTag: \"{{ etcd_image_tag }}\"\n{% else %}\n external:\n endpoints:\n{% for endpoint in etcd_access_addresses.split(',') %}\n - {{ endpoint }}\n{% endfor %}\n{% endif %}\ndns:\n imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }}\n imageTag: {{ coredns_image_tag }}\n" }, { "path": "roles/dynamic_groups/tasks/main.yml", "content": "---\n- name: Match needed groups by their old names or definition\n vars:\n group_mappings:\n kube_control_plane:\n - kube-master\n kube_node:\n - kube-node\n calico_rr:\n - calico-rr\n no_floating:\n - no-floating\n k8s_cluster:\n - kube_node\n - kube_control_plane\n - calico_rr\n group_by:\n key: \"{{ item.key }}\"\n when: group_names | intersect(item.value) | length > 0\n loop: \"{{ group_mappings | dict2items }}\"\n" }, { "path": "roles/etcd/handlers/backup.yml", "content": "---\n- name: Refresh Time Fact\n setup:\n filter: ansible_date_time\n listen: Restart etcd\n when: etcd_cluster_is_healthy.rc == 0\n\n- name: Set Backup Directory\n set_fact:\n etcd_backup_directory: \"{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}\"\n listen: Restart etcd\n\n- name: Create Backup Directory\n file:\n path: \"{{ etcd_backup_directory }}\"\n state: directory\n owner: root\n group: root\n mode: \"0600\"\n listen: Restart etcd\n when: etcd_cluster_is_healthy.rc == 0\n\n- name: Stat etcd v2 data directory\n stat:\n path: \"{{ etcd_data_dir }}/member\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: etcd_data_dir_member\n listen: Restart etcd\n when: etcd_cluster_is_healthy.rc == 0\n\n- name: Backup etcd v2 data\n when:\n - etcd_data_dir_member.stat.exists\n - etcd_cluster_is_healthy.rc == 0\n - etcd_version is version('3.6.0', '<')\n command: >-\n {{ bin_dir }}/etcdctl backup\n --data-dir {{ etcd_data_dir }}\n --backup-dir {{ etcd_backup_directory }}\n environment:\n ETCDCTL_API: \"2\"\n retries: 3\n register: backup_v2_command\n until: backup_v2_command.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n listen: Restart etcd\n\n- name: Backup etcd v3 data\n command: >-\n {{ bin_dir }}/etcdctl\n snapshot save {{ etcd_backup_directory }}/snapshot.db\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses.split(',') | first }}\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n retries: 3\n register: etcd_backup_v3_command\n until: etcd_backup_v3_command.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n listen: Restart etcd\n when: etcd_cluster_is_healthy.rc == 0\n" }, { "path": "roles/etcd/handlers/backup_cleanup.yml", "content": "---\n- name: Find old etcd backups\n ansible.builtin.find:\n file_type: directory\n recurse: false\n paths: \"{{ etcd_backup_prefix }}\"\n patterns: \"etcd-*\"\n register: _etcd_backups\n when: etcd_backup_retention_count >= 0\n listen: Restart etcd\n\n- name: Remove old etcd backups\n ansible.builtin.file:\n state: absent\n path: \"{{ item }}\"\n loop: \"{{ (_etcd_backups.files | sort(attribute='ctime', reverse=True))[etcd_backup_retention_count:] | map(attribute='path') }}\"\n when: etcd_backup_retention_count >= 0\n listen: Restart etcd\n" }, { "path": "roles/etcd/handlers/main.yml", "content": "---\n- name: Backup etcd\n import_tasks: backup.yml\n\n- name: Restart etcd\n systemd_service:\n name: etcd\n state: restarted\n daemon_reload: true\n when: ('etcd' in group_names)\n throttle: \"{{ groups['etcd'] | length // 2 }}\"\n # Etcd cluster MUST have an odd number of members\n # Truncated integer division by 2 will always return (majority - 1) which\n # means the cluster will keep quorum and stay available\n\n- name: Restart etcd-events\n systemd_service:\n name: etcd-events\n state: restarted\n daemon_reload: true\n # TODO: this seems odd. etcd-events should be a different group possibly ?\n when: ('etcd' in group_names)\n throttle: \"{{ groups['etcd'] | length // 2 }}\"\n\n- name: Wait for etcd up\n uri:\n url: \"https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2379/health\"\n validate_certs: false\n client_cert: \"{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\"\n client_key: \"{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\"\n register: result\n until: result.status is defined and result.status == 200\n retries: 60\n delay: 1\n listen: Restart etcd\n\n- name: Cleanup etcd backups\n import_tasks: backup_cleanup.yml\n\n- name: Wait for etcd-events up\n uri:\n url: \"https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2383/health\"\n validate_certs: false\n client_cert: \"{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\"\n client_key: \"{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\"\n register: result\n until: result.status is defined and result.status == 200\n retries: 60\n delay: 1\n listen: Restart etcd-events\n\n- name: Set etcd_secret_changed\n set_fact:\n etcd_secret_changed: true\n" }, { "path": "roles/etcd/meta/main.yml", "content": "---\ndependencies:\n - role: adduser\n user: \"{{ addusers.etcd }}\"\n when: not (ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\", \"ClearLinux\"] or is_fedora_coreos)\n - role: adduser\n user: \"{{ addusers.kube }}\"\n when: not (ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\", \"ClearLinux\"] or is_fedora_coreos)\n - role: etcd_defaults\n" }, { "path": "roles/etcd/tasks/check_certs.yml", "content": "---\n- name: \"Check_certs | Register certs that have already been generated on first etcd node\"\n find:\n paths: \"{{ etcd_cert_dir }}\"\n patterns: \"ca.pem,node*.pem,member*.pem,admin*.pem\"\n get_checksum: true\n delegate_to: \"{{ groups['etcd'][0] }}\"\n register: etcdcert_master\n run_once: true\n\n- name: \"Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false\"\n set_fact:\n sync_certs: false\n gen_certs: false\n etcd_secret_changed: false\n\n- name: \"Check certs | Register ca and etcd admin/member certs on etcd hosts\"\n stat:\n path: \"{{ etcd_cert_dir }}/{{ item }}\"\n get_attributes: false\n get_checksum: true\n get_mime: false\n register: etcd_member_certs\n when: ('etcd' in group_names)\n with_items:\n - ca.pem\n - member-{{ inventory_hostname }}.pem\n - member-{{ inventory_hostname }}-key.pem\n - admin-{{ inventory_hostname }}.pem\n - admin-{{ inventory_hostname }}-key.pem\n\n- name: \"Check certs | Register ca and etcd node certs on kubernetes hosts\"\n stat:\n path: \"{{ etcd_cert_dir }}/{{ item }}\"\n register: etcd_node_certs\n when: ('k8s_cluster' in group_names)\n with_items:\n - ca.pem\n - node-{{ inventory_hostname }}.pem\n - node-{{ inventory_hostname }}-key.pem\n\n- name: \"Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(1/2)\"\n set_fact:\n gen_certs: true\n when: force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list\n run_once: true\n with_items: \"{{ expected_files }}\"\n vars:\n expected_files: >-\n ['{{ etcd_cert_dir }}/ca.pem',\n {% set etcd_members = groups['etcd'] %}\n {% for host in etcd_members %}\n '{{ etcd_cert_dir }}/admin-{{ host }}.pem',\n '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem',\n '{{ etcd_cert_dir }}/member-{{ host }}.pem',\n '{{ etcd_cert_dir }}/member-{{ host }}-key.pem',\n {% endfor %}\n {% set k8s_nodes = groups['kube_control_plane'] %}\n {% for host in k8s_nodes %}\n '{{ etcd_cert_dir }}/node-{{ host }}.pem',\n '{{ etcd_cert_dir }}/node-{{ host }}-key.pem'\n {% if not loop.last %}{{ ',' }}{% endif %}\n {% endfor %}]\n\n- name: \"Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(2/2)\"\n set_fact:\n gen_certs: true\n run_once: true\n with_items: \"{{ expected_files }}\"\n vars:\n expected_files: >-\n ['{{ etcd_cert_dir }}/ca.pem',\n {% set etcd_members = groups['etcd'] %}\n {% for host in etcd_members %}\n '{{ etcd_cert_dir }}/admin-{{ host }}.pem',\n '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem',\n '{{ etcd_cert_dir }}/member-{{ host }}.pem',\n '{{ etcd_cert_dir }}/member-{{ host }}-key.pem',\n {% endfor %}\n {% set k8s_nodes = groups['k8s_cluster'] | unique | sort %}\n {% for host in k8s_nodes %}\n '{{ etcd_cert_dir }}/node-{{ host }}.pem',\n '{{ etcd_cert_dir }}/node-{{ host }}-key.pem'\n {% if not loop.last %}{{ ',' }}{% endif %}\n {% endfor %}]\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list\n\n- name: \"Check_certs | Set 'gen_*_certs' groups to track which nodes needs to have certs generated on first etcd node\"\n vars:\n existing_certs: etcdcert_master.files | map(attribute='path')\n ansible.builtin.group_by:\n key: \"gen_{{ item.node_type }}_certs_{{ force_etcd_cert_refresh or item.certs is not subset(existing_certs) }}\"\n loop: \"{{ cert_files | dict2items(key_name='node_type', value_name='certs') }}\"\n\n- name: \"Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match\"\n set_fact:\n etcd_member_requires_sync: true\n when:\n - ('etcd' in group_names)\n - (not etcd_member_certs.results[0].stat.exists | default(false)) or\n (not etcd_member_certs.results[1].stat.exists | default(false)) or\n (not etcd_member_certs.results[2].stat.exists | default(false)) or\n (not etcd_member_certs.results[3].stat.exists | default(false)) or\n (not etcd_member_certs.results[4].stat.exists | default(false)) or\n (etcd_member_certs.results[0].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_member_certs.results[0].stat.path) | map(attribute=\"checksum\") | first | default('')) or\n (etcd_member_certs.results[1].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_member_certs.results[1].stat.path) | map(attribute=\"checksum\") | first | default('')) or\n (etcd_member_certs.results[2].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_member_certs.results[2].stat.path) | map(attribute=\"checksum\") | first | default('')) or\n (etcd_member_certs.results[3].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_member_certs.results[3].stat.path) | map(attribute=\"checksum\") | first | default('')) or\n (etcd_member_certs.results[4].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_member_certs.results[4].stat.path) | map(attribute=\"checksum\") | first | default(''))\n\n- name: \"Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match\"\n set_fact:\n kubernetes_host_requires_sync: true\n when:\n - ('k8s_cluster' in group_names) and\n inventory_hostname not in groups['etcd']\n - (not etcd_node_certs.results[0].stat.exists | default(false)) or\n (not etcd_node_certs.results[1].stat.exists | default(false)) or\n (not etcd_node_certs.results[2].stat.exists | default(false)) or\n (etcd_node_certs.results[0].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_node_certs.results[0].stat.path) | map(attribute=\"checksum\") | first | default('')) or\n (etcd_node_certs.results[1].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_node_certs.results[1].stat.path) | map(attribute=\"checksum\") | first | default('')) or\n (etcd_node_certs.results[2].stat.checksum | default('') != etcdcert_master.files | selectattr(\"path\", \"equalto\", etcd_node_certs.results[2].stat.path) | map(attribute=\"checksum\") | first | default(''))\n\n- name: \"Check_certs | Set 'sync_certs' to true\"\n set_fact:\n sync_certs: true\n when:\n - etcd_member_requires_sync | default(false) or\n kubernetes_host_requires_sync | default(false) or\n 'gen_master_certs_True' in group_names or\n 'gen_node_certs_True' in group_names\n" }, { "path": "roles/etcd/tasks/clean_v2_store.yml", "content": "---\n# When upgrading from etcd 3.5 to 3.6, need to clean up v2 store before upgrading.\n# Without this, etcd 3.6 will crash with following error:\n# \"panic: detected disallowed v2 WAL for stage --v2-deprecation=write-only [recovered]\"\n- name: Cleanup v2 store when upgrade etcd from <3.6 to >=3.6\n when:\n - etcd_cluster_setup\n - etcd_current_version != ''\n - etcd_current_version is version('3.6.0', '<')\n - etcd_version is version('3.6.0', '>=')\n block:\n - name: Ensure etcd version is >=3.5.26\n when:\n - etcd_current_version is version('3.5.26', '<')\n fail:\n msg: \"You need to upgrade etcd to 3.5.26 or later before upgrade to 3.6. Current version is {{ etcd_current_version }}.\"\n\n # Workarounds:\n # Disable --enable-v2 (recommended in 20289) and do workaround of 20231 (MAX_WALS=1 and SNAPSHOT_COUNT=1)\n # - https://github.com/etcd-io/etcd/issues/20809\n # - https://github.com/etcd-io/etcd/discussions/20231#discussioncomment-13958051\n - name: Change etcd configuration temporally to limit number of WALs and snapshots to clean up v2 store\n ansible.builtin.lineinfile:\n path: /etc/etcd.env\n regexp: \"{{ item.regexp }}\"\n line: \"{{ item.line }}\"\n loop:\n - { regexp: '^ETCD_SNAPSHOT_COUNT=', line: 'ETCD_SNAPSHOT_COUNT=1' }\n - { regexp: '^ETCD_MAX_WALS=', line: 'ETCD_MAX_WALS=1' }\n - { regexp: '^ETCD_MAX_SNAPSHOTS=', line: 'ETCD_MAX_SNAPSHOTS=1' }\n - { regexp: '^ETCD_ENABLE_V2=', line: 'ETCD_ENABLE_V2=false' }\n\n # Restart etcd to apply temporal configuration and prevent some upgrade failures\n # See also: https://etcd.io/blog/2025/upgrade_from_3.5_to_3.6_issue_followup/\n - name: Stop etcd\n service:\n name: etcd\n state: stopped\n\n - name: Start etcd\n service:\n name: etcd\n state: started\n" }, { "path": "roles/etcd/tasks/configure.yml", "content": "---\n- name: Configure | Check if etcd cluster is healthy\n shell: \"set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null\"\n args:\n executable: /bin/bash\n register: etcd_cluster_is_healthy\n failed_when: false\n changed_when: false\n check_mode: false\n run_once: true\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n\n- name: Configure | Check if etcd-events cluster is healthy\n shell: \"set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null\"\n args:\n executable: /bin/bash\n register: etcd_events_cluster_is_healthy\n failed_when: false\n changed_when: false\n check_mode: false\n run_once: true\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_events_access_addresses }}\"\n\n- name: Configure | Refresh etcd config\n include_tasks: refresh_config.yml\n when: ('etcd' in group_names)\n\n- name: Configure | Copy etcd.service systemd file\n template:\n src: \"etcd-{{ etcd_deployment_type }}.service.j2\"\n dest: /etc/systemd/system/etcd.service\n backup: true\n mode: \"0644\"\n # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)\n # Remove once we drop support for systemd < 250\n validate: \"sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:etcd-{{ etcd_deployment_type }}.service'\"\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n\n- name: Configure | Copy etcd-events.service systemd file\n template:\n src: \"etcd-events-{{ etcd_deployment_type }}.service.j2\"\n dest: /etc/systemd/system/etcd-events.service\n backup: true\n mode: \"0644\"\n validate: \"sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:etcd-events-{{ etcd_deployment_type }}.service'\"\n # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)\n # Remove once we drop support for systemd < 250\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n\n- name: Configure | reload systemd\n systemd_service:\n daemon_reload: true\n when: ('etcd' in group_names)\n\n# when scaling new etcd will fail to start\n- name: Configure | Ensure etcd is running\n service:\n name: etcd\n state: started\n enabled: true\n ignore_errors: \"{{ etcd_cluster_is_healthy.rc == 0 }}\" # noqa ignore-errors\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n\n# when scaling new etcd will fail to start\n- name: Configure | Ensure etcd-events is running\n service:\n name: etcd-events\n state: started\n enabled: true\n ignore_errors: \"{{ etcd_events_cluster_is_healthy.rc != 0 }}\" # noqa ignore-errors\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n\n- name: Configure | Wait for etcd cluster to be healthy\n shell: \"set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null\"\n args:\n executable: /bin/bash\n register: etcd_cluster_is_healthy\n until: etcd_cluster_is_healthy.rc == 0\n retries: \"{{ etcd_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n changed_when: false\n check_mode: false\n run_once: true\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n\n- name: Configure | Wait for etcd-events cluster to be healthy\n shell: \"set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null\"\n args:\n executable: /bin/bash\n register: etcd_events_cluster_is_healthy\n until: etcd_events_cluster_is_healthy.rc == 0\n retries: \"{{ etcd_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n changed_when: false\n check_mode: false\n run_once: true\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_events_access_addresses }}\"\n\n- name: Configure | Check if member is in etcd cluster\n shell: \"{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address | replace('[', '') | replace(']', '') }}\"\n register: etcd_member_in_cluster\n ignore_errors: true # noqa ignore-errors\n changed_when: false\n check_mode: false\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n\n- name: Configure | Check if member is in etcd-events cluster\n shell: \"{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address | replace('[', '') | replace(']', '') }}\"\n register: etcd_events_member_in_cluster\n ignore_errors: true # noqa ignore-errors\n changed_when: false\n check_mode: false\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_events_access_addresses }}\"\n\n- name: Configure | Join member(s) to etcd cluster one at a time\n include_tasks: join_etcd_member.yml\n with_items: \"{{ groups['etcd'] }}\"\n when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0\n\n- name: Configure | Join member(s) to etcd-events cluster one at a time\n include_tasks: join_etcd-events_member.yml\n with_items: \"{{ groups['etcd'] }}\"\n when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0\n" }, { "path": "roles/etcd/tasks/gen_certs_script.yml", "content": "---\n- name: Gen_certs | create etcd cert dir\n file:\n path: \"{{ etcd_cert_dir }}\"\n group: \"{{ etcd_cert_group }}\"\n state: directory\n owner: \"{{ etcd_owner }}\"\n mode: \"0700\"\n\n- name: \"Gen_certs | create etcd script dir (on {{ groups['etcd'][0] }})\"\n file:\n path: \"{{ etcd_script_dir }}\"\n state: directory\n owner: root\n mode: \"0700\"\n run_once: true\n when: inventory_hostname == groups['etcd'][0]\n\n- name: Gen_certs | write openssl config\n template:\n src: \"openssl.conf.j2\"\n dest: \"{{ etcd_config_dir }}/openssl.conf\"\n mode: \"0640\"\n run_once: true\n delegate_to: \"{{ groups['etcd'][0] }}\"\n when:\n - gen_certs | default(false)\n - inventory_hostname == groups['etcd'][0]\n\n- name: Gen_certs | copy certs generation script\n template:\n src: \"make-ssl-etcd.sh.j2\"\n dest: \"{{ etcd_script_dir }}/make-ssl-etcd.sh\"\n mode: \"0700\"\n run_once: true\n when:\n - inventory_hostname == groups['etcd'][0]\n\n- name: Gen_certs | run cert generation script for etcd and kube control plane nodes\n command: \"bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}\"\n environment:\n MASTERS: \"{{ groups['gen_master_certs_True'] | ansible.builtin.intersect(groups['etcd']) | join(' ') }}\"\n HOSTS: \"{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['kube_control_plane']) | join(' ') }}\"\n run_once: true\n delegate_to: \"{{ groups['etcd'][0] }}\"\n when: gen_certs | default(false)\n notify: Set etcd_secret_changed\n\n- name: Gen_certs | run cert generation script for all clients\n command: \"bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}\"\n environment:\n HOSTS: \"{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['k8s_cluster']) | join(' ') }}\"\n run_once: true\n delegate_to: \"{{ groups['etcd'][0] }}\"\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - gen_certs | default(false)\n notify: Set etcd_secret_changed\n\n- name: Gen_certs | Gather etcd member/admin and kube_control_plane client certs from first etcd node\n slurp:\n src: \"{{ item }}\"\n register: etcd_master_certs\n with_items:\n - \"{{ etcd_cert_dir }}/ca.pem\"\n - \"{{ etcd_cert_dir }}/ca-key.pem\"\n - \"[{% for node in groups['etcd'] %}\n '{{ etcd_cert_dir }}/admin-{{ node }}.pem',\n '{{ etcd_cert_dir }}/admin-{{ node }}-key.pem',\n '{{ etcd_cert_dir }}/member-{{ node }}.pem',\n '{{ etcd_cert_dir }}/member-{{ node }}-key.pem',\n {% endfor %}]\"\n - \"[{% for node in (groups['kube_control_plane']) %}\n '{{ etcd_cert_dir }}/node-{{ node }}.pem',\n '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',\n {% endfor %}]\"\n delegate_to: \"{{ groups['etcd'][0] }}\"\n when:\n - ('etcd' in group_names)\n - sync_certs | default(false)\n - inventory_hostname != groups['etcd'][0]\n notify: Set etcd_secret_changed\n\n- name: Gen_certs | Write etcd member/admin and kube_control_plane client certs to other etcd nodes\n copy:\n dest: \"{{ item.item }}\"\n content: \"{{ item.content | b64decode }}\"\n group: \"{{ etcd_cert_group }}\"\n owner: \"{{ etcd_owner }}\"\n mode: \"0640\"\n with_items: \"{{ etcd_master_certs.results }}\"\n when:\n - ('etcd' in group_names)\n - sync_certs | default(false)\n - inventory_hostname != groups['etcd'][0]\n loop_control:\n label: \"{{ item.item }}\"\n\n- name: Gen_certs | Gather node certs from first etcd node\n slurp:\n src: \"{{ item }}\"\n register: etcd_master_node_certs\n with_items:\n - \"[{% for node in groups['k8s_cluster'] %}\n '{{ etcd_cert_dir }}/node-{{ node }}.pem',\n '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',\n {% endfor %}]\"\n delegate_to: \"{{ groups['etcd'][0] }}\"\n when:\n - ('etcd' in group_names)\n - inventory_hostname != groups['etcd'][0]\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n notify: Set etcd_secret_changed\n\n- name: Gen_certs | Write node certs to other etcd nodes\n copy:\n dest: \"{{ item.item }}\"\n content: \"{{ item.content | b64decode }}\"\n group: \"{{ etcd_cert_group }}\"\n owner: \"{{ etcd_owner }}\"\n mode: \"0640\"\n with_items: \"{{ etcd_master_node_certs.results }}\"\n when:\n - ('etcd' in group_names)\n - inventory_hostname != groups['etcd'][0]\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n loop_control:\n label: \"{{ item.item }}\"\n\n- name: Gen_certs | Generate etcd certs\n include_tasks: gen_nodes_certs_script.yml\n when:\n - ('kube_control_plane' in group_names) and\n sync_certs | default(false) and inventory_hostname not in groups['etcd']\n\n- name: Gen_certs | Generate etcd certs on nodes if needed\n include_tasks: gen_nodes_certs_script.yml\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - ('k8s_cluster' in group_names) and\n sync_certs | default(false) and inventory_hostname not in groups['etcd']\n\n# This is a hack around the fact kubeadm expect the same certs path on all kube_control_plane\n# TODO: fix certs generation to have the same file everywhere\n# OR work with kubeadm on node-specific config\n- name: Gen_certs | Pretend all control plane have all certs (with symlinks)\n file:\n state: link\n src: \"{{ etcd_cert_dir }}/node-{{ inventory_hostname }}{{ item[0] }}.pem\"\n dest: \"{{ etcd_cert_dir }}/node-{{ item[1] }}{{ item[0] }}.pem\"\n mode: \"0640\"\n loop: \"{{ suffixes | product(groups['kube_control_plane']) }}\"\n vars:\n suffixes:\n - ''\n - '-key'\n when:\n - ('kube_control_plane' in group_names)\n - item[1] != inventory_hostname\n register: symlink_created\n failed_when:\n - symlink_created is failed\n - ('refusing to convert from file to symlink' not in symlink_created.msg)\n" }, { "path": "roles/etcd/tasks/gen_nodes_certs_script.yml", "content": "---\n- name: Gen_certs | Set cert names per node\n set_fact:\n my_etcd_node_certs: [ 'ca.pem',\n 'node-{{ inventory_hostname }}.pem',\n 'node-{{ inventory_hostname }}-key.pem']\n tags:\n - facts\n\n- name: \"Check_certs | Set 'sync_certs' to true on nodes\"\n set_fact:\n sync_certs: true\n with_items:\n - \"{{ my_etcd_node_certs }}\"\n\n- name: Gen_certs | Gather node certs\n vars:\n ansible_ssh_retries: 10\n shell: \"set -o pipefail && tar cfz - -C {{ etcd_cert_dir }} {{ my_etcd_node_certs | join(' ') }} | base64 --wrap=0\"\n args:\n executable: /bin/bash\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n register: etcd_node_certs\n check_mode: false\n delegate_to: \"{{ groups['etcd'][0] }}\"\n changed_when: false\n\n- name: Gen_certs | Copy certs on nodes\n shell: \"set -o pipefail && base64 -d <<< '{{ etcd_node_certs.stdout | quote }}' | tar xz -C {{ etcd_cert_dir }}\"\n args:\n executable: /bin/bash\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n changed_when: false\n" }, { "path": "roles/etcd/tasks/install_docker.yml", "content": "---\n- name: Get currently-deployed etcd version\n shell: \"{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}' --format='{{ '{{ .Image }}' }}'\"\n register: etcd_current_docker_image\n when: etcd_cluster_setup\n\n- name: Get currently-deployed etcd-events version\n shell: \"{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}-events' --format='{{ '{{ .Image }}' }}'\"\n register: etcd_events_current_docker_image\n when: etcd_events_cluster_setup\n\n- name: Restart etcd if necessary\n command: /bin/true\n notify: Restart etcd\n when:\n - etcd_cluster_setup\n - etcd_image_tag not in etcd_current_docker_image.stdout | default('')\n\n- name: Restart etcd-events if necessary\n command: /bin/true\n notify: Restart etcd-events\n when:\n - etcd_events_cluster_setup\n - etcd_image_tag not in etcd_events_current_docker_image.stdout | default('')\n\n- name: Get currently-deployed etcd version as x.y.z format\n set_fact:\n etcd_current_version: \"{{ (etcd_current_docker_image.stdout | regex_search('.*:v([0-9]+\\\\.[0-9]+\\\\.[0-9]+)', '\\\\1'))[0] | default('') }}\"\n when: etcd_cluster_setup\n\n- name: Cleanup v2 store data\n import_tasks: clean_v2_store.yml\n\n- name: Install etcd launch script\n template:\n src: etcd.j2\n dest: \"{{ bin_dir }}/etcd\"\n owner: 'root'\n mode: \"0750\"\n backup: true\n when: etcd_cluster_setup\n\n- name: Install etcd-events launch script\n template:\n src: etcd-events.j2\n dest: \"{{ bin_dir }}/etcd-events\"\n owner: 'root'\n mode: \"0750\"\n backup: true\n when: etcd_events_cluster_setup\n" }, { "path": "roles/etcd/tasks/install_host.yml", "content": "---\n- name: Get currently-deployed etcd version\n command: \"{{ bin_dir }}/etcd --version\"\n register: etcd_current_host_version\n # There's a chance this play could run before etcd is installed at all\n # TODO: figure out whether this happens. \"A chance\" is not enough information\n ignore_errors: true\n when: etcd_cluster_setup\n\n- name: Restart etcd if necessary\n command: /bin/true\n notify: Restart etcd\n when:\n - etcd_cluster_setup\n - etcd_version not in etcd_current_host_version.stdout | default('')\n\n- name: Restart etcd-events if necessary\n command: /bin/true\n notify: Restart etcd-events\n when:\n - etcd_events_cluster_setup\n - etcd_version not in etcd_current_host_version.stdout | default('')\n\n- name: Get currently-deployed etcd version as x.y.z format\n set_fact:\n etcd_current_version: \"{{ (etcd_current_host_version.stdout | regex_search('etcd Version: ([0-9]+\\\\.[0-9]+\\\\.[0-9]+)', '\\\\1'))[0] | default('') }}\"\n when: etcd_cluster_setup\n\n- name: Cleanup v2 store data\n import_tasks: clean_v2_store.yml\n\n- name: Install | Copy etcd binary from download dir\n copy:\n src: \"{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}\"\n dest: \"{{ bin_dir }}/{{ item }}\"\n mode: \"0755\"\n remote_src: true\n with_items:\n - etcd\n when: etcd_cluster_setup\n" }, { "path": "roles/etcd/tasks/join_etcd-events_member.yml", "content": "---\n- name: Join Member | Add member to etcd-events cluster\n command: \"{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}\"\n register: member_add_result\n until: member_add_result.rc == 0\n retries: \"{{ etcd_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_events_access_addresses }}\"\n\n- name: Join Member | Refresh etcd config\n include_tasks: refresh_config.yml\n vars:\n # noqa: jinja[spacing]\n etcd_events_peer_addresses: >-\n {% for host in groups['etcd'] -%}\n {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}\n {{ \"etcd\" + loop.index | string }}=https://{{ hostvars[host].etcd_events_access_address | default(hostvars[host]['main_ip']) | ansible.utils.ipwrap }}:2382,\n {%- endif -%}\n {%- if loop.last -%}\n {{ etcd_member_name }}={{ etcd_events_peer_url }}\n {%- endif -%}\n {%- endfor -%}\n\n- name: Join Member | Ensure member is in etcd-events cluster\n shell: \"set -o pipefail && {{ bin_dir }}/etcdctl member list | grep -w {{ etcd_events_access_address }} >/dev/null\"\n args:\n executable: /bin/bash\n register: etcd_events_member_in_cluster\n changed_when: false\n check_mode: false\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_events_access_addresses }}\"\n\n- name: Configure | Ensure etcd-events is running\n service:\n name: etcd-events\n state: started\n enabled: true\n" }, { "path": "roles/etcd/tasks/join_etcd_member.yml", "content": "---\n- name: Join Member | Add member to etcd cluster\n command: \"{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}\"\n register: member_add_result\n until: member_add_result.rc == 0 or 'Peer URLs already exists' in member_add_result.stderr\n failed_when: member_add_result.rc != 0 and 'Peer URLs already exists' not in member_add_result.stderr\n retries: \"{{ etcd_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n\n- name: Join Member | Refresh etcd config\n include_tasks: refresh_config.yml\n vars:\n # noqa: jinja[spacing]\n etcd_peer_addresses: >-\n {% for host in groups['etcd'] -%}\n {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}\n {{ \"etcd\" + loop.index | string }}=https://{{ hostvars[host].etcd_access_address | default(hostvars[host]['main_ip']) | ansible.utils.ipwrap }}:2380,\n {%- endif -%}\n {%- if loop.last -%}\n {{ etcd_member_name }}={{ etcd_peer_url }}\n {%- endif -%}\n {%- endfor -%}\n\n- name: Join Member | Ensure member is in etcd cluster\n shell: \"set -o pipefail && {{ bin_dir }}/etcdctl member list | grep -w {{ etcd_access_address }} >/dev/null\"\n args:\n executable: /bin/bash\n register: etcd_member_in_cluster\n changed_when: false\n check_mode: false\n retries: \"{{ etcd_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n until: etcd_member_in_cluster.rc == 0\n tags:\n - facts\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n\n- name: Configure | Ensure etcd is running\n service:\n name: etcd\n state: started\n enabled: true\n" }, { "path": "roles/etcd/tasks/main.yml", "content": "---\n- name: Check etcd certs\n include_tasks: check_certs.yml\n when: cert_management == \"script\"\n tags:\n - etcd-secrets\n - facts\n\n- name: Generate etcd certs\n include_tasks: \"gen_certs_script.yml\"\n when:\n - cert_management == \"script\"\n tags:\n - etcd-secrets\n\n- name: Trust etcd CA\n include_tasks: upd_ca_trust.yml\n when:\n - ('etcd' in group_names) or ('kube_control_plane' in group_names)\n tags:\n - etcd-secrets\n\n- name: Trust etcd CA on nodes if needed\n include_tasks: upd_ca_trust.yml\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - ('k8s_cluster' in group_names)\n tags:\n - etcd-secrets\n\n- name: \"Gen_certs | Get etcd certificate serials\"\n command: \"openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial\"\n register: \"etcd_client_cert_serial_result\"\n changed_when: false\n check_mode: false\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - ('k8s_cluster' in group_names)\n tags:\n - control-plane\n - network\n\n- name: Set etcd_client_cert_serial\n set_fact:\n etcd_client_cert_serial: \"{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}\"\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - ('k8s_cluster' in group_names)\n tags:\n - control-plane\n - network\n\n- name: Install etcd\n include_tasks: \"install_{{ etcd_deployment_type }}.yml\"\n when: ('etcd' in group_names)\n tags:\n - upgrade\n\n- name: Install etcdctl and etcdutl binary\n import_role:\n name: etcdctl_etcdutl\n tags:\n - etcdctl\n - etcdutl\n - upgrade\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n\n- name: Configure etcd\n include_tasks: configure.yml\n when: ('etcd' in group_names)\n\n- name: Refresh etcd config\n include_tasks: refresh_config.yml\n when: ('etcd' in group_names)\n\n- name: Restart etcd if certs changed\n command: /bin/true\n notify: Restart etcd\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n - etcd_secret_changed | default(false)\n\n- name: Restart etcd-events if certs changed\n command: /bin/true\n notify: Restart etcd\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n - etcd_secret_changed | default(false)\n\n# After etcd cluster is assembled, make sure that\n# initial state of the cluster is in `existing`\n# state instead of `new`.\n- name: Refresh etcd config again for idempotency\n include_tasks: refresh_config.yml\n when: ('etcd' in group_names)\n" }, { "path": "roles/etcd/tasks/refresh_config.yml", "content": "---\n- name: Refresh config | Create etcd config file\n template:\n src: etcd.env.j2\n dest: /etc/etcd.env\n mode: \"0640\"\n notify: Restart etcd\n when:\n - ('etcd' in group_names)\n - etcd_cluster_setup\n\n- name: Refresh config | Create etcd-events config file\n template:\n src: etcd-events.env.j2\n dest: /etc/etcd-events.env\n mode: \"0640\"\n notify: Restart etcd-events\n when:\n - ('etcd' in group_names)\n - etcd_events_cluster_setup\n" }, { "path": "roles/etcd/tasks/upd_ca_trust.yml", "content": "---\n- name: Gen_certs | target ca-certificate store file\n set_fact:\n ca_cert_path: |-\n {% if ansible_os_family == \"Debian\" -%}\n /usr/local/share/ca-certificates/etcd-ca.crt\n {%- elif ansible_os_family == \"RedHat\" -%}\n /etc/pki/ca-trust/source/anchors/etcd-ca.crt\n {%- elif ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"] -%}\n /etc/ssl/certs/etcd-ca.pem\n {%- elif ansible_os_family == \"Suse\" -%}\n /etc/pki/trust/anchors/etcd-ca.pem\n {%- elif ansible_os_family == \"ClearLinux\" -%}\n /usr/share/ca-certs/etcd-ca.pem\n {%- endif %}\n tags:\n - facts\n\n- name: Gen_certs | add CA to trusted CA dir\n copy:\n src: \"{{ etcd_cert_dir }}/ca.pem\"\n dest: \"{{ ca_cert_path }}\"\n remote_src: true\n mode: \"0640\"\n register: etcd_ca_cert\n\n- name: Gen_certs | update ca-certificates (Debian/Ubuntu/SUSE/Flatcar) # noqa no-handler\n command: update-ca-certificates\n when: etcd_ca_cert.changed and ansible_os_family in [\"Debian\", \"Flatcar\", \"Flatcar Container Linux by Kinvolk\", \"Suse\"]\n\n- name: Gen_certs | update ca-certificates (RedHat) # noqa no-handler\n command: update-ca-trust extract\n when: etcd_ca_cert.changed and ansible_os_family == \"RedHat\"\n\n- name: Gen_certs | update ca-certificates (ClearLinux) # noqa no-handler\n command: clrtrust add \"{{ ca_cert_path }}\"\n when: etcd_ca_cert.changed and ansible_os_family == \"ClearLinux\"\n" }, { "path": "roles/etcd/templates/etcd-docker.service.j2", "content": "[Unit]\nDescription=etcd docker wrapper\nWants=docker.socket\nAfter=docker.service\n\n[Service]\nUser=root\nPermissionsStartOnly=true\nEnvironmentFile=-/etc/etcd.env\nExecStart={{ bin_dir }}/etcd\nExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name | default(\"etcd\") }}\nExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name | default(\"etcd\") }}\nRestart=always\nRestartSec=15s\nTimeoutStartSec=30s\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/etcd/templates/etcd-events-docker.service.j2", "content": "[Unit]\nDescription=etcd docker wrapper\nWants=docker.socket\nAfter=docker.service\n\n[Service]\nUser=root\nPermissionsStartOnly=true\nEnvironmentFile=-/etc/etcd-events.env\nExecStart={{ bin_dir }}/etcd-events\nExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events\nExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events\nRestart=always\nRestartSec=15s\nTimeoutStartSec=30s\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/etcd/templates/etcd-events-host.service.j2", "content": "[Unit]\nDescription=etcd\nAfter=network.target\n\n[Service]\nType=notify\nUser=root\nEnvironmentFile=/etc/etcd-events.env\nExecStart={{ bin_dir }}/etcd\nNotifyAccess=all\nRestart=always\nRestartSec=10s\nLimitNOFILE=40000\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/etcd/templates/etcd-events.env.j2", "content": "ETCD_DATA_DIR={{ etcd_events_data_dir }}\nETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}\nETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}\nETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}\n\nETCD_METRICS={{ etcd_metrics }}\nETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2383,https://127.0.0.1:2383\nETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}\nETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}\nETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd\nETCD_LISTEN_PEER_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2382\nETCD_NAME={{ etcd_member_name }}-events\nETCD_PROXY=off\nETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }}\nETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}\nETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }}\nETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}\nETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}\n\n# TLS settings\nETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem\nETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\nETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\nETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}\n\nETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem\nETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\nETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\nETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}\n\n{% if etcd_tls_cipher_suites is defined %}\nETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ \",\" if not loop.last else \"\" }}{% endfor %}\n{% endif %}\n\n{% for key, value in etcd_extra_vars.items() %}\n{{ key }}={{ value }}\n{% endfor %}\n" }, { "path": "roles/etcd/templates/etcd-events.j2", "content": "#!/bin/bash\n{{ docker_bin_dir }}/docker run \\\n --restart=on-failure:5 \\\n --env-file=/etc/etcd-events.env \\\n --net=host \\\n -v /etc/ssl/certs:/etc/ssl/certs:ro \\\n -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \\\n -v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \\\n {% if etcd_memory_limit is defined %}\n --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \\\n {% endif %}\n {% if etcd_cpu_limit is defined %}\n --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \\\n {% endif %}\n {% if etcd_blkio_weight is defined %}\n --blkio-weight={{ etcd_blkio_weight }} \\\n {% endif %}\n --name={{ etcd_member_name }}-events \\\n {{ etcd_image_repo }}:{{ etcd_image_tag }} \\\n /usr/local/bin/etcd \\\n \"$@\"\n" }, { "path": "roles/etcd/templates/etcd-host.service.j2", "content": "[Unit]\nDescription=etcd\nAfter=network.target\n\n[Service]\nType=notify\nUser=root\nEnvironmentFile=/etc/etcd.env\nExecStart={{ bin_dir }}/etcd\nNotifyAccess=all\nRestart=always\nRestartSec=10s\nLimitNOFILE=40000\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/etcd/templates/etcd.env.j2", "content": "# Environment file for etcd {{ etcd_version }}\nETCD_DATA_DIR={{ etcd_data_dir }}\nETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}\nETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}\nETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}\n\nETCD_METRICS={{ etcd_metrics }}\n{% if etcd_listen_metrics_urls is defined %}\nETCD_LISTEN_METRICS_URLS={{ etcd_listen_metrics_urls }}\n{% elif etcd_metrics_port is defined %}\nETCD_LISTEN_METRICS_URLS=http://{{ etcd_address | ansible.utils.ipwrap }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }}\n{% endif %}\nETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2379,https://127.0.0.1:2379\nETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}\nETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}\nETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd\nETCD_LISTEN_PEER_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2380\nETCD_NAME={{ etcd_member_name }}\nETCD_PROXY=off\nETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}\nETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}\nETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }}\nETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}\nETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}\nETCD_LOG_LEVEL={{ etcd_log_level }}\nETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }}\nETCD_MAX_WALS={{ etcd_max_wals }}\n\n# TLS settings\nETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem\nETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\nETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\nETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}\n\nETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem\nETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\nETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\nETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}\n\n{% if etcd_tls_cipher_suites is defined %}\nETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ \",\" if not loop.last else \"\" }}{% endfor %}\n{% endif %}\n\n{% for key, value in etcd_extra_vars.items() %}\n{{ key }}={{ value }}\n{% endfor %}\n\n# CLI settings\nETCDCTL_ENDPOINTS=https://127.0.0.1:2379\nETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem\nETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\nETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\n\n# ETCD 3.5.x issue\n# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer\nETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK={{ etcd_experimental_initial_corrupt_check }}\n\n{% if etcd_experimental_enable_distributed_tracing %}\nETCD_EXPERIMENTAL_ENABLE_DISTRIBUTED_TRACING=true\nETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SAMPLING_RATE={{ etcd_experimental_distributed_tracing_sample_rate }}\nETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_ADDRESS={{ etcd_experimental_distributed_tracing_address }}\nETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SERVICE_NAME={{ etcd_experimental_distributed_tracing_service_name }}\nETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_INSTANCE_ID={{ etcd_member_name }}\n{% endif %}\n\nETCD_EXPERIMENTAL_WATCH_PROGRESS_NOTIFY_INTERVAL={{ etcd_experimental_watch_progress_notify_interval }}\n" }, { "path": "roles/etcd/templates/etcd.j2", "content": "#!/bin/bash\n{{ docker_bin_dir }}/docker run \\\n --restart=on-failure:5 \\\n --env-file=/etc/etcd.env \\\n --net=host \\\n -v /etc/ssl/certs:/etc/ssl/certs:ro \\\n -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \\\n -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:rw \\\n{% if etcd_memory_limit is defined %}\n --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \\\n{% endif %}\n{% if etcd_cpu_limit is defined %}\n --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \\\n{% endif %}\n{% if etcd_blkio_weight is defined %}\n --blkio-weight={{ etcd_blkio_weight }} \\\n{% endif %}\n --name={{ etcd_member_name | default(\"etcd\") }} \\\n {{ etcd_image_repo }}:{{ etcd_image_tag }} \\\n /usr/local/bin/etcd \\\n \"$@\"\n" }, { "path": "roles/etcd/templates/make-ssl-etcd.sh.j2", "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\nusage()\n{\n cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) -f [-d ]\n -h | --help : Show this message\n -f | --config : Openssl configuration file\n -d | --ssldir : Directory where the certificates will be installed\n\n ex :\n $(basename $0) -f openssl.conf -d /srv/ssl\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n case \"$1\" in\n -h | --help) usage; exit 0;;\n -f | --config) CONFIG=${2}; shift 2;;\n -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n *)\n usage\n echo \"ERROR : Unknown option\"\n exit 3\n ;;\n esac\ndone\n\nif [ -z ${CONFIG} ]; then\n echo \"ERROR: the openssl configuration file is missing. option -f\"\n exit 1\nfi\nif [ -z ${SSLDIR} ]; then\n SSLDIR=\"/etc/ssl/etcd\"\nfi\n\ntmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p \"${SSLDIR}\"\n\n# Root CA\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n # Reuse existing CA\n cp $SSLDIR/{ca.pem,ca-key.pem} .\nelse\n openssl genrsa -out ca-key.pem {{certificates_key_size}}\n openssl req -x509 -new -nodes -key ca-key.pem -days {{certificates_duration}} -out ca.pem -subj \"/CN=etcd-ca\"\nfi\n\n# ETCD member\nif [ -n \"$MASTERS\" ]; then\n for host in $MASTERS; do\n cn=\"${host%%.*}\"\n # Member key\n openssl genrsa -out member-${host}-key.pem {{certificates_key_size}}\n openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj \"/CN=etcd-member-${cn}\" -config ${CONFIG}\n openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG}\n\n # Admin key\n openssl genrsa -out admin-${host}-key.pem {{certificates_key_size}}\n openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj \"/CN=etcd-admin-${cn}\"\n openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG}\n done\nfi\n\n# Node keys\nif [ -n \"$HOSTS\" ]; then\n for host in $HOSTS; do\n cn=\"${host%%.*}\"\n openssl genrsa -out node-${host}-key.pem {{certificates_key_size}}\n openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj \"/CN=etcd-node-${cn}\"\n openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG}\n done\nfi\n\n# Install certs\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n # No pass existing CA\n rm -f ca.pem ca-key.pem\nfi\n\nif [ -n \"$(ls -A *.pem)\" ]; then\n mv *.pem ${SSLDIR}/\nfi\n" }, { "path": "roles/etcd/templates/openssl.conf.j2", "content": "{% set counter = {'dns': 2,'ip': 1,} %}{% macro increment(dct, key, inc=1)%}{% if dct.update({key: dct[key] + inc}) %} {% endif %}{% endmacro %}[req]\nreq_extensions = v3_req\ndistinguished_name = req_distinguished_name\n\n[req_distinguished_name]\n\n[ v3_req ]\nbasicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = @alt_names\n\n[ ssl_client ]\nextendedKeyUsage = clientAuth, serverAuth\nbasicConstraints = CA:FALSE\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid,issuer\nsubjectAltName = @alt_names\n\n[ v3_ca ]\nbasicConstraints = CA:TRUE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\nsubjectAltName = @alt_names\nauthorityKeyIdentifier=keyid:always,issuer\n\n[alt_names]\nDNS.1 = localhost\n{% for host in groups['etcd'] %}\n{% if hostvars[host]['etcd_access_address'] is defined and not (hostvars[host]['etcd_access_address'] | ansible.utils.ipaddr) %}\n{# If defined, the address which etcd uses to access its members must be included in the SAN, otherwise etcd will fail with a TLS error upon startup. #}\nDNS.{{ counter[\"dns\"] }} = {{ hostvars[host]['etcd_access_address'] }}{{ increment(counter, 'dns') }}\n{% endif %}\n{# This will always expand to inventory_hostname, which can be a completely arbitrary name, that etcd will not know or care about, hence this line is (probably) redundant. #}\nDNS.{{ counter[\"dns\"] }} = {{ host }}{{ increment(counter, 'dns') }}\n{% endfor %}\n{% for etcd_alt_name in etcd_cert_alt_names %}\nDNS.{{ counter[\"dns\"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }}\n{% endfor %}\n{% for host in groups['etcd'] %}\n{% for address in hostvars[host]['main_access_ips'] %}\nIP.{{ counter[\"ip\"] }} = {{ address }}{{ increment(counter, 'ip') }}\n{% endfor %}\n{% for address in hostvars[host]['main_ips'] %}\nIP.{{ counter[\"ip\"] }} = {{ address }}{{ increment(counter, 'ip') }}\n{% endfor %}\n{% endfor %}\n{% for cert_alt_ip in etcd_cert_alt_ips %}\nIP.{{ counter[\"ip\"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }}\n{% endfor %}\nIP.{{ counter[\"ip\"] }} = 127.0.0.1{{ increment(counter, 'ip') }}\nIP.{{ counter[\"ip\"] }} = ::1\n" }, { "path": "roles/etcd_defaults/defaults/main.yml", "content": "---\n# Set etcd user\netcd_owner: etcd\n\n# Set to false to only do certificate management\netcd_cluster_setup: true\netcd_events_cluster_setup: false\n\n# Set to true to separate k8s events to a different etcd cluster\netcd_events_cluster_enabled: false\n\netcd_backup_prefix: \"/var/backups\"\netcd_data_dir: \"/var/lib/etcd\"\n\n# Number of etcd backups to retain. Set to a value < 0 to retain all backups\netcd_backup_retention_count: -1\n\nforce_etcd_cert_refresh: true\netcd_config_dir: /etc/ssl/etcd\netcd_cert_dir: \"{{ etcd_config_dir }}/ssl\"\netcd_cert_group: root\n# Note: This does not set up DNS entries. It simply adds the following DNS\n# entries to the certificate\netcd_cert_alt_names:\n - \"etcd.kube-system.svc.{{ dns_domain }}\"\n - \"etcd.kube-system.svc\"\n - \"etcd.kube-system\"\n - \"etcd\"\netcd_cert_alt_ips: []\n\netcd_script_dir: \"{{ bin_dir }}/etcd-scripts\"\n\netcd_heartbeat_interval: \"250\"\netcd_election_timeout: \"5000\"\n\n## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.\netcd_metrics: \"basic\"\n\n# Define in inventory to set a separate port for etcd to expose metrics on\n# etcd_metrics_port: 2381\n\n## A dictionary of extra environment variables to add to etcd.env, formatted like:\n## etcd_extra_vars:\n## ETCD_VAR1: \"value1\"\n## ETCD_VAR2: \"value2\"\netcd_extra_vars: {}\n\n# Limits\n## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.\n## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.\n## This value is only relevant when deploying etcd with `etcd_deployment_type: docker`\netcd_memory_limit: \"{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}\"\n\n## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than\n## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check\n## etcd documentation for more information.\n# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.\netcd_quota_backend_bytes: \"2147483648\"\n\n# Maximum client request size in bytes the server will accept.\n# etcd is designed to handle small key value pairs typical for metadata.\n# Larger requests will work, but may increase the latency of other requests\netcd_max_request_bytes: \"1572864\"\n\n# Uncomment to set CPU share for etcd\n# etcd_cpu_limit: 300m\n\netcd_blkio_weight: 1000\n\netcd_node_cert_hosts: \"{{ groups['k8s_cluster'] }}\"\n\n## Etcd auto compaction retention for mvcc key value store in hour\netcd_compaction_retention: \"8\"\n\n# Force clients like etcdctl to use TLS certs (different than peer security)\netcd_secure_client: true\n\n# Enable peer client cert authentication\netcd_peer_client_auth: true\n\n# Maximum number of snapshot files to retain (0 is unlimited)\netcd_max_snapshots: 5\n\n# Maximum number of wal files to retain (0 is unlimited)\netcd_max_wals: 5\n\n# Number of loop retries\netcd_retries: 4\n\n## Support tls cipher suites.\n# etcd_tls_cipher_suites: {}\n# - TLS_RSA_WITH_RC4_128_SHA\n# - TLS_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA\n# - TLS_RSA_WITH_AES_256_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_RC4_128_SHA\n# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n\n# ETCD 3.5.x issue\n# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer\netcd_experimental_initial_corrupt_check: true\n\n# Enable distributed tracing\n# https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing\netcd_experimental_enable_distributed_tracing: false\netcd_experimental_distributed_tracing_sample_rate: 100 # Per million spans\netcd_experimental_distributed_tracing_address: \"localhost:4317\"\netcd_experimental_distributed_tracing_service_name: etcd\n\n# The interval for etcd watch progress notify events\netcd_experimental_watch_progress_notify_interval: 5s\n\netcd_log_level: info\n" }, { "path": "roles/etcd_defaults/vars/main.yml", "content": "---\ncert_files:\n master:\n - \"{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem\"\n - \"{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem\"\n - \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n - \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n node:\n - \"{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem\"\n - \"{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem\"\n" }, { "path": "roles/etcdctl_etcdutl/tasks/main.yml", "content": "---\n- name: Copy etcdctl and etcdutl binary from docker container\n command: sh -c \"{{ docker_bin_dir }}/docker rm -f etcdxtl-binarycopy;\n {{ docker_bin_dir }}/docker create --name etcdxtl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&\n {{ docker_bin_dir }}/docker cp etcdxtl-binarycopy:/usr/local/bin/{{ item }} {{ bin_dir }}/{{ item }} &&\n {{ docker_bin_dir }}/docker rm -f etcdxtl-binarycopy\"\n with_items:\n - etcdctl\n - etcdutl\n register: etcdxtl_install_result\n until: etcdxtl_install_result.rc == 0\n retries: 4\n delay: \"{{ retry_stagger | random + 3 }}\"\n changed_when: false\n when: container_manager == \"docker\"\n\n- name: Download etcd binary\n include_tasks: \"../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.etcd) }}\"\n when: container_manager in ['crio', 'containerd']\n\n- name: Copy etcd binary\n unarchive:\n src: \"{{ downloads.etcd.dest }}\"\n dest: \"{{ local_release_dir }}/\"\n remote_src: true\n when: container_manager in ['crio', 'containerd']\n\n- name: Copy etcdctl and etcdutl binary from download dir\n copy:\n src: \"{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}\"\n dest: \"{{ bin_dir }}/{{ item }}\"\n mode: \"0755\"\n remote_src: true\n with_items:\n - etcdctl\n - etcdutl\n when: container_manager in ['crio', 'containerd']\n\n- name: Create etcdctl wrapper script\n template:\n src: etcdctl.sh.j2\n dest: \"{{ bin_dir }}/etcdctl.sh\"\n mode: \"0755\"\n" }, { "path": "roles/etcdctl_etcdutl/templates/etcdctl.sh.j2", "content": "#!/bin/bash\n# {{ ansible_managed }}\n# example invocation: etcdctl.sh get --keys-only --from-key \"\"\n\netcdctl \\\n{% if etcd_deployment_type == \"kubeadm\" %}\n --cacert {{ kube_cert_dir }}/etcd/ca.crt \\\n --cert {{ kube_cert_dir }}/etcd/server.crt \\\n --key {{ kube_cert_dir }}/etcd/server.key \"$@\"\n{% else %}\n --cacert {{ etcd_cert_dir }}/ca.pem \\\n --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem \\\n --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem \"$@\"\n{% endif %}\n" }, { "path": "roles/helm-apps/README.md", "content": "Role Name\n=========\n\nThis role is intended to be used to fetch and deploy Helm Charts as part of\ncluster installation or upgrading with kubespray.\n\nRequirements\n------------\n\nThe role needs to be executed on a host with access to the Kubernetes API, and\nwith the helm binary in place.\n\nRole Variables\n--------------\n\nSee meta/argument_specs.yml\n\nPlaybook example:\n\n```yaml\n---\n- hosts: kube_control_plane[0]\n gather_facts: no\n roles:\n - name: helm-apps\n releases:\n - name: app\n namespace: app\n chart_ref: simple-app/simple-app\n - name: app2\n namespace: app\n chart_ref: simple-app/simple-app\n wait_timeout: \"10m\" # override the same option in `release_common_opts`\n repositories: \"{{ repos }}\"\n - name: simple-app\n url: \"https://blog.leiwang.info/simple-app\"\n release_common_opts: \"{{ helm_params }}\"\n wait_timeout: \"5m\"\n```\n" }, { "path": "roles/helm-apps/meta/argument_specs.yml", "content": "---\nargument_specs:\n main:\n short_description: Install a list of Helm charts.\n options:\n releases:\n type: list\n elements: dict\n required: true\n description: |\n List of dictionaries passed as arguments to kubernetes.core.helm.\n Arguments passed here will override those in `helm_settings`. For\n structure of the dictionary, see the documentation for\n kubernetes.core.helm ansible module.\n options:\n chart_ref:\n type: path\n required: true\n chart_version:\n type: str\n name:\n type: str\n required: true\n namespace:\n type: str\n required: true\n values:\n type: dict\n # Possibly general options\n create_namespace:\n type: bool\n chart_repo_url:\n type: str\n disable_hook:\n type: bool\n history_max:\n type: int\n purge:\n type: bool\n replace:\n type: bool\n skip_crds:\n type: bool\n wait:\n type: bool\n default: true\n wait_timeout:\n type: str\n atomic:\n type: bool\n default: true\n\n repositories:\n type: list\n elements: dict\n description: |\n List of dictionaries passed as arguments to\n kubernetes.core.helm_repository.\n default: []\n options:\n name:\n type: str\n required: true\n password:\n type: str\n username:\n type: str\n url:\n type: str\n release_common_opts:\n type: dict\n description: |\n Common arguments for every helm invocation.\n default: {}\n options:\n create_namespace:\n type: bool\n default: true\n chart_repo_url:\n type: str\n disable_hook:\n type: bool\n history_max:\n type: int\n purge:\n type: bool\n replace:\n type: bool\n skip_crds:\n type: bool\n wait:\n type: bool\n default: true\n wait_timeout:\n type: str\n default: \"5m\"\n atomic:\n type: bool\n default: true\n" }, { "path": "roles/helm-apps/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/helm\n" }, { "path": "roles/helm-apps/tasks/main.yml", "content": "---\n- name: Add Helm repositories\n kubernetes.core.helm_repository: \"{{ helm_repository_defaults | combine(item) }}\" # noqa args[module]\n loop: \"{{ repositories }}\"\n\n- name: Update Helm repositories\n kubernetes.core.helm:\n state: absent\n binary_path: \"{{ bin_dir }}/helm\"\n release_name: dummy # trick needed to refresh in separate step\n release_namespace: kube-system\n update_repo_cache: true\n when:\n - repositories != []\n - helm_update\n\n- name: Install Helm Applications\n kubernetes.core.helm: \"{{ helm_defaults | combine(release_common_opts, item) }}\" # noqa args[module]\n loop: \"{{ releases }}\"\n" }, { "path": "roles/helm-apps/vars/main.yml", "content": "---\nhelm_update: true\n\nhelm_defaults:\n atomic: true\n binary_path: \"{{ bin_dir }}/helm\"\n\nhelm_repository_defaults:\n binary_path: \"{{ bin_dir }}/helm\"\n force_update: true\n" }, { "path": "roles/kubernetes/client/defaults/main.yml", "content": "---\nkubeconfig_localhost: false\nkubeconfig_localhost_ansible_host: false\nkubectl_localhost: false\nartifacts_dir: \"{{ inventory_dir }}/artifacts\"\n\nkube_config_dir: \"/etc/kubernetes\"\nkube_apiserver_port: \"6443\"\n" }, { "path": "roles/kubernetes/client/tasks/main.yml", "content": "---\n- name: Set external kube-apiserver endpoint\n set_fact:\n # noqa: jinja[spacing]\n external_apiserver_address: >-\n {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%}\n {{ loadbalancer_apiserver.address }}\n {%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%}\n {{ hostvars[groups['kube_control_plane'][0]].ansible_host }}\n {%- else -%}\n {{ kube_apiserver_access_address }}\n {%- endif -%}\n # noqa: jinja[spacing]\n external_apiserver_port: >-\n {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%}\n {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}\n {%- else -%}\n {{ kube_apiserver_port }}\n {%- endif -%}\n tags:\n - facts\n\n- name: Create kube config dir for current/ansible become user\n file:\n path: \"{{ ansible_env.HOME | default('/root') }}/.kube\"\n mode: \"0700\"\n state: directory\n\n- name: Write admin kubeconfig to current/ansible become user home\n copy:\n src: \"{{ kube_config_dir }}/admin.conf\"\n dest: \"{{ ansible_env.HOME | default('/root') }}/.kube/config\"\n remote_src: true\n mode: \"0600\"\n backup: true\n\n- name: Create kube artifacts dir\n file:\n path: \"{{ artifacts_dir }}\"\n mode: \"0750\"\n state: directory\n delegate_to: localhost\n connection: local\n become: false\n run_once: true\n when: kubeconfig_localhost\n\n- name: Wait for k8s apiserver\n wait_for:\n host: \"{{ kube_apiserver_access_address }}\"\n port: \"{{ kube_apiserver_port }}\"\n timeout: 180\n\n- name: Create kubeconfig localhost artifacts\n when: kubeconfig_localhost\n block:\n - name: Generate admin kubeconfig using kubeadm\n command: >-\n {{ bin_dir }}/kubeadm kubeconfig user\n --client-name=kubernetes-admin-{{ cluster_name }}\n --org=kubeadm:cluster-admins\n --config {{ kube_config_dir }}/kubeadm-config.yaml\n register: kubeadm_admin_kubeconfig\n changed_when: false\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n - name: Write admin kubeconfig on ansible host\n copy:\n content: \"{{ kubeadm_admin_kubeconfig.stdout | from_yaml | combine(override, recursive=true) | to_nice_yaml(indent=2) }}\"\n dest: \"{{ artifacts_dir }}/admin.conf\"\n mode: \"0600\"\n vars:\n admin_kubeconfig: \"{{ kubeadm_admin_kubeconfig.stdout | from_yaml }}\"\n context: \"kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}\"\n override:\n clusters:\n - \"{{ admin_kubeconfig['clusters'][0] | combine({'name': cluster_name, 'cluster': admin_kubeconfig['clusters'][0]['cluster'] | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})}, recursive=true) }}\"\n contexts:\n - \"{{ admin_kubeconfig['contexts'][0] | combine({'name': context, 'context': admin_kubeconfig['contexts'][0]['context'] | combine({'cluster': cluster_name})}, recursive=true) }}\"\n current-context: \"{{ context }}\"\n delegate_to: localhost\n connection: local\n become: false\n run_once: true\n\n- name: Copy kubectl binary to ansible host\n fetch:\n src: \"{{ bin_dir }}/kubectl\"\n dest: \"{{ artifacts_dir }}/kubectl\"\n flat: true\n validate_checksum: false\n register: copy_binary_result\n until: copy_binary_result is not failed\n retries: 20\n become: false\n run_once: true\n when: kubectl_localhost\n\n- name: Create helper script kubectl.sh on ansible host\n copy:\n content: |\n #!/bin/bash\n ${BASH_SOURCE%/*}/kubectl --kubeconfig=${BASH_SOURCE%/*}/admin.conf \"$@\"\n dest: \"{{ artifacts_dir }}/kubectl.sh\"\n mode: \"0755\"\n become: false\n run_once: true\n delegate_to: localhost\n connection: local\n when: kubectl_localhost and kubeconfig_localhost\n" }, { "path": "roles/kubernetes/control-plane/defaults/main/etcd.yml", "content": "---\n# Set etcd user/group\netcd_owner: etcd\n\n# Note: This does not set up DNS entries. It simply adds the following DNS\n# entries to the certificate\netcd_cert_alt_names:\n - \"etcd.kube-system.svc.{{ dns_domain }}\"\n - \"etcd.kube-system.svc\"\n - \"etcd.kube-system\"\n - \"etcd\"\netcd_cert_alt_ips: []\n\netcd_heartbeat_interval: \"250\"\netcd_election_timeout: \"5000\"\n\netcd_metrics: \"basic\"\n\n## A dictionary of extra environment variables to add to etcd.env, formatted like:\n## etcd_extra_vars:\n## var1: \"value1\"\n## var2: \"value2\"\n## Note this is different from the etcd role with ETCD_ prfexi, caps, and underscores\netcd_extra_vars: {}\n\n# etcd_quota_backend_bytes: \"2147483648\"\n# etcd_max_request_bytes: \"1572864\"\n\netcd_compaction_retention: \"8\"\n" }, { "path": "roles/kubernetes/control-plane/defaults/main/kube-proxy.yml", "content": "---\n# bind address for kube-proxy\nkube_proxy_bind_address: '0.0.0.0'\n\n# acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the\n# default value of 'application/json'. This field will control all connections to the server used by a particular\n# client.\nkube_proxy_client_accept_content_types: ''\n\n# burst allows extra queries to accumulate when a client is exceeding its rate.\nkube_proxy_client_burst: 10\n\n# contentType is the content type used when sending data to the server from this client.\nkube_proxy_client_content_type: application/vnd.kubernetes.protobuf\n\n# kubeconfig is the path to a KubeConfig file.\n# Leave as empty string to generate from other fields\nkube_proxy_client_kubeconfig: ''\n\n# qps controls the number of queries per second allowed for this connection.\nkube_proxy_client_qps: 5\n\n# How often configuration from the apiserver is refreshed. Must be greater than 0.\nkube_proxy_config_sync_period: 15m0s\n\n### Conntrack\n# maxPerCore is the maximum number of NAT connections to track\n# per CPU core (0 to leave the limit as-is and ignore min).\nkube_proxy_conntrack_max_per_core: 32768\n\n# min is the minimum value of connect-tracking records to allocate,\n# regardless of conntrackMaxPerCore (set maxPerCore=0 to leave the limit as-is).\nkube_proxy_conntrack_min: 131072\n\n# tcpCloseWaitTimeout is how long an idle conntrack entry\n# in CLOSE_WAIT state will remain in the conntrack\n# table. (e.g. '60s'). Must be greater than 0 to set.\nkube_proxy_conntrack_tcp_close_wait_timeout: 1h0m0s\n\n# tcpEstablishedTimeout is how long an idle TCP connection will be kept open\n# (e.g. '2s'). Must be greater than 0 to set.\nkube_proxy_conntrack_tcp_established_timeout: 24h0m0s\n\n# Enables profiling via web interface on /debug/pprof handler.\n# Profiling handlers will be handled by metrics server.\nkube_proxy_enable_profiling: false\n\n# bind address for kube-proxy health check\nkube_proxy_healthz_bind_address: 0.0.0.0:10256\n\n# If using the pure iptables proxy, SNAT everything. Note that it breaks any\n# policy engine.\nkube_proxy_masquerade_all: false\n\n# If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with.\n# Must be within the range [0, 31].\nkube_proxy_masquerade_bit: 14\n\n# The minimum interval of how often the iptables or ipvs rules can be refreshed as\n# endpoints and services change (e.g. '5s', '1m', '2h22m').\nkube_proxy_min_sync_period: 0s\n\n# The maximum interval of how often iptables or ipvs rules are refreshed (e.g. '5s', '1m', '2h22m').\n# Must be greater than 0.\nkube_proxy_sync_period: 30s\n\n# A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.\nkube_proxy_exclude_cidrs: []\n\n# The ipvs scheduler type when proxy mode is ipvs\n# rr: round-robin\n# lc: least connection\n# dh: destination hashing\n# sh: source hashing\n# sed: shortest expected delay\n# nq: never queue\nkube_proxy_scheduler: rr\n\n# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface\n# must be set to true for MetalLB, kube-vip(ARP enabled) to work\nkube_proxy_strict_arp: false\n\n# kube_proxy_tcp_timeout is the timeout value used for idle IPVS TCP sessions.\n# The default value is 0, which preserves the current timeout value on the system.\nkube_proxy_tcp_timeout: 0s\n\n# kube_proxy_tcp_fin_timeout is the timeout value used for IPVS TCP sessions after receiving a FIN.\n# The default value is 0, which preserves the current timeout value on the system.\nkube_proxy_tcp_fin_timeout: 0s\n\n# kube_proxy_udp_timeout is the timeout value used for IPVS UDP packets.\n# The default value is 0, which preserves the current timeout value on the system.\nkube_proxy_udp_timeout: 0s\n\n# The IP address and port for the metrics server to serve on\n# (set to 0.0.0.0 for all IPv4 interfaces and `::` for all IPv6 interfaces)\nkube_proxy_metrics_bind_address: 127.0.0.1:10249\n\n# A string slice of values which specify the addresses to use for NodePorts.\n# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).\n# The default empty string slice ([]) means to use all local addresses.\nkube_proxy_nodeport_addresses: >-\n {%- if kube_proxy_nodeport_addresses_cidr is defined -%}\n [{{ kube_proxy_nodeport_addresses_cidr }}]\n {%- else -%}\n []\n {%- endif -%}\n\n# oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]\nkube_proxy_oom_score_adj: -999\n\n# portRange is the range of host ports (beginPort-endPort, inclusive) that may be consumed\n# in order to proxy service traffic. If unspecified, 0, or (0-0) then ports will be randomly chosen.\nkube_proxy_port_range: ''\n" }, { "path": "roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml", "content": "---\n# Extra args passed by kubeadm\nkube_kubeadm_scheduler_extra_args: {}\n\n# Associated interface must be reachable by the rest of the cluster, and by\n# CLI/web clients.\nkube_scheduler_bind_address: \"::\"\n\n# ClientConnection options (e.g. Burst, QPS) except from kubeconfig.\nkube_scheduler_client_conn_extra_opts: {}\n\n# Additional KubeSchedulerConfiguration settings (e.g. metricsBindAddress).\nkube_scheduler_config_extra_opts: {}\n\n# List of scheduler extenders (dicts), each holding the values of how to\n# communicate with the extender.\nkube_scheduler_extenders: []\n\n# Leader Election options (e.g. ResourceName, RetryPerion) except from\n# LeaseDuration and Renew deadline which are defined in following vars.\nkube_scheduler_leader_elect_extra_opts: {}\n\n# Leader election lease duration\nkube_scheduler_leader_elect_lease_duration: 15s\n\n# Leader election lease timeout\nkube_scheduler_leader_elect_renew_deadline: 10s\n\n# Lisf of scheduling profiles (ditcs) supported by kube-scheduler\nkube_scheduler_profiles: []\n\n# Extra volume mounts\nscheduler_extra_volumes: {}\n" }, { "path": "roles/kubernetes/control-plane/defaults/main/main.yml", "content": "---\n# disable upgrade cluster\nupgrade_cluster_setup: false\n\n# Number of retries (with 5 seconds interval) to check that new control plane nodes\n# are in Ready condition after joining\ncontrol_plane_node_become_ready_tries: 24\n# By default the external API listens on all interfaces, this can be changed to\n# listen on a specific address/interface.\n# NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost\n# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.\nkube_apiserver_bind_address: \"::\"\n\n# A port range to reserve for services with NodePort visibility.\n# Inclusive at both ends of the range.\nkube_apiserver_node_port_range: \"30000-32767\"\n\n# ETCD backend for k8s data\nkube_apiserver_storage_backend: etcd3\n\n# The interval of compaction requests. If 0, the compaction request from apiserver is disabled.\nkube_apiserver_etcd_compaction_interval: \"5m0s\"\n\n# CIS 1.2.26\n# Validate that the service account token\n# in the request is actually present in etcd.\nkube_apiserver_service_account_lookup: true\n\nkube_etcd_cacert_file: ca.pem\nkube_etcd_cert_file: node-{{ inventory_hostname }}.pem\nkube_etcd_key_file: node-{{ inventory_hostname }}-key.pem\n\n# Associated interfaces must be reachable by the rest of the cluster, and by\n# CLI/web clients.\nkube_controller_manager_bind_address: \"::\"\n\n## Control plane health check settings\ncontrol_plane_health_retries: 60 # Default retries for apiserver, scheduler, controller-manager health checks\n\n# Leader election lease durations and timeouts for controller-manager\nkube_controller_manager_leader_elect_lease_duration: 15s\nkube_controller_manager_leader_elect_renew_deadline: 10s\n\n# discovery_timeout modifies the discovery timeout\ndiscovery_timeout: 5m0s\n\n# Instruct first control plane node to refresh kubeadm token\nkubeadm_refresh_token: true\n\n# Scale down coredns replicas to 0 if not using coredns dns_mode\nkubeadm_scale_down_coredns_enabled: true\n\n# audit support\nkubernetes_audit: false\n# path to audit log file\naudit_log_path: /var/log/audit/kube-apiserver-audit.log\n# num days\naudit_log_maxage: 30\n# the num of audit logs to retain\naudit_log_maxbackups: 10\n# the max size in MB to retain\naudit_log_maxsize: 100\n# policy file\naudit_policy_file: \"{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml\"\n# custom audit policy rules (to replace the default ones)\n# audit_policy_custom_rules: |\n# - level: None\n# users: []\n# verbs: []\n# resources: []\n\n# audit log hostpath\naudit_log_name: audit-logs\naudit_log_hostpath: /var/log/kubernetes/audit\naudit_log_mountpath: \"{{ audit_log_path | dirname }}\"\n\n# audit policy hostpath\naudit_policy_name: audit-policy\naudit_policy_hostpath: \"{{ audit_policy_file | dirname }}\"\naudit_policy_mountpath: \"{{ audit_policy_hostpath }}\"\n\n# audit webhook support\nkubernetes_audit_webhook: false\n\n# path to audit webhook config file\naudit_webhook_config_file: \"{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml\"\naudit_webhook_server_url: \"https://audit.app\"\naudit_webhook_server_extra_args: {}\naudit_webhook_mode: batch\naudit_webhook_batch_max_size: 100\naudit_webhook_batch_max_wait: 1s\n\nkube_controller_node_monitor_grace_period: 40s\nkube_controller_node_monitor_period: 5s\nkube_controller_terminated_pod_gc_threshold: 12500\nkube_apiserver_request_timeout: \"1m0s\"\nkube_apiserver_pod_eviction_not_ready_timeout_seconds: \"300\"\nkube_apiserver_pod_eviction_unreachable_timeout_seconds: \"300\"\n\n# 1.10+ admission plugins\nkube_apiserver_enable_admission_plugins: []\n\n# enable admission plugins configuration\nkube_apiserver_admission_control_config_file: false\n\n# data structure to configure EventRateLimit admission plugin\n# this should have the following structure:\n# kube_apiserver_admission_event_rate_limits:\n# :\n# type: \n# qps: \n# burst: \n# cache_size: \nkube_apiserver_admission_event_rate_limits: {}\n\n## PodSecurityAdmission plugin configuration\nkube_pod_security_use_default: false\nkube_pod_security_default_enforce: baseline\nkube_pod_security_default_enforce_version: \"v{{ kube_major_version }}\"\nkube_pod_security_default_audit: restricted\nkube_pod_security_default_audit_version: \"v{{ kube_major_version }}\"\nkube_pod_security_default_warn: restricted\nkube_pod_security_default_warn_version: \"v{{ kube_major_version }}\"\nkube_pod_security_exemptions_usernames: []\nkube_pod_security_exemptions_runtime_class_names: []\nkube_pod_security_exemptions_namespaces:\n - kube-system\n\n## ResourceQuota plugin configuration\n## Resources that ResourceQuota should limit by default if no quota exists\n## Example below enforces quota on all storage classes\n# kube_resource_quota_limited_resources:\n# - apiGroup: \"\"\n# resource: persistentvolumeclaims\n# matchContains:\n# - .storageclass.storage.k8s.io/requests.storage\nkube_resource_quota_limited_resources: []\n\n# 1.10+ list of disabled admission plugins\nkube_apiserver_disable_admission_plugins: []\n\n# extra runtime config\nkube_api_runtime_config: []\n\n## Enable/Disable Kube API Server Authentication Methods\nkube_token_auth: false\nkube_oidc_auth: false\n\n## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication\nkube_webhook_token_auth: false\nkube_webhook_token_auth_url_skip_tls_verify: false\n# kube_webhook_token_auth_url: https://...\n## base64-encoded string of the webhook's CA certificate\n# kube_webhook_token_auth_ca_data: \"LS0t...\"\n\n## Variables for webhook token authz https://kubernetes.io/docs/reference/access-authn-authz/webhook/\n# kube_webhook_authorization_url: https://...\nkube_webhook_authorization: false\nkube_webhook_authorization_url_skip_tls_verify: false\n\n# Default podnodeselector\nkube_apiserver_admission_plugins_podnodeselector_default_node_selector: \"\"\n\n## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/\n## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)\n\n# kube_oidc_url: https:// ...\n# kube_oidc_client_id: kubernetes\n## Optional settings for OIDC\n# kube_oidc_username_claim: sub\n# kube_oidc_username_prefix: 'oidc:'\n# kube_oidc_groups_claim: groups\n# kube_oidc_groups_prefix: 'oidc:'\n# Copy oidc CA file to the following path if needed\n# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem\n# Optionally include a base64-encoded oidc CA cert\n# kube_oidc_ca_cert: c3RhY2thYnVzZS5jb20...\n\n# List of the preferred NodeAddressTypes to use for kubelet connections.\nkubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP'\n\n## Extra args for k8s components passing by kubeadm\nkube_kubeadm_apiserver_extra_args: {}\nkube_kubeadm_controller_extra_args: {}\n\n## Extra control plane host volume mounts\n## Example:\n# apiserver_extra_volumes:\n# - name: name\n# hostPath: /host/path\n# mountPath: /mount/path\n# readOnly: true\napiserver_extra_volumes: {}\ncontroller_manager_extra_volumes: {}\n\n## Encrypting Secret Data at Rest\nkube_encrypt_secret_data: false\nkube_encrypt_token: \"{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}\"\n# Must be either: aescbc, secretbox or aesgcm\nkube_encryption_algorithm: \"secretbox\"\n# Which kubernetes resources to encrypt\nkube_encryption_resources: [secrets]\n\nsecrets_encryption_query: \"resources[*].providers[0].{{ kube_encryption_algorithm }}.keys[0].secret\"\n\n## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.\n# tls_min_version: \"\"\n\n## Support tls cipher suites.\n# tls_cipher_suites:\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\n# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_RSA_WITH_RC4_128_SHA\n# - TLS_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_RSA_WITH_AES_256_CBC_SHA\n# - TLS_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_RSA_WITH_RC4_128_SHA\n\n## Amount of time to retain events. (default 1h0m0s)\nevent_ttl_duration: \"1h0m0s\"\n\n## Automatically renew K8S control plane certificates on first Monday of each month\nauto_renew_certificates: false\n# First Monday of each month\nauto_renew_certificates_systemd_calendar: \"Mon *-*-1,2,3,4,5,6,7 03:00:00\"\n# kubeadm renews all the certificates during control plane upgrade.\n# If we have requirement like without renewing certs upgrade the cluster,\n# we can opt out from the default behavior by setting kubeadm_upgrade_auto_cert_renewal to false\nkubeadm_upgrade_auto_cert_renewal: true\n\n# Add Subject Alternative Names to the Kubernetes apiserver certificates.\n# Useful if you access the API from multiples load balancers, for instance.\nsupplementary_addresses_in_ssl_keys: []\n\n# Bash alias of kubectl to interact with Kubernetes cluster much easier\n# kubectl_alias: k\n\n## Enable distributed tracing for kube-apiserver\nkube_apiserver_tracing: false\nkube_apiserver_tracing_endpoint: \"[::]:4317\"\nkube_apiserver_tracing_sampling_rate_per_million: 100\n\n# Enable kubeadm file discovery if anonymous access has been removed\nkubeadm_use_file_discovery: \"{{ remove_anonymous_access }}\"\n\n# imagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel. Default: true\nkubeadm_image_pull_serial: true\n\n# Supported asymmetric encryption algorithm types for the cluster's keys and certificates.\n# can be one of RSA-2048(default), RSA-3072, RSA-4096, ECDSA-P256\n# ref: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/#kubeadm-k8s-io-v1beta4-ClusterConfiguration\nkube_asymmetric_encryption_algorithm: \"RSA-2048\"\n\n# certificates validity period configuration\n# non-CA certificate validity period, default 1 year (365d × 24h = 8760h)\nkube_cert_validity_period: 8760h\n# CA certificate validity period, default 10 years (365d × 24h × 10 = 87600h)\nkube_ca_cert_validity_period: 87600h\n" }, { "path": "roles/kubernetes/control-plane/handlers/main.yml", "content": "---\n- name: Control plane | reload systemd\n systemd_service:\n daemon_reload: true\n listen: Control plane | restart kubelet\n\n- name: Control plane | reload kubelet\n service:\n name: kubelet\n state: restarted\n listen: Control plane | restart kubelet\n\n- name: Control plane | Remove apiserver container docker\n shell: \"set -o pipefail && docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f\"\n args:\n executable: /bin/bash\n register: remove_apiserver_container\n retries: 10\n until: remove_apiserver_container.rc == 0\n delay: 1\n when: container_manager == \"docker\"\n listen: Control plane | Restart apiserver\n\n- name: Control plane | Remove apiserver container containerd/crio\n shell: \"set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-apiserver*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'\"\n args:\n executable: /bin/bash\n register: remove_apiserver_container\n retries: 10\n until: remove_apiserver_container.rc == 0\n delay: 1\n when: container_manager in ['containerd', 'crio']\n listen: Control plane | Restart apiserver\n\n- name: Control plane | Remove scheduler container docker\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f\"\n args:\n executable: /bin/bash\n register: remove_scheduler_container\n retries: 10\n until: remove_scheduler_container.rc == 0\n delay: 1\n when: container_manager == \"docker\"\n listen: Control plane | Restart kube-scheduler\n\n- name: Control plane | Remove scheduler container containerd/crio\n shell: \"set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-scheduler*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'\"\n args:\n executable: /bin/bash\n register: remove_scheduler_container\n retries: 10\n until: remove_scheduler_container.rc == 0\n delay: 1\n when: container_manager in ['containerd', 'crio']\n listen: Control plane | Restart kube-scheduler\n\n- name: Control plane | Remove controller manager container docker\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f\"\n args:\n executable: /bin/bash\n register: remove_cm_container\n retries: 10\n until: remove_cm_container.rc == 0\n delay: 1\n when: container_manager == \"docker\"\n listen: Control plane | Restart kube-controller-manager\n\n- name: Control plane | Remove controller manager container containerd/crio\n shell: \"set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-controller-manager*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'\"\n args:\n executable: /bin/bash\n register: remove_cm_container\n retries: 10\n until: remove_cm_container.rc == 0\n delay: 1\n when: container_manager in ['containerd', 'crio']\n listen: Control plane | Restart kube-controller-manager\n\n- name: Control plane | wait for kube-scheduler\n vars:\n endpoint: \"{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '::' else 'localhost' }}\"\n uri:\n url: https://{{ endpoint }}:10259/healthz\n validate_certs: false\n register: scheduler_result\n until: scheduler_result.status == 200\n retries: \"{{ control_plane_health_retries }}\"\n delay: 1\n listen:\n - Control plane | restart kubelet\n - Control plane | Restart kube-scheduler\n\n- name: Control plane | wait for kube-controller-manager\n vars:\n endpoint: \"{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '::' else 'localhost' }}\"\n uri:\n url: https://{{ endpoint }}:10257/healthz\n validate_certs: false\n register: controller_manager_result\n until: controller_manager_result.status == 200\n retries: \"{{ control_plane_health_retries }}\"\n delay: 1\n listen:\n - Control plane | restart kubelet\n - Control plane | Restart kube-controller-manager\n\n- name: Control plane | wait for the apiserver to be running\n uri:\n url: \"{{ kube_apiserver_endpoint }}/healthz\"\n validate_certs: false\n register: result\n until: result.status == 200\n retries: \"{{ control_plane_health_retries }}\"\n delay: 1\n listen:\n - Control plane | restart kubelet\n - Control plane | Restart apiserver\n" }, { "path": "roles/kubernetes/control-plane/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes/kubeadm_common\n - role: adduser\n user: \"{{ addusers.etcd }}\"\n when:\n - etcd_deployment_type == \"kubeadm\"\n - not (ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\", \"ClearLinux\"] or is_fedora_coreos)\n - role: network_plugin/calico_defaults\n - role: etcd_defaults\n" }, { "path": "roles/kubernetes/control-plane/tasks/check-api.yml", "content": "---\n- name: Kubeadm | Check api is up\n uri:\n url: \"https://{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}/healthz\"\n validate_certs: false\n when: ('kube_control_plane' in group_names)\n register: _result\n retries: 60\n delay: 5\n until: _result.status == 200\n" }, { "path": "roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml", "content": "---\n- name: Check if secret for encrypting data at rest already exist\n stat:\n path: \"{{ kube_cert_dir }}/secrets_encryption.yaml\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: secrets_encryption_file\n\n- name: Slurp secrets_encryption file if it exists\n slurp:\n src: \"{{ kube_cert_dir }}/secrets_encryption.yaml\"\n register: secret_file_encoded\n when: secrets_encryption_file.stat.exists\n\n- name: Base 64 Decode slurped secrets_encryption.yaml file\n set_fact:\n secret_file_decoded: \"{{ secret_file_encoded['content'] | b64decode | from_yaml }}\"\n when: secrets_encryption_file.stat.exists\n\n- name: Extract secret value from secrets_encryption.yaml\n set_fact:\n kube_encrypt_token_extracted: \"{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}\"\n when: secrets_encryption_file.stat.exists\n\n- name: Set kube_encrypt_token across control plane nodes\n set_fact:\n kube_encrypt_token: \"{{ kube_encrypt_token_extracted }}\"\n delegate_to: \"{{ item }}\"\n delegate_facts: true\n with_inventory_hostnames: kube_control_plane\n when: kube_encrypt_token_extracted is defined\n\n- name: Write secrets for encrypting secret data at rest\n template:\n src: secrets_encryption.yaml.j2\n dest: \"{{ kube_cert_dir }}/secrets_encryption.yaml\"\n owner: root\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n" }, { "path": "roles/kubernetes/control-plane/tasks/kubeadm-backup.yml", "content": "---\n- name: Backup old certs and keys\n copy:\n src: \"{{ kube_cert_dir }}/{{ item }}\"\n dest: \"{{ kube_cert_dir }}/{{ item }}.old\"\n mode: preserve\n remote_src: true\n with_items:\n - apiserver.crt\n - apiserver.key\n - apiserver-kubelet-client.crt\n - apiserver-kubelet-client.key\n - front-proxy-client.crt\n - front-proxy-client.key\n ignore_errors: true # noqa ignore-errors\n\n- name: Backup old confs\n copy:\n src: \"{{ kube_config_dir }}/{{ item }}\"\n dest: \"{{ kube_config_dir }}/{{ item }}.old\"\n mode: preserve\n remote_src: true\n with_items:\n - admin.conf\n - controller-manager.conf\n - kubelet.conf\n - scheduler.conf\n ignore_errors: true # noqa ignore-errors\n" }, { "path": "roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml", "content": "---\n- name: Calculate etcd cert serial\n command: \"openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial\"\n register: \"etcd_client_cert_serial_result\"\n changed_when: false\n tags:\n - network\n\n- name: Set etcd_client_cert_serial\n set_fact:\n etcd_client_cert_serial: \"{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}\"\n tags:\n - network\n\n- name: Ensure etcdctl and etcdutl script is installed\n import_role:\n name: etcdctl_etcdutl\n when: etcd_deployment_type == \"kubeadm\"\n tags:\n - etcdctl\n - etcdutl\n\n- name: Set ownership for etcd data directory\n file:\n path: \"{{ etcd_data_dir }}\"\n owner: \"{{ etcd_owner }}\"\n group: \"{{ etcd_owner }}\"\n mode: \"0700\"\n when: etcd_deployment_type == \"kubeadm\"\n" }, { "path": "roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml", "content": "---\n- name: Set kubeadm_discovery_address\n set_fact:\n # noqa: jinja[spacing]\n kubeadm_discovery_address: >-\n {%- if \"127.0.0.1\" in kube_apiserver_endpoint or \"localhost\" in kube_apiserver_endpoint -%}\n {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}\n {%- else -%}\n {{ kube_apiserver_endpoint | regex_replace('https://', '') }}\n {%- endif %}\n tags:\n - facts\n\n- name: Obtain kubeadm certificate key for joining control planes nodes\n when:\n - not kube_external_ca_mode\n run_once: true\n block:\n - name: Upload certificates so they are fresh and not expired\n command: >-\n {{ bin_dir }}/kubeadm init phase\n --config {{ kube_config_dir }}/kubeadm-config.yaml\n upload-certs\n --upload-certs\n register: kubeadm_upload_cert\n delegate_to: \"{{ first_kube_control_plane }}\"\n\n - name: Parse certificate key if not set\n set_fact:\n kubeadm_certificate_key: \"{{ kubeadm_upload_cert.stdout_lines[-1] | trim }}\"\n\n- name: Wait for k8s apiserver\n wait_for:\n host: \"{{ kubeadm_discovery_address | regex_replace('\\\\]?:\\\\d+$', '') | regex_replace('^\\\\[', '') }}\"\n port: \"{{ kubeadm_discovery_address.split(':')[-1] }}\"\n timeout: 180\n\n\n- name: Check already run\n debug:\n msg: \"{{ kubeadm_already_run.stat.exists }}\"\n\n- name: Reset cert directory\n shell: >-\n if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then\n {{ bin_dir }}/kubeadm reset -f --cert-dir {{ kube_cert_dir }};\n fi\n environment:\n PATH: \"{{ bin_dir }}:{{ ansible_env.PATH }}\"\n when:\n - inventory_hostname != first_kube_control_plane\n - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists\n - not kube_external_ca_mode\n\n- name: Get kubeconfig for join discovery process\n command: \"{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'\"\n register: kubeconfig_file_discovery\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n when:\n - kubeadm_use_file_discovery\n - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists\n\n- name: Copy discovery kubeconfig\n copy:\n dest: \"{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml\"\n content: \"{{ kubeconfig_file_discovery.stdout }}\"\n owner: \"root\"\n mode: \"0644\"\n when:\n - inventory_hostname != first_kube_control_plane\n - kubeadm_use_file_discovery\n - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists\n\n- name: Create kubeadm ControlPlane config\n template:\n src: \"kubeadm-controlplane.yaml.j2\"\n dest: \"{{ kube_config_dir }}/kubeadm-controlplane.yaml\"\n mode: \"0640\"\n backup: true\n validate: \"{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}\"\n when:\n - inventory_hostname != first_kube_control_plane\n - not kubeadm_already_run.stat.exists\n\n- name: Joining control plane node to the cluster.\n command: >-\n {{ bin_dir }}/kubeadm join\n --config {{ kube_config_dir }}/kubeadm-controlplane.yaml\n --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}\n --skip-phases={{ kubeadm_join_phases_skip | join(',') }}\n environment:\n PATH: \"{{ bin_dir }}:{{ ansible_env.PATH }}\"\n register: kubeadm_join_control_plane\n retries: 3\n throttle: 1\n until: kubeadm_join_control_plane is succeeded\n when:\n - inventory_hostname != first_kube_control_plane\n - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists\n\n- name: Wait for new control plane nodes to be Ready\n when: kubeadm_already_run.stat.exists\n run_once: true\n command: >\n {{ kubectl }} get nodes --selector node-role.kubernetes.io/control-plane\n -o jsonpath-as-json=\"{.items[*].status.conditions[?(@.type == 'Ready')]}\"\n register: control_plane_node_ready_conditions\n retries: \"{{ control_plane_node_become_ready_tries }}\"\n delay: 5\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n until: >\n control_plane_node_ready_conditions.stdout\n | from_json | selectattr('status', '==', 'True')\n | length == (groups['kube_control_plane'] | length)\n" }, { "path": "roles/kubernetes/control-plane/tasks/kubeadm-setup.yml", "content": "---\n- name: Install OIDC certificate\n copy:\n content: \"{{ kube_oidc_ca_cert | b64decode }}\"\n dest: \"{{ kube_oidc_ca_file }}\"\n owner: root\n group: root\n mode: \"0644\"\n when:\n - kube_oidc_auth\n - kube_oidc_ca_cert is defined\n\n- name: Kubeadm | Check if kubeadm has already run\n stat:\n path: \"/var/lib/kubelet/config.yaml\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kubeadm_already_run\n\n- name: Kubeadm | Backup kubeadm certs / kubeconfig\n import_tasks: kubeadm-backup.yml\n when:\n - kubeadm_already_run.stat.exists\n\n- name: Kubeadm | aggregate all SANs\n set_fact:\n apiserver_sans: \"{{ _apiserver_sans | flatten | select | unique }}\"\n vars:\n _apiserver_sans:\n - \"kubernetes\"\n - \"kubernetes.default\"\n - \"kubernetes.default.svc\"\n - \"kubernetes.default.svc.{{ dns_domain }}\"\n - \"{{ kube_apiserver_ip }}\"\n - \"localhost\"\n - \"127.0.0.1\"\n - \"::1\"\n - \"{{ apiserver_loadbalancer_domain_name | d('') }}\"\n - \"{{ loadbalancer_apiserver.address | d('') }}\"\n - \"{{ supplementary_addresses_in_ssl_keys }}\"\n - \"{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') }}\"\n - \"{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') }}\"\n - \"{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | select('defined') }}\"\n - \"{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'address']) | select('defined') }}\"\n - \"{{ groups['kube_control_plane'] | map('extract', hostvars, 'ansible_hostname') }}\"\n - \"{{ groups['kube_control_plane'] | map('extract', hostvars, 'ansible_fqdn') }}\"\n - \"{{ kube_override_hostname }}\"\n - \"{{ kube_vip_address }}\"\n tags: facts\n\n- name: Create audit-policy directory\n file:\n path: \"{{ audit_policy_file | dirname }}\"\n state: directory\n mode: \"0640\"\n when: kubernetes_audit or kubernetes_audit_webhook\n\n- name: Write api audit policy yaml\n template:\n src: apiserver-audit-policy.yaml.j2\n dest: \"{{ audit_policy_file }}\"\n mode: \"0640\"\n when: kubernetes_audit or kubernetes_audit_webhook\n notify: Control plane | Restart apiserver\n\n- name: Write api audit webhook config yaml\n template:\n src: apiserver-audit-webhook-config.yaml.j2\n dest: \"{{ audit_webhook_config_file }}\"\n mode: \"0640\"\n when: kubernetes_audit_webhook\n notify: Control plane | Restart apiserver\n\n- name: Create apiserver tracing config directory\n file:\n path: \"{{ kube_config_dir }}/tracing\"\n state: directory\n mode: \"0640\"\n when: kube_apiserver_tracing\n\n- name: Write apiserver tracing config yaml\n template:\n src: apiserver-tracing.yaml.j2\n dest: \"{{ kube_config_dir }}/tracing/apiserver-tracing.yaml\"\n mode: \"0640\"\n when: kube_apiserver_tracing\n notify: Control plane | Restart apiserver\n\n# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.\n- name: Set kubeadm_config_api_fqdn define\n set_fact:\n kubeadm_config_api_fqdn: \"{{ apiserver_loadbalancer_domain_name }}\"\n when: loadbalancer_apiserver is defined\n\n- name: Kubeadm | Create kubeadm config\n template:\n src: \"kubeadm-config.v1beta4.yaml.j2\"\n dest: \"{{ kube_config_dir }}/kubeadm-config.yaml\"\n mode: \"0640\"\n validate: \"{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}\"\n\n- name: Kubeadm | Create directory to store admission control configurations\n file:\n path: \"{{ kube_config_dir }}/admission-controls\"\n state: directory\n mode: \"0640\"\n when: kube_apiserver_admission_control_config_file\n\n- name: Kubeadm | Push admission control config file\n template:\n src: \"admission-controls.yaml.j2\"\n dest: \"{{ kube_config_dir }}/admission-controls/admission-controls.yaml\"\n mode: \"0640\"\n when: kube_apiserver_admission_control_config_file\n notify: Control plane | Restart apiserver\n\n- name: Kubeadm | Push admission control config files\n template:\n src: \"{{ item | lower }}.yaml.j2\"\n dest: \"{{ kube_config_dir }}/admission-controls/{{ item | lower }}.yaml\"\n mode: \"0640\"\n when:\n - kube_apiserver_admission_control_config_file\n - item in kube_apiserver_admission_plugins_needs_configuration\n loop: \"{{ kube_apiserver_enable_admission_plugins }}\"\n notify: Control plane | Restart apiserver\n\n- name: Kubeadm | Check apiserver.crt SANs\n vars:\n apiserver_ips: \"{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}\"\n apiserver_hosts: \"{{ apiserver_sans | difference(apiserver_ips) }}\"\n when:\n - kubeadm_already_run.stat.exists\n - not kube_external_ca_mode\n block:\n - name: Kubeadm | Check apiserver.crt SAN IPs\n command:\n cmd: \"openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkip {{ item }}\"\n loop: \"{{ apiserver_ips }}\"\n register: apiserver_sans_ip_check\n changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate')\n failed_when: apiserver_sans_ip_check.rc != 0 and apiserver_sans_ip_check.stdout is not search('does NOT match certificate')\n - name: Kubeadm | Check apiserver.crt SAN hosts\n command:\n cmd: \"openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}\"\n loop: \"{{ apiserver_hosts }}\"\n register: apiserver_sans_host_check\n changed_when: apiserver_sans_host_check.stdout is not search('does match certificate')\n failed_when: apiserver_sans_host_check.rc != 0 and apiserver_sans_host_check.stdout is not search('does NOT match certificate')\n\n- name: Kubeadm | regenerate apiserver cert 1/2\n file:\n state: absent\n path: \"{{ kube_cert_dir }}/{{ item }}\"\n with_items:\n - apiserver.crt\n - apiserver.key\n when:\n - kubeadm_already_run.stat.exists\n - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed\n - not kube_external_ca_mode\n\n- name: Kubeadm | regenerate apiserver cert 2/2\n command: >-\n {{ bin_dir }}/kubeadm\n init phase certs apiserver\n --config={{ kube_config_dir }}/kubeadm-config.yaml\n when:\n - kubeadm_already_run.stat.exists\n - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed\n - not kube_external_ca_mode\n\n # TODO: Remove --skip-phases from command when v1beta4 UpgradeConfiguration supports skipPhases\n- name: Kubeadm | Initialize first control plane node\n when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists\n vars:\n kubeadm_init_first_control_plane_cmd: >-\n timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}\n {{ bin_dir }}/kubeadm init\n --config={{ kube_config_dir }}/kubeadm-config.yaml\n --ignore-preflight-errors={{ _ignore_errors | flatten | join(',') }}\n --skip-phases={{ kubeadm_init_phases_skip | join(',') }}\n {{ kube_external_ca_mode | ternary('', '--upload-certs') }}\n _ignore_errors: \"{{ kubeadm_ignore_preflight_errors }}\"\n environment:\n PATH: \"{{ bin_dir }}:{{ ansible_env.PATH }}\"\n notify: Control plane | restart kubelet\n block:\n - name: Kubeadm | Initialize first control plane node (1st try)\n command: \"{{ kubeadm_init_first_control_plane_cmd }}\"\n register: kubeadm_init\n failed_when: kubeadm_init.rc != 0 and \"field is immutable\" not in kubeadm_init.stderr\n rescue:\n # Retry is because upload config sometimes fails\n # This retry task is separated from 1st task to show log of failure of 1st task.\n - name: Kubeadm | Initialize first control plane node (retry)\n command: \"{{ kubeadm_init_first_control_plane_cmd }}\"\n vars:\n _errors_from_first_try:\n - 'FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml'\n - 'FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml'\n - 'FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml'\n - 'Port-10250'\n _ignore_errors:\n - \"{{ kubeadm_ignore_preflight_errors }}\"\n - \"{{ _errors_from_first_try if 'all' not in kubeadm_ignore_preflight_errors else [] }}\"\n register: kubeadm_init\n retries: 2\n until: kubeadm_init is succeeded or \"field is immutable\" in kubeadm_init.stderr\n failed_when: kubeadm_init.rc != 0 and \"field is immutable\" not in kubeadm_init.stderr\n\n- name: Set kubeadm certificate key\n set_fact:\n kubeadm_certificate_key: \"{{ item | regex_search('--certificate-key ([^ ]+)', '\\\\1') | first }}\"\n with_items: \"{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_init'].stdout_lines | default([]) }}\"\n when:\n - kubeadm_certificate_key is not defined\n - (item | trim) is match('.*--certificate-key.*')\n\n- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)\n shell: >-\n {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token delete {{ kubeadm_token }} || :;\n {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }}\n changed_when: false\n when:\n - inventory_hostname == first_kube_control_plane\n - kubeadm_token is defined\n - kubeadm_refresh_token\n tags:\n - kubeadm_token\n\n- name: Remove binding to anonymous user\n command: \"{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found\"\n when: inventory_hostname == first_kube_control_plane and remove_anonymous_access\n\n- name: Create kubeadm token for joining nodes with 24h expiration (default)\n command: \"{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create\"\n changed_when: false\n register: temp_token\n retries: 5\n delay: 5\n until: temp_token is succeeded\n delegate_to: \"{{ first_kube_control_plane }}\"\n when: kubeadm_token is not defined\n tags:\n - kubeadm_token\n\n- name: Set kubeadm_token\n set_fact:\n kubeadm_token: \"{{ temp_token.stdout }}\"\n when: temp_token.stdout is defined\n tags:\n - kubeadm_token\n\n- name: Kubeadm | Join other control plane nodes\n include_tasks: kubeadm-secondary.yml\n\n- name: Kubeadm | upgrade kubernetes cluster to {{ kube_version }}\n include_tasks: kubeadm-upgrade.yml\n when:\n - upgrade_cluster_setup\n - kubeadm_already_run.stat.exists\n\n# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.\n- name: Kubeadm | Remove taint for control plane node with node role\n command: \"{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}\"\n delegate_to: \"{{ first_kube_control_plane }}\"\n with_items:\n - \"node-role.kubernetes.io/control-plane:NoSchedule-\"\n when: ('kube_node' in group_names)\n failed_when: false\n" }, { "path": "roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml", "content": "---\n- name: Ensure kube-apiserver is up before upgrade\n import_tasks: check-api.yml\n\n- name: Kubeadm | Upgrade first control plane node to {{ kube_version }}\n command: >-\n timeout -k 600s 600s\n {{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}\n --config={{ kube_config_dir }}/kubeadm-config.yaml\n register: kubeadm_upgrade\n when: inventory_hostname == first_kube_control_plane\n failed_when: kubeadm_upgrade.rc != 0 and \"field is immutable\" not in kubeadm_upgrade.stderr\n environment:\n PATH: \"{{ bin_dir }}:{{ ansible_env.PATH }}\"\n\n- name: Kubeadm | Upgrade other control plane nodes to {{ kube_version }}\n command: >-\n {{ bin_dir }}/kubeadm upgrade node\n --config={{ kube_config_dir }}/kubeadm-config.yaml\n register: kubeadm_upgrade\n when: inventory_hostname != first_kube_control_plane\n failed_when: kubeadm_upgrade.rc != 0 and \"field is immutable\" not in kubeadm_upgrade.stderr\n environment:\n PATH: \"{{ bin_dir }}:{{ ansible_env.PATH }}\"\n\n# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied\n- name: Update kubeadm and kubelet configmaps after upgrade\n command: \"{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml\"\n register: kubeadm_upload_config\n # Retry is because upload config sometimes fails\n retries: 3\n until: kubeadm_upload_config.rc == 0\n when:\n - inventory_hostname == first_kube_control_plane\n\n- name: Update kube-proxy configmap after upgrade\n command: \"{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml\"\n register: kube_proxy_upload_config\n # Retry is because upload config sometimes fails\n retries: 3\n until: kube_proxy_upload_config.rc == 0\n when:\n - inventory_hostname == first_kube_control_plane\n - ('addon/kube-proxy' not in kubeadm_init_phases_skip)\n\n- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap\n command: \"{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml\"\n when:\n - etcd_deployment_type == \"kubeadm\"\n notify: Control plane | restart kubelet\n\n- name: Rewrite kubernetes control plane static pod manifests with updated configmap\n command: \"{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml\"\n notify: Control plane | restart kubelet\n\n- name: Flush kubelet handlers\n meta: flush_handlers\n\n- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates\n import_tasks: check-api.yml\n\n- name: Kubeadm | Remove binding to anonymous user\n command: \"{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found\"\n when: remove_anonymous_access\n\n- name: Kubeadm | clean kubectl cache to refresh api types\n file:\n path: \"{{ item }}\"\n state: absent\n with_items:\n - /root/.kube/cache\n - /root/.kube/http-cache\n\n# FIXME: https://github.com/kubernetes/kubeadm/issues/1318\n- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode\n command: >-\n {{ kubectl }}\n -n kube-system\n scale deployment/coredns --replicas 0\n register: scale_down_coredns\n retries: 6\n delay: 5\n until: scale_down_coredns is succeeded\n run_once: true\n when:\n - kubeadm_scale_down_coredns_enabled\n - dns_mode not in ['coredns', 'coredns_dual']\n changed_when: false\n" }, { "path": "roles/kubernetes/control-plane/tasks/main.yml", "content": "---\n- name: Pre-upgrade control plane\n import_tasks: pre-upgrade.yml\n tags:\n - k8s-pre-upgrade\n\n- name: Create webhook token auth config\n template:\n src: webhook-token-auth-config.yaml.j2\n dest: \"{{ kube_config_dir }}/webhook-token-auth-config.yaml\"\n mode: \"0640\"\n when: kube_webhook_token_auth | default(false)\n\n- name: Create webhook authorization config\n template:\n src: webhook-authorization-config.yaml.j2\n dest: \"{{ kube_config_dir }}/webhook-authorization-config.yaml\"\n mode: \"0640\"\n when: kube_webhook_authorization | default(false)\n\n- name: Create structured AuthorizationConfiguration file\n copy:\n content: \"{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}\"\n dest: \"{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml\"\n mode: \"0640\"\n vars:\n authz_config:\n apiVersion: apiserver.config.k8s.io/{{ kube_apiserver_authorization_config_api_version }}\n kind: AuthorizationConfiguration\n authorizers: \"{{ kube_apiserver_authorization_config_authorizers }}\"\n when: kube_apiserver_use_authorization_config_file\n\n- name: Create kube-scheduler config\n template:\n src: kubescheduler-config.yaml.j2\n dest: \"{{ kube_config_dir }}/kubescheduler-config.yaml\"\n mode: \"0644\"\n\n- name: Apply Kubernetes encrypt at rest config\n import_tasks: encrypt-at-rest.yml\n when:\n - kube_encrypt_secret_data\n tags:\n - kube-apiserver\n\n- name: Install | Copy kubectl binary from download dir\n copy:\n src: \"{{ downloads.kubectl.dest }}\"\n dest: \"{{ bin_dir }}/kubectl\"\n mode: \"0755\"\n remote_src: true\n tags:\n - kubectl\n - upgrade\n\n- name: Install kubectl bash completion\n shell: \"{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh\"\n when: ansible_os_family in [\"Debian\",\"RedHat\", \"Suse\"]\n tags:\n - kubectl\n ignore_errors: true # noqa ignore-errors\n\n- name: Set kubectl bash completion file permissions\n file:\n path: /etc/bash_completion.d/kubectl.sh\n owner: root\n group: root\n mode: \"0755\"\n when: ansible_os_family in [\"Debian\",\"RedHat\", \"Suse\"]\n tags:\n - kubectl\n - upgrade\n ignore_errors: true # noqa ignore-errors\n\n- name: Set bash alias for kubectl\n blockinfile:\n path: /etc/bash_completion.d/kubectl.sh\n block: |-\n alias {{ kubectl_alias }}=kubectl\n if [[ $(type -t compopt) = \"builtin\" ]]; then\n complete -o default -F __start_kubectl {{ kubectl_alias }}\n else\n complete -o default -o nospace -F __start_kubectl {{ kubectl_alias }}\n fi\n state: present\n marker: \"# Ansible entries {mark}\"\n when:\n - ansible_os_family in [\"Debian\",\"RedHat\", \"Suse\"]\n - kubectl_alias is defined and kubectl_alias != \"\"\n tags:\n - kubectl\n - upgrade\n ignore_errors: true # noqa ignore-errors\n\n- name: Include kubeadm setup\n import_tasks: kubeadm-setup.yml\n\n- name: Include kubeadm etcd extra tasks\n include_tasks: kubeadm-etcd.yml\n when: etcd_deployment_type == \"kubeadm\"\n\n- name: Cleanup unused AuthorizationConfiguration file versions\n file:\n path: \"{{ kube_config_dir }}/apiserver-authorization-config-{{ item }}.yaml\"\n state: absent\n loop: \"{{ ['v1alpha1', 'v1beta1', 'v1'] | reject('equalto', kube_apiserver_authorization_config_api_version) | list }}\"\n when: kube_apiserver_use_authorization_config_file\n\n- name: Install script to renew K8S control plane certificates\n template:\n src: k8s-certs-renew.sh.j2\n dest: \"{{ bin_dir }}/k8s-certs-renew.sh\"\n mode: \"0755\"\n\n- name: Renew K8S control plane certificates monthly 1/2\n template:\n src: \"{{ item }}.j2\"\n dest: \"/etc/systemd/system/{{ item }}\"\n mode: \"0644\"\n validate: \"sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{item}}'\"\n # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)\n # Remove once we drop support for systemd < 250\n with_items:\n - k8s-certs-renew.service\n - k8s-certs-renew.timer\n register: k8s_certs_units\n when: auto_renew_certificates\n\n- name: Renew K8S control plane certificates monthly 2/2\n systemd_service:\n name: k8s-certs-renew.timer\n enabled: true\n state: started\n daemon_reload: \"{{ k8s_certs_units is changed }}\"\n when: auto_renew_certificates\n" }, { "path": "roles/kubernetes/control-plane/tasks/pre-upgrade.yml", "content": "---\n- name: \"Pre-upgrade | Delete control plane manifests if etcd secrets changed\"\n file:\n path: \"/etc/kubernetes/manifests/{{ item }}.manifest\"\n state: absent\n with_items:\n - [\"kube-apiserver\", \"kube-controller-manager\", \"kube-scheduler\"]\n register: kube_apiserver_manifest_replaced\n when: etcd_secret_changed | default(false)\n\n- name: \"Pre-upgrade | Delete control plane containers forcefully\" # noqa no-handler\n shell: \"set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f\"\n args:\n executable: /bin/bash\n with_items:\n - [\"kube-apiserver\", \"kube-controller-manager\", \"kube-scheduler\"]\n when: kube_apiserver_manifest_replaced.changed\n register: remove_control_plane_container\n retries: 10\n until: remove_control_plane_container.rc == 0\n delay: 1\n" }, { "path": "roles/kubernetes/control-plane/templates/admission-controls.yaml.j2", "content": "apiVersion: apiserver.config.k8s.io/v1\nkind: AdmissionConfiguration\nplugins:\n{% for plugin in kube_apiserver_enable_admission_plugins %}\n{% if plugin in kube_apiserver_admission_plugins_needs_configuration %}\n- name: {{ plugin }}\n path: {{ kube_config_dir }}/{{ plugin | lower }}.yaml\n{% endif %}\n{% endfor %}\n" }, { "path": "roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2", "content": "apiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n{% if audit_policy_custom_rules is defined and audit_policy_custom_rules != \"\" %}\n{{ audit_policy_custom_rules | indent(2, true) }}\n{% else %}\n # The following requests were manually identified as high-volume and low-risk,\n # so drop them.\n - level: None\n users: [\"system:kube-proxy\"]\n verbs: [\"watch\"]\n resources:\n - group: \"\" # core\n resources: [\"endpoints\", \"services\", \"services/status\"]\n - level: None\n # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.\n # TODO(#46983): Change this to the ingress controller service account.\n users: [\"system:unsecured\"]\n namespaces: [\"kube-system\"]\n verbs: [\"get\"]\n resources:\n - group: \"\" # core\n resources: [\"configmaps\"]\n - level: None\n users: [\"kubelet\"] # legacy kubelet identity\n verbs: [\"get\"]\n resources:\n - group: \"\" # core\n resources: [\"nodes\", \"nodes/status\"]\n - level: None\n userGroups: [\"system:nodes\"]\n verbs: [\"get\"]\n resources:\n - group: \"\" # core\n resources: [\"nodes\", \"nodes/status\"]\n - level: None\n users:\n - system:kube-controller-manager\n - system:kube-scheduler\n - system:serviceaccount:kube-system:endpoint-controller\n verbs: [\"get\", \"update\"]\n namespaces: [\"kube-system\"]\n resources:\n - group: \"\" # core\n resources: [\"endpoints\"]\n - level: None\n users: [\"system:apiserver\"]\n verbs: [\"get\"]\n resources:\n - group: \"\" # core\n resources: [\"namespaces\", \"namespaces/status\", \"namespaces/finalize\"]\n # Don't log HPA fetching metrics.\n - level: None\n users:\n - system:kube-controller-manager\n verbs: [\"get\", \"list\"]\n resources:\n - group: \"metrics.k8s.io\"\n # Don't log these read-only URLs.\n - level: None\n nonResourceURLs:\n - /healthz*\n - /version\n - /swagger*\n # Don't log events requests.\n - level: None\n resources:\n - group: \"\" # core\n resources: [\"events\"]\n # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,\n # so only log at the Metadata level.\n - level: Metadata\n resources:\n - group: \"\" # core\n resources: [\"secrets\", \"configmaps\", \"serviceaccounts/token\"]\n - group: authentication.k8s.io\n resources: [\"tokenreviews\"]\n omitStages:\n - \"RequestReceived\"\n # Get responses can be large; skip them.\n - level: Request\n verbs: [\"get\", \"list\", \"watch\"]\n resources:\n - group: \"\" # core\n - group: \"admissionregistration.k8s.io\"\n - group: \"apiextensions.k8s.io\"\n - group: \"apiregistration.k8s.io\"\n - group: \"apps\"\n - group: \"authentication.k8s.io\"\n - group: \"authorization.k8s.io\"\n - group: \"autoscaling\"\n - group: \"batch\"\n - group: \"certificates.k8s.io\"\n - group: \"extensions\"\n - group: \"metrics.k8s.io\"\n - group: \"networking.k8s.io\"\n - group: \"policy\"\n - group: \"rbac.authorization.k8s.io\"\n - group: \"settings.k8s.io\"\n - group: \"storage.k8s.io\"\n omitStages:\n - \"RequestReceived\"\n # Default level for known APIs\n - level: RequestResponse\n resources:\n - group: \"\" # core\n - group: \"admissionregistration.k8s.io\"\n - group: \"apiextensions.k8s.io\"\n - group: \"apiregistration.k8s.io\"\n - group: \"apps\"\n - group: \"authentication.k8s.io\"\n - group: \"authorization.k8s.io\"\n - group: \"autoscaling\"\n - group: \"batch\"\n - group: \"certificates.k8s.io\"\n - group: \"extensions\"\n - group: \"metrics.k8s.io\"\n - group: \"networking.k8s.io\"\n - group: \"policy\"\n - group: \"rbac.authorization.k8s.io\"\n - group: \"settings.k8s.io\"\n - group: \"storage.k8s.io\"\n omitStages:\n - \"RequestReceived\"\n # Default level for all other requests.\n - level: Metadata\n omitStages:\n - \"RequestReceived\"\n{% endif %}\n" }, { "path": "roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2", "content": "apiVersion: v1\nkind: Config\nclusters:\n- cluster:\n server: {{ audit_webhook_server_url }}\n{% for key in audit_webhook_server_extra_args %}\n {{ key }}: \"{{ audit_webhook_server_extra_args[key] }}\"\n{% endfor %}\n name: auditsink\ncontexts:\n- context:\n cluster: auditsink\n user: \"\"\n name: default-context\ncurrent-context: default-context\npreferences: {}\nusers: []\n" }, { "path": "roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2", "content": "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: TracingConfiguration\nendpoint: \"{{ kube_apiserver_tracing_endpoint }}\"\nsamplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }}\n" }, { "path": "roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2", "content": "apiVersion: eventratelimit.admission.k8s.io/v1alpha1\nkind: Configuration\nlimits:\n{% for limit in kube_apiserver_admission_event_rate_limits.values() %}\n- type: {{ limit.type }}\n qps: {{ limit.qps }}\n burst: {{ limit.burst }}\n{% if limit.cache_size is defined %}\n cacheSize: {{ limit.cache_size }}\n{% endif %}\n{% endfor %}\n" }, { "path": "roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2", "content": "[Unit]\nDescription=Renew K8S control plane certificates\n\n[Service]\nType=oneshot\nExecStart={{ bin_dir }}/k8s-certs-renew.sh\n" }, { "path": "roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2", "content": "#!/bin/bash\n\necho \"## Check Expiration before renewal ##\"\n\n{{ bin_dir }}/kubeadm certs check-expiration\n\ndays_buffer=7 # set a time margin, because we should not renew at the last moment\nnext_time=$(systemctl show k8s-certs-renew.timer -p NextElapseUSecRealtime --value)\n\nif [ \"${next_time}\" == \"\" ]; then\n\techo \"## Skip expiry comparison due to fail to parse next elapse from systemd calendar,do renewal directly ##\"\nelse\n\tcurrent_time=$(date +%s)\n\ttarget_time=$(date -d \"${next_time} + ${days_buffer} days\" +%s) # $next_time - $days_buffer days\n\texpiry_threshold=$(( ${target_time} - ${current_time} ))\n\texpired_certs=$({{ bin_dir }}/kubeadm certs check-expiration -o jsonpath=\"{.certificates[?(@.residualTime<${expiry_threshold}.0)]}\")\n if [ \"${expired_certs}\" == \"\" ];then\n\t\techo \"## Skip cert renew and K8S container restart, since all residualTimes are beyond threshold ##\"\n\t\texit 0\n\tfi\nfi\n\necho \"## Renewing certificates managed by kubeadm ##\"\n{{ bin_dir }}/kubeadm certs renew all\n\necho \"## Restarting control plane pods managed by kubeadm ##\"\n{% if container_manager == \"docker\" %}\n{{ docker_bin_dir }}/docker ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs {{ docker_bin_dir }}/docker rm -f\n{% else %}\n{{ bin_dir }}/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs {{ bin_dir }}/crictl rmp -f\n{% endif %}\n\necho \"## Updating /root/.kube/config ##\"\ncp {{ kube_config_dir }}/admin.conf /root/.kube/config\n\necho \"## Waiting for apiserver to be up again ##\"\nuntil printf \"\" 2>>/dev/null >>/dev/tcp/127.0.0.1/{{ kube_apiserver_port | default(6443) }}; do sleep 1; done\n\necho \"## Expiration after renewal ##\"\n{{ bin_dir }}/kubeadm certs check-expiration\n" }, { "path": "roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2", "content": "[Unit]\nDescription=Timer to renew K8S control plane certificates\n\n[Timer]\nOnCalendar={{ auto_renew_certificates_systemd_calendar }}\nRandomizedDelaySec={{ 10 * (groups['kube_control_plane'] | length) }}min\nFixedRandomDelay=yes\nPersistent=yes\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2", "content": "apiVersion: kubeadm.k8s.io/v1beta4\nkind: InitConfiguration\n{% if kubeadm_token is defined %}\nbootstrapTokens:\n- token: \"{{ kubeadm_token }}\"\n description: \"kubespray kubeadm bootstrap token\"\n ttl: \"24h\"\n{% endif %}\nlocalAPIEndpoint:\n advertiseAddress: \"{{ kube_apiserver_address }}\"\n bindPort: {{ kube_apiserver_port }}\n{% if kubeadm_certificate_key is defined %}\ncertificateKey: {{ kubeadm_certificate_key }}\n{% endif %}\nnodeRegistration:\n{% if kube_override_hostname | default('') %}\n name: \"{{ kube_override_hostname }}\"\n{% endif %}\n{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}\n taints:\n - effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n{% else %}\n taints: []\n{% endif %}\n criSocket: {{ cri_socket }}\n{% if cloud_provider == \"external\" %}\n kubeletExtraArgs:\n - name: cloud-provider\n value: external\n{% endif %}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n imagePullSerial: {{ kubeadm_image_pull_serial | lower }}\n{% if kubeadm_patches | length > 0 %}\npatches:\n directory: {{ kubeadm_patches_dir }}\n{% endif %}\n---\napiVersion: kubeadm.k8s.io/v1beta4\nkind: ClusterConfiguration\nclusterName: {{ cluster_name }}\nencryptionAlgorithm: {{ kube_asymmetric_encryption_algorithm }}\ncertificateValidityPeriod: {{ kube_cert_validity_period }}\ncaCertificateValidityPeriod: {{ kube_ca_cert_validity_period }}\netcd:\n{% if etcd_deployment_type != \"kubeadm\" %}\n external:\n endpoints:\n{% for endpoint in etcd_access_addresses.split(',') %}\n - \"{{ endpoint }}\"\n{% endfor %}\n caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}\n certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}\n keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}\n{% elif etcd_deployment_type == \"kubeadm\" %}\n local:\n imageRepository: \"{{ etcd_image_repo | regex_replace(\"/etcd$\",\"\") }}\"\n imageTag: \"{{ etcd_image_tag }}\"\n dataDir: \"{{ etcd_data_dir }}\"\n extraArgs:\n - name: metrics\n value: {{ etcd_metrics }}\n - name: election-timeout\n value: \"{{ etcd_election_timeout }}\"\n - name: heartbeat-interval\n value: \"{{ etcd_heartbeat_interval }}\"\n - name: auto-compaction-retention\n value: \"{{ etcd_compaction_retention }}\"\n{% if etcd_listen_metrics_urls is defined %}\n - name: listen-metrics-urls\n value: \"{{ etcd_listen_metrics_urls }}\"\n{% endif %}\n - name: snapshot-count\n value: \"{{ etcd_snapshot_count }}\"\n - name: quota-backend-bytes\n value: \"{{ etcd_quota_backend_bytes }}\"\n - name: max-request-bytes\n value: \"{{ etcd_max_request_bytes }}\"\n - name: log-level\n value: \"{{ etcd_log_level }}\"\n{% for key, value in etcd_extra_vars.items() %}\n - name: {{ key }}\n value: \"{{ value }}\"\n{% endfor %}\n serverCertSANs:\n{% for san in etcd_cert_alt_names %}\n - \"{{ san }}\"\n{% endfor %}\n{% for san in etcd_cert_alt_ips %}\n - \"{{ san }}\"\n{% endfor %}\n peerCertSANs:\n{% for san in etcd_cert_alt_names %}\n - \"{{ san }}\"\n{% endfor %}\n{% for san in etcd_cert_alt_ips %}\n - \"{{ san }}\"\n{% endfor %}\n{% endif %}\ndns:\n{% if 'addon/coredns' in kubeadm_init_phases_skip %}\n disabled: true\n{% endif %}\n imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }}\n imageTag: {{ coredns_image_tag }}\nnetworking:\n dnsDomain: {{ dns_domain }}\n serviceSubnet: \"{{ kube_service_subnets }}\"\n{% if kube_network_plugin is defined and kube_network_plugin not in [\"kube-ovn\"] %}\n podSubnet: \"{{ kube_pods_subnets }}\"\n{% endif %}\n{% if kubeadm_feature_gates %}\nfeatureGates:\n{% for feature in kubeadm_feature_gates %}\n {{ feature | replace(\"=\", \": \") }}\n{% endfor %}\n{% endif %}\nkubernetesVersion: v{{ kube_version }}\n{% if kubeadm_config_api_fqdn is defined %}\ncontrolPlaneEndpoint: \"{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}\"\n{% else %}\ncontrolPlaneEndpoint: \"{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}\"\n{% endif %}\ncertificatesDir: {{ kube_cert_dir }}\nimageRepository: {{ kubeadm_image_repo }}\napiServer:\n extraArgs:\n - name: etcd-compaction-interval\n value: \"{{ kube_apiserver_etcd_compaction_interval }}\"\n - name: default-not-ready-toleration-seconds\n value: \"{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}\"\n - name: default-unreachable-toleration-seconds\n value: \"{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}\"\n{% if kube_api_anonymous_auth is defined %}\n{# TODO: rework once suppport for structured auth lands #}\n - name: anonymous-auth\n value: \"{{ kube_api_anonymous_auth }}\"\n{% endif %}\n{% if kube_apiserver_use_authorization_config_file %}\n - name: authorization-config\n value: \"{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml\"\n{% else %}\n - name: authorization-mode\n value: \"{{ authorization_modes | join(',') }}\"\n{% endif %}\n - name: bind-address\n value: \"{{ kube_apiserver_bind_address }}\"\n{% if kube_apiserver_enable_admission_plugins | length > 0 %}\n - name: enable-admission-plugins\n value: \"{{ kube_apiserver_enable_admission_plugins | join(',') }}\"\n{% endif %}\n{% if kube_apiserver_admission_control_config_file %}\n - name: admission-control-config-file\n value: \"{{ kube_config_dir }}/admission-controls.yaml\"\n{% endif %}\n{% if kube_apiserver_disable_admission_plugins | length > 0 %}\n - name: disable-admission-plugins\n value: \"{{ kube_apiserver_disable_admission_plugins | join(',') }}\"\n{% endif %}\n - name: apiserver-count\n value: \"{{ kube_apiserver_count }}\"\n - name: endpoint-reconciler-type\n value: lease\n{% if etcd_events_cluster_enabled %}\n - name: etcd-servers-overrides\n value: \"/events#{{ etcd_events_access_addresses_semicolon }}\"\n{% endif %}\n - name: service-node-port-range\n value: \"{{ kube_apiserver_node_port_range }}\"\n - name: service-cluster-ip-range\n value: \"{{ kube_service_subnets }}\"\n - name: kubelet-preferred-address-types\n value: \"{{ kubelet_preferred_address_types }}\"\n - name: profiling\n value: \"{{ kube_profiling }}\"\n - name: request-timeout\n value: \"{{ kube_apiserver_request_timeout }}\"\n - name: enable-aggregator-routing\n value: \"{{ kube_api_aggregator_routing }}\"\n{% if kube_apiserver_service_account_lookup %}\n - name: service-account-lookup\n value: \"{{ kube_apiserver_service_account_lookup }}\"\n{% endif %}\n{% if kube_oidc_auth and kube_oidc_url is defined and kube_oidc_client_id is defined %}\n - name: oidc-issuer-url\n value: \"{{ kube_oidc_url }}\"\n - name: oidc-client-id\n value: \"{{ kube_oidc_client_id }}\"\n{% if kube_oidc_ca_file is defined %}\n - name: oidc-ca-file\n value: \"{{ kube_oidc_ca_file }}\"\n{% endif %}\n{% if kube_oidc_username_claim is defined %}\n - name: oidc-username-claim\n value: \"{{ kube_oidc_username_claim }}\"\n{% endif %}\n{% if kube_oidc_groups_claim is defined %}\n - name: oidc-groups-claim\n value: \"{{ kube_oidc_groups_claim }}\"\n{% endif %}\n{% if kube_oidc_username_prefix is defined %}\n - name: oidc-username-prefix\n value: \"{{ kube_oidc_username_prefix }}\"\n{% endif %}\n{% if kube_oidc_groups_prefix is defined %}\n - name: oidc-groups-prefix\n value: \"{{ kube_oidc_groups_prefix }}\"\n{% endif %}\n{% endif %}\n{% if kube_webhook_token_auth %}\n - name: authentication-token-webhook-config-file\n value: \"{{ kube_config_dir }}/webhook-token-auth-config.yaml\"\n{% endif %}\n{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}\n - name: authorization-webhook-config-file\n value: \"{{ kube_config_dir }}/webhook-authorization-config.yaml\"\n{% endif %}\n{% if kube_encrypt_secret_data %}\n - name: encryption-provider-config\n value: \"{{ kube_cert_dir }}/secrets_encryption.yaml\"\n{% endif %}\n - name: storage-backend\n value: \"{{ kube_apiserver_storage_backend }}\"\n{% if kube_api_runtime_config | length > 0 %}\n - name: runtime-config\n value: \"{{ kube_api_runtime_config | join(',') }}\"\n{% endif %}\n - name: allow-privileged\n value: \"true\"\n{% if kubernetes_audit or kubernetes_audit_webhook %}\n - name: audit-policy-file\n value: \"{{ audit_policy_file }}\"\n{% endif %}\n{% if kubernetes_audit %}\n - name: audit-log-path\n value: \"{{ audit_log_path }}\"\n - name: audit-log-maxage\n value: \"{{ audit_log_maxage }}\"\n - name: audit-log-maxbackup\n value: \"{{ audit_log_maxbackups }}\"\n - name: audit-log-maxsize\n value: \"{{ audit_log_maxsize }}\"\n{% endif %}\n{% if kubernetes_audit_webhook %}\n - name: audit-webhook-config-file\n value: \"{{ audit_webhook_config_file }}\"\n - name: audit-webhook-mode\n value: \"{{ audit_webhook_mode }}\"\n{% if audit_webhook_mode == \"batch\" %}\n - name: audit-webhook-batch-max-size\n value: \"{{ audit_webhook_batch_max_size }}\"\n - name: audit-webhook-batch-max-wait\n value: \"{{ audit_webhook_batch_max_wait }}\"\n{% endif %}\n{% endif %}\n{% for key in kube_kubeadm_apiserver_extra_args %}\n - name: \"{{ key }}\"\n value: \"{{ kube_kubeadm_apiserver_extra_args[key] }}\"\n{% endfor %}\n{% if kube_apiserver_feature_gates or kube_feature_gates %}\n - name: feature-gates\n value: \"{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}\"\n{% endif %}\n{% if tls_min_version is defined %}\n - name: tls-min-version\n value: \"{{ tls_min_version }}\"\n{% endif %}\n{% if tls_cipher_suites is defined %}\n - name: tls-cipher-suites\n value: \"{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}\"\n{% endif %}\n - name: event-ttl\n value: \"{{ event_ttl_duration }}\"\n{% if kubelet_rotate_server_certificates %}\n - name: kubelet-certificate-authority\n value: \"{{ kube_cert_dir }}/ca.crt\"\n{% endif %}\n{% if kube_apiserver_tracing %}\n - name: tracing-config-file\n value: \"{{ kube_config_dir }}/tracing/apiserver-tracing.yaml\"\n{% endif %}\n{% if kubernetes_audit or kube_token_auth or kube_webhook_token_auth or apiserver_extra_volumes or ssl_ca_dirs | length %}\n extraVolumes:\n{% if kube_token_auth %}\n - name: token-auth-config\n hostPath: {{ kube_token_dir }}\n mountPath: {{ kube_token_dir }}\n{% endif %}\n{% if kube_webhook_token_auth %}\n - name: webhook-token-auth-config\n hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml\n mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml\n{% endif %}\n{% if kube_webhook_authorization %}\n - name: webhook-authorization-config\n hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml\n mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml\n{% endif %}\n{% if kube_apiserver_use_authorization_config_file %}\n - name: authorization-config\n hostPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml\n mountPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml\n{% endif %}\n{% if kubernetes_audit or kubernetes_audit_webhook %}\n - name: {{ audit_policy_name }}\n hostPath: {{ audit_policy_hostpath }}\n mountPath: {{ audit_policy_mountpath }}\n{% if audit_log_path != \"-\" %}\n - name: {{ audit_log_name }}\n hostPath: {{ audit_log_hostpath }}\n mountPath: {{ audit_log_mountpath }}\n readOnly: false\n{% endif %}\n{% endif %}\n{% if kube_apiserver_admission_control_config_file %}\n - name: admission-control-configs\n hostPath: {{ kube_config_dir }}/admission-controls\n mountPath: {{ kube_config_dir }}\n readOnly: false\n pathType: DirectoryOrCreate\n{% endif %}\n{% if kube_apiserver_tracing %}\n - name: tracing\n hostPath: {{ kube_config_dir }}/tracing\n mountPath: {{ kube_config_dir }}/tracing\n readOnly: true\n pathType: DirectoryOrCreate\n{% endif %}\n{% for volume in apiserver_extra_volumes %}\n - name: {{ volume.name }}\n hostPath: {{ volume.hostPath }}\n mountPath: {{ volume.mountPath }}\n readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}\n{% endfor %}\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n hostPath: {{ dir }}\n mountPath: {{ dir }}\n readOnly: true\n{% endfor %}\n{% endif %}\n{% endif %}\n certSANs:\n{% for san in apiserver_sans %}\n - \"{{ san }}\"\n{% endfor %}\ncontrollerManager:\n extraArgs:\n - name: node-monitor-grace-period\n value: \"{{ kube_controller_node_monitor_grace_period }}\"\n - name: node-monitor-period\n value: \"{{ kube_controller_node_monitor_period }}\"\n{% if kube_network_plugin is defined and kube_network_plugin not in [\"kube-ovn\"] %}\n - name: cluster-cidr\n value: \"{{ kube_pods_subnets }}\"\n{% endif %}\n - name: service-cluster-ip-range\n value: \"{{ kube_service_subnets }}\"\n{% if kube_network_plugin is defined and kube_network_plugin == \"calico\" and not calico_ipam_host_local %}\n - name: allocate-node-cidrs\n value: \"false\"\n{% else %}\n{% if ipv4_stack %}\n - name: node-cidr-mask-size-ipv4\n value: \"{{ kube_network_node_prefix }}\"\n{% endif %}\n{% if ipv6_stack %}\n - name: node-cidr-mask-size-ipv6\n value: \"{{ kube_network_node_prefix_ipv6 }}\"\n{% endif %}\n{% endif %}\n - name: profiling\n value: \"{{ kube_profiling }}\"\n - name: terminated-pod-gc-threshold\n value: \"{{ kube_controller_terminated_pod_gc_threshold }}\"\n - name: bind-address\n value: \"{{ kube_controller_manager_bind_address }}\"\n - name: leader-elect-lease-duration\n value: \"{{ kube_controller_manager_leader_elect_lease_duration }}\"\n - name: leader-elect-renew-deadline\n value: \"{{ kube_controller_manager_leader_elect_renew_deadline }}\"\n{% if kube_controller_feature_gates or kube_feature_gates %}\n - name: feature-gates\n value: \"{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}\"\n{% endif %}\n{% for key in kube_kubeadm_controller_extra_args %}\n - name: \"{{ key }}\"\n value: \"{{ kube_kubeadm_controller_extra_args[key] }}\"\n{% endfor %}\n{% if kube_network_plugin is defined and kube_network_plugin not in [\"cloud\"] %}\n - name: configure-cloud-routes\n value: \"false\"\n{% endif %}\n{% if kubelet_flexvolumes_plugins_dir is defined %}\n - name: flex-volume-plugin-dir\n value: \"{{ kubelet_flexvolumes_plugins_dir }}\"\n{% endif %}\n{% if tls_min_version is defined %}\n - name: tls-min-version\n value: \"{{ tls_min_version }}\"\n{% endif %}\n{% if tls_cipher_suites is defined %}\n - name: tls-cipher-suites\n value: \"{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}\"\n{% endif %}\n{% if controller_manager_extra_volumes %}\n extraVolumes:\n{% for volume in controller_manager_extra_volumes %}\n - name: {{ volume.name }}\n hostPath: {{ volume.hostPath }}\n mountPath: {{ volume.mountPath }}\n readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}\n{% endfor %}\n{% endif %}\nscheduler:\n extraArgs:\n - name: bind-address\n value: \"{{ kube_scheduler_bind_address }}\"\n - name: config\n value: \"{{ kube_config_dir }}/kubescheduler-config.yaml\"\n{% if kube_scheduler_feature_gates or kube_feature_gates %}\n - name: feature-gates\n value: \"{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}\"\n{% endif %}\n - name: profiling\n value: \"{{ kube_profiling }}\"\n{% if kube_kubeadm_scheduler_extra_args | length > 0 %}\n{% for key in kube_kubeadm_scheduler_extra_args %}\n - name: \"{{ key }}\"\n value: \"{{ kube_kubeadm_scheduler_extra_args[key] }}\"\n{% endfor %}\n{% endif %}\n{% if tls_min_version is defined %}\n - name: tls-min-version\n value: \"{{ tls_min_version }}\"\n{% endif %}\n{% if tls_cipher_suites is defined %}\n - name: tls-cipher-suites\n value: \"{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}\"\n{% endif %}\n extraVolumes:\n - name: kubescheduler-config\n hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml\n mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml\n readOnly: true\n{% if scheduler_extra_volumes %}\n{% for volume in scheduler_extra_volumes %}\n - name: {{ volume.name }}\n hostPath: {{ volume.hostPath }}\n mountPath: {{ volume.mountPath }}\n readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}\n{% endfor %}\n{% endif %}\n---\napiVersion: kubeadm.k8s.io/v1beta4\nkind: UpgradeConfiguration\napply:\n kubernetesVersion: v{{ kube_version }}\n allowExperimentalUpgrades: true\n certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}\n etcdUpgrade: {{ (etcd_deployment_type == \"kubeadm\") | lower }}\n forceUpgrade: true\n{% if kubeadm_ignore_preflight_errors | length > 0 %}\n ignorePreflightErrors:\n{% for ignore_error in kubeadm_ignore_preflight_errors %}\n - \"{{ ignore_error }}\"\n{% endfor %}\n{% endif %}\n{% if kubeadm_patches | length > 0 %}\n patches:\n directory: {{ kubeadm_patches_dir }}\n{% endif %}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n imagePullSerial: {{ kubeadm_image_pull_serial | lower }}\n{% for skip_phase in kubeadm_init_phases_skip %}\n{% if loop.first %}\n skipPhases:\n{% endif %}\n - \"{{ skip_phase }}\"\n{% endfor %}\nnode:\n certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}\n etcdUpgrade: {{ (etcd_deployment_type == \"kubeadm\") | lower }}\n{% if kubeadm_ignore_preflight_errors | length > 0 %}\n ignorePreflightErrors:\n{% for ignore_error in kubeadm_ignore_preflight_errors %}\n - \"{{ ignore_error }}\"\n{% endfor %}\n{% endif %}\n{% if kubeadm_patches | length > 0 %}\n patches:\n directory: {{ kubeadm_patches_dir }}\n{% endif %}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n imagePullSerial: {{ kubeadm_image_pull_serial | lower }}\n{% for skip_phase in kubeadm_upgrade_node_phases_skip %}\n{% if loop.first %}\n skipPhases:\n{% endif %}\n - \"{{ skip_phase }}\"\n{% endfor %}\n---\napiVersion: kubeproxy.config.k8s.io/v1alpha1\nkind: KubeProxyConfiguration\nbindAddress: \"{{ kube_proxy_bind_address }}\"\nclientConnection:\n acceptContentTypes: {{ kube_proxy_client_accept_content_types }}\n burst: {{ kube_proxy_client_burst }}\n contentType: {{ kube_proxy_client_content_type }}\n kubeconfig: {{ kube_proxy_client_kubeconfig }}\n qps: {{ kube_proxy_client_qps }}\n{% if kube_network_plugin is defined and kube_network_plugin not in [\"kube-ovn\"] %}\nclusterCIDR: \"{{ kube_pods_subnets }}\"\n{% endif %}\nconfigSyncPeriod: {{ kube_proxy_config_sync_period }}\nconntrack:\n maxPerCore: {{ kube_proxy_conntrack_max_per_core }}\n min: {{ kube_proxy_conntrack_min }}\n tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}\n tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}\nenableProfiling: {{ kube_proxy_enable_profiling }}\nhealthzBindAddress: \"{{ kube_proxy_healthz_bind_address }}\"\nhostnameOverride: \"{{ kube_override_hostname }}\"\niptables:\n masqueradeAll: {{ kube_proxy_masquerade_all }}\n masqueradeBit: {{ kube_proxy_masquerade_bit }}\n minSyncPeriod: {{ kube_proxy_min_sync_period }}\n syncPeriod: {{ kube_proxy_sync_period }}\nipvs:\n excludeCIDRs: {{ kube_proxy_exclude_cidrs }}\n minSyncPeriod: {{ kube_proxy_min_sync_period }}\n scheduler: {{ kube_proxy_scheduler }}\n syncPeriod: {{ kube_proxy_sync_period }}\n strictARP: {{ kube_proxy_strict_arp }}\n tcpTimeout: {{ kube_proxy_tcp_timeout }}\n tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}\n udpTimeout: {{ kube_proxy_udp_timeout }}\nmetricsBindAddress: \"{{ kube_proxy_metrics_bind_address }}\"\nmode: {{ kube_proxy_mode }}\nnodePortAddresses: {{ kube_proxy_nodeport_addresses }}\noomScoreAdj: {{ kube_proxy_oom_score_adj }}\nportRange: {{ kube_proxy_port_range }}\n{% if kube_proxy_feature_gates or kube_feature_gates %}\n{% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %}\nfeatureGates:\n{% for feature in feature_gates %}\n {{ feature | replace(\"=\", \": \") }}\n{% endfor %}\n{% endif %}\n{# DNS settings for kubelet #}\n{% if enable_nodelocaldns %}\n{% set kubelet_cluster_dns = [nodelocaldns_ip] %}\n{% elif dns_mode in ['coredns'] %}\n{% set kubelet_cluster_dns = [skydns_server] %}\n{% elif dns_mode == 'coredns_dual' %}\n{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}\n{% elif dns_mode == 'manual' %}\n{% set kubelet_cluster_dns = [manual_dns_server] %}\n{% else %}\n{% set kubelet_cluster_dns = [] %}\n{% endif %}\n---\napiVersion: kubelet.config.k8s.io/v1beta1\nkind: KubeletConfiguration\n{% if kube_version is version('1.35.0', '>=') %}\nfailCgroupV1: {{ kubelet_fail_cgroup_v1 }}\n{% endif %}\nclusterDNS:\n{% for dns_address in kubelet_cluster_dns %}\n- {{ dns_address }}\n{% endfor %}\n{% if kubelet_feature_gates or kube_feature_gates %}\n{% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %}\nfeatureGates:\n{% for feature in feature_gates %}\n {{ feature | replace(\"=\", \": \") }}\n{% endfor %}\n{% endif %}\n" }, { "path": "roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2", "content": "apiVersion: kubeadm.k8s.io/v1beta4\nkind: JoinConfiguration\ndiscovery:\n{% if kubeadm_use_file_discovery %}\n file:\n kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml\n{% else %}\n bootstrapToken:\n{% if kubeadm_config_api_fqdn is defined %}\n apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}\n{% else %}\n apiServerEndpoint: \"{{ kubeadm_discovery_address }}\"\n{% endif %}\n token: {{ kubeadm_token }}\n unsafeSkipCAVerification: true\n{% endif %}\n tlsBootstrapToken: {{ kubeadm_token }}\ntimeouts:\n discovery: {{ discovery_timeout }}\ncontrolPlane:\n localAPIEndpoint:\n advertiseAddress: \"{{ kube_apiserver_address }}\"\n bindPort: {{ kube_apiserver_port }}\n certificateKey: {{ kubeadm_certificate_key }}\nnodeRegistration:\n name: {{ kube_override_hostname | default(inventory_hostname) }}\n criSocket: {{ cri_socket }}\n{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}\n taints:\n - effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n{% else %}\n taints: []\n{% endif %}\n{% if kubeadm_patches | length > 0 %}\npatches:\n directory: {{ kubeadm_patches_dir }}\n{% endif %}\n" }, { "path": "roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2", "content": "apiVersion: kubescheduler.config.k8s.io/v1\nkind: KubeSchedulerConfiguration\nclientConnection:\n kubeconfig: \"{{ kube_config_dir }}/scheduler.conf\"\n{% for key in kube_scheduler_client_conn_extra_opts %}\n {{ key }}: {{ kube_scheduler_client_conn_extra_opts[key] }}\n{% endfor %}\n{% if kube_scheduler_extenders %}\nextenders:\n{{ kube_scheduler_extenders | to_nice_yaml(indent=2, width=256) }}\n{% endif %}\nleaderElection:\n leaseDuration: {{ kube_scheduler_leader_elect_lease_duration }}\n renewDeadline: {{ kube_scheduler_leader_elect_renew_deadline }}\n{% for key in kube_scheduler_leader_elect_extra_opts %}\n {{ key }}: {{ kube_scheduler_leader_elect_extra_opts[key] }}\n{% endfor %}\n{% if kube_scheduler_profiles %}\nprofiles:\n{{ kube_scheduler_profiles | to_nice_yaml(indent=2, width=256) }}\n{% endif %}\n{% for key in kube_scheduler_config_extra_opts %}\n{{ key }}: {{ kube_scheduler_config_extra_opts[key] }}\n{% endfor %}\n" }, { "path": "roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2", "content": "podNodeSelectorPluginConfig:\n clusterDefaultNodeSelector: {{ kube_apiserver_admission_plugins_podnodeselector_default_node_selector }}\n" }, { "path": "roles/kubernetes/control-plane/templates/podsecurity.yaml.j2", "content": "apiVersion: pod-security.admission.config.k8s.io/v1\nkind: PodSecurityConfiguration\n{% if kube_pod_security_use_default %}\ndefaults:\n enforce: \"{{ kube_pod_security_default_enforce }}\"\n enforce-version: \"{{ kube_pod_security_default_enforce_version }}\"\n audit: \"{{ kube_pod_security_default_audit }}\"\n audit-version: \"{{ kube_pod_security_default_audit_version }}\"\n warn: \"{{ kube_pod_security_default_warn }}\"\n warn-version: \"{{ kube_pod_security_default_warn_version }}\"\nexemptions:\n usernames: {{ kube_pod_security_exemptions_usernames | to_json }}\n runtimeClasses: {{ kube_pod_security_exemptions_runtime_class_names | to_json }}\n namespaces: {{ kube_pod_security_exemptions_namespaces | to_json }}\n{% else %}\n# This file is intentinally left empty as kube_pod_security_use_default={{ kube_pod_security_use_default }}\n{% endif %}\n" }, { "path": "roles/kubernetes/control-plane/templates/resourcequota.yaml.j2", "content": "apiVersion: apiserver.config.k8s.io/v1\nkind: ResourceQuotaConfiguration\n{% if kube_resource_quota_limited_resources | d(false) -%}\nlimitedResources:\n{{ kube_resource_quota_limited_resources | to_nice_yaml(indent=2, sort_keys=false) }}\n{% else %}\n# No limitedResources configured. If limitedResources are required, please set kube_resource_quota_limited_resources.\n{%- endif %}\n" }, { "path": "roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2", "content": "apiVersion: apiserver.config.k8s.io/v1\nkind: EncryptionConfiguration\nresources:\n - resources:\n{{ kube_encryption_resources | to_nice_yaml | indent(4, True) }}\n providers:\n - {{ kube_encryption_algorithm }}:\n keys:\n - name: key\n secret: {{ kube_encrypt_token | b64encode }}\n - identity: {}\n" }, { "path": "roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2", "content": "# clusters refers to the remote service.\nclusters:\n- name: webhook-token-authz-cluster\n cluster:\n server: {{ kube_webhook_authorization_url }}\n insecure-skip-tls-verify: {{ kube_webhook_authorization_url_skip_tls_verify }}\n\n# users refers to the API server's webhook configuration.\nusers:\n- name: webhook-token-authz-user\n\n# kubeconfig files require a context. Provide one for the API server.\ncurrent-context: webhook-token-authz\ncontexts:\n- context:\n cluster: webhook-token-authz-cluster\n user: webhook-token-authz-user\n name: webhook-token-authz\n" }, { "path": "roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2", "content": "# clusters refers to the remote service.\nclusters:\n- name: webhook-token-auth-cluster\n cluster:\n server: {{ kube_webhook_token_auth_url }}\n insecure-skip-tls-verify: {{ kube_webhook_token_auth_url_skip_tls_verify }}\n{% if kube_webhook_token_auth_ca_data is defined %}\n certificate-authority-data: {{ kube_webhook_token_auth_ca_data }}\n{% endif %}\n\n# users refers to the API server's webhook configuration.\nusers:\n- name: webhook-token-auth-user\n\n# kubeconfig files require a context. Provide one for the API server.\ncurrent-context: webhook-token-auth\ncontexts:\n- context:\n cluster: webhook-token-auth-cluster\n user: webhook-token-auth-user\n name: webhook-token-auth\n" }, { "path": "roles/kubernetes/control-plane/vars/main.yaml", "content": "---\n# list of admission plugins that needs to be configured\n# https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/\nkube_apiserver_admission_plugins_needs_configuration:\n- EventRateLimit\n- ImagePolicyWebhook\n- PodSecurity\n- PodNodeSelector\n- ResourceQuota\n" }, { "path": "roles/kubernetes/kubeadm/defaults/main.yml", "content": "---\n# discovery_timeout modifies the discovery timeout\n# This value must be smaller than kubeadm_join_timeout\ndiscovery_timeout: 60s\nkubeadm_join_timeout: 120s\n\n# Enable kubeadm file discovery if anonymous access has been removed\nkubeadm_use_file_discovery: \"{{ remove_anonymous_access }}\"\n" }, { "path": "roles/kubernetes/kubeadm/handlers/main.yml", "content": "---\n- name: Kubeadm | reload systemd\n systemd_service:\n daemon_reload: true\n listen: Kubeadm | restart kubelet\n\n- name: Kubeadm | reload kubelet\n service:\n name: kubelet\n state: restarted\n listen: Kubeadm | restart kubelet\n" }, { "path": "roles/kubernetes/kubeadm/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes/kubeadm_common\n" }, { "path": "roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml", "content": "---\n- name: Parse certificate key if not set\n set_fact:\n kubeadm_certificate_key: \"{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_certificate_key'] }}\"\n when: kubeadm_certificate_key is undefined\n\n- name: Create kubeadm cert controlplane config\n template:\n src: \"kubeadm-client.conf.j2\"\n dest: \"{{ kube_config_dir }}/kubeadm-cert-controlplane.conf\"\n mode: \"0640\"\n validate: \"{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}\"\n vars:\n kubeadm_cert_controlplane: true\n\n- name: Pull control plane certs down\n shell: >-\n {{ bin_dir }}/kubeadm join phase\n control-plane-prepare download-certs\n --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf\n &&\n {{ bin_dir }}/kubeadm join phase\n control-plane-prepare certs\n --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf\n args:\n creates: \"{{ kube_cert_dir }}/apiserver-etcd-client.key\"\n\n- name: Delete unneeded certificates\n file:\n path: \"{{ item }}\"\n state: absent\n with_items:\n - \"{{ kube_cert_dir }}/apiserver.crt\"\n - \"{{ kube_cert_dir }}/apiserver.key\"\n - \"{{ kube_cert_dir }}/ca.key\"\n - \"{{ kube_cert_dir }}/etcd/ca.key\"\n - \"{{ kube_cert_dir }}/etcd/healthcheck-client.crt\"\n - \"{{ kube_cert_dir }}/etcd/healthcheck-client.key\"\n - \"{{ kube_cert_dir }}/etcd/peer.crt\"\n - \"{{ kube_cert_dir }}/etcd/peer.key\"\n - \"{{ kube_cert_dir }}/etcd/server.crt\"\n - \"{{ kube_cert_dir }}/etcd/server.key\"\n - \"{{ kube_cert_dir }}/front-proxy-ca.crt\"\n - \"{{ kube_cert_dir }}/front-proxy-ca.key\"\n - \"{{ kube_cert_dir }}/front-proxy-client.crt\"\n - \"{{ kube_cert_dir }}/front-proxy-client.key\"\n - \"{{ kube_cert_dir }}/sa.key\"\n - \"{{ kube_cert_dir }}/sa.pub\"\n\n- name: Calculate etcd cert serial\n command: \"openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial\"\n register: \"etcd_client_cert_serial_result\"\n changed_when: false\n when:\n - group_names | intersect(['k8s_cluster', 'calico_rr']) | length > 0\n tags:\n - network\n\n- name: Set etcd_client_cert_serial\n set_fact:\n etcd_client_cert_serial: \"{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}\"\n tags:\n - network\n" }, { "path": "roles/kubernetes/kubeadm/tasks/main.yml", "content": "---\n- name: Set kubeadm_discovery_address\n set_fact:\n # noqa: jinja[spacing]\n kubeadm_discovery_address: >-\n {%- if \"127.0.0.1\" in kube_apiserver_endpoint or \"localhost\" in kube_apiserver_endpoint -%}\n {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}\n {%- else -%}\n {{ kube_apiserver_endpoint | replace(\"https://\", \"\") }}\n {%- endif %}\n tags:\n - facts\n\n- name: Check if kubelet.conf exists\n stat:\n path: \"{{ kube_config_dir }}/kubelet.conf\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kubelet_conf\n\n- name: Check if kubeadm CA cert is accessible\n stat:\n path: \"{{ kube_cert_dir }}/ca.crt\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kubeadm_ca_stat\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n\n- name: Fetch CA certificate from control plane node\n slurp:\n src: \"{{ kube_cert_dir }}/ca.crt\"\n register: ca_cert_content\n when:\n - kubeadm_ca_stat.stat is defined\n - kubeadm_ca_stat.stat.exists\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n\n- name: Create kubeadm token for joining nodes with 24h expiration (default)\n command: \"{{ bin_dir }}/kubeadm token create\"\n register: temp_token\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when: kubeadm_token is not defined\n changed_when: false\n\n- name: Set kubeadm_token to generated token\n set_fact:\n kubeadm_token: \"{{ temp_token.stdout }}\"\n when: kubeadm_token is not defined\n\n- name: Get kubeconfig for join discovery process\n command: \"{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'\"\n register: kubeconfig_file_discovery\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n when: kubeadm_use_file_discovery\n\n- name: Check if discovery kubeconfig exists\n stat:\n path: \"{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml\"\n register: cluster_info_discovery_kubeconfig\n\n- name: Copy discovery kubeconfig\n copy:\n dest: \"{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml\"\n content: \"{{ kubeconfig_file_discovery.stdout }}\"\n owner: \"root\"\n mode: \"0644\"\n when:\n - ('kube_control_plane' not in group_names)\n - not kubelet_conf.stat.exists or not cluster_info_discovery_kubeconfig.stat.exists\n - kubeadm_use_file_discovery\n\n- name: Create kubeadm client config\n template:\n src: \"kubeadm-client.conf.j2\"\n dest: \"{{ kube_config_dir }}/kubeadm-client.conf\"\n backup: true\n mode: \"0640\"\n validate: \"{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}\"\n when: ('kube_control_plane' not in group_names)\n\n- name: Join to cluster if needed\n environment:\n PATH: \"{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin\"\n when:\n - ('kube_control_plane' not in group_names)\n - not kubelet_conf.stat.exists\n vars:\n ignored:\n - \"{{ 'DirAvailable--etc-kubernetes-manifests' if 'all' not in kubeadm_ignore_preflight_errors }}\"\n - \"{{ kubeadm_ignore_preflight_errors }}\"\n command: >-\n timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}\n {{ bin_dir }}/kubeadm join\n --config {{ kube_config_dir }}/kubeadm-client.conf\n --ignore-preflight-errors={{ ignored | select | flatten | join(',') }}\n --skip-phases={{ kubeadm_join_phases_skip | join(',') }}\n\n- name: Update server field in kubelet kubeconfig\n lineinfile:\n dest: \"{{ kube_config_dir }}/kubelet.conf\"\n regexp: 'server:'\n line: ' server: {{ kube_apiserver_endpoint }}'\n backup: true\n when:\n - kubeadm_config_api_fqdn is not defined\n - ('kube_control_plane' not in group_names)\n - kubeadm_discovery_address != kube_apiserver_endpoint | replace(\"https://\", \"\")\n notify: Kubeadm | restart kubelet\n\n- name: Update server field in kubelet kubeconfig - external lb\n lineinfile:\n dest: \"{{ kube_config_dir }}/kubelet.conf\"\n regexp: '^ server: https'\n line: ' server: {{ kube_apiserver_endpoint }}'\n backup: true\n when:\n - ('kube_control_plane' not in group_names)\n - loadbalancer_apiserver is defined\n notify: Kubeadm | restart kubelet\n\n- name: Get current resourceVersion of kube-proxy configmap\n command: \"{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'\"\n register: original_configmap_resource_version\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n delegate_facts: false\n when:\n - kube_proxy_deployed\n tags:\n - kube-proxy\n\n# FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point\n# incorrectly to first control plane node, creating SPoF.\n- name: Update server field in kube-proxy kubeconfig\n shell: >-\n set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml\n | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g'\n | {{ kubectl }} replace -f -\n args:\n executable: /bin/bash\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n delegate_facts: false\n when:\n - kubeadm_config_api_fqdn is not defined\n - kubeadm_discovery_address != kube_apiserver_endpoint | replace(\"https://\", \"\")\n - kube_proxy_deployed\n - loadbalancer_apiserver_localhost\n tags:\n - kube-proxy\n\n- name: Update server field in kube-proxy kubeconfig - external lb\n shell: >-\n set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml\n | sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g'\n | {{ kubectl }} replace -f -\n args:\n executable: /bin/bash\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n delegate_facts: false\n when:\n - kube_proxy_deployed\n - loadbalancer_apiserver is defined\n tags:\n - kube-proxy\n\n- name: Get new resourceVersion of kube-proxy configmap\n command: \"{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'\"\n register: new_configmap_resource_version\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n delegate_facts: false\n when:\n - kube_proxy_deployed\n tags:\n - kube-proxy\n\n- name: Set ca.crt file permission\n file:\n path: \"{{ kube_cert_dir }}/ca.crt\"\n owner: root\n group: root\n mode: \"0644\"\n\n- name: Restart all kube-proxy pods to ensure that they load the new configmap\n command: \"{{ kubectl }} delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n delegate_facts: false\n when:\n - kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined\n - kubeadm_discovery_address != kube_apiserver_endpoint | replace(\"https://\", \"\") or loadbalancer_apiserver is defined\n - kube_proxy_deployed\n - original_configmap_resource_version.stdout != new_configmap_resource_version.stdout\n tags:\n - kube-proxy\n\n- name: Extract etcd certs from control plane if using etcd kubeadm mode\n include_tasks: kubeadm_etcd_node.yml\n when:\n - etcd_deployment_type == \"kubeadm\"\n - inventory_hostname not in groups['kube_control_plane']\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\"] or cilium_deploy_additionally\n - kube_network_plugin != \"calico\" or calico_datastore == \"etcd\"\n - kube_network_plugin != \"cilium\" or cilium_identity_allocation_mode != 'crd'\n" }, { "path": "roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2", "content": "---\napiVersion: kubeadm.k8s.io/v1beta4\nkind: JoinConfiguration\ndiscovery:\n{% if kubeadm_use_file_discovery %}\n file:\n kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml\n{% else %}\n bootstrapToken:\n{% if kubeadm_config_api_fqdn is defined %}\n apiServerEndpoint: \"{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}\"\n{% else %}\n apiServerEndpoint: \"{{ kubeadm_discovery_address }}\"\n{% endif %}\n token: {{ kubeadm_token }}\n{% if ca_cert_content is defined %}\n caCertHashes:\n - sha256:{{ (ca_cert_content.content | b64decode | community.crypto.x509_certificate_info).public_key_fingerprints.sha256.replace(':', '') }}\n{% else %}\n unsafeSkipCAVerification: true\n{% endif %}\n{% endif %}\n tlsBootstrapToken: {{ kubeadm_token }}\ntimeouts:\n discovery: {{ discovery_timeout }}\ncaCertPath: {{ kube_cert_dir }}/ca.crt\n{% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %}\ncontrolPlane:\n localAPIEndpoint:\n advertiseAddress: \"{{ kube_apiserver_address }}\"\n bindPort: {{ kube_apiserver_port }}\n certificateKey: {{ kubeadm_certificate_key }}\n{% endif %}\nnodeRegistration:\n name: '{{ kube_override_hostname }}'\n criSocket: {{ cri_socket }}\n{% if 'calico_rr' in group_names and 'kube_node' not in group_names %}\n taints:\n - effect: NoSchedule\n key: node-role.kubernetes.io/calico-rr\n{% endif %}\n{% if kubeadm_patches | length > 0 %}\npatches:\n directory: {{ kubeadm_patches_dir }}\n{% endif %}\n" }, { "path": "roles/kubernetes/kubeadm_common/defaults/main.yml", "content": "---\nkubeadm_patches_dir: \"{{ kube_config_dir }}/patches\"\nkubeadm_patches: []\n# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches\n# Correspondance with this link\n# patchtype = type\n# target = target\n# suffix -> managed automatically\n# extension -> always \"yaml\"\n# kubeadm_patches:\n# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration\n# type: strategic(default)|json|merge\n# patch:\n# metadata:\n# annotations:\n# example.com/test: \"true\"\n# labels:\n# example.com/prod_level: \"{{ prod_level }}\"\n# - ...\n# Patches are applied in the order they are specified.\n\n# List of errors to ignore during kubeadm preflight checks\nkubeadm_ignore_preflight_errors: []\n" }, { "path": "roles/kubernetes/kubeadm_common/tasks/main.yml", "content": "---\n- name: Kubeadm | Create directory to store kubeadm patches\n file:\n path: \"{{ kubeadm_patches_dir }}\"\n state: directory\n mode: \"0750\"\n when: kubeadm_patches | length > 0\n\n- name: Kubeadm | List existing kubeadm patches\n find:\n paths:\n - \"{{ kubeadm_patches_dir }}\"\n file_type: file\n use_regex: true\n patterns:\n - '^(kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration)[0-9]+\\+(strategic|json|merge).yaml$'\n register: existing_kubeadm_patches\n\n- name: Kubeadm | Copy kubeadm patches from inventory files\n copy:\n content: \"{{ item.patch | to_yaml }}\"\n dest: \"{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml\"\n owner: \"root\"\n mode: \"0644\"\n loop: \"{{ kubeadm_patches }}\"\n loop_control:\n index_var: suffix\n register: current_kubeadm_patches\n\n- name: Kubeadm | Delete old patches\n loop: \"{{ existing_kubeadm_patches.files | map(attribute='path') |\n difference(\n current_kubeadm_patches.results | map(attribute='dest')\n ) }}\"\n file:\n state: absent\n path: \"{{ item }}\"\n" }, { "path": "roles/kubernetes/node/defaults/main.yml", "content": "---\n# advertised host IP for kubelet. This affects network plugin config. Take caution\n# add ipv6 manual for dualstack mode because ipv4 priority in main_ip for dualstack\nkubelet_address: \"{{ main_ips | join(',') }}\"\n\n# bind address for kubelet. Set to :: to listen on all interfaces\nkubelet_bind_address: \"{{ main_ip | default('::') }}\"\n\n# resolv.conf to base dns config\nkube_resolv_conf: \"/etc/resolv.conf\"\n\n# Set to empty to avoid cgroup creation\nkubelet_enforce_node_allocatable: \"\\\"\\\"\"\n\n# Set runtime and kubelet cgroups when using systemd as cgroup driver (default)\nkube_service_cgroups: \"{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}\"\nkubelet_runtime_cgroups: \"/{{ kube_service_cgroups }}/{{ container_manager }}.service\"\nkubelet_kubelet_cgroups: \"/{{ kube_service_cgroups }}/kubelet.service\"\n\n# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver\nkubelet_runtime_cgroups_cgroupfs: \"/system.slice/{{ container_manager }}.service\"\nkubelet_kubelet_cgroups_cgroupfs: \"/system.slice/kubelet.service\"\n\n# Set systemd service hardening features\nkubelet_systemd_hardening: false\n\n# Kubelet service dependencies other than container runtime\nkubelet_systemd_wants_dependencies: []\n\n# List of secure IPs for kubelet\n# don't forget ipv6 addresses for dualstack(because \"main_ip\" prioritizes ipv4)\nkube_node_addresses: >-\n {%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%}\n {{ hostvars[host]['main_ips'] | join(' ') }}{{ ' ' if not loop.last else '' }}\n {%- endfor -%}\nkubelet_secure_addresses: \"localhost link-local {{ kube_pods_subnets | regex_replace(',', ' ') }} {{ kube_node_addresses }}\"\n\n# Reserve this space for kube resources\n# Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations).\nkube_reserved: false\nkube_reserved_cgroups: \"/{{ kube_reserved_cgroups_for_service_slice }}\"\nkube_memory_reserved: \"256Mi\"\nkube_cpu_reserved: \"100m\"\nkube_ephemeral_storage_reserved: \"500Mi\"\nkube_pid_reserved: \"1000\"\n\n# Set to true to reserve resources for system daemons\nsystem_reserved: false\nsystem_reserved_cgroups_for_service_slice: system.slice\nsystem_reserved_cgroups: \"/{{ system_reserved_cgroups_for_service_slice }}\"\nsystem_memory_reserved: \"512Mi\"\nsystem_cpu_reserved: \"500m\"\nsystem_ephemeral_storage_reserved: \"500Mi\"\nsystem_pid_reserved: 1000\n\n## Eviction Thresholds to avoid system OOMs\n# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds\neviction_hard: {}\neviction_hard_control_plane: {}\n\nkubelet_status_update_frequency: 10s\n\n# kube-vip\nkube_vip_arp_enabled: false\nkube_vip_interface:\nkube_vip_services_interface:\nkube_vip_cidr: 32\nkube_vip_dns_mode: first\nkube_vip_controlplane_enabled: false\nkube_vip_ddns_enabled: false\nkube_vip_cp_detect: false\nkube_vip_services_enabled: false\nkube_vip_leader_election_enabled: \"{{ kube_vip_arp_enabled }}\"\nkube_vip_bgp_enabled: false\nkube_vip_bgp_routerid:\nkube_vip_local_as: 65000\nkube_vip_bgp_peeraddress:\nkube_vip_bgp_peerpass:\nkube_vip_bgp_peeras: 65000\nkube_vip_bgppeers:\nkube_vip_enableServicesElection: false\nkube_vip_lb_enable: false\nkube_vip_leasename: plndr-cp-lock\nkube_vip_svc_leasename: plndr-svcs-lock\nkube_vip_leaseduration: 5\nkube_vip_renewdeadline: 3\nkube_vip_retryperiod: 1\nkube_vip_enable_node_labeling: false\nkube_vip_bgp_sourceip:\nkube_vip_bgp_sourceif:\n\n# Requests for load balancer app\nloadbalancer_apiserver_memory_requests: 32M\nloadbalancer_apiserver_cpu_requests: 25m\n\nloadbalancer_apiserver_keepalive_timeout: 5m\nloadbalancer_apiserver_pod_name: \"{% if loadbalancer_apiserver_type == 'nginx' %}nginx-proxy{% else %}haproxy{% endif %}\"\n\n# Uncomment if you need to enable deprecated runtimes\n# kube_api_runtime_config:\n# - apps/v1beta1=true\n# - apps/v1beta2=true\n# - extensions/v1beta1/daemonsets=true\n# - extensions/v1beta1/deployments=true\n# - extensions/v1beta1/replicasets=true\n# - extensions/v1beta1/networkpolicies=true\n\n# A port range to reserve for services with NodePort visibility.\n# Inclusive at both ends of the range.\nkube_apiserver_node_port_range: \"30000-32767\"\n\n# Configure the amount of pods able to run on single node\n# default is equal to application default\nkubelet_max_pods: 110\n\n# Sets the maximum number of processes running per Pod\n# Default value -1 = unlimited\nkubelet_pod_pids_limit: -1\n\n## Support parameters to be passed to kubelet via kubelet-config.yaml\nkubelet_config_extra_args: {}\n\n## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver\nkubelet_config_extra_args_cgroupfs:\n systemCgroups: /system.slice\n cgroupRoot: /\n\n# Maximum number of container log files that can be present for a container.\nkubelet_logfiles_max_nr: 5\n\n# Maximum size of the container log file before it is rotated\nkubelet_logfiles_max_size: 10Mi\n\n## Support custom flags to be passed to kubelet\nkubelet_custom_flags: []\n\n# The read-only port for the Kubelet to serve on with no authentication/authorization.\nkube_read_only_port: 0\n\n# Port for healthz for Kubelet\nkubelet_healthz_port: 10248\n\n# Bind address for healthz for Kubelet\nkubelet_healthz_bind_address: 127.0.0.1\n\n# sysctl_file_path to add sysctl conf to\nsysctl_file_path: \"/etc/sysctl.d/99-sysctl.conf\"\n\n## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.\n# tls_min_version: \"\"\n\n## Support tls cipher suites.\n# tls_cipher_suites:\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\n# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\n# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n# - TLS_ECDHE_RSA_WITH_RC4_128_SHA\n# - TLS_RSA_WITH_3DES_EDE_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA\n# - TLS_RSA_WITH_AES_128_CBC_SHA256\n# - TLS_RSA_WITH_AES_128_GCM_SHA256\n# - TLS_RSA_WITH_AES_256_CBC_SHA\n# - TLS_RSA_WITH_AES_256_GCM_SHA384\n# - TLS_RSA_WITH_RC4_128_SHA\n\nkube_proxy_ipvs_modules:\n - ip_vs\n - ip_vs_rr\n - ip_vs_wrr\n - ip_vs_sh\n - ip_vs_wlc\n - ip_vs_lc\n\n## Enable distributed tracing for kubelet\nkubelet_tracing: false\nkubelet_tracing_endpoint: \"[::]:4317\"\nkubelet_tracing_sampling_rate_per_million: 100\n\n# The maximum number of image pulls in parallel. Set it to a integer great than 1 to enable image pulling in parallel.\nkubelet_max_parallel_image_pulls: 1\n" }, { "path": "roles/kubernetes/node/handlers/main.yml", "content": "---\n- name: Kubelet | reload systemd\n systemd_service:\n daemon_reload: true\n listen: Node | restart kubelet\n\n- name: Kubelet | restart kubelet\n service:\n name: kubelet\n state: restarted\n listen: Node | restart kubelet\n" }, { "path": "roles/kubernetes/node/tasks/facts.yml", "content": "---\n- name: Gather cgroups facts for docker\n when: container_manager == 'docker'\n block:\n - name: Look up docker cgroup driver\n shell: \"set -o pipefail && docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'\"\n args:\n executable: /bin/bash\n register: docker_cgroup_driver_result\n changed_when: false\n check_mode: false\n\n - name: Set kubelet_cgroup_driver_detected fact for docker\n set_fact:\n kubelet_cgroup_driver_detected: \"{{ docker_cgroup_driver_result.stdout }}\"\n\n- name: Gather cgroups facts for crio\n when: container_manager == 'crio'\n block:\n - name: Look up crio cgroup driver\n shell: \"set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'\"\n args:\n executable: /bin/bash\n register: crio_cgroup_driver_result\n changed_when: false\n\n - name: Set kubelet_cgroup_driver_detected fact for crio\n set_fact:\n kubelet_cgroup_driver_detected: \"{{ crio_cgroup_driver_result.stdout }}\"\n\n- name: Set kubelet_cgroup_driver_detected fact for containerd\n when: container_manager == 'containerd'\n set_fact:\n kubelet_cgroup_driver_detected: >-\n {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}\n\n- name: Set kubelet_cgroup_driver\n set_fact:\n kubelet_cgroup_driver: \"{{ kubelet_cgroup_driver_detected }}\"\n when: kubelet_cgroup_driver is undefined\n\n- name: Set kubelet_cgroups options when cgroupfs is used\n set_fact:\n kubelet_runtime_cgroups: \"{{ kubelet_runtime_cgroups_cgroupfs }}\"\n kubelet_kubelet_cgroups: \"{{ kubelet_kubelet_cgroups_cgroupfs }}\"\n when: kubelet_cgroup_driver == 'cgroupfs'\n\n- name: Set kubelet_config_extra_args options when cgroupfs is used\n set_fact:\n kubelet_config_extra_args: \"{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}\"\n when: kubelet_cgroup_driver == 'cgroupfs'\n\n- name: Os specific vars\n include_vars: \"{{ item }}\"\n with_first_found:\n - files:\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}.yml\"\n - \"{{ ansible_os_family | lower }}.yml\"\n skip: true\n" }, { "path": "roles/kubernetes/node/tasks/install.yml", "content": "---\n- name: Install | Copy kubeadm binary from download dir\n copy:\n src: \"{{ downloads.kubeadm.dest }}\"\n dest: \"{{ bin_dir }}/kubeadm\"\n mode: \"0755\"\n remote_src: true\n tags:\n - kubeadm\n when:\n - not ('kube_control_plane' in group_names)\n\n- name: Install | Copy kubelet binary from download dir\n copy:\n src: \"{{ downloads.kubelet.dest }}\"\n dest: \"{{ bin_dir }}/kubelet\"\n mode: \"0755\"\n remote_src: true\n tags:\n - kubelet\n - upgrade\n notify: Node | restart kubelet\n" }, { "path": "roles/kubernetes/node/tasks/kubelet.yml", "content": "---\n- name: Set kubelet api version to v1beta1\n set_fact:\n kubeletConfig_api_version: v1beta1\n tags:\n - kubelet\n - kubeadm\n\n- name: Write kubelet environment config file (kubeadm)\n template:\n src: \"kubelet.env.{{ kubeletConfig_api_version }}.j2\"\n dest: \"{{ kube_config_dir }}/kubelet.env\"\n setype: \"{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}\"\n backup: true\n mode: \"0600\"\n notify: Node | restart kubelet\n tags:\n - kubelet\n - kubeadm\n\n- name: Write kubelet config file\n template:\n src: \"kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2\"\n dest: \"{{ kube_config_dir }}/kubelet-config.yaml\"\n mode: \"0600\"\n notify: Kubelet | restart kubelet\n tags:\n - kubelet\n - kubeadm\n\n- name: Write kubelet systemd init file\n template:\n src: \"kubelet.service.j2\"\n dest: \"/etc/systemd/system/kubelet.service\"\n backup: true\n mode: \"0600\"\n validate: \"sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:kubelet.service'\"\n # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)\n # Remove once we drop support for systemd < 250\n notify: Node | restart kubelet\n tags:\n - kubelet\n - kubeadm\n\n- name: Flush_handlers and reload-systemd\n meta: flush_handlers\n\n- name: Enable kubelet\n service:\n name: kubelet\n enabled: true\n state: started\n tags:\n - kubelet\n notify: Kubelet | restart kubelet\n" }, { "path": "roles/kubernetes/node/tasks/loadbalancer/haproxy.yml", "content": "---\n- name: Haproxy | Cleanup potentially deployed nginx-proxy\n file:\n path: \"{{ kube_manifest_dir }}/nginx-proxy.yml\"\n state: absent\n\n- name: Haproxy | Make haproxy directory\n file:\n path: \"{{ haproxy_config_dir }}\"\n state: directory\n mode: \"0755\"\n owner: root\n\n- name: Haproxy | Write haproxy configuration\n template:\n src: \"loadbalancer/haproxy.cfg.j2\"\n dest: \"{{ haproxy_config_dir }}/haproxy.cfg\"\n owner: root\n mode: \"0755\"\n backup: true\n register: haproxy_conf\n\n- name: Haproxy | Write static pod\n template:\n src: manifests/haproxy.manifest.j2\n dest: \"{{ kube_manifest_dir }}/haproxy.yml\"\n mode: \"0640\"\n" }, { "path": "roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml", "content": "---\n- name: Kube-vip | Check cluster settings for kube-vip\n fail:\n msg: \"kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md\"\n when:\n - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp\n - kube_vip_arp_enabled\n\n- name: Kube-vip | Check mutually exclusive BGP source settings\n vars:\n kube_vip_bgp_sourceip_normalized: \"{{ kube_vip_bgp_sourceip | default('', true) | string | trim }}\"\n kube_vip_bgp_sourceif_normalized: \"{{ kube_vip_bgp_sourceif | default('', true) | string | trim }}\"\n assert:\n that:\n - kube_vip_bgp_sourceip_normalized == '' or kube_vip_bgp_sourceif_normalized == ''\n fail_msg: \"kube-vip allows only one of kube_vip_bgp_sourceip or kube_vip_bgp_sourceif.\"\n when:\n - kube_vip_bgp_enabled | default(false)\n\n- name: Kube-vip | Check if super-admin.conf exists\n stat:\n path: \"{{ kube_config_dir }}/super-admin.conf\"\n failed_when: false\n changed_when: false\n register: stat_kube_vip_super_admin\n\n- name: Kube-vip | Check if kubeadm has already run\n stat:\n path: \"/var/lib/kubelet/config.yaml\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kubeadm_already_run\n\n- name: Kube-vip | Set admin.conf\n set_fact:\n kube_vip_admin_conf: admin.conf\n\n- name: Kube-vip | Set admin.conf for first Control Plane\n set_fact:\n kube_vip_admin_conf: super-admin.conf\n when:\n - inventory_hostname == groups['kube_control_plane'] | first\n - (stat_kube_vip_super_admin.stat.exists and stat_kube_vip_super_admin.stat.isreg) or (not kubeadm_already_run.stat.exists )\n\n- name: Kube-vip | Write static pod\n template:\n src: manifests/kube-vip.manifest.j2\n dest: \"{{ kube_manifest_dir }}/kube-vip.yml\"\n mode: \"0640\"\n" }, { "path": "roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml", "content": "---\n- name: Haproxy | Cleanup potentially deployed haproxy\n file:\n path: \"{{ kube_manifest_dir }}/haproxy.yml\"\n state: absent\n\n- name: Nginx-proxy | Make nginx directory\n file:\n path: \"{{ nginx_config_dir }}\"\n state: directory\n mode: \"0700\"\n owner: root\n\n- name: Nginx-proxy | Write nginx-proxy configuration\n template:\n src: \"loadbalancer/nginx.conf.j2\"\n dest: \"{{ nginx_config_dir }}/nginx.conf\"\n owner: root\n mode: \"0755\"\n backup: true\n register: nginx_conf\n\n- name: Nginx-proxy | Write static pod\n template:\n src: manifests/nginx-proxy.manifest.j2\n dest: \"{{ kube_manifest_dir }}/nginx-proxy.yml\"\n mode: \"0640\"\n" }, { "path": "roles/kubernetes/node/tasks/main.yml", "content": "---\n- name: Fetch facts\n import_tasks: facts.yml\n tags:\n - facts\n - kubelet\n\n- name: Ensure /var/lib/cni exists\n file:\n path: /var/lib/cni\n state: directory\n mode: \"0755\"\n\n- name: Install kubelet binary\n import_tasks: install.yml\n tags:\n - kubelet\n\n- name: Install kube-vip\n import_tasks: loadbalancer/kube-vip.yml\n when:\n - ('kube_control_plane' in group_names)\n - kube_vip_enabled\n tags:\n - kube-vip\n\n- name: Install nginx-proxy\n import_tasks: loadbalancer/nginx-proxy.yml\n when:\n - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')\n - loadbalancer_apiserver_localhost\n - loadbalancer_apiserver_type == 'nginx'\n tags:\n - nginx\n\n- name: Install haproxy\n import_tasks: loadbalancer/haproxy.yml\n when:\n - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')\n - loadbalancer_apiserver_localhost\n - loadbalancer_apiserver_type == 'haproxy'\n tags:\n - haproxy\n\n- name: Ensure nodePort range is reserved\n ansible.posix.sysctl:\n name: net.ipv4.ip_local_reserved_ports\n value: \"{{ kube_apiserver_node_port_range }}\"\n sysctl_set: true\n sysctl_file: \"{{ sysctl_file_path }}\"\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n when: kube_apiserver_node_port_range is defined\n tags:\n - kube-proxy\n\n- name: Verify if br_netfilter module exists\n command: \"modinfo br_netfilter\"\n environment:\n PATH: \"{{ ansible_env.PATH }}:/sbin\" # Make sure we can workaround RH's conservative path management\n register: modinfo_br_netfilter\n failed_when: modinfo_br_netfilter.rc not in [0, 1]\n changed_when: false\n check_mode: false\n\n- name: Verify br_netfilter module path exists\n file:\n path: \"{{ item }}\"\n state: directory\n mode: \"0755\"\n loop:\n - /etc/modules-load.d\n - /etc/modprobe.d\n\n- name: Enable br_netfilter module\n community.general.modprobe:\n name: br_netfilter\n state: present\n when: modinfo_br_netfilter.rc == 0\n\n- name: Persist br_netfilter module\n copy:\n dest: /etc/modules-load.d/kubespray-br_netfilter.conf\n content: br_netfilter\n mode: \"0644\"\n when: modinfo_br_netfilter.rc == 0\n\n# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module\n- name: Check if bridge-nf-call-iptables key exists\n command: \"sysctl net.bridge.bridge-nf-call-iptables\"\n failed_when: false\n changed_when: false\n check_mode: false\n register: sysctl_bridge_nf_call_iptables\n\n- name: Enable bridge-nf-call tables\n ansible.posix.sysctl:\n name: \"{{ item }}\"\n state: present\n sysctl_file: \"{{ sysctl_file_path }}\"\n value: \"1\"\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n when: sysctl_bridge_nf_call_iptables.rc == 0\n with_items:\n - net.bridge.bridge-nf-call-iptables\n - net.bridge.bridge-nf-call-arptables\n - net.bridge.bridge-nf-call-ip6tables\n\n- name: Modprobe Kernel Module for IPVS\n community.general.modprobe:\n name: \"{{ item }}\"\n state: present\n persistent: present\n loop: \"{{ kube_proxy_ipvs_modules }}\"\n when: kube_proxy_mode == 'ipvs'\n tags:\n - kube-proxy\n\n- name: Modprobe conntrack module\n community.general.modprobe:\n name: \"{{ item }}\"\n state: present\n persistent: present\n register: modprobe_conntrack_module\n ignore_errors: true # noqa ignore-errors\n loop:\n - nf_conntrack\n - nf_conntrack_ipv4\n when:\n - kube_proxy_mode == 'ipvs'\n - modprobe_conntrack_module is not defined or modprobe_conntrack_module is ansible.builtin.failed # loop until first success\n tags:\n - kube-proxy\n\n- name: Modprobe Kernel Module for nftables\n community.general.modprobe:\n name: \"nf_tables\"\n state: present\n persistent: present\n when: kube_proxy_mode == 'nftables'\n tags:\n - kube-proxy\n\n- name: Install kubelet\n import_tasks: kubelet.yml\n tags:\n - kubelet\n - kubeadm\n" }, { "path": "roles/kubernetes/node/templates/http-proxy.conf.j2", "content": "[Service]\nEnvironment={% if http_proxy %}\"HTTP_PROXY={{ http_proxy }}\"{% endif %} {% if https_proxy %}\"HTTPS_PROXY={{ https_proxy }}\"{% endif %} {% if no_proxy %}\"NO_PROXY={{ no_proxy }}\"{% endif %}\n" }, { "path": "roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2", "content": "apiVersion: kubelet.config.k8s.io/v1beta1\nkind: KubeletConfiguration\nnodeStatusUpdateFrequency: \"{{ kubelet_status_update_frequency }}\"\nfailSwapOn: {{ kubelet_fail_swap_on }}\nauthentication:\n anonymous:\n enabled: false\n webhook:\n enabled: {{ kubelet_authentication_token_webhook }}\n x509:\n clientCAFile: {{ kube_cert_dir }}/ca.crt\nauthorization:\n{% if kubelet_authorization_mode_webhook %}\n mode: Webhook\n{% else %}\n mode: AlwaysAllow\n{% endif %}\n{% if kube_version is version('1.35.0', '>=') %}\nfailCgroupV1: {{ kubelet_fail_cgroup_v1 }}\n{% endif %}\n{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != \"\\\"\\\"\" %}\n{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split(\",\") %}\nenforceNodeAllocatable:\n{% for item in kubelet_enforce_node_allocatable_list %}\n- {{ item }}\n{% endfor %}\n{% endif %}\nstaticPodPath: {{ kube_manifest_dir }}\ncgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}\ncontainerLogMaxFiles: {{ kubelet_logfiles_max_nr }}\ncontainerLogMaxSize: {{ kubelet_logfiles_max_size }}\ncontainerRuntimeEndpoint : {{ cri_socket }}\nmaxPods: {{ kubelet_max_pods }}\npodPidsLimit: {{ kubelet_pod_pids_limit }}\naddress: \"{{ kubelet_bind_address }}\"\nreadOnlyPort: {{ kube_read_only_port }}\nhealthzPort: {{ kubelet_healthz_port }}\nhealthzBindAddress: \"{{ kubelet_healthz_bind_address }}\"\nkubeletCgroups: {{ kubelet_kubelet_cgroups }}\nclusterDomain: {{ dns_domain }}\n{% if kubelet_protect_kernel_defaults | bool %}\nprotectKernelDefaults: true\n{% endif %}\n{% if kubelet_rotate_certificates | bool %}\nrotateCertificates: true\n{% endif %}\n{% if kubelet_rotate_server_certificates | bool %}\nserverTLSBootstrap: true\n{% endif %}\n{# DNS settings for kubelet #}\n{% if enable_nodelocaldns %}\n{% set kubelet_cluster_dns = [nodelocaldns_ip] %}\n{% elif dns_mode in ['coredns'] %}\n{% set kubelet_cluster_dns = [skydns_server] %}\n{% elif dns_mode == 'coredns_dual' %}\n{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}\n{% elif dns_mode == 'manual' %}\n{% set kubelet_cluster_dns = [manual_dns_server] %}\n{% else %}\n{% set kubelet_cluster_dns = [] %}\n{% endif %}\nclusterDNS:\n{% for dns_address in kubelet_cluster_dns %}\n- {{ dns_address }}\n{% endfor %}\n{# Node reserved CPU/memory #}\n{% for scope in \"kube\", \"system\" %}\n{% if lookup('ansible.builtin.vars', scope + \"_reserved\") | bool %}\n{{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }}\n{% endif %}\n{{ scope }}Reserved:\n{% for resource in \"cpu\", \"memory\", \"ephemeral-storage\", \"pid\" %}\n {{ resource }}: \"{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}\"\n{% endfor %}\n{% endfor %}\n{% if eviction_hard is defined and eviction_hard %}\nevictionHard:\n {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }}\n{% endif %}\nresolvConf: \"{{ kube_resolv_conf }}\"\n{% if kubelet_config_extra_args %}\n{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}\n{% endif %}\n{% if kubelet_feature_gates or kube_feature_gates %}\nfeatureGates:\n{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %}\n {{ feature | replace(\"=\", \": \") }}\n{% endfor %}\n{% endif %}\n{% if tls_min_version is defined %}\ntlsMinVersion: {{ tls_min_version }}\n{% endif %}\n{% if tls_cipher_suites is defined %}\ntlsCipherSuites:\n{% for tls in tls_cipher_suites %}\n- {{ tls }}\n{% endfor %}\n{% endif %}\neventRecordQPS: {{ kubelet_event_record_qps }}\nshutdownGracePeriod: {{ kubelet_shutdown_grace_period }}\nshutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }}\n{% if not kubelet_fail_swap_on %}\nmemorySwap:\n swapBehavior: {{ kubelet_swap_behavior }}\n{% endif %}\n{% if kubelet_streaming_connection_idle_timeout is defined %}\nstreamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }}\n{% endif %}\n{% if kubelet_image_gc_high_threshold is defined %}\nimageGCHighThresholdPercent: {{ kubelet_image_gc_high_threshold }}\n{% endif %}\n{% if kubelet_image_gc_low_threshold is defined %}\nimageGCLowThresholdPercent: {{ kubelet_image_gc_low_threshold }}\n{% endif %}\n{% if kubelet_make_iptables_util_chains is defined %}\nmakeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }}\n{% endif %}\n{% if kubelet_seccomp_default is defined %}\nseccompDefault: {{ kubelet_seccomp_default | bool }}\n{% endif %}\n{% if kubelet_cpu_manager_policy is defined %}\ncpuManagerPolicy: {{ kubelet_cpu_manager_policy }}\n{% endif %}\n{% if kubelet_cpu_manager_policy_options is defined %}\ncpuManagerPolicyOptions:\n {{ kubelet_cpu_manager_policy_options | to_nice_yaml(indent=2) | indent(width=2) }}\n{% endif %}\n{% if kubelet_topology_manager_policy is defined %}\ntopologyManagerPolicy: {{ kubelet_topology_manager_policy }}\n{% endif %}\n{% if kubelet_topology_manager_scope is defined %}\ntopologyManagerScope: {{ kubelet_topology_manager_scope }}\n{% endif %}\n{% if kubelet_tracing %}\ntracing:\n endpoint: \"{{ kubelet_tracing_endpoint }}\"\n samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }}\n{% endif %}\nmaxParallelImagePulls: {{ kubelet_max_parallel_image_pulls }}\n" }, { "path": "roles/kubernetes/node/templates/kubelet.env.v1beta1.j2", "content": "KUBE_LOG_LEVEL=\"--v={{ kube_log_level }}\"\nKUBELET_ADDRESS=\"--node-ip={{ kubelet_address }}\"\n{% if kube_override_hostname|default('') %}\nKUBELET_HOSTNAME=\"--hostname-override={{ kube_override_hostname }}\"\n{% endif %}\n\n{# Base kubelet args #}\n{% set kubelet_args_base -%}\n{# start kubeadm specific settings #}\n--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n--config={{ kube_config_dir }}/kubelet-config.yaml \\\n--kubeconfig={{ kube_config_dir }}/kubelet.conf \\\n{# end kubeadm specific settings #}\n--runtime-cgroups={{ kubelet_runtime_cgroups }} \\\n{% endset %}\n\nKUBELET_ARGS=\"{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}\"\n{% if kubelet_flexvolumes_plugins_dir is defined %}\nKUBELET_VOLUME_PLUGIN=\"--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}\"\n{% endif %}\n{% if kube_network_plugin is defined and kube_network_plugin == \"cloud\" %}\nKUBELET_NETWORK_PLUGIN=\"--hairpin-mode=promiscuous-bridge --network-plugin=kubenet\"\n{% endif %}\n{% if cloud_provider == \"external\" %}\nKUBELET_CLOUDPROVIDER=\"--cloud-provider={{ cloud_provider }}\"\n{% else %}\nKUBELET_CLOUDPROVIDER=\"\"\n{% endif %}\n\nPATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n" }, { "path": "roles/kubernetes/node/templates/kubelet.service.j2", "content": "[Unit]\nDescription=Kubernetes Kubelet Server\nDocumentation=https://github.com/GoogleCloudPlatform/kubernetes\nAfter={{ container_manager }}.service\n{% if container_manager == 'docker' %}\nWants=docker.socket\n{% else %}\nWants={{ container_manager }}.service\n{% endif %}\n{% for kubelet_dependency in kubelet_systemd_wants_dependencies|default([]) %}\n{% if kubelet_dependency|length > 0 %}\nWants={{ kubelet_dependency }}\n{% endif %}\n{% endfor %}\n\n[Service]\nEnvironmentFile=-{{ kube_config_dir }}/kubelet.env\n{% if system_reserved|bool %}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ system_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ system_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ system_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ system_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ system_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ system_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ system_reserved_cgroups_for_service_slice }}\n{% endif %}\n{% if kube_reserved|bool %}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ kube_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ kube_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ kube_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ kube_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ kube_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ kube_reserved_cgroups_for_service_slice }}\nExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ kube_reserved_cgroups_for_service_slice }}\n{% endif %}\nExecStart={{ bin_dir }}/kubelet \\\n\t\t$KUBE_LOGTOSTDERR \\\n\t\t$KUBE_LOG_LEVEL \\\n\t\t$KUBELET_API_SERVER \\\n\t\t$KUBELET_ADDRESS \\\n\t\t$KUBELET_PORT \\\n\t\t$KUBELET_HOSTNAME \\\n\t\t$KUBELET_ARGS \\\n\t\t$DOCKER_SOCKET \\\n\t\t$KUBELET_NETWORK_PLUGIN \\\n\t\t$KUBELET_VOLUME_PLUGIN \\\n\t\t$KUBELET_CLOUDPROVIDER\nRestart=always\nRestartSec=10s\n{% if kubelet_systemd_hardening %}\n# Hardening setup\nIPAddressDeny=any\nIPAddressAllow={{ kubelet_secure_addresses }}\n{% endif %}\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2", "content": "global\n maxconn 4000\n log 127.0.0.1 local0\n\ndefaults\n mode http\n log global\n option httplog\n option dontlognull\n option http-server-close\n option redispatch\n retries 5\n timeout http-request 5m\n timeout queue 5m\n timeout connect 30s\n timeout client {{ loadbalancer_apiserver_keepalive_timeout }}\n timeout server 15m\n timeout http-keep-alive 30s\n timeout check 30s\n maxconn 4000\n\n{% if loadbalancer_apiserver_healthcheck_port is defined -%}\nfrontend healthz\n bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }}\n {% if ipv6_stack -%}\n bind :::{{ loadbalancer_apiserver_healthcheck_port }}\n {% endif -%}\n mode http\n monitor-uri /healthz\n{% endif %}\n\nfrontend kube_api_frontend\n bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}\n {% if ipv6_stack -%}\n bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}\n {% endif -%}\n mode tcp\n option tcplog\n default_backend kube_api_backend\n\nbackend kube_api_backend\n mode tcp\n balance leastconn\n default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100\n option httpchk GET /healthz\n http-check expect status 200\n {% for host in groups['kube_control_plane'] -%}\n server {{ host }} {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} check check-ssl verify none\n {% endfor -%}\n" }, { "path": "roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2", "content": "error_log stderr notice;\n\nworker_processes 2;\nworker_rlimit_nofile 130048;\nworker_shutdown_timeout 10s;\n\nevents {\n multi_accept on;\n use epoll;\n worker_connections 16384;\n}\n\nstream {\n upstream kube_apiserver {\n least_conn;\n {% for host in groups['kube_control_plane'] -%}\n server {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }};\n {% endfor -%}\n }\n\n server {\n listen 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};\n {% if ipv6_stack -%}\n listen [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};\n {% endif -%}\n proxy_pass kube_apiserver;\n proxy_timeout 10m;\n proxy_connect_timeout 1s;\n }\n}\n\nhttp {\n aio threads;\n aio_write on;\n tcp_nopush on;\n tcp_nodelay on;\n\n keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }};\n keepalive_requests 100;\n reset_timedout_connection on;\n server_tokens off;\n autoindex off;\n\n {% if loadbalancer_apiserver_healthcheck_port is defined -%}\n server {\n listen {{ loadbalancer_apiserver_healthcheck_port }};\n {% if ipv6_stack -%}\n listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};\n {% endif -%}\n location /healthz {\n access_log off;\n return 200;\n }\n location /stub_status {\n stub_status on;\n access_log off;\n }\n }\n {% endif %}\n}\n" }, { "path": "roles/kubernetes/node/templates/manifests/haproxy.manifest.j2", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n name: {{ loadbalancer_apiserver_pod_name }}\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n k8s-app: kube-haproxy\n annotations:\n haproxy-cfg-checksum: \"{{ haproxy_conf.checksum }}\"\nspec:\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n nodeSelector:\n kubernetes.io/os: linux\n priorityClassName: system-node-critical\n containers:\n - name: haproxy\n image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n resources:\n requests:\n cpu: {{ loadbalancer_apiserver_cpu_requests }}\n memory: {{ loadbalancer_apiserver_memory_requests }}\n {% if loadbalancer_apiserver_healthcheck_port is defined -%}\n livenessProbe:\n httpGet:\n path: /healthz\n port: {{ loadbalancer_apiserver_healthcheck_port }}\n readinessProbe:\n httpGet:\n path: /healthz\n port: {{ loadbalancer_apiserver_healthcheck_port }}\n {% endif -%}\n volumeMounts:\n - mountPath: /usr/local/etc/haproxy/\n name: etc-haproxy\n readOnly: true\n volumes:\n - name: etc-haproxy\n hostPath:\n path: {{ haproxy_config_dir }}\n" }, { "path": "roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2", "content": "# Inspired by https://github.com/kube-vip/kube-vip/blob/v1.0.3/pkg/kubevip/config_generator.go#L103\napiVersion: v1\nkind: Pod\nmetadata:\n name: kube-vip\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n k8s-app: kube-vip\nspec:\n containers:\n - args:\n - manager\n env:\n - name: vip_arp\n value: {{ kube_vip_arp_enabled | string | to_json }}\n - name: port\n value: {{ kube_apiserver_port | string | to_json }}\n - name: vip_nodename\n value: {{ inventory_hostname }}\n{% if kube_vip_interface %}\n - name: vip_interface\n value: {{ kube_vip_interface | string | to_json }}\n{% endif %}\n{% if kube_vip_services_interface %}\n - name: vip_servicesinterface\n value: {{ kube_vip_services_interface | string | to_json }}\n{% endif %}\n{% if kube_vip_cidr %}\n - name: vip_{{ \"subnet\" if kube_vip_version is version('0.9.0', '>=') else \"cidr\" }}\n value: {{ kube_vip_cidr | string | to_json }}\n{% endif %}\n{% if kube_vip_dns_mode %}\n - name: dns_mode\n value: {{ kube_vip_dns_mode | string | to_json }}\n{% endif %}\n{% if kube_vip_controlplane_enabled %}\n - name: cp_enable\n value: \"true\"\n - name: cp_namespace\n value: kube-system\n - name: vip_ddns\n value: {{ kube_vip_ddns_enabled | string | to_json }}\n - name: cp_detect\n value: {{ kube_vip_cp_detect | string | to_json }}\n{% endif %}\n{% if kube_vip_services_enabled %}\n - name: svc_enable\n value: \"true\"\n{% endif %}\n{% if kube_vip_svc_leasename %}\n - name: svc_leasename\n value: {{ kube_vip_svc_leasename | string | to_json }}\n{% endif %}\n{% if kube_vip_enableServicesElection %}\n - name: svc_election\n value: \"true\"\n{% endif %}\n{% if kube_vip_leader_election_enabled %}\n - name: vip_leaderelection\n value: \"true\"\n - name: vip_leasename\n value: {{ kube_vip_leasename | string | to_json }}\n - name: vip_leaseduration\n value: {{ kube_vip_leaseduration | string | to_json }}\n - name: vip_renewdeadline\n value: {{ kube_vip_renewdeadline | string | to_json }}\n - name: vip_retryperiod\n value: {{ kube_vip_retryperiod | string | to_json }}\n{% endif %}\n{% if kube_vip_enable_node_labeling %}\n - name: enable_node_labeling\n value: {{ kube_vip_enable_node_labeling | string | to_json }}\n{% endif %}\n{% if kube_vip_bgp_enabled %}\n - name: bgp_enable\n value: \"true\"\n - name: bgp_routerid\n value: {{ kube_vip_bgp_routerid | string | to_json }}\n - name: bgp_as\n value: {{ kube_vip_local_as | string | to_json }}\n - name: bgp_peeraddress\n value: {{ kube_vip_bgp_peeraddress | to_json }}\n - name: bgp_peerpass\n value: {{ kube_vip_bgp_peerpass | to_json }}\n - name: bgp_peeras\n value: {{ kube_vip_bgp_peeras | string | to_json }}\n{% set kube_vip_bgp_sourceip_normalized = kube_vip_bgp_sourceip | default('', true) | string | trim %}\n{% if kube_vip_bgp_sourceip_normalized %}\n - name: bgp_sourceip\n value: {{ kube_vip_bgp_sourceip_normalized | to_json }}\n{% endif %}\n{% set kube_vip_bgp_sourceif_normalized = kube_vip_bgp_sourceif | default('', true) | string | trim %}\n{% if kube_vip_bgp_sourceif_normalized %}\n - name: bgp_sourceif\n value: {{ kube_vip_bgp_sourceif_normalized | to_json }}\n{% endif %}\n{% if kube_vip_bgppeers %}\n - name: bgp_peers\n value: {{ kube_vip_bgppeers | join(',') | to_json }}\n{% endif %}\n{% endif %}\n - name: address\n value: {{ kube_vip_address | to_json }}\n{% if kube_vip_lb_enable %}\n - name: lb_enable\n value: \"true\"\n{% endif %}\n{% if kube_vip_lb_fwdmethod %}\n - name: lb_fwdmethod\n value: {{ kube_vip_lb_fwdmethod | string | to_json }}\n{% endif %}\n image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n name: kube-vip\n resources: {}\n{% if kube_vip_lb_fwdmethod == \"masquerade\" %}\n securityContext:\n privileged: true\n{% else %}\n securityContext:\n capabilities:\n add:\n - NET_ADMIN\n - NET_RAW\n drop:\n - ALL\n{% endif %}\n volumeMounts:\n - mountPath: /etc/kubernetes/admin.conf\n name: kubeconfig\n hostAliases:\n - hostnames:\n - kubernetes\n ip: 127.0.0.1\n hostNetwork: true\n volumes:\n - hostPath:\n path: /etc/kubernetes/{{kube_vip_admin_conf}}\n name: kubeconfig\nstatus: {}\n" }, { "path": "roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n name: {{ loadbalancer_apiserver_pod_name }}\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n k8s-app: kube-nginx\n annotations:\n nginx-cfg-checksum: \"{{ nginx_conf.checksum }}\"\nspec:\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n nodeSelector:\n kubernetes.io/os: linux\n priorityClassName: system-node-critical\n containers:\n - name: nginx-proxy\n image: {{ nginx_image_repo }}:{{ nginx_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n resources:\n requests:\n cpu: {{ loadbalancer_apiserver_cpu_requests }}\n memory: {{ loadbalancer_apiserver_memory_requests }}\n {% if loadbalancer_apiserver_healthcheck_port is defined -%}\n livenessProbe:\n httpGet:\n path: /healthz\n port: {{ loadbalancer_apiserver_healthcheck_port }}\n readinessProbe:\n httpGet:\n path: /healthz\n port: {{ loadbalancer_apiserver_healthcheck_port }}\n {% endif -%}\n volumeMounts:\n - mountPath: /etc/nginx\n name: etc-nginx\n readOnly: true\n volumes:\n - name: etc-nginx\n hostPath:\n path: {{ nginx_config_dir }}\n" }, { "path": "roles/kubernetes/node/vars/fedora.yml", "content": "---\nkube_resolv_conf: \"/run/systemd/resolve/resolv.conf\"\n" }, { "path": "roles/kubernetes/node/vars/ubuntu-18.yml", "content": "---\nkube_resolv_conf: \"/run/systemd/resolve/resolv.conf\"\n" }, { "path": "roles/kubernetes/node/vars/ubuntu-20.yml", "content": "---\nkube_resolv_conf: \"/run/systemd/resolve/resolv.conf\"\n" }, { "path": "roles/kubernetes/node/vars/ubuntu-22.yml", "content": "---\nkube_resolv_conf: \"/run/systemd/resolve/resolv.conf\"\n" }, { "path": "roles/kubernetes/node/vars/ubuntu-24.yml", "content": "---\nkube_resolv_conf: \"/run/systemd/resolve/resolv.conf\"\n" }, { "path": "roles/kubernetes/node-label/tasks/main.yml", "content": "---\n- name: Kubernetes Apps | Wait for kube-apiserver\n uri:\n url: \"{{ kube_apiserver_endpoint }}/healthz\"\n validate_certs: false\n client_cert: \"{{ kube_apiserver_client_cert }}\"\n client_key: \"{{ kube_apiserver_client_key }}\"\n register: result\n until: result.status == 200\n retries: 10\n delay: 6\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Set role node label to empty list\n set_fact:\n role_node_labels: []\n\n- name: Node label for nvidia GPU nodes\n set_fact:\n role_node_labels: \"{{ role_node_labels + ['nvidia.com/gpu=true'] }}\"\n when:\n - nvidia_gpu_nodes is defined\n - nvidia_accelerator_enabled | bool\n - inventory_hostname in nvidia_gpu_nodes\n\n- name: Set inventory node label to empty list\n set_fact:\n inventory_node_labels: []\n\n- name: Populate inventory node label\n set_fact:\n inventory_node_labels: \"{{ inventory_node_labels + ['%s=%s' | format(item.key, item.value)] }}\"\n loop: \"{{ node_labels | d({}) | dict2items }}\"\n when:\n - node_labels is defined\n - node_labels is mapping\n\n- debug: # noqa name[missing]\n var: role_node_labels\n- debug: # noqa name[missing]\n var: inventory_node_labels\n\n- name: Set label to node\n command: >-\n {{ kubectl }} label node {% if kube_override_hostname %}{{ kube_override_hostname }}{% else %}{{ inventory_hostname }}{% endif %} {{ item }} --overwrite=true\n loop: \"{{ role_node_labels + inventory_node_labels }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n changed_when: false\n...\n" }, { "path": "roles/kubernetes/node-taint/defaults/main.yml", "content": "---\nnode_taints: []\n" }, { "path": "roles/kubernetes/node-taint/tasks/main.yml", "content": "---\n- name: Set role and inventory node taint to empty list\n set_fact:\n role_node_taints: []\n inventory_node_taints: []\n\n- name: Node taint for nvidia GPU nodes\n set_fact:\n role_node_taints: \"{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}\"\n when:\n - nvidia_gpu_nodes is defined\n - nvidia_accelerator_enabled | bool\n - inventory_hostname in nvidia_gpu_nodes\n\n- name: Populate inventory node taint\n set_fact:\n inventory_node_taints: \"{{ inventory_node_taints + node_taints }}\"\n when:\n - node_taints is defined\n - node_taints is not string\n - node_taints is not mapping\n - node_taints is iterable\n\n- debug: # noqa name[missing]\n var: role_node_taints\n- debug: # noqa name[missing]\n var: inventory_node_taints\n\n- name: Set taint to node\n command: >-\n {{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n changed_when: false\n when:\n - (role_node_taints + inventory_node_taints) | length > 0\n" }, { "path": "roles/kubernetes/preinstall/defaults/main.yml", "content": "---\n# Set to true to allow pre-checks to fail and continue deployment\nignore_assert_errors: false\n# Set to false to disable the backup parameter, set to true to accumulate backups of config files.\nleave_etc_backup_files: true\nnameservers: []\ncloud_resolver: []\ndisable_host_nameservers: false\n# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf\ndns_late: false\n\n# Set to true if your network does not support IPv6\n# This may be necessary for pulling Docker images from\n# GCE docker repository\ndisable_ipv6_dns: false\n\n# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).\nremove_default_searchdomains: false\n\nkube_owner: kube\nkube_cert_group: kube-cert\nkube_config_dir: /etc/kubernetes\nkube_cert_dir: \"{{ kube_config_dir }}/ssl\"\nkube_cert_compat_dir: /etc/kubernetes/pki\nkubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec\n\n# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content\n# for hostnet pods and infra needs\nresolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf\n\n# sysctl_file_path to add sysctl conf to\nsysctl_file_path: \"/etc/sysctl.d/99-sysctl.conf\"\n\n# Minimal memory requirement in MB for safety checks\nminimal_node_memory_mb: 1024\nminimal_master_memory_mb: 1500\n\n## NTP Settings\n\n# Manage the NTP configuration file.\nntp_manage_config: false\n# Specify the NTP servers\n# Only takes effect when ntp_manage_config is true.\nntp_servers:\n - \"0.pool.ntp.org iburst\"\n - \"1.pool.ntp.org iburst\"\n - \"2.pool.ntp.org iburst\"\n - \"3.pool.ntp.org iburst\"\n# Restrict NTP access to these hosts.\n# Only takes effect when ntp_manage_config is true.\nntp_restrict:\n - \"127.0.0.1\"\n - \"::1\"\n# Specify whether to filter interfaces\nntp_filter_interface: false\n# Specify the interfaces\n# Only takes effect when ntp_filter_interface is true\n# ntp_interfaces:\n# - ignore wildcard\n# - listen xxx\n# The NTP driftfile path\n# Only takes effect when ntp_manage_config is true.\n# Default value is `/var/lib/ntp/ntp.drift`, for ntpsec use '/var/lib/ntpsec/ntp.drift'\nntp_driftfile: >-\n {% if ntp_package == \"ntpsec\" -%}\n /var/lib/ntpsec/ntp.drift\n {%- else -%}\n /var/lib/ntp/ntp.drift\n {%- endif -%}\n# Only takes effect when ntp_manage_config is true.\nntp_tinker_panic: false\n\n# Force sync time immediately after the ntp installed, which is useful in a newly installed system.\nntp_force_sync_immediately: false\n\n# Set the timezone for your server. eg: \"Etc/UTC\",\"Etc/GMT-8\". If not set, the timezone will not change.\nntp_timezone: \"\"\n\n# Currently known os distributions\nsupported_os_distributions:\n - 'RedHat'\n - 'CentOS'\n - 'Fedora'\n - 'Ubuntu'\n - 'Debian'\n - 'Flatcar'\n - 'Flatcar Container Linux by Kinvolk'\n - 'Suse'\n - 'openSUSE Leap'\n - 'openSUSE Tumbleweed'\n - 'ClearLinux'\n - 'OracleLinux'\n - 'AlmaLinux'\n - 'Rocky'\n - 'Amazon'\n - 'Kylin Linux Advanced Server'\n - 'UnionTech'\n - 'UniontechOS'\n - 'openEuler'\n\n# Extending some distributions into the redhat os family\nredhat_os_family_extensions:\n - \"UnionTech\"\n - \"UniontechOS\"\n\n# Sets DNSStubListener=no, useful if you get \"0.0.0.0:53: bind: address already in use\"\nsystemd_resolved_disable_stub_listener: \"{{ ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] }}\"\n\n# Used to disable File Access Policy Daemon service.\n# If service is enabled, the CNI plugin installation will fail\ndisable_fapolicyd: true\n" }, { "path": "roles/kubernetes/preinstall/files/dhclient_nodnsupdate", "content": "#!/bin/sh\nmake_resolv_conf() {\n :\n}\n" }, { "path": "roles/kubernetes/preinstall/handlers/main.yml", "content": "---\n- name: Preinstall | apply resolvconf cloud-init\n command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n listen: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk\n\n- name: Preinstall | reload NetworkManager\n service:\n name: NetworkManager.service\n state: restarted\n listen: Preinstall | update resolvconf for networkmanager\n\n- name: Preinstall | reload kubelet\n service:\n name: kubelet\n state: restarted\n notify:\n - Preinstall | kube-controller configured\n - Preinstall | kube-apiserver configured\n - Preinstall | restart kube-controller-manager docker\n - Preinstall | restart kube-controller-manager crio/containerd\n - Preinstall | restart kube-apiserver docker\n - Preinstall | restart kube-apiserver crio/containerd\n when: not dns_early | bool\n listen:\n - Preinstall | propagate resolvconf to k8s components\n - Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk\n - Preinstall | update resolvconf for networkmanager\n\n# FIXME(mattymo): Also restart for kubeadm mode\n- name: Preinstall | kube-apiserver configured\n stat:\n path: \"{{ kube_manifest_dir }}/kube-apiserver.yaml\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kube_apiserver_set\n when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'\n listen: Preinstall | propagate resolvconf to k8s components\n\n# FIXME(mattymo): Also restart for kubeadm mode\n- name: Preinstall | kube-controller configured\n stat:\n path: \"{{ kube_manifest_dir }}/kube-controller-manager.yaml\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kube_controller_set\n when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'\n listen: Preinstall | propagate resolvconf to k8s components\n\n- name: Preinstall | restart kube-controller-manager docker\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f\"\n args:\n executable: /bin/bash\n when:\n - container_manager == \"docker\"\n - ('kube_control_plane' in group_names)\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - kube_controller_set.stat.exists\n listen: Preinstall | propagate resolvconf to k8s components\n\n- name: Preinstall | restart kube-controller-manager crio/containerd\n shell: \"set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'\"\n args:\n executable: /bin/bash\n register: preinstall_restart_controller_manager\n retries: 10\n delay: 1\n until: preinstall_restart_controller_manager.rc == 0\n when:\n - container_manager in ['crio', 'containerd']\n - ('kube_control_plane' in group_names)\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - kube_controller_set.stat.exists\n listen: Preinstall | propagate resolvconf to k8s components\n\n- name: Preinstall | restart kube-apiserver docker\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f\"\n args:\n executable: /bin/bash\n when:\n - container_manager == \"docker\"\n - ('kube_control_plane' in group_names)\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - kube_apiserver_set.stat.exists\n listen: Preinstall | propagate resolvconf to k8s components\n\n- name: Preinstall | restart kube-apiserver crio/containerd\n shell: \"set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'\"\n args:\n executable: /bin/bash\n register: preinstall_restart_apiserver\n retries: 10\n until: preinstall_restart_apiserver.rc == 0\n delay: 1\n when:\n - container_manager in ['crio', 'containerd']\n - ('kube_control_plane' in group_names)\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - kube_apiserver_set.stat.exists\n listen: Preinstall | propagate resolvconf to k8s components\n\n# When running this as the last phase ensure we wait for kube-apiserver to come up\n- name: Preinstall | wait for the apiserver to be running\n uri:\n url: \"{{ kube_apiserver_endpoint }}/healthz\"\n validate_certs: false\n register: result\n until: result.status == 200\n retries: 60\n delay: 1\n when:\n - dns_late\n - ('kube_control_plane' in group_names)\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"] and not is_fedora_coreos\n listen: Preinstall | propagate resolvconf to k8s components\n\n- name: Preinstall | Restart systemd-resolved\n service:\n name: systemd-resolved\n state: restarted\n\n- name: Preinstall | restart ntp\n service:\n name: \"{{ ntp_service_name }}\"\n state: restarted\n when: ntp_enabled\n" }, { "path": "roles/kubernetes/preinstall/meta/main.yml", "content": "---\ndependencies:\n - role: adduser\n user: \"{{ addusers.kube }}\"\n when:\n - not is_fedora_coreos\n tags:\n - kubelet\n" }, { "path": "roles/kubernetes/preinstall/tasks/0010-swapoff.yml", "content": "---\n- name: Check if /etc/fstab exists\n stat:\n path: \"/etc/fstab\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: fstab_file\n\n- name: Remove swapfile from /etc/fstab\n ansible.posix.mount:\n name: \"{{ item }}\"\n fstype: swap\n state: absent\n loop:\n - swap\n - none\n when: fstab_file.stat.exists\n\n- name: Mask swap.target (persist swapoff)\n ansible.builtin.systemd_service:\n name: swap.target\n masked: true\n\n- name: Disable swap\n command: /sbin/swapoff -a\n" }, { "path": "roles/kubernetes/preinstall/tasks/0020-set_facts.yml", "content": "---\n- name: Set os_family fact for other redhat-based operating systems\n set_fact:\n ansible_os_family: \"RedHat\"\n ansible_distribution_major_version: \"8\"\n when: ansible_distribution in redhat_os_family_extensions\n tags:\n - facts\n\n- name: Check resolvconf\n command: which resolvconf\n register: resolvconf\n failed_when: false\n changed_when: false\n check_mode: false\n\n- name: Check existence of /etc/resolvconf/resolv.conf.d\n stat:\n path: /etc/resolvconf/resolv.conf.d\n get_attributes: false\n get_checksum: false\n get_mime: false\n failed_when: false\n register: resolvconfd_path\n\n- name: Check status of /etc/resolv.conf\n stat:\n path: /etc/resolv.conf\n follow: false\n get_attributes: false\n get_checksum: false\n get_mime: false\n failed_when: false\n register: resolvconf_stat\n\n # Used in vars/\n- name: Fetch resolv.conf\n when: resolvconf_stat.stat.exists\n slurp:\n src: /etc/resolv.conf\n register: resolvconf_slurp\n\n- name: NetworkManager | Check if host has NetworkManager\n # noqa command-instead-of-module - Should we use service_facts for this?\n command: systemctl is-active --quiet NetworkManager.service\n register: networkmanager_enabled\n failed_when: false\n changed_when: false\n check_mode: false\n\n- name: Check systemd-resolved\n # noqa command-instead-of-module - Should we use service_facts for this?\n command: systemctl is-active systemd-resolved\n register: systemd_resolved_enabled\n failed_when: false\n changed_when: false\n check_mode: false\n\n- name: Set default dns if remove_default_searchdomains is false\n set_fact:\n default_searchdomains: [\"default.svc.{{ dns_domain }}\", \"svc.{{ dns_domain }}\"]\n when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)\n\n- name: Set dns facts\n set_fact:\n resolvconf: >-\n {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}\n\n- name: Check if kubelet is configured\n stat:\n path: \"{{ kube_config_dir }}/kubelet.env\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kubelet_configured\n changed_when: false\n\n- name: Check if early DNS configuration stage\n set_fact:\n dns_early: \"{{ not kubelet_configured.stat.exists }}\"\n\n- name: Target resolv.conf files\n set_fact:\n resolvconffile: /etc/resolv.conf\n base: >-\n {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}\n head: >-\n {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}\n when: not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"] and not is_fedora_coreos\n\n- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS)\n set_fact:\n resolvconffile: /tmp/resolveconf_cloud_init_conf\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"] or is_fedora_coreos\n\n- name: Check if /etc/dhclient.conf exists\n stat:\n path: /etc/dhclient.conf\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: dhclient_stat\n\n- name: Target dhclient conf file for /etc/dhclient.conf\n set_fact:\n dhclientconffile: /etc/dhclient.conf\n when: dhclient_stat.stat.exists\n\n- name: Check if /etc/dhcp/dhclient.conf exists\n stat:\n path: /etc/dhcp/dhclient.conf\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: dhcp_dhclient_stat\n\n- name: Target dhclient conf file for /etc/dhcp/dhclient.conf\n set_fact:\n dhclientconffile: /etc/dhcp/dhclient.conf\n when: dhcp_dhclient_stat.stat.exists\n\n- name: Target dhclient hook file for Red Hat family\n set_fact:\n dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh\n when: ansible_os_family == \"RedHat\"\n\n- name: Target dhclient hook file for Debian family\n set_fact:\n dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate\n when: ansible_os_family == \"Debian\"\n\n- name: Set etcd vars if using kubeadm mode\n set_fact:\n etcd_cert_dir: \"{{ kube_cert_dir }}\"\n kube_etcd_cacert_file: \"etcd/ca.crt\"\n kube_etcd_cert_file: \"apiserver-etcd-client.crt\"\n kube_etcd_key_file: \"apiserver-etcd-client.key\"\n when:\n - etcd_deployment_type == \"kubeadm\"\n\n- name: Check /usr readonly\n stat:\n path: \"/usr\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: usr\n\n- name: Set alternate flexvolume path\n set_fact:\n kubelet_flexvolumes_plugins_dir: /var/lib/kubelet/volumeplugins\n when: not usr.stat.writeable\n" }, { "path": "roles/kubernetes/preinstall/tasks/0040-verify-settings.yml", "content": "---\n- name: Stop if any host not in '--limit' does not have a fact cache\n vars:\n uncached_hosts: \"{{ hostvars | dict2items | selectattr('value.ansible_default_ipv6', 'undefined') | selectattr('value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}\"\n excluded_hosts: \"{{ groups['k8s_cluster'] | difference(query('inventory_hostnames', ansible_limit)) }}\"\n assert:\n that: uncached_hosts | intersect(excluded_hosts) == []\n fail_msg: |\n Kubespray does not support '--limit' without a populated facts cache for the excluded hosts.\n Please run the facts.yml playbook first without '--limit'.\n The following excluded hosts are not cached: {{ uncached_hosts | intersect(excluded_hosts) }}\n run_once: true\n when:\n - ansible_limit is defined\n - not ignore_assert_errors\n- name: Stop if non systemd OS type\n assert:\n that: ansible_service_mgr == \"systemd\"\n when: not ignore_assert_errors\n\n- name: Stop if the os does not support\n assert:\n that: (allow_unsupported_distribution_setup | default(false)) or ansible_distribution in supported_os_distributions\n msg: \"{{ ansible_distribution }} is not a known OS\"\n when: not ignore_assert_errors\n\n- name: Stop if memory is too small for control plane nodes\n assert:\n that: ansible_memtotal_mb >= minimal_master_memory_mb\n when:\n - not ignore_assert_errors\n - ('kube_control_plane' in group_names)\n\n- name: Stop if memory is too small for nodes\n assert:\n that: ansible_memtotal_mb >= minimal_node_memory_mb\n when:\n - not ignore_assert_errors\n - ('kube_node' in group_names)\n\n# This command will fail if cgroups are not enabled on the node.\n# For reference: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version\n- name: Stop if cgroups are not enabled on nodes\n command: stat -fc %T /sys/fs/cgroup/\n changed_when: false\n when: not ignore_assert_errors\n\n- name: Stop if ip var does not match local ips\n assert:\n that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses)\n msg: \"IPv4: '{{ ansible_all_ipv4_addresses }}' and IPv6: '{{ ansible_all_ipv6_addresses }}' do not contain '{{ ip }}'\"\n when:\n - not ignore_assert_errors\n - ip is defined\n\n- name: Stop if access_ip is not pingable\n command: ping -c1 {{ main_access_ip }}\n when:\n - main_access_ip is defined\n - not ignore_assert_errors\n - ping_access_ip\n changed_when: false\n\n- name: Stop if kernel version is too low for cilium\n assert:\n that: ansible_kernel.split('-')[0] is version('4.9.17', '>=')\n when:\n - kube_network_plugin == 'cilium' or cilium_deploy_additionally\n - not ignore_assert_errors\n\n- name: Stop if kernel version is too low for nftables\n assert:\n that: ansible_kernel.split('-')[0] is version('5.13', '>=')\n when:\n - kube_proxy_mode == 'nftables'\n - not ignore_assert_errors\n\n- name: Stop if bad hostname\n assert:\n that: inventory_hostname is match(\"[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\")\n msg: \"Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character\"\n when: not ignore_assert_errors\n\n- name: Stop if /etc/resolv.conf has no configured nameservers\n assert:\n that: configured_nameservers | length>0\n fail_msg: \"nameserver should not be empty in /etc/resolv.conf\"\n when:\n - upstream_dns_servers | length == 0\n - not disable_host_nameservers\n - dns_mode in ['coredns', 'coredns_dual']\n\n- name: Stop if download_localhost is enabled for Flatcar Container Linux\n assert:\n that: ansible_os_family not in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n msg: \"download_run_once not supported for Flatcar Container Linux\"\n when: download_run_once or download_force_cache\n" }, { "path": "roles/kubernetes/preinstall/tasks/0050-create_directories.yml", "content": "---\n- name: Create kubernetes directories\n file:\n path: \"{{ item }}\"\n state: directory\n owner: \"{{ kube_owner }}\"\n mode: \"0755\"\n when: ('k8s_cluster' in group_names)\n become: true\n tags:\n - kubelet\n - kube-controller-manager\n - kube-apiserver\n - bootstrap_os\n - apps\n - network\n - control-plane\n - node\n with_items:\n - \"{{ kube_config_dir }}\"\n - \"{{ kube_manifest_dir }}\"\n - \"{{ kube_script_dir }}\"\n - \"{{ kubelet_flexvolumes_plugins_dir }}\"\n\n- name: Create other directories of root owner\n file:\n path: \"{{ item }}\"\n state: directory\n owner: root\n mode: \"0755\"\n when: ('k8s_cluster' in group_names)\n become: true\n tags:\n - kubelet\n - kube-controller-manager\n - kube-apiserver\n - bootstrap_os\n - apps\n - network\n - control-plane\n - node\n with_items:\n - \"{{ kube_cert_dir }}\"\n - \"{{ bin_dir }}\"\n\n- name: Check if kubernetes kubeadm compat cert dir exists\n stat:\n path: \"{{ kube_cert_compat_dir }}\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kube_cert_compat_dir_check\n when:\n - ('k8s_cluster' in group_names)\n - kube_cert_dir != kube_cert_compat_dir\n\n- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498)\n file:\n src: \"{{ kube_cert_dir }}\"\n dest: \"{{ kube_cert_compat_dir }}\"\n state: link\n mode: \"0755\"\n when:\n - ('k8s_cluster' in group_names)\n - kube_cert_dir != kube_cert_compat_dir\n - not kube_cert_compat_dir_check.stat.exists\n\n- name: Create cni directories\n file:\n path: \"{{ item }}\"\n state: directory\n owner: \"{{ kube_owner }}\"\n mode: \"0755\"\n with_items:\n - \"/etc/cni/net.d\"\n - \"/opt/cni/bin\"\n when:\n - kube_network_plugin in [\"calico\", \"flannel\", \"cilium\", \"kube-ovn\", \"kube-router\", \"macvlan\"]\n - ('k8s_cluster' in group_names)\n tags:\n - network\n - cilium\n - calico\n - kube-ovn\n - kube-router\n - bootstrap_os\n\n- name: Create calico cni directories\n file:\n path: \"{{ item }}\"\n state: directory\n owner: \"{{ kube_owner }}\"\n mode: \"0755\"\n with_items:\n - \"/var/lib/calico\"\n when:\n - kube_network_plugin == \"calico\"\n - ('k8s_cluster' in group_names)\n tags:\n - network\n - calico\n - bootstrap_os\n\n- name: Create local volume provisioner directories\n file:\n path: \"{{ local_volume_provisioner_storage_classes[item].host_dir }}\"\n state: directory\n owner: root\n group: root\n mode: \"{{ local_volume_provisioner_directory_mode }}\"\n with_items: \"{{ local_volume_provisioner_storage_classes.keys() | list }}\"\n when:\n - ('k8s_cluster' in group_names)\n - local_volume_provisioner_enabled\n tags:\n - persistent_volumes\n" }, { "path": "roles/kubernetes/preinstall/tasks/0060-resolvconf.yml", "content": "---\n- name: Create temporary resolveconf cloud init file\n command: cp -f /etc/resolv.conf \"{{ resolvconffile }}\"\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n\n- name: Add domain/search/nameservers/options to resolv.conf\n blockinfile:\n path: \"{{ resolvconffile }}\"\n block: |-\n domain {{ dns_domain }}\n search {{ (default_searchdomains + searchdomains) | join(' ') }}\n {% for item in nameserverentries %}\n nameserver {{ item }}\n {% endfor %}\n options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }}\n state: present\n insertbefore: BOF\n create: true\n backup: \"{{ not resolvconf_stat.stat.islnk }}\"\n marker: \"# Ansible entries {mark}\"\n mode: \"0644\"\n notify: Preinstall | propagate resolvconf to k8s components\n\n- name: Remove search/domain/nameserver options before block\n replace:\n path: \"{{ item[0] }}\"\n regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)'\n backup: \"{{ not resolvconf_stat.stat.islnk }}\"\n with_nested:\n - \"{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}\"\n - [ 'search\\s', 'nameserver\\s', 'domain\\s', 'options\\s' ]\n notify: Preinstall | propagate resolvconf to k8s components\n\n- name: Remove search/domain/nameserver options after block\n replace:\n path: \"{{ item[0] }}\"\n regexp: '(# Ansible entries END\\n(?:(?!^{{ item[1] }}).*\\n)*)(?:^{{ item[1] }}.*\\n?)+'\n replace: '\\1'\n backup: \"{{ not resolvconf_stat.stat.islnk }}\"\n with_nested:\n - \"{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}\"\n - [ 'search\\s', 'nameserver\\s', 'domain\\s', 'options\\s' ]\n notify: Preinstall | propagate resolvconf to k8s components\n\n- name: Get temporary resolveconf cloud init file content\n command: cat {{ resolvconffile }}\n register: cloud_config\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n\n- name: Persist resolvconf cloud init file\n template:\n dest: \"{{ resolveconf_cloud_init_conf }}\"\n src: resolvconf.j2\n owner: root\n mode: \"0644\"\n notify: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n" }, { "path": "roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml", "content": "---\n- name: Create systemd-resolved drop-in directory\n file:\n state: directory\n name: /etc/systemd/resolved.conf.d/\n mode: \"0755\"\n\n- name: Write Kubespray DNS settings to systemd-resolved\n template:\n src: resolved.conf.j2\n dest: /etc/systemd/resolved.conf.d/kubespray.conf\n owner: root\n group: root\n mode: \"0644\"\n notify: Preinstall | Restart systemd-resolved\n" }, { "path": "roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml", "content": "---\n- name: NetworkManager | Ensure NetworkManager conf.d dir\n file:\n path: \"/etc/NetworkManager/conf.d\"\n state: directory\n recurse: true\n\n- name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico)\n copy:\n content: |\n [keyfile]\n unmanaged-devices+=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico\n dest: /etc/NetworkManager/conf.d/calico.conf\n mode: \"0644\"\n when:\n - kube_network_plugin == \"calico\"\n notify: Preinstall | reload NetworkManager\n\n# TODO: add other network_plugin interfaces\n\n- name: NetworkManager | Prevent NetworkManager from managing K8S interfaces (kube-ipvs0/nodelocaldns)\n copy:\n content: |\n [keyfile]\n unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns\n dest: /etc/NetworkManager/conf.d/k8s.conf\n mode: \"0644\"\n notify: Preinstall | reload NetworkManager\n" }, { "path": "roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml", "content": "---\n- name: NetworkManager | Add nameservers to NM configuration\n community.general.ini_file:\n path: /etc/NetworkManager/conf.d/dns.conf\n section: global-dns-domain-*\n option: servers\n value: \"{{ nameserverentries | join(',') }}\"\n mode: '0600'\n backup: \"{{ leave_etc_backup_files }}\"\n when:\n - ('127.0.0.53' not in nameserverentries\n or systemd_resolved_enabled.rc != 0)\n notify: Preinstall | update resolvconf for networkmanager\n\n- name: Set default dns if remove_default_searchdomains is false\n set_fact:\n default_searchdomains: [\"default.svc.{{ dns_domain }}\", \"svc.{{ dns_domain }}\"]\n when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0)\n\n- name: NetworkManager | Add DNS search to NM configuration\n community.general.ini_file:\n path: /etc/NetworkManager/conf.d/dns.conf\n section: global-dns\n option: searches\n value: \"{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}\"\n mode: '0600'\n backup: \"{{ leave_etc_backup_files }}\"\n notify: Preinstall | update resolvconf for networkmanager\n\n- name: NetworkManager | Add DNS options to NM configuration\n community.general.ini_file:\n path: /etc/NetworkManager/conf.d/dns.conf\n section: global-dns\n option: options\n value: \"ndots:{{ ndots }},timeout:{{ dns_timeout | default('2') }},attempts:{{ dns_attempts | default('2') }}\"\n mode: '0600'\n backup: \"{{ leave_etc_backup_files }}\"\n notify: Preinstall | update resolvconf for networkmanager\n" }, { "path": "roles/kubernetes/preinstall/tasks/0080-system-configurations.yml", "content": "---\n# Todo : selinux configuration\n- name: Confirm selinux deployed\n stat:\n path: /etc/selinux/config\n get_attributes: false\n get_checksum: false\n get_mime: false\n when:\n - ansible_os_family == \"RedHat\"\n - \"'Amazon' not in ansible_distribution\"\n register: slc\n\n- name: Set selinux policy\n ansible.posix.selinux:\n policy: targeted\n state: \"{{ preinstall_selinux_state }}\"\n when:\n - ansible_os_family == \"RedHat\"\n - \"'Amazon' not in ansible_distribution\"\n - slc.stat.exists\n tags:\n - bootstrap_os\n\n- name: Disable IPv6 DNS lookup\n lineinfile:\n dest: /etc/gai.conf\n line: \"precedence ::ffff:0:0/96 100\"\n state: present\n create: true\n backup: \"{{ leave_etc_backup_files }}\"\n mode: \"0644\"\n when:\n - disable_ipv6_dns\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n tags:\n - bootstrap_os\n\n- name: Clean previously used sysctl file locations\n file:\n path: \"/etc/sysctl.d/{{ item }}\"\n state: absent\n with_items:\n - ipv4-ip_forward.conf\n - bridge-nf-call.conf\n\n- name: Stat sysctl file configuration\n stat:\n path: \"{{ sysctl_file_path }}\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: sysctl_file_stat\n tags:\n - bootstrap_os\n\n- name: Change sysctl file path to link source if linked\n set_fact:\n sysctl_file_path: \"{{ sysctl_file_stat.stat.lnk_source }}\"\n when:\n - sysctl_file_stat.stat.islnk is defined\n - sysctl_file_stat.stat.islnk\n tags:\n - bootstrap_os\n\n- name: Make sure sysctl file path folder exists\n file:\n name: \"{{ sysctl_file_path | dirname }}\"\n state: directory\n mode: \"0755\"\n\n- name: Enable ip forwarding\n ansible.posix.sysctl:\n sysctl_file: \"{{ sysctl_file_path }}\"\n name: net.ipv4.ip_forward\n value: \"1\"\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n when: ipv4_stack | bool\n\n- name: Enable ipv6 forwarding\n ansible.posix.sysctl:\n sysctl_file: \"{{ sysctl_file_path }}\"\n name: net.ipv6.conf.all.forwarding\n value: \"1\"\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n when: ipv6_stack | bool\n\n- name: Check if we need to set fs.may_detach_mounts\n stat:\n path: /proc/sys/fs/may_detach_mounts\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: fs_may_detach_mounts\n ignore_errors: true # noqa ignore-errors\n\n- name: Set fs.may_detach_mounts if needed\n ansible.posix.sysctl:\n sysctl_file: \"{{ sysctl_file_path }}\"\n name: fs.may_detach_mounts\n value: 1\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n when: fs_may_detach_mounts.stat.exists | d(false)\n\n- name: Ensure kubelet expected parameters are set\n ansible.posix.sysctl:\n sysctl_file: \"{{ sysctl_file_path }}\"\n name: \"{{ item.name }}\"\n value: \"{{ item.value }}\"\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n with_items:\n - { name: kernel.keys.root_maxbytes, value: 25000000 }\n - { name: kernel.keys.root_maxkeys, value: 1000000 }\n - { name: kernel.panic, value: 10 }\n - { name: kernel.panic_on_oops, value: 1 }\n - { name: vm.overcommit_memory, value: 1 }\n - { name: vm.panic_on_oom, value: 0 }\n when: kubelet_protect_kernel_defaults | bool\n\n- name: Check dummy module\n community.general.modprobe:\n name: dummy\n state: present\n params: 'numdummies=0'\n when: enable_nodelocaldns\n\n- name: Set additional sysctl variables\n ansible.posix.sysctl:\n sysctl_file: \"{{ sysctl_file_path }}\"\n name: \"{{ item.name }}\"\n value: \"{{ item.value }}\"\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n with_items: \"{{ additional_sysctl }}\"\n\n- name: Disable fapolicyd service\n failed_when: false\n systemd_service:\n name: fapolicyd\n state: stopped\n enabled: false\n when: disable_fapolicyd\n" }, { "path": "roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml", "content": "---\n\n- name: Disable systemd-timesyncd\n service:\n name: systemd-timesyncd.service\n enabled: false\n state: stopped\n failed_when: false\n\n- name: Set fact NTP settings\n set_fact:\n # noqa: jinja[spacing]\n ntp_config_file: >-\n {% if ntp_package == \"ntp\" -%}\n /etc/ntp.conf\n {%- elif ntp_package == \"ntpsec\" -%}\n /etc/ntpsec/ntp.conf\n {%- elif ansible_os_family in ['RedHat', 'Suse'] -%}\n /etc/chrony.conf\n {%- else -%}\n /etc/chrony/chrony.conf\n {%- endif -%}\n # noqa: jinja[spacing]\n ntp_service_name: >-\n {% if ntp_package == \"chrony\" -%}\n chronyd\n {%- elif ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\", \"RedHat\", \"Suse\"] -%}\n ntpd\n {%- else -%}\n ntp\n {%- endif %}\n\n- name: Generate NTP configuration file.\n template:\n src: \"{{ ntp_config_file | basename }}.j2\"\n dest: \"{{ ntp_config_file }}\"\n mode: \"0644\"\n notify: Preinstall | restart ntp\n when:\n - ntp_manage_config\n\n- name: Stop the NTP Deamon For Sync Immediately # `ntpd -gq`,`chronyd -q` requires the ntp daemon stop\n service:\n name: \"{{ ntp_service_name }}\"\n state: stopped\n when:\n - ntp_force_sync_immediately\n\n- name: Force Sync NTP Immediately\n # noqa: jinja[spacing]\n command: >-\n timeout -k 60s 60s\n {% if ntp_package == \"chrony\" -%}\n chronyd -q\n {%- else -%}\n ntpd -gq\n {%- endif -%}\n when:\n - ntp_force_sync_immediately\n\n- name: Ensure NTP service is started and enabled\n service:\n name: \"{{ ntp_service_name }}\"\n state: started\n enabled: true\n\n- name: Ensure tzdata package\n package:\n name:\n - tzdata\n state: present\n when:\n - ntp_timezone\n - not is_fedora_coreos\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n\n- name: Gather selinux facts\n ansible.builtin.setup:\n gather_subset: selinux\n when:\n - ntp_timezone\n - ansible_os_family == \"RedHat\"\n\n- name: Put SELinux in permissive mode, logging actions that would be blocked.\n ansible.posix.selinux:\n policy: targeted\n state: permissive\n when:\n - ntp_timezone\n - ansible_os_family == \"RedHat\"\n - ansible_facts.selinux.status == 'enabled'\n - ansible_facts.selinux.mode == 'enforcing'\n\n- name: Set ntp_timezone\n community.general.timezone:\n name: \"{{ ntp_timezone }}\"\n when:\n - ntp_timezone\n\n- name: Re-enable SELinux\n ansible.posix.selinux:\n policy: targeted\n state: \"{{ preinstall_selinux_state }}\"\n when:\n - ntp_timezone\n - ansible_os_family == \"RedHat\"\n - ansible_facts.selinux.status == 'enabled'\n" }, { "path": "roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml", "content": "---\n- name: Configure dhclient to supersede search/domain/nameservers\n blockinfile:\n # 1 is the 2nd item of a tuple in items()\n block: |-\n {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%}\n {% if key == \"domain-name-servers\" -%}\n supersede {{ key }} {{ val | join(',') }};\n {% else -%}\n supersede {{ key }} \"{{ val | join('\",\"') }}\";\n {% endif -%}\n {% endfor %}\n path: \"{{ dhclientconffile }}\"\n create: true\n state: present\n insertbefore: BOF\n backup: \"{{ leave_etc_backup_files }}\"\n marker: \"# Ansible entries {mark}\"\n mode: \"0644\"\n notify: Preinstall | propagate resolvconf to k8s components\n\n- name: Configure dhclient hooks for resolv.conf (non-RH)\n template:\n src: dhclient_dnsupdate.sh.j2\n dest: \"{{ dhclienthookfile }}\"\n owner: root\n mode: \"0755\"\n notify: Preinstall | propagate resolvconf to k8s components\n when: ansible_os_family not in [ \"RedHat\", \"Suse\" ]\n\n- name: Configure dhclient hooks for resolv.conf (RH-only)\n template:\n src: dhclient_dnsupdate_rh.sh.j2\n dest: \"{{ dhclienthookfile }}\"\n owner: root\n mode: \"0755\"\n notify: Preinstall | propagate resolvconf to k8s components\n when: ansible_os_family == \"RedHat\"\n" }, { "path": "roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml", "content": "---\n\n# These tasks will undo changes done by kubespray in the past if needed (e.g. when upgrading from kubespray 2.0.x\n# or when changing resolvconf_mode)\n\n- name: Remove kubespray specific config from dhclient config\n blockinfile:\n path: \"{{ dhclientconffile }}\"\n state: absent\n backup: \"{{ leave_etc_backup_files }}\"\n marker: \"# Ansible entries {mark}\"\n notify: Preinstall | propagate resolvconf to k8s components\n\n- name: Remove kubespray specific dhclient hook\n file:\n path: \"{{ dhclienthookfile }}\"\n state: absent\n notify: Preinstall | propagate resolvconf to k8s components\n" }, { "path": "roles/kubernetes/preinstall/tasks/main.yml", "content": "---\n# Disable swap\n- name: Disable swap\n import_tasks: 0010-swapoff.yml\n when:\n - not dns_late\n - kubelet_fail_swap_on\n\n- name: Set facts\n import_tasks: 0020-set_facts.yml\n tags:\n - resolvconf\n - facts\n\n- name: Check settings\n import_tasks: 0040-verify-settings.yml\n when:\n - not dns_late\n tags:\n - asserts\n\n- name: Create directories\n import_tasks: 0050-create_directories.yml\n when:\n - not dns_late\n\n- name: Apply resolvconf settings\n import_tasks: 0060-resolvconf.yml\n when:\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - systemd_resolved_enabled.rc != 0\n - networkmanager_enabled.rc != 0\n tags:\n - bootstrap_os\n - resolvconf\n\n- name: Apply systemd-resolved settings\n import_tasks: 0061-systemd-resolved.yml\n when:\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - systemd_resolved_enabled.rc == 0\n tags:\n - bootstrap_os\n - resolvconf\n\n- name: Apply networkmanager unmanaged devices settings\n import_tasks: 0062-networkmanager-unmanaged-devices.yml\n when:\n - networkmanager_enabled.rc == 0\n tags:\n - bootstrap_os\n\n- name: Apply networkmanager DNS settings\n import_tasks: 0063-networkmanager-dns.yml\n when:\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - networkmanager_enabled.rc == 0\n tags:\n - bootstrap_os\n - resolvconf\n\n- name: Apply system configurations\n import_tasks: 0080-system-configurations.yml\n when:\n - not dns_late\n tags:\n - bootstrap_os\n\n- name: Configure NTP\n import_tasks: 0081-ntp-configurations.yml\n when:\n - not dns_late\n - ntp_enabled\n tags:\n - bootstrap_os\n\n- name: Configure dhclient\n import_tasks: 0100-dhclient-hooks.yml\n when:\n - dns_mode != 'none'\n - resolvconf_mode == 'host_resolvconf'\n - dhclientconffile is defined\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n tags:\n - bootstrap_os\n - resolvconf\n\n- name: Configure dhclient dhclient hooks\n import_tasks: 0110-dhclient-hooks-undo.yml\n when:\n - dns_mode != 'none'\n - resolvconf_mode != 'host_resolvconf'\n - dhclientconffile is defined\n - not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n tags:\n - bootstrap_os\n - resolvconf\n\n# We need to make sure the network is restarted early enough so that docker can later pick up the correct system\n# nameservers and search domains\n- name: Flush handlers\n meta: flush_handlers\n\n- name: Check if we are running inside a Azure VM\n stat:\n path: /var/lib/waagent/\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: azure_check\n when:\n - not dns_late\n tags:\n - bootstrap_os\n\n- name: Run calico checks\n include_role:\n name: network_plugin/calico\n tasks_from: check\n when:\n - kube_network_plugin == 'calico'\n - not ignore_assert_errors\n" }, { "path": "roles/kubernetes/preinstall/templates/ansible_git.j2", "content": "; This file contains the information which identifies the deployment state relative to the git repo\n[default]\n{{ gitinfo.stdout }}\n" }, { "path": "roles/kubernetes/preinstall/templates/chrony.conf.j2", "content": "# {{ ansible_managed }}\n\n# Specify one or more NTP servers.\n# Use public servers from the pool.ntp.org project.\n# Please consider joining the pool (http://www.pool.ntp.org/join.html).\n{% for server in ntp_servers %}\nserver {{ server }}\n{% endfor %}\n\n# Record the rate at which the system clock gains/losses time.\ndriftfile /var/lib/chrony/drift\n\n{% if ntp_tinker_panic is sameas true %}\n# Force time sync if the drift exceeds the threshold specified\n# Useful for VMs that can be paused and much later resumed.\nmakestep 1.0 -1\n{% else %}\n# Allow the system clock to be stepped in the first three updates\n# if its offset is larger than 1 second.\nmakestep 1.0 3\n{% endif %}\n\n# Enable kernel synchronization of the real-time clock (RTC).\nrtcsync\n\n# Specify directory for log files.\nlogdir /var/log/chrony\n" }, { "path": "roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2", "content": "#!/bin/sh\n#\n# Prepend resolver options to /etc/resolv.conf after dhclient`\n# regenerates the file. See man (5) resolver for more details.\n#\nif [ $reason = \"BOUND\" ]; then\n if [ -n \"$new_domain_search\" -o -n \"$new_domain_name_servers\" ]; then\n RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')\n OPTIONS=\"options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}\"\n\n printf \"%b\\n\" \"$RESOLV_CONF\\n$OPTIONS\" > /etc/resolv.conf\n fi\nfi\n" }, { "path": "roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2", "content": "#!/bin/sh\n#\n# Prepend resolver options to /etc/resolv.conf after dhclient`\n# regenerates the file. See man (5) resolver for more details.\n#\nzdnsupdate_config() {\n if [ -n \"$new_domain_search\" -o -n \"$new_domain_name_servers\" ]; then\n RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')\n OPTIONS=\"options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}\"\n\n echo -e \"$RESOLV_CONF\\n$OPTIONS\" > /etc/resolv.conf\n fi\n}\n\nzdnsupdate_restore() {\n :\n}\n" }, { "path": "roles/kubernetes/preinstall/templates/ntp.conf.j2", "content": "# {{ ansible_managed }}\n\n# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help\n\ndriftfile {{ ntp_driftfile }}\n\n{% if ntp_tinker_panic is sameas true %}\n# Always reset the clock, even if the new time is more than 1000s away\n# from the current system time. Useful for VMs that can be paused\n# and much later resumed.\ntinker panic 0\n{% endif %}\n\n# Specify one or more NTP servers.\n# Use public servers from the pool.ntp.org project.\n# Please consider joining the pool (http://www.pool.ntp.org/join.html).\n{% for item in ntp_servers %}\npool {{ item }}\n{% endfor %}\n\n# Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for\n# details. The web page \n# might also be helpful.\n#\n# Note that \"restrict\" applies to both servers and clients, so a configuration\n# that might be intended to block requests from certain clients could also end\n# up blocking replies from your own upstream servers.\n\n# By default, exchange time with everybody, but don't allow configuration.\nrestrict -4 default kod notrap nomodify nopeer noquery limited\nrestrict -6 default kod notrap nomodify nopeer noquery limited\n\n# Local users may interrogate the ntp server more closely.\n{% for item in ntp_restrict %}\nrestrict {{ item }}\n{% endfor %}\n\n# Needed for filtering interfaces\n{% if ntp_filter_interface %}\n{% for item in ntp_interfaces %}\ninterface {{ item }}\n{% endfor %}\n{% endif %}\n\n# Needed for adding pool entries\nrestrict source notrap nomodify noquery\n\n# Disable the monitoring facility to prevent amplification attacks using ntpdc\n# monlist command when default restrict does not include the noquery flag. See\n# CVE-2013-5211 for more details.\n# Note: Monitoring will not be disabled with the limited restriction flag.\ndisable monitor\n" }, { "path": "roles/kubernetes/preinstall/templates/resolvconf.j2", "content": "#cloud-config\nwrite_files:\n - path: \"/etc/resolv.conf\"\n permissions: \"0644\"\n owner: \"root\"\n content: |\n {% for l in cloud_config.stdout_lines %}\n {{ l }}\n {% endfor %}\n #\n" }, { "path": "roles/kubernetes/preinstall/templates/resolved.conf.j2", "content": "[Resolve]\n{% if not dns_early and dns_late %}\nDNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }}\n{% endif %}\nFallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }}\n{% if remove_default_searchdomains and searchdomains | length != 0 %}\nDomains={{ searchdomains | join(' ') }}\n{% else %}\nDomains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }}\n{% endif %}\nDNSSEC=no\nCache=no-negative\n{% if systemd_resolved_disable_stub_listener | bool %}\nDNSStubListener=no\n{% endif %}\n" }, { "path": "roles/kubernetes/preinstall/vars/main.yml", "content": "---\ncoredns_server_by_mode:\n coredns: \"{{ [skydns_server] }}\"\n coredns_dual: \"{{ [skydns_server, skydns_server_secondary] }}\"\n manual: \"{{ manual_dns_server.split(',') }}\"\n none: []\ncoredns_server: \"{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}\"\n\n_nameserverentries:\n late:\n - \"{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}\"\n early:\n - \"{{ nameservers }}\"\n - \"{{ cloud_resolver }}\"\n - \"{{ configured_nameservers if not disable_host_nameservers else [] }}\"\nnameserverentries: \"{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}\"\ndhclient_supersede:\n domain-name-servers: \"{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}\"\n domain-name: \"{{ [dns_domain] }}\"\n domain-search: \"{{ default_searchdomains + searchdomains }}\"\nconfigured_nameservers: \"{{ (resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\\\s*(\\\\S*)', multiline=True) | ansible.utils.ipaddr)\n if resolvconf_stat.stat.exists else [] }}\"\n" }, { "path": "roles/kubernetes-apps/ansible/defaults/main.yml", "content": "---\n# Limits for coredns\n\n# uncomment the line below to customize the DNS cpu limit value\n# dns_cpu_limit: 300m\ndns_memory_limit: 300Mi\ndns_cpu_requests: 100m\ndns_memory_requests: 70Mi\ndns_min_replicas: \"{{ [2, groups['k8s_cluster'] | length] | min }}\"\ndns_nodes_per_replica: 16\ndns_cores_per_replica: 256\ndns_prevent_single_point_failure: \"{{ 'true' if dns_min_replicas | int > 1 else 'false' }}\"\nenable_coredns_reverse_dns_lookups: true\ncoredns_svc_name: \"coredns\"\ncoredns_ordinal_suffix: \"\"\n# dns_extra_tolerations: [{effect: NoSchedule, operator: \"Exists\"}]\ncoredns_affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 100\n podAffinityTerm:\n labelSelector:\n matchExpressions:\n - key: k8s-app\n operator: In\n values: [\"kube-dns\"]\n topologyKey: kubernetes.io/hostname\ncoredns_deployment_nodeselector: \"kubernetes.io/os: linux\"\ncoredns_default_zone_cache_block: |\n cache 30\n\ncoredns_pod_disruption_budget: false\n# when enable_dns_autoscaler is false, coredns_replicas is used to set the number of replicas\ncoredns_replicas: 2\n# value for coredns pdb\ncoredns_pod_disruption_budget_max_unavailable: \"30%\"\ndeploy_coredns: true\n# coredns_additional_configs adds any extra configuration to coredns\n# coredns_additional_configs: |\n# whoami\n# local\n\n# coredns_rewrite_block: |\n# rewrite stop {\n# name regex (.*)\\.my\\.domain {1}.svc.cluster.local\n# answer name (.*)\\.svc\\.cluster\\.local {1}.my.domain\n# }\n\n# coredns_additional_error_config: |\n# consolidate 5m \".* i/o timeout$\" warning\n\n# Configure coredns and nodelocaldns to correctly answer DNS queries when you changed\n# your 'dns_domain' and some workloads used it directly.\nold_dns_domains: []\n\n# dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section\n# dns_upstream_forward_extra_opts:\n# policy: sequential\n\n# Apply extra options to coredns kubernetes plugin\n# coredns_kubernetes_extra_opts:\n# - 'fallthrough example.local'\n\n# nodelocaldns\nnodelocaldns_cpu_requests: 100m\nnodelocaldns_memory_limit: 200Mi\nnodelocaldns_memory_requests: 70Mi\nnodelocaldns_ds_nodeselector: \"kubernetes.io/os: linux\"\nnodelocaldns_prometheus_port: 9253\nnodelocaldns_secondary_prometheus_port: 9255\n\n# nodelocaldns_additional_configs adds any extra configuration to coredns\n# nodelocaldns_additional_configs: |\n# whoami\n# local\n\n# Limits for dns-autoscaler\ndns_autoscaler_cpu_requests: 20m\ndns_autoscaler_memory_requests: 10Mi\ndns_autoscaler_deployment_nodeselector: \"kubernetes.io/os: linux\"\n# dns_autoscaler_extra_tolerations: [{effect: NoSchedule, operator: \"Exists\"}]\ndns_autoscaler_affinity: {}\n\n# etcd metrics\n# etcd_metrics_service_labels:\n# k8s-app: etcd\n# app.kubernetes.io/managed-by: Kubespray\n# app: kube-prometheus-stack-kube-etcd\n# release: prometheus-stack\n\n# Policy Controllers\n# policy_controller_extra_tolerations: [{effect: NoSchedule, operator: \"Exists\"}]\n" }, { "path": "roles/kubernetes-apps/ansible/tasks/main.yml", "content": "---\n- name: Kubernetes Apps | Wait for kube-apiserver\n uri:\n url: \"{{ kube_apiserver_endpoint }}/healthz\"\n validate_certs: false\n client_cert: \"{{ kube_apiserver_client_cert }}\"\n client_key: \"{{ kube_apiserver_client_key }}\"\n register: result\n until: result.status == 200\n retries: 20\n delay: 1\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Apps | CoreDNS\n command:\n cmd: \"{{ kubectl_apply_stdin }}\"\n stdin: \"{{ lookup('template', item) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n loop: \"{{ coredns_manifests | flatten }}\"\n tags:\n - coredns\n vars:\n clusterIP: \"{{ skydns_server }}\"\n when:\n - dns_mode in ['coredns', 'coredns_dual']\n - deploy_coredns\n\n- name: Kubernetes Apps | CoreDNS Secondary\n command:\n cmd: \"{{ kubectl_apply_stdin }}\"\n stdin: \"{{ lookup('template', item) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n loop: \"{{ coredns_manifests | flatten }}\"\n tags:\n - coredns\n vars:\n clusterIP: \"{{ skydns_server_secondary }}\"\n coredns_ordinal_suffix: \"-secondary\"\n when:\n - dns_mode == 'coredns_dual'\n - deploy_coredns\n\n- name: Kubernetes Apps | nodelocalDNS\n command:\n cmd: \"{{ kubectl_apply_stdin }}\"\n stdin: \"{{ lookup('template', item) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n loop: \"{{ nodelocaldns_manifests | flatten }}\"\n when:\n - enable_nodelocaldns\n tags:\n - nodelocaldns\n - coredns\n vars:\n primaryClusterIP: >-\n {%- if dns_mode in ['coredns', 'coredns_dual'] -%}\n {{ skydns_server }}\n {%- elif dns_mode == 'manual' -%}\n {{ manual_dns_server }}\n {%- endif -%}\n secondaryclusterIP: \"{{ skydns_server_secondary }}\"\n forwardTarget: >-\n {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%}\n {{ primaryClusterIP }} {{ secondaryclusterIP }}\n {%- else -%}\n {{ primaryClusterIP }}\n {%- endif -%}\n upstreamForwardTarget: >-\n {%- if upstream_dns_servers | length > 0 -%}\n {{ upstream_dns_servers | join(' ') }}\n {%- else -%}\n /etc/resolv.conf\n {%- endif -%}\n\n- name: Kubernetes Apps | Etcd metrics endpoints\n command:\n cmd: \"{{ kubectl_apply_stdin }}\"\n stdin: \"{{ lookup('template', item) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n loop:\n - etcd_metrics-endpoints.yml.j2\n - etcd_metrics-service.yml.j2\n when: etcd_metrics_port is defined and etcd_metrics_service_labels is defined\n tags:\n - etcd_metrics\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n addonmanager.kubernetes.io/mode: Reconcile\n name: system:coredns\nrules:\n- apiGroups:\n - \"\"\n resources:\n - endpoints\n - services\n - pods\n - namespaces\n verbs:\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - get\n- apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - list\n - watch\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n annotations:\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n addonmanager.kubernetes.io/mode: EnsureExists\n name: system:coredns\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:coredns\nsubjects:\n - kind: ServiceAccount\n name: coredns\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2", "content": "---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: coredns\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: EnsureExists\ndata:\n Corefile: |\n{% if coredns_external_zones is defined and coredns_external_zones | length > 0 %}\n{% for block in coredns_external_zones %}\n {{ block['zones'] | join(' ') }} {\n log\n errors {\n{% if coredns_additional_error_config is defined %}\n {{ coredns_additional_error_config | indent(width=10, first=False) }}\n{% endif %}\n }\n{% if block['rewrite'] is defined and block['rewrite'] | length > 0 %}\n{% for rewrite_match in block['rewrite'] %}\n rewrite {{ rewrite_match }}\n{% endfor %}\n{% endif %}\n forward . {{ block['nameservers'] | join(' ') }}\n loadbalance\n cache {{ block['cache'] | default(5) }}\n reload\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n{% endfor %}\n{% endif %}\n .:53 {\n{% if coredns_additional_configs is defined %}\n {{ coredns_additional_configs | indent(width=8, first=False) }}\n{% endif %}\n errors {\n{% if coredns_additional_error_config is defined %}\n {{ coredns_additional_error_config | indent(width=10, first=False) }}\n{% endif %}\n }\n health {\n lameduck 5s\n }\n{% if coredns_rewrite_block is defined %}\n {{ coredns_rewrite_block | indent(width=8, first=False) }}\n{% endif %}\n{% for old_dns_domain in old_dns_domains %}\n rewrite name suffix {{ old_dns_domain }} {{ dns_domain }} answer auto\n{% endfor %}\n ready\n kubernetes {{ dns_domain }} {% if coredns_kubernetes_extra_domains is defined %}{{ coredns_kubernetes_extra_domains }} {% endif %}{% if enable_coredns_reverse_dns_lookups %}in-addr.arpa ip6.arpa {% endif %}{\n pods insecure\n{% if enable_coredns_k8s_endpoint_pod_names %}\n endpoint_pod_names\n{% endif %}\n{% if enable_coredns_reverse_dns_lookups %}\n fallthrough in-addr.arpa ip6.arpa\n{% endif %}\n{% if coredns_kubernetes_extra_opts is defined %}\n{% for opt in coredns_kubernetes_extra_opts %}\n {{ opt }}\n{% endfor %}\n{% endif %}\n }\n prometheus :9153\n forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} {\n prefer_udp\n max_concurrent 1000\n{% if dns_upstream_forward_extra_opts is defined %}\n{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %}\n {{ (optname ~ ' ' ~ optvalue) | trim }}\n {# do not add a trailing space when optvalue == ''\n workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #}\n{% endfor %}\n{% endif %}\n }\n{% if enable_coredns_k8s_external %}\n k8s_external {{ coredns_k8s_external_zone }}\n{% endif %}\n {{ coredns_default_zone_cache_block | indent(width=8, first=False) }}\n loop\n reload\n loadbalance\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n{% if dns_etchosts | default(None) %}\n hosts: |\n {{ dns_etchosts | indent(width=4, first=False) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2", "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: \"coredns{{ coredns_ordinal_suffix }}\"\n namespace: kube-system\n labels:\n k8s-app: \"kube-dns{{ coredns_ordinal_suffix }}\"\n addonmanager.kubernetes.io/mode: Reconcile\n kubernetes.io/name: \"coredns{{ coredns_ordinal_suffix }}\"\nspec:\n{% if not enable_dns_autoscaler %}\n replicas: {{ coredns_replicas }}\n{% endif %}\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n maxSurge: 10%\n selector:\n matchLabels:\n k8s-app: kube-dns{{ coredns_ordinal_suffix }}\n template:\n metadata:\n labels:\n k8s-app: kube-dns{{ coredns_ordinal_suffix }}\n annotations:\n createdby: 'kubespray'\n checksum/config: \"{{ lookup('template', 'coredns-config.yml.j2') | checksum }}\"\n spec:\n securityContext:\n seccompProfile:\n type: RuntimeDefault\n nodeSelector:\n {{ coredns_deployment_nodeselector }}\n priorityClassName: system-cluster-critical\n serviceAccountName: coredns\n tolerations:\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n{% if dns_extra_tolerations is defined %}\n {{ dns_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}\n{% endif %}\n affinity:\n {{ coredns_affinity | to_nice_yaml(indent=2) | indent(8) }}\n containers:\n - name: coredns\n image: \"{{ coredns_image_repo }}:{{ coredns_image_tag }}\"\n imagePullPolicy: {{ k8s_image_pull_policy }}\n resources:\n # TODO: Set memory limits when we've profiled the container for large\n # clusters, then set request = limit to keep this container in\n # guaranteed class. Currently, this container falls into the\n # \"burstable\" category so the kubelet doesn't backoff from restarting it.\n limits:\n{% if dns_cpu_limit is defined %}\n cpu: {{ dns_cpu_limit }}\n{% endif %}\n memory: {{ dns_memory_limit }}\n requests:\n cpu: {{ dns_cpu_requests }}\n memory: {{ dns_memory_requests }}\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n ports:\n - containerPort: 53\n name: dns\n protocol: UDP\n - containerPort: 53\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 10\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 10\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: coredns\n items:\n - key: Corefile\n path: Corefile\n{% if dns_etchosts | default(None) %}\n - key: hosts\n path: hosts\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2", "content": "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: coredns{{ coredns_ordinal_suffix }}\nspec:\n maxUnavailable: {{ coredns_pod_disruption_budget_max_unavailable }}\n selector:\n matchLabels:\n k8s-app: kube-dns{{ coredns_ordinal_suffix }}\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: coredns\n namespace: kube-system\n labels:\n kubernetes.io/cluster-service: \"true\"\n addonmanager.kubernetes.io/mode: Reconcile\n" }, { "path": "roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2", "content": "---\napiVersion: v1\nkind: Service\nmetadata:\n name: {{ coredns_svc_name }}{{ coredns_ordinal_suffix }}\n namespace: kube-system\n labels:\n k8s-app: kube-dns{{ coredns_ordinal_suffix }}\n kubernetes.io/name: \"coredns{{ coredns_ordinal_suffix }}\"\n addonmanager.kubernetes.io/mode: Reconcile\n annotations:\n prometheus.io/port: \"9153\"\n prometheus.io/scrape: \"true\"\n createdby: 'kubespray'\nspec:\n selector:\n k8s-app: kube-dns{{ coredns_ordinal_suffix }}\n clusterIP: {{ clusterIP }}\n ports:\n - name: dns\n port: 53\n protocol: UDP\n - name: dns-tcp\n port: 53\n protocol: TCP\n - name: metrics\n port: 9153\n protocol: TCP\n" }, { "path": "roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2", "content": "---\n# Copyright 2016 The Kubernetes Authors. All rights reserved\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:dns-autoscaler\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nrules:\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"replicationcontrollers/scale\"]\n verbs: [\"get\", \"update\"]\n - apiGroups: [\"extensions\", \"apps\"]\n resources: [\"deployments/scale\", \"replicasets/scale\"]\n verbs: [\"get\", \"update\"]\n - apiGroups: [\"\"]\n resources: [\"configmaps\"]\n verbs: [\"get\", \"create\"]\n" }, { "path": "roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2", "content": "---\n# Copyright 2016 The Kubernetes Authors. All rights reserved\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:dns-autoscaler\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nsubjects:\n - kind: ServiceAccount\n name: dns-autoscaler\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: system:dns-autoscaler\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2", "content": "---\n# Copyright 2016 The Kubernetes Authors. All rights reserved\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nkind: ServiceAccount\napiVersion: v1\nmetadata:\n name: dns-autoscaler\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n" }, { "path": "roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2", "content": "---\n# Copyright 2016 The Kubernetes Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: dns-autoscaler{{ coredns_ordinal_suffix }}\n namespace: kube-system\n labels:\n k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n selector:\n matchLabels:\n k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}\n template:\n metadata:\n labels:\n k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}\n annotations:\n spec:\n nodeSelector:\n {{ dns_autoscaler_deployment_nodeselector }}\n priorityClassName: system-cluster-critical\n securityContext:\n seccompProfile:\n type: RuntimeDefault\n supplementalGroups: [ 65534 ]\n fsGroup: 65534\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n - effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n{% if dns_autoscaler_extra_tolerations is defined %}\n {{ dns_autoscaler_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}\n{% endif %}\n affinity:\n {{ dns_autoscaler_affinity | to_nice_yaml(indent=2) | indent(8) }}\n containers:\n - name: autoscaler\n image: \"{{ dnsautoscaler_image_repo }}:{{ dnsautoscaler_image_tag }}\"\n resources:\n requests:\n cpu: {{ dns_autoscaler_cpu_requests }}\n memory: {{ dns_autoscaler_memory_requests }}\n readinessProbe:\n httpGet:\n path: /healthz\n port: 8080\n scheme: HTTP\n command:\n - /cluster-proportional-autoscaler\n - --namespace=kube-system\n - --default-params={\"linear\":{\"preventSinglePointFailure\":{{ dns_prevent_single_point_failure }},\"coresPerReplica\":{{ dns_cores_per_replica }},\"nodesPerReplica\":{{ dns_nodes_per_replica }},\"min\":{{ dns_min_replicas }}}}\n - --logtostderr=true\n - --v=2\n - --configmap=dns-autoscaler{{ coredns_ordinal_suffix }}\n - --target=Deployment/coredns{{ coredns_ordinal_suffix }}\n serviceAccountName: dns-autoscaler\n" }, { "path": "roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2", "content": "apiVersion: v1\nkind: Endpoints\nmetadata:\n name: etcd-metrics\n namespace: kube-system\n labels:\n k8s-app: etcd\n app.kubernetes.io/managed-by: Kubespray\nsubsets:\n{% for etcd_metrics_address, etcd_host in etcd_metrics_addresses.split(',') | zip(etcd_hosts) %}\n - addresses:\n - ip: {{ etcd_metrics_address | urlsplit('hostname') }}\n targetRef:\n kind: Node\n name: {{ etcd_host }}\n ports:\n - name: http-metrics\n port: {{ etcd_metrics_address | urlsplit('port') }}\n protocol: TCP\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: etcd-metrics\n namespace: kube-system\n labels:\n {{ etcd_metrics_service_labels | to_yaml(indent=2, width=1337) | indent(width=4) }}\nspec:\n ports:\n - name: http-metrics\n protocol: TCP\n port: {{ etcd_metrics_port }}\n # targetPort:\n" }, { "path": "roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: nodelocaldns\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: EnsureExists\n\ndata:\n Corefile: |\n{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %}\n{% for block in nodelocaldns_external_zones %}\n {{ block['zones'] | join(' ') }} {\n errors\n cache {{ block['cache'] | default(30) }}\n reload\n{% if block['rewrite'] is defined and block['rewrite'] | length > 0 %}\n{% for rewrite_match in block['rewrite'] %}\n rewrite {{ rewrite_match }}\n{% endfor %}\n{% endif %}\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ block['nameservers'] | join(' ') }}\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}\n log\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n{% endfor %}\n{% endif %}\n {{ ([dns_domain] + old_dns_domains) | join(' ') }}:53 {\n errors\n cache {\n success 9984 30\n denial 9984 5\n }\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ forwardTarget }} {\n force_tcp\n }\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}\n health {{ nodelocaldns_ip | ansible.utils.ipwrap }}:{{ nodelocaldns_health_port }}\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n in-addr.arpa:53 {\n errors\n cache 30\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ forwardTarget }} {\n force_tcp\n }\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}\n }\n ip6.arpa:53 {\n errors\n cache 30\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ forwardTarget }} {\n force_tcp\n }\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}\n }\n .:53 {\n{% if nodelocaldns_additional_configs is defined %}\n {{ nodelocaldns_additional_configs | indent(width=8, first=False) }}\n{% endif %}\n errors\n cache 30\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} {\n{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %}\n {{ (optname ~ ' ' ~ optvalue) | trim }}\n {# do not add a trailing space when optvalue == ''\n workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #}\n{% endfor %}\n }{% endif %}\n\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n{% if enable_nodelocaldns_secondary %}\n Corefile-second: |\n{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %}\n{% for block in nodelocaldns_external_zones %}\n {{ block['zones'] | join(' ') }} {\n errors\n cache {{ block['cache'] | default(30) }}\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ block['nameservers'] | join(' ') }}\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}\n log\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n{% endfor %}\n{% endif %}\n {{ dns_domain }}:53 {\n errors\n cache {\n success 9984 30\n denial 9984 5\n }\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ forwardTarget }} {\n force_tcp\n }\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}\n health {{ nodelocaldns_ip | ansible.utils.ipwrap }}:{{ nodelocaldns_second_health_port }}\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n in-addr.arpa:53 {\n errors\n cache 30\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ forwardTarget }} {\n force_tcp\n }\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}\n }\n ip6.arpa:53 {\n errors\n cache 30\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ forwardTarget }} {\n force_tcp\n }\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}\n }\n .:53 {\n{% if nodelocaldns_additional_configs is defined %}\n {{ nodelocaldns_additional_configs | indent(width=8, first=False) }}\n{% endif %}\n errors\n cache 30\n reload\n loop\n bind {{ nodelocaldns_ip }}\n forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} {\n{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %}\n {{ (optname ~ ' ' ~ optvalue) | trim }}\n {# do not add a trailing space when optvalue == ''\n workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #}\n{% endfor %}\n }{% endif %}\n\n prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}\n{% if dns_etchosts | default(None) %}\n hosts /etc/coredns/hosts {\n fallthrough\n }\n{% endif %}\n }\n{% endif %}\n{% if dns_etchosts | default(None) %}\n hosts: |\n {{ dns_etchosts | indent(width=4, first=False) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: nodelocaldns\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n selector:\n matchLabels:\n k8s-app: node-local-dns\n template:\n metadata:\n labels:\n k8s-app: node-local-dns\n annotations:\n prometheus.io/scrape: 'true'\n prometheus.io/port: '{{ nodelocaldns_prometheus_port }}'\n checksum/config: \"{{ lookup('template', 'nodelocaldns-config.yml.j2') | checksum }}\"\n spec:\n nodeSelector:\n {{ nodelocaldns_ds_nodeselector }}\n priorityClassName: system-node-critical\n serviceAccountName: nodelocaldns\n hostNetwork: true\n dnsPolicy: Default # Don't use cluster DNS.\n tolerations:\n - effect: NoSchedule\n operator: \"Exists\"\n - effect: NoExecute\n operator: \"Exists\"\n containers:\n - name: node-cache\n image: \"{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}\"\n resources:\n limits:\n memory: {{ nodelocaldns_memory_limit }}\n requests:\n cpu: {{ nodelocaldns_cpu_requests }}\n memory: {{ nodelocaldns_memory_requests }}\n args:\n - -localip\n - {{ nodelocaldns_ip }}\n - -conf\n - /etc/coredns/Corefile\n - -upstreamsvc\n - coredns\n{% if enable_nodelocaldns_secondary %}\n - -skipteardown\n{% endif %}\n ports:\n - containerPort: 53\n name: dns\n protocol: UDP\n - containerPort: 53\n name: dns-tcp\n protocol: TCP\n - containerPort: {{ nodelocaldns_prometheus_port }}\n name: metrics\n protocol: TCP\n securityContext:\n capabilities:\n add:\n - NET_ADMIN\n{% if nodelocaldns_bind_metrics_host_ip %}\n env:\n - name: MY_HOST_IP\n valueFrom:\n fieldRef:\n fieldPath: status.hostIP\n{% endif %}\n livenessProbe:\n httpGet:\n host: {{ nodelocaldns_ip }}\n path: /health\n port: {{ nodelocaldns_health_port }}\n scheme: HTTP\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 10\n readinessProbe:\n httpGet:\n host: {{ nodelocaldns_ip }}\n path: /health\n port: {{ nodelocaldns_health_port }}\n scheme: HTTP\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 10\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n - name: xtables-lock\n mountPath: /run/xtables.lock\n volumes:\n - name: config-volume\n configMap:\n name: nodelocaldns\n items:\n - key: Corefile\n path: Corefile\n{% if dns_etchosts | default(None) %}\n - key: hosts\n path: hosts\n{% endif %}\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a \"force\n # deletion\": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.\n terminationGracePeriodSeconds: 0\n updateStrategy:\n rollingUpdate:\n maxUnavailable: {{ serial | default('20%') }}\n type: RollingUpdate\n" }, { "path": "roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2", "content": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: nodelocaldns\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n" }, { "path": "roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: nodelocaldns-second\n namespace: kube-system\n labels:\n k8s-app: kube-dns\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n selector:\n matchLabels:\n k8s-app: node-local-dns-second\n template:\n metadata:\n labels:\n k8s-app: node-local-dns-second\n annotations:\n prometheus.io/scrape: 'true'\n prometheus.io/port: '{{ nodelocaldns_secondary_prometheus_port }}'\n checksum/config: \"{{ lookup('template', 'nodelocaldns-config.yml.j2') | checksum }}\"\n spec:\n nodeSelector:\n {{ nodelocaldns_ds_nodeselector }}\n priorityClassName: system-cluster-critical\n serviceAccountName: nodelocaldns\n hostNetwork: true\n dnsPolicy: Default # Don't use cluster DNS.\n tolerations:\n - effect: NoSchedule\n operator: \"Exists\"\n - effect: NoExecute\n operator: \"Exists\"\n containers:\n - name: node-cache\n image: \"{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}\"\n resources:\n limits:\n memory: {{ nodelocaldns_memory_limit }}\n requests:\n cpu: {{ nodelocaldns_cpu_requests }}\n memory: {{ nodelocaldns_memory_requests }}\n args: [ \"-localip\", \"{{ nodelocaldns_ip }}\", \"-conf\", \"/etc/coredns/Corefile\", \"-upstreamsvc\", \"coredns\", \"-skipteardown\" ]\n ports:\n - containerPort: {{ nodelocaldns_secondary_prometheus_port }}\n name: metrics\n protocol: TCP\n securityContext:\n capabilities:\n add:\n - NET_ADMIN\n{% if nodelocaldns_bind_metrics_host_ip %}\n env:\n - name: MY_HOST_IP\n valueFrom:\n fieldRef:\n fieldPath: status.hostIP\n{% endif %}\n livenessProbe:\n httpGet:\n host: {{ nodelocaldns_ip }}\n path: /health\n port: {{ nodelocaldns_health_port }}\n scheme: HTTP\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 10\n readinessProbe:\n httpGet:\n host: {{ nodelocaldns_ip }}\n path: /health\n port: {{ nodelocaldns_health_port }}\n scheme: HTTP\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 10\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n - name: xtables-lock\n mountPath: /run/xtables.lock\n lifecycle:\n preStop:\n exec:\n command:\n - sh\n - -c\n - sleep {{ nodelocaldns_secondary_skew_seconds }} && kill -9 1\n volumes:\n - name: config-volume\n configMap:\n name: nodelocaldns\n items:\n - key: Corefile-second\n path: Corefile\n{% if dns_etchosts | default(None) %}\n - key: hosts\n path: hosts\n{% endif %}\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n # Implement a time skew between the main nodelocaldns and this secondary.\n # Since the two nodelocaldns instances share the :53 port, we want to keep\n # at least one running at any time even if the manifests are replaced simultaneously\n terminationGracePeriodSeconds: {{ nodelocaldns_secondary_skew_seconds }}\n updateStrategy:\n rollingUpdate:\n maxUnavailable: {{ serial | default('20%') }}\n type: RollingUpdate\n" }, { "path": "roles/kubernetes-apps/ansible/vars/main.yml", "content": "---\ndns_autoscaler_manifests:\n- dns-autoscaler-sa.yml.j2\n- dns-autoscaler.yml.j2\n- dns-autoscaler-clusterrole.yml.j2\n- dns-autoscaler-clusterrolebinding.yml.j2\n\ncoredns_manifests:\n- coredns-clusterrole.yml.j2\n- coredns-clusterrolebinding.yml.j2\n- coredns-config.yml.j2\n- coredns-deployment.yml.j2\n- coredns-sa.yml.j2\n- coredns-svc.yml.j2\n- \"{{ dns_autoscaler_manifests if enable_dns_autoscaler else [] }}\"\n- \"{{ 'coredns-poddisruptionbudget.yml.j2' if coredns_pod_disruption_budget else [] }}\"\n\nnodelocaldns_manifests:\n- nodelocaldns-config.yml.j2\n- nodelocaldns-daemonset.yml.j2\n- nodelocaldns-sa.yml.j2\n- \"{{ 'nodelocaldns-second-daemonset.yml.j2' if enable_nodelocaldns_secondary else [] }}\"\n" }, { "path": "roles/kubernetes-apps/argocd/defaults/main.yml", "content": "---\nargocd_enabled: false\nargocd_version: 2.14.5\nargocd_namespace: argocd\n# argocd_admin_password:\n" }, { "path": "roles/kubernetes-apps/argocd/tasks/main.yml", "content": "---\n- name: Kubernetes Apps | Download yq\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.yq) }}\"\n\n- name: Kubernetes Apps | Copy yq binary from download dir\n ansible.posix.synchronize:\n src: \"{{ downloads.yq.dest }}\"\n dest: \"{{ bin_dir }}/yq\"\n compress: false\n perms: true\n owner: false\n group: false\n delegate_to: \"{{ inventory_hostname }}\"\n\n- name: Kubernetes Apps | Set ArgoCD template list\n set_fact:\n argocd_templates:\n - name: namespace\n file: argocd-namespace.yml\n - name: install\n file: \"{{ downloads.argocd_install.dest | basename }}\"\n namespace: \"{{ argocd_namespace }}\"\n download: \"{{ downloads.argocd_install }}\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n- name: Kubernetes Apps | Download ArgoCD remote manifests\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(item.download) }}\"\n with_items: \"{{ argocd_templates | selectattr('download', 'defined') | list }}\"\n loop_control:\n label: \"{{ item.file }}\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n- name: Kubernetes Apps | Copy ArgoCD remote manifests from download dir\n ansible.posix.synchronize:\n src: \"{{ local_release_dir }}/{{ item.file }}\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n compress: false\n perms: true\n owner: false\n group: false\n delegate_to: \"{{ inventory_hostname }}\"\n with_items: \"{{ argocd_templates | selectattr('download', 'defined') | list }}\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n- name: Kubernetes Apps | Set ArgoCD namespace for remote manifests\n become: true\n command: |\n {{ bin_dir }}/yq eval-all -i '.metadata.namespace=\"{{ argocd_namespace }}\"' {{ kube_config_dir }}/{{ item.file }}\n with_items: \"{{ argocd_templates | selectattr('download', 'defined') | list }}\"\n loop_control:\n label: \"{{ item.file }}\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n- name: Kubernetes Apps | Create ArgoCD manifests from templates\n become: true\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ argocd_templates | selectattr('download', 'undefined') | list }}\"\n loop_control:\n label: \"{{ item.file }}\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n- name: Kubernetes Apps | Install ArgoCD\n become: true\n kube:\n name: ArgoCD\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.file }}\"\n state: latest\n with_items: \"{{ argocd_templates }}\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n# https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it\n- name: Kubernetes Apps | Set ArgoCD custom admin password\n become: true\n shell: |\n {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n {{ argocd_namespace }} patch secret argocd-secret -p \\\n '{\n \"stringData\": {\n \"admin.password\": \"{{ argocd_admin_password | password_hash('bcrypt') }}\",\n \"admin.passwordMtime\": \"'$(date +%FT%T%Z)'\"\n }\n }'\n when:\n - argocd_admin_password is defined\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n" }, { "path": "roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ argocd_namespace }}\n labels:\n app: argocd\n" }, { "path": "roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml", "content": "---\napiVersion: scheduling.k8s.io/v1\nkind: PriorityClass\nmetadata:\n name: k8s-cluster-critical\nvalue: 1000000000\nglobalDefault: false\ndescription: \"This priority class should only be used by the pods installed using kubespray.\"\n" }, { "path": "roles/kubernetes-apps/cluster_roles/tasks/main.yml", "content": "---\n- name: Kubernetes Apps | Wait for kube-apiserver\n uri:\n url: \"{{ kube_apiserver_endpoint }}/healthz\"\n validate_certs: false\n client_cert: \"{{ kube_apiserver_client_cert }}\"\n client_key: \"{{ kube_apiserver_client_key }}\"\n register: result\n until: result.status == 200\n retries: 10\n delay: 6\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes\n template:\n src: \"node-crb.yml.j2\"\n dest: \"{{ kube_config_dir }}/node-crb.yml\"\n mode: \"0640\"\n register: node_crb_manifest\n when:\n - rbac_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Apply workaround to allow all nodes with cert O=system:nodes to register\n kube:\n name: \"kubespray:system:node\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"clusterrolebinding\"\n filename: \"{{ kube_config_dir }}/node-crb.yml\"\n state: latest\n register: result\n until: result is succeeded\n retries: 10\n delay: 6\n when:\n - rbac_enabled\n - node_crb_manifest.changed\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Apps | Remove old webhook ClusterRole\n kube:\n name: \"system:node-webhook\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"clusterrole\"\n state: absent\n when:\n - rbac_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags: node-webhook\n\n- name: Kubernetes Apps | Remove old webhook ClusterRoleBinding\n kube:\n name: \"system:node-webhook\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"clusterrolebinding\"\n state: absent\n when:\n - rbac_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags: node-webhook\n\n- name: PriorityClass | Copy k8s-cluster-critical-pc.yml file\n copy:\n src: k8s-cluster-critical-pc.yml\n dest: \"{{ kube_config_dir }}/k8s-cluster-critical-pc.yml\"\n mode: \"0640\"\n when: inventory_hostname == groups['kube_control_plane'] | last\n\n- name: PriorityClass | Create k8s-cluster-critical\n kube:\n name: k8s-cluster-critical\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"PriorityClass\"\n filename: \"{{ kube_config_dir }}/k8s-cluster-critical-pc.yml\"\n state: latest\n register: result\n until: result is succeeded\n retries: 10\n delay: 6\n when: inventory_hostname == groups['kube_control_plane'] | last\n" }, { "path": "roles/kubernetes-apps/cluster_roles/templates/namespace.j2", "content": "apiVersion: v1\nkind: Namespace\nmetadata:\n name: \"kube-system\"\n" }, { "path": "roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n annotations:\n rbac.authorization.kubernetes.io/autoupdate: \"true\"\n labels:\n kubernetes.io/bootstrapping: rbac-defaults\n name: kubespray:system:node\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:node\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: Group\n name: system:nodes\n" }, { "path": "roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: system:vsphere-cloud-provider\nrules:\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n - update\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: system:vsphere-cloud-provider\nroleRef:\n kind: ClusterRole\n name: system:vsphere-cloud-provider\n apiGroup: rbac.authorization.k8s.io\nsubjects:\n- kind: ServiceAccount\n name: vsphere-cloud-provider\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/common_crds/gateway_api/defaults/main.yml", "content": "---\ngateway_api_enabled: false\n\n# `gateway_api_channel` default is \"standard\".\n# \"standard\" release channel includes all resources that have graduated to GA or beta, including GatewayClass, Gateway, HTTPRoute, and ReferenceGrant.\n# \"experimental\" for some experimental resources and fields. Note that future releases of the API could include breaking changes to experimental resources and fields. For example, any experimental resource or field could be removed in a future release.\n# https://gateway-api.sigs.k8s.io/guides/#install-experimental-channel\ngateway_api_channel: \"standard\"\n" }, { "path": "roles/kubernetes-apps/common_crds/gateway_api/tasks/main.yml", "content": "---\n- name: Gateway API | Download YAML\n include_tasks: \"../../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.gateway_api_crds) }}\"\n\n- name: Gateway API | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/gateway_api\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Gateway API | Copy YAML from download dir\n copy:\n src: \"{{ local_release_dir }}/gateway-api-{{ gateway_api_channel }}-install.yaml\"\n dest: \"{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml\"\n mode: \"0644\"\n remote_src: true\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n\n- name: Gateway API | Install Gateway API\n kube:\n name: Gateway API\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml\"\n state: latest\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n" }, { "path": "roles/kubernetes-apps/common_crds/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/common_crds/gateway_api\n when: gateway_api_enabled\n tags:\n - gateway_api\n\n - role: kubernetes-apps/common_crds/prometheus_operator_crds\n when: prometheus_operator_crds_enabled\n tags:\n - prometheus_operator_crds\n" }, { "path": "roles/kubernetes-apps/common_crds/prometheus_operator_crds/tasks/main.yml", "content": "---\n- name: Prometheus Operator CRDs | Download YAML\n include_tasks: \"../../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.prometheus_operator_crds) }}\"\n\n- name: Prometheus Operator CRDs | Install\n command:\n cmd: \"{{ bin_dir }}/kubectl apply -f {{ local_release_dir }}/prometheus-operator-crds.yaml\"\n when:\n - \"inventory_hostname == groups['kube_control_plane'][0]\"\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/container_engine_accelerator/nvidia_gpu\n when: nvidia_accelerator_enabled\n tags:\n - apps\n - nvidia_gpu\n - container_engine_accelerator\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml", "content": "---\nnvidia_accelerator_enabled: false\nnvidia_driver_version: \"390.87\"\nnvidia_gpu_tesla_base_url: https://us.download.nvidia.com/tesla/\nnvidia_gpu_gtx_base_url: http://us.download.nvidia.com/XFree86/Linux-x86_64/\nnvidia_gpu_flavor: tesla\nnvidia_url_end: \"{{ nvidia_driver_version }}/NVIDIA-Linux-x86_64-{{ nvidia_driver_version }}.run\"\nnvidia_driver_install_container: false\nnvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2\nnvidia_driver_install_ubuntu_container: registry.k8s.io/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63\nnvidia_driver_install_supported: false\nnvidia_gpu_device_plugin_container: \"registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e\"\nnvidia_gpu_nodes: []\nnvidia_gpu_device_plugin_memory: 30Mi\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml", "content": "---\n\n- name: Container Engine Acceleration Nvidia GPU | gather os specific variables\n include_vars: \"{{ item }}\"\n with_first_found:\n - files:\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}.yml\"\n - \"{{ ansible_os_family | lower }}.yml\"\n skip: true\n\n- name: Container Engine Acceleration Nvidia GPU | Set fact of download url Tesla\n set_fact:\n nvidia_driver_download_url_default: \"{{ nvidia_gpu_tesla_base_url }}{{ nvidia_url_end }}\"\n when: nvidia_gpu_flavor | lower == \"tesla\"\n\n- name: Container Engine Acceleration Nvidia GPU | Set fact of download url GTX\n set_fact:\n nvidia_driver_download_url_default: \"{{ nvidia_gpu_gtx_base_url }}{{ nvidia_url_end }}\"\n when: nvidia_gpu_flavor | lower == \"gtx\"\n\n- name: Container Engine Acceleration Nvidia GPU | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/container_engine_accelerator\"\n owner: root\n group: root\n mode: \"0755\"\n recurse: true\n\n- name: Container Engine Acceleration Nvidia GPU | Create manifests for nvidia accelerators\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - { name: nvidia-driver-install-daemonset, file: nvidia-driver-install-daemonset.yml, type: daemonset }\n - { name: k8s-device-plugin-nvidia-daemonset, file: k8s-device-plugin-nvidia-daemonset.yml, type: daemonset }\n register: container_engine_accelerator_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container\n\n- name: Container Engine Acceleration Nvidia GPU | Apply manifests for nvidia accelerators\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"kube-system\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ container_engine_accelerator_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container and nvidia_driver_install_supported\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: nvidia-gpu-device-plugin\n namespace: kube-system\n labels:\n k8s-app: nvidia-gpu-device-plugin\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n selector:\n matchLabels:\n k8s-app: nvidia-gpu-device-plugin\n template:\n metadata:\n labels:\n k8s-app: nvidia-gpu-device-plugin\n spec:\n priorityClassName: system-node-critical\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: \"nvidia.com/gpu\"\n operator: Exists\n tolerations:\n - operator: \"Exists\"\n effect: \"NoExecute\"\n - operator: \"Exists\"\n effect: \"NoSchedule\"\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n hostPID: true\n volumes:\n - name: device-plugin\n hostPath:\n path: /var/lib/kubelet/device-plugins\n - name: dev\n hostPath:\n path: /dev\n containers:\n - image: \"{{ nvidia_gpu_device_plugin_container }}\"\n command: [\"/usr/bin/nvidia-gpu-device-plugin\", \"-logtostderr\"]\n name: nvidia-gpu-device-plugin\n resources:\n requests:\n cpu: 50m\n memory: {{ nvidia_gpu_device_plugin_memory }}\n limits:\n cpu: 50m\n memory: {{ nvidia_gpu_device_plugin_memory }}\n securityContext:\n privileged: true\n volumeMounts:\n - name: device-plugin\n mountPath: /device-plugin\n - name: dev\n mountPath: /dev\n updateStrategy:\n type: RollingUpdate\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2", "content": "# Copyright 2017 Google Inc. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: nvidia-driver-installer\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n name: nvidia-driver-installer\n template:\n metadata:\n labels:\n name: nvidia-driver-installer\n spec:\n priorityClassName: system-node-critical\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: \"nvidia.com/gpu\"\n operator: Exists\n tolerations:\n - key: \"nvidia.com/gpu\"\n effect: \"NoSchedule\"\n operator: \"Exists\"\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n hostPID: true\n volumes:\n - name: dev\n hostPath:\n path: /dev\n - name: nvidia-install-dir-host\n hostPath:\n path: /home/kubernetes/bin/nvidia\n - name: root-mount\n hostPath:\n path: /\n initContainers:\n - image: \"{{ nvidia_driver_install_container }}\"\n name: nvidia-driver-installer\n resources:\n requests:\n cpu: 0.15\n securityContext:\n privileged: true\n env:\n - name: NVIDIA_INSTALL_DIR_HOST\n value: /home/kubernetes/bin/nvidia\n - name: NVIDIA_INSTALL_DIR_CONTAINER\n value: /usr/local/nvidia\n - name: ROOT_MOUNT_DIR\n value: /root\n - name: NVIDIA_DRIVER_VERSION\n value: \"{{ nvidia_driver_version }}\"\n - name: NVIDIA_DRIVER_DOWNLOAD_URL\n value: \"{{ nvidia_driver_download_url_default }}\"\n volumeMounts:\n - name: nvidia-install-dir-host\n mountPath: /usr/local/nvidia\n - name: dev\n mountPath: /dev\n - name: root-mount\n mountPath: /root\n containers:\n - image: \"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}\"\n name: pause\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml", "content": "---\nnvidia_driver_install_container: \"{{ nvidia_driver_install_ubuntu_container }}\"\nnvidia_driver_install_supported: true\n" }, { "path": "roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml", "content": "---\nnvidia_driver_install_container: \"{{ nvidia_driver_install_ubuntu_container }}\"\nnvidia_driver_install_supported: true\n" }, { "path": "roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml", "content": "---\n\n- name: Crun | Copy runtime class manifest\n template:\n src: runtimeclass-crun.yml\n dest: \"{{ kube_config_dir }}/runtimeclass-crun.yml\"\n mode: \"0664\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Crun | Apply manifests\n kube:\n name: \"runtimeclass-crun\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"runtimeclass\"\n filename: \"{{ kube_config_dir }}/runtimeclass-crun.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml", "content": "---\nkind: RuntimeClass\napiVersion: node.k8s.io/v1\nmetadata:\n name: crun\nhandler: crun\n" }, { "path": "roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml", "content": "---\n- name: GVisor | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/gvisor\"\n owner: root\n group: root\n mode: \"0755\"\n recurse: true\n\n- name: GVisor | Templates List\n set_fact:\n gvisor_templates:\n - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass }\n\n- name: GVisort | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/gvisor/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ gvisor_templates }}\"\n register: gvisor_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: GVisor | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ gvisor_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2", "content": "---\nkind: RuntimeClass\napiVersion: node.k8s.io/v1\nmetadata:\n name: gvisor\nhandler: runsc\n" }, { "path": "roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml", "content": "---\n\nkata_containers_qemu_overhead: true\nkata_containers_qemu_overhead_fixed_cpu: 250m\nkata_containers_qemu_overhead_fixed_memory: 160Mi\n" }, { "path": "roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml", "content": "---\n\n- name: Kata Containers | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/kata_containers\"\n owner: root\n group: root\n mode: \"0755\"\n recurse: true\n\n- name: Kata Containers | Templates list\n set_fact:\n kata_containers_templates:\n - { name: runtimeclass-kata-qemu, file: runtimeclass-kata-qemu.yml, type: runtimeclass }\n\n- name: Kata Containers | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/kata_containers/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ kata_containers_templates }}\"\n register: kata_containers_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kata Containers | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/kata_containers/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ kata_containers_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2", "content": "---\nkind: RuntimeClass\napiVersion: node.k8s.io/v1\nmetadata:\n name: kata-qemu\nhandler: kata-qemu\n{% if kata_containers_qemu_overhead %}\noverhead:\n podFixed:\n cpu: {{ kata_containers_qemu_overhead_fixed_cpu }}\n memory: {{ kata_containers_qemu_overhead_fixed_memory }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/container_runtimes/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/container_runtimes/kata_containers\n when: kata_containers_enabled\n tags:\n - apps\n - kata-containers\n - container-runtimes\n\n - role: kubernetes-apps/container_runtimes/gvisor\n when: gvisor_enabled\n tags:\n - apps\n - gvisor\n - container-runtimes\n\n - role: kubernetes-apps/container_runtimes/crun\n when: crun_enabled\n tags:\n - apps\n - crun\n - container-runtimes\n\n - role: kubernetes-apps/container_runtimes/youki\n when:\n - youki_enabled\n - container_manager == 'crio'\n tags:\n - apps\n - youki\n - container-runtimes\n" }, { "path": "roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml", "content": "---\n\n- name: Youki | Copy runtime class manifest\n template:\n src: runtimeclass-youki.yml\n dest: \"{{ kube_config_dir }}/runtimeclass-youki.yml\"\n mode: \"0664\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Youki | Apply manifests\n kube:\n name: \"runtimeclass-youki\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"runtimeclass\"\n filename: \"{{ kube_config_dir }}/runtimeclass-youki.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml", "content": "---\nkind: RuntimeClass\napiVersion: node.k8s.io/v1\nmetadata:\n name: youki\nhandler: youki\n" }, { "path": "roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml", "content": "---\naws_ebs_csi_enable_volume_scheduling: true\naws_ebs_csi_enable_volume_snapshot: false\naws_ebs_csi_enable_volume_resizing: false\naws_ebs_csi_controller_replicas: 1\naws_ebs_csi_plugin_image_tag: latest\n\n# Add annotions to ebs_csi_controller. Useful if using kube2iam for role assumption\n# aws_ebs_csi_annotations:\n# - key: iam.amazonaws.com/role\n# value: your-ebs-role-arn\n" }, { "path": "roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml", "content": "---\n- name: AWS CSI Driver | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml}\n - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml}\n - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml}\n - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml}\n register: aws_csi_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: AWS CSI Driver | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ aws_csi_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2", "content": "# Controller Service\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: ebs-csi-controller-sa\n namespace: kube-system\n\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-external-provisioner-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"list\", \"watch\", \"create\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-csi-provisioner-binding\nsubjects:\n - kind: ServiceAccount\n name: ebs-csi-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: ebs-external-provisioner-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n\n# The permissions in this ClusterRole are tightly coupled with the version of csi-attacher used. More information about this can be found in kubernetes-csi/external-attacher.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-external-attacher-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments/status\"]\n verbs: [\"patch\"]\n\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-csi-attacher-binding\nsubjects:\n - kind: ServiceAccount\n name: ebs-csi-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: ebs-external-attacher-role\n apiGroup: rbac.authorization.k8s.io\n\n{% if aws_ebs_csi_enable_volume_snapshot %}\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-external-snapshotter-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"create\", \"list\", \"watch\", \"delete\"]\n\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-csi-snapshotter-binding\nsubjects:\n - kind: ServiceAccount\n name: ebs-csi-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: ebs-external-snapshotter-role\n apiGroup: rbac.authorization.k8s.io\n\n{% endif %}\n\n{% if aws_ebs_csi_enable_volume_resizing %}\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-external-resizer-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: ebs-csi-resizer-binding\nsubjects:\n - kind: ServiceAccount\n name: ebs-csi-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: ebs-external-resizer-role\n apiGroup: rbac.authorization.k8s.io\n\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2", "content": "---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: ebs-csi-controller\n namespace: kube-system\nspec:\n replicas: {{ aws_ebs_csi_controller_replicas }}\n selector:\n matchLabels:\n app: ebs-csi-controller\n app.kubernetes.io/name: aws-ebs-csi-driver\n template:\n metadata:\n labels:\n app: ebs-csi-controller\n app.kubernetes.io/name: aws-ebs-csi-driver\n{% if aws_ebs_csi_annotations is defined %}\n annotations:\n{% for annotation in aws_ebs_csi_annotations %}\n {{ annotation.key }}: {{ annotation.value }}\n{% endfor %}\n{% endif %}\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n serviceAccountName: ebs-csi-controller-sa\n priorityClassName: system-cluster-critical\n containers:\n - name: ebs-plugin\n image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }}\n args:\n - --endpoint=$(CSI_ENDPOINT)\n{% if aws_ebs_csi_extra_volume_tags is defined %}\n - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }}\n{% endif %}\n - --logtostderr\n - --v=5\n env:\n - name: CSI_ENDPOINT\n value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock\n - name: AWS_ACCESS_KEY_ID\n valueFrom:\n secretKeyRef:\n name: aws-secret\n key: key_id\n optional: true\n - name: AWS_SECRET_ACCESS_KEY\n valueFrom:\n secretKeyRef:\n name: aws-secret\n key: access_key\n optional: true\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n ports:\n - name: healthz\n containerPort: 9808\n protocol: TCP\n livenessProbe:\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 10\n timeoutSeconds: 3\n periodSeconds: 10\n failureThreshold: 5\n - name: csi-provisioner\n image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }}\n args:\n - --csi-address=$(ADDRESS)\n - --v=5\n{% if aws_ebs_csi_enable_volume_scheduling %}\n - --feature-gates=Topology=true\n{% endif %}\n - --leader-election=true\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-attacher\n image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }}\n args:\n - --csi-address=$(ADDRESS)\n - --v=5\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n{% if aws_ebs_csi_enable_volume_snapshot %}\n - name: csi-snapshotter\n image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }}\n args:\n - --csi-address=$(ADDRESS)\n - --timeout=15s\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n{% endif %}\n{% if aws_ebs_csi_enable_volume_resizing %}\n - name: csi-resizer\n image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --csi-address=$(ADDRESS)\n - --v=5\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n{% endif %}\n - name: liveness-probe\n image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}\n args:\n - --csi-address=/csi/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n volumes:\n - name: socket-dir\n emptyDir: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2", "content": "---\napiVersion: storage.k8s.io/v1\nkind: CSIDriver\nmetadata:\n name: ebs.csi.aws.com\nspec:\n attachRequired: true\n podInfoOnMount: false\n" }, { "path": "roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2", "content": "---\n# Node Service\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: ebs-csi-node\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n app: ebs-csi-node\n app.kubernetes.io/name: aws-ebs-csi-driver\n template:\n metadata:\n labels:\n app: ebs-csi-node\n app.kubernetes.io/name: aws-ebs-csi-driver\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n priorityClassName: system-node-critical\n containers:\n - name: ebs-plugin\n securityContext:\n privileged: true\n image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }}\n args:\n - --endpoint=$(CSI_ENDPOINT)\n{% if aws_ebs_csi_extra_volume_tags is defined %}\n - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }}\n{% endif %}\n - --logtostderr\n - --v=5\n env:\n - name: CSI_ENDPOINT\n value: unix:/csi/csi.sock\n volumeMounts:\n - name: kubelet-dir\n mountPath: /var/lib/kubelet\n mountPropagation: \"Bidirectional\"\n - name: plugin-dir\n mountPath: /csi\n - name: device-dir\n mountPath: /dev\n ports:\n - name: healthz\n containerPort: 9808\n protocol: TCP\n livenessProbe:\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 10\n timeoutSeconds: 3\n periodSeconds: 10\n failureThreshold: 5\n - name: node-driver-registrar\n image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}\n args:\n - --csi-address=$(ADDRESS)\n - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)\n - --v=5\n lifecycle:\n preStop:\n exec:\n command: [\"/bin/sh\", \"-c\", \"rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock\"]\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n - name: DRIVER_REG_SOCK_PATH\n value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n - name: registration-dir\n mountPath: /registration\n - name: liveness-probe\n image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}\n args:\n - --csi-address=/csi/csi.sock\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n volumes:\n - name: kubelet-dir\n hostPath:\n path: /var/lib/kubelet\n type: Directory\n - name: plugin-dir\n hostPath:\n path: /var/lib/kubelet/plugins/ebs.csi.aws.com/\n type: DirectoryOrCreate\n - name: registration-dir\n hostPath:\n path: /var/lib/kubelet/plugins_registry/\n type: Directory\n - name: device-dir\n hostPath:\n path: /dev\n type: Directory\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml", "content": "---\nazure_csi_use_instance_metadata: true\nazure_csi_controller_replicas: 2\nazure_csi_plugin_image_tag: latest\nazure_csi_controller_affinity: {}\nazure_csi_node_affinity: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml", "content": "---\n- name: Azure CSI Driver | check azure_csi_tenant_id value\n fail:\n msg: \"azure_csi_tenant_id is missing\"\n when: azure_csi_tenant_id is not defined or not azure_csi_tenant_id\n\n- name: Azure CSI Driver | check azure_csi_subscription_id value\n fail:\n msg: \"azure_csi_subscription_id is missing\"\n when: azure_csi_subscription_id is not defined or not azure_csi_subscription_id\n\n- name: Azure CSI Driver | check azure_csi_aad_client_id value\n fail:\n msg: \"azure_csi_aad_client_id is missing\"\n when: azure_csi_aad_client_id is not defined or not azure_csi_aad_client_id\n\n- name: Azure CSI Driver | check azure_csi_aad_client_secret value\n fail:\n msg: \"azure_csi_aad_client_secret is missing\"\n when: azure_csi_aad_client_secret is not defined or not azure_csi_aad_client_secret\n\n- name: Azure CSI Driver | check azure_csi_resource_group value\n fail:\n msg: \"azure_csi_resource_group is missing\"\n when: azure_csi_resource_group is not defined or not azure_csi_resource_group\n\n- name: Azure CSI Driver | check azure_csi_location value\n fail:\n msg: \"azure_csi_location is missing\"\n when: azure_csi_location is not defined or not azure_csi_location\n\n- name: Azure CSI Driver | check azure_csi_subnet_name value\n fail:\n msg: \"azure_csi_subnet_name is missing\"\n when: azure_csi_subnet_name is not defined or not azure_csi_subnet_name\n\n- name: Azure CSI Driver | check azure_csi_security_group_name value\n fail:\n msg: \"azure_csi_security_group_name is missing\"\n when: azure_csi_security_group_name is not defined or not azure_csi_security_group_name\n\n- name: Azure CSI Driver | check azure_csi_vnet_name value\n fail:\n msg: \"azure_csi_vnet_name is missing\"\n when: azure_csi_vnet_name is not defined or not azure_csi_vnet_name\n\n- name: Azure CSI Driver | check azure_csi_vnet_resource_group value\n fail:\n msg: \"azure_csi_vnet_resource_group is missing\"\n when: azure_csi_vnet_resource_group is not defined or not azure_csi_vnet_resource_group\n\n- name: \"Azure CSI Driver | check azure_csi_use_instance_metadata is a bool\"\n assert:\n that: azure_csi_use_instance_metadata | type_debug == 'bool'\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml", "content": "---\n- name: Azure CSI Driver | Check Azure credentials\n include_tasks: azure-credential-check.yml\n\n- name: Azure CSI Driver | Write Azure CSI cloud-config\n template:\n src: \"azure-csi-cloud-config.j2\"\n dest: \"{{ kube_config_dir }}/azure_csi_cloud_config\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Azure CSI Driver | Get base64 cloud-config\n slurp:\n src: \"{{ kube_config_dir }}/azure_csi_cloud_config\"\n register: cloud_config_secret\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Azure CSI Driver | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml}\n - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml}\n - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller-rbac.yml}\n - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller.yml}\n - {name: azure-csi-azuredisk-node-rbac, file: azure-csi-azuredisk-node-rbac.yml}\n - {name: azure-csi-azuredisk-node, file: azure-csi-azuredisk-node.yml}\n register: azure_csi_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Azure CSI Driver | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ azure_csi_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-azuredisk-controller-sa\n namespace: kube-system\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-external-provisioner-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"get\", \"list\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-csi-provisioner-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: azuredisk-external-provisioner-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-external-attacher-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"csi.storage.k8s.io\"]\n resources: [\"csinodeinfos\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments/status\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-csi-attacher-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: azuredisk-external-attacher-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-cluster-driver-registrar-role\nrules:\n - apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"create\", \"list\", \"watch\", \"delete\"]\n - apiGroups: [\"csi.storage.k8s.io\"]\n resources: [\"csidrivers\"]\n verbs: [\"create\", \"delete\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-csi-driver-registrar-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: azuredisk-cluster-driver-registrar-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-external-snapshotter-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents/status\"]\n verbs: [\"update\"]\n - apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"create\", \"list\", \"watch\", \"delete\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-csi-snapshotter-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: azuredisk-external-snapshotter-role\n apiGroup: rbac.authorization.k8s.io\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-external-resizer-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: azuredisk-csi-resizer-role\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: azuredisk-external-resizer-role\n apiGroup: rbac.authorization.k8s.io\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-azuredisk-controller-secret-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-azuredisk-controller-secret-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-azuredisk-controller-secret-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2", "content": "---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: csi-azuredisk-controller\n namespace: kube-system\nspec:\n replicas: {{ azure_csi_controller_replicas }}\n selector:\n matchLabels:\n app: csi-azuredisk-controller\n template:\n metadata:\n labels:\n app: csi-azuredisk-controller\n spec:\n hostNetwork: true\n serviceAccountName: csi-azuredisk-controller-sa\n nodeSelector:\n kubernetes.io/os: linux\n priorityClassName: system-cluster-critical\n tolerations:\n - key: \"node-role.kubernetes.io/control-plane\"\n effect: \"NoSchedule\"\n{% if azure_csi_controller_affinity %}\n affinity:\n {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }}\n{% endif %}\n containers:\n - name: csi-provisioner\n image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--feature-gates=Topology=true\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--v=2\"\n - \"--timeout=15s\"\n - \"--leader-election\"\n - \"--worker-threads=40\"\n - \"--extra-create-metadata=true\"\n - \"--strict-topology=true\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n resources:\n limits:\n memory: 500Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: csi-attacher\n image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"-v=2\"\n - \"-csi-address=$(ADDRESS)\"\n - \"-timeout=600s\"\n - \"-leader-election\"\n - \"-worker-threads=500\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n resources:\n limits:\n memory: 500Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: csi-snapshotter\n image: {{ azure_csi_image_repo }}/csi-snapshotter:{{ azure_csi_snapshotter_image_tag }}\n args:\n - \"-csi-address=$(ADDRESS)\"\n - \"-leader-election\"\n - \"-v=2\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n resources:\n limits:\n memory: 100Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: csi-resizer\n image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }}\n args:\n - \"-csi-address=$(ADDRESS)\"\n - \"-v=2\"\n - \"-leader-election\"\n - '-handle-volume-inuse-error=false'\n - \"-timeout=60s\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n resources:\n limits:\n memory: 500Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: liveness-probe\n image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}\n args:\n - --csi-address=/csi/csi.sock\n - --probe-timeout=3s\n - --health-port=29602\n - --v=2\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n resources:\n limits:\n memory: 100Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: azuredisk\n image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--v=5\"\n - \"--endpoint=$(CSI_ENDPOINT)\"\n - \"--metrics-address=0.0.0.0:29604\"\n - \"--disable-avset-nodes=true\"\n - \"--drivername=disk.csi.azure.com\"\n - \"--cloud-config-secret-name=cloud-config\"\n - \"--cloud-config-secret-namespace=kube-system\"\n ports:\n - containerPort: 29602\n name: healthz\n protocol: TCP\n - containerPort: 29604\n name: metrics\n protocol: TCP\n livenessProbe:\n failureThreshold: 5\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 30\n timeoutSeconds: 10\n periodSeconds: 30\n env:\n - name: AZURE_CREDENTIAL_FILE\n value: \"/etc/kubernetes/azure.json\"\n - name: CSI_ENDPOINT\n value: unix:///csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n - mountPath: /etc/kubernetes/\n name: azure-cred\n readOnly: true\n resources:\n limits:\n memory: 500Mi\n requests:\n cpu: 10m\n memory: 20Mi\n volumes:\n - name: socket-dir\n emptyDir: {}\n - name: azure-cred\n secret:\n secretName: cloud-config\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2", "content": "---\napiVersion: storage.k8s.io/v1\nkind: CSIDriver\nmetadata:\n name: disk.csi.azure.com\nspec:\n attachRequired: true\n podInfoOnMount: true\n volumeLifecycleModes: # added in Kubernetes 1.16\n - Persistent\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-azuredisk-node-sa\n namespace: kube-system\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-azuredisk-node-secret-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-azuredisk-node-secret-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-azuredisk-node-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-azuredisk-node-secret-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2", "content": "---\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: csi-azuredisk-node\n namespace: kube-system\nspec:\n updateStrategy:\n rollingUpdate:\n maxUnavailable: 1\n type: RollingUpdate\n selector:\n matchLabels:\n app: csi-azuredisk-node\n template:\n metadata:\n labels:\n app: csi-azuredisk-node\n spec:\n hostNetwork: true\n dnsPolicy: Default\n serviceAccountName: csi-azuredisk-node-sa\n nodeSelector:\n kubernetes.io/os: linux\n{% if azure_csi_node_affinity %}\n affinity:\n {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }}\n{% endif %}\n priorityClassName: system-node-critical\n tolerations:\n - operator: Exists\n containers:\n - name: liveness-probe\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --csi-address=/csi/csi.sock\n - --probe-timeout=3s\n - --health-port=29603\n - --v=2\n resources:\n limits:\n memory: 100Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: node-driver-registrar\n image: {{ azure_csi_image_repo }}/csi-node-driver-registrar:{{ azure_csi_node_registrar_image_tag }}\n args:\n - --csi-address=$(ADDRESS)\n - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)\n - --v=2\n livenessProbe:\n exec:\n command:\n - /csi-node-driver-registrar\n - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)\n - --mode=kubelet-registration-probe\n initialDelaySeconds: 30\n timeoutSeconds: 15\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n - name: DRIVER_REG_SOCK_PATH\n value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: registration-dir\n mountPath: /registration\n resources:\n limits:\n memory: 100Mi\n requests:\n cpu: 10m\n memory: 20Mi\n - name: azuredisk\n image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--v=5\"\n - \"--endpoint=$(CSI_ENDPOINT)\"\n - \"--nodeid=$(KUBE_NODE_NAME)\"\n - \"--metrics-address=0.0.0.0:29605\"\n - \"--enable-perf-optimization=true\"\n - \"--drivername=disk.csi.azure.com\"\n - \"--volume-attach-limit=-1\"\n - \"--cloud-config-secret-name=cloud-config\"\n - \"--cloud-config-secret-namespace=kube-system\"\n ports:\n - containerPort: 29603\n name: healthz\n protocol: TCP\n - containerPort: 29605\n name: metrics\n protocol: TCP\n livenessProbe:\n failureThreshold: 5\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 30\n timeoutSeconds: 10\n periodSeconds: 30\n env:\n - name: AZURE_CREDENTIAL_FILE\n value: \"/etc/kubernetes/azure.json\"\n - name: CSI_ENDPOINT\n value: unix:///csi/csi.sock\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: spec.nodeName\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n - mountPath: /var/lib/kubelet/\n mountPropagation: Bidirectional\n name: mountpoint-dir\n - mountPath: /etc/kubernetes/\n name: azure-cred\n - mountPath: /dev\n name: device-dir\n - mountPath: /sys/bus/scsi/devices\n name: sys-devices-dir\n - mountPath: /sys/class/scsi_host/\n name: scsi-host-dir\n resources:\n limits:\n memory: 200Mi\n requests:\n cpu: 10m\n memory: 20Mi\n volumes:\n - hostPath:\n path: /var/lib/kubelet/plugins/disk.csi.azure.com\n type: DirectoryOrCreate\n name: socket-dir\n - hostPath:\n path: /var/lib/kubelet/\n type: DirectoryOrCreate\n name: mountpoint-dir\n - hostPath:\n path: /var/lib/kubelet/plugins_registry/\n type: DirectoryOrCreate\n name: registration-dir\n - secret:\n defaultMode: 0644\n secretName: cloud-config\n name: azure-cred\n - hostPath:\n path: /dev\n type: Directory\n name: device-dir\n - hostPath:\n path: /sys/bus/scsi/devices\n type: Directory\n name: sys-devices-dir\n - hostPath:\n path: /sys/class/scsi_host/\n type: Directory\n name: scsi-host-dir\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2", "content": "kind: Secret\napiVersion: v1\nmetadata:\n name: cloud-config\n namespace: kube-system\ndata:\n azure.json: {{ cloud_config_secret.content }}\n" }, { "path": "roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2", "content": "{\n \"cloud\":\"AzurePublicCloud\",\n \"tenantId\": \"{{ azure_csi_tenant_id }}\",\n \"subscriptionId\": \"{{ azure_csi_subscription_id }}\",\n \"aadClientId\": \"{{ azure_csi_aad_client_id }}\",\n \"aadClientSecret\": \"{{ azure_csi_aad_client_secret }}\",\n \"location\": \"{{ azure_csi_location }}\",\n \"resourceGroup\": \"{{ azure_csi_resource_group }}\",\n \"vnetName\": \"{{ azure_csi_vnet_name }}\",\n \"vnetResourceGroup\": \"{{ azure_csi_vnet_resource_group }}\",\n \"subnetName\": \"{{ azure_csi_subnet_name }}\",\n \"securityGroupName\": \"{{ azure_csi_security_group_name }}\",\n \"useInstanceMetadata\": {{ azure_csi_use_instance_metadata }},\n}\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml", "content": "---\ncinder_csi_attacher_image_tag: \"v4.4.2\"\ncinder_csi_provisioner_image_tag: \"v3.6.2\"\ncinder_csi_snapshotter_image_tag: \"v6.3.2\"\ncinder_csi_resizer_image_tag: \"v1.9.2\"\ncinder_csi_livenessprobe_image_tag: \"v2.11.0\"\n\n# To access Cinder, the CSI controller will need credentials to access\n# openstack apis. Per default this values will be\n# read from the environment.\ncinder_auth_url: \"{{ lookup('env', 'OS_AUTH_URL') }}\"\ncinder_username: \"{{ lookup('env', 'OS_USERNAME') }}\"\ncinder_password: \"{{ lookup('env', 'OS_PASSWORD') }}\"\ncinder_application_credential_id: \"{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}\"\ncinder_application_credential_name: \"{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}\"\ncinder_application_credential_secret: \"{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}\"\ncinder_region: \"{{ lookup('env', 'OS_REGION_NAME') }}\"\ncinder_tenant_id: \"{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}\"\ncinder_tenant_name: \"{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}\"\ncinder_domain_name: \"{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}\"\ncinder_domain_id: \"{{ lookup('env', 'OS_USER_DOMAIN_ID') }}\"\ncinder_cacert: \"{{ lookup('env', 'OS_CACERT') }}\"\n\n# For now, only Cinder v3 is supported in Cinder CSI driver\ncinder_blockstorage_version: \"v3\"\ncinder_csi_controller_replicas: 1\n\n# Optional. Set to true, to rescan block device and verify its size before expanding\n# the filesystem.\n# Not all hypervizors have a /sys/class/block/XXX/device/rescan location, therefore if\n# you enable this option and your hypervizor doesn't support this, you'll get a warning\n# log on resize event. It is recommended to disable this option in this case.\n# Defaults to false\n# cinder_csi_rescan_on_resize: true\n\ncinder_tolerations: []\n\n## Dictionaries of extra arguments to add to the cinder CSI plugin containers\n## Format:\n## cinder_csi_attacher_extra_args:\n## arg1: \"value1\"\n## arg2: \"value2\"\ncinder_csi_attacher_extra_args: {}\ncinder_csi_provisioner_extra_args: {}\ncinder_csi_snapshotter_extra_args: {}\ncinder_csi_resizer_extra_args: {}\ncinder_csi_plugin_extra_args: {}\ncinder_liveness_probe_extra_args: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml", "content": "---\n- name: Cinder CSI Driver | check cinder_auth_url value\n fail:\n msg: \"cinder_auth_url is missing\"\n when: cinder_auth_url is not defined or not cinder_auth_url\n\n- name: Cinder CSI Driver | check cinder_username value cinder_application_credential_name value\n fail:\n msg: \"you must either set cinder_username or cinder_application_credential_name\"\n when:\n - cinder_username is not defined or not cinder_username\n - cinder_application_credential_name is not defined or not cinder_application_credential_name\n\n- name: Cinder CSI Driver | check cinder_application_credential_id value\n fail:\n msg: \"cinder_application_credential_id is missing\"\n when:\n - cinder_application_credential_name is defined\n - cinder_application_credential_name | length > 0\n - cinder_application_credential_id is not defined or not cinder_application_credential_id\n\n- name: Cinder CSI Driver | check cinder_application_credential_secret value\n fail:\n msg: \"cinder_application_credential_secret is missing\"\n when:\n - cinder_application_credential_name is defined\n - cinder_application_credential_name | length > 0\n - cinder_application_credential_secret is not defined or not cinder_application_credential_secret\n\n- name: Cinder CSI Driver | check cinder_password value\n fail:\n msg: \"cinder_password is missing\"\n when:\n - cinder_username is defined\n - cinder_username | length > 0\n - cinder_application_credential_name is not defined or not cinder_application_credential_name\n - cinder_application_credential_secret is not defined or not cinder_application_credential_secret\n - cinder_password is not defined or not cinder_password\n\n- name: Cinder CSI Driver | check cinder_region value\n fail:\n msg: \"cinder_region is missing\"\n when: cinder_region is not defined or not cinder_region\n\n- name: Cinder CSI Driver | check cinder_tenant_id value\n fail:\n msg: \"one of cinder_tenant_id or cinder_tenant_name must be specified\"\n when:\n - cinder_tenant_id is not defined or not cinder_tenant_id\n - cinder_tenant_name is not defined or not cinder_tenant_name\n - cinder_application_credential_name is not defined or not cinder_application_credential_name\n\n- name: Cinder CSI Driver | check cinder_domain_id value\n fail:\n msg: \"one of cinder_domain_id or cinder_domain_name must be specified\"\n when:\n - cinder_domain_id is not defined or not cinder_domain_id\n - cinder_domain_name is not defined or not cinder_domain_name\n - cinder_application_credential_name is not defined or not cinder_application_credential_name\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml", "content": "---\n# include to workaround mitogen issue\n# https://github.com/dw/mitogen/issues/663\n\n- name: Cinder CSI Driver | Write cacert file\n copy:\n src: \"{{ cinder_cacert }}\"\n dest: \"{{ kube_config_dir }}/cinder-cacert.pem\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n delegate_to: \"{{ delegate_host_to_write_cacert }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml", "content": "---\n- name: Cinder CSI Driver | Check Cinder credentials\n include_tasks: cinder-credential-check.yml\n\n- name: Cinder CSI Driver | Write cacert file\n include_tasks: cinder-write-cacert.yml\n run_once: true\n loop: \"{{ groups['k8s_cluster'] }}\"\n loop_control:\n loop_var: delegate_host_to_write_cacert\n when:\n - ('k8s_cluster' in group_names)\n - cinder_cacert is defined\n - cinder_cacert | length > 0\n\n- name: Cinder CSI Driver | Write Cinder cloud-config\n template:\n src: \"cinder-csi-cloud-config.j2\"\n dest: \"{{ kube_config_dir }}/cinder_cloud_config\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cinder CSI Driver | Get base64 cloud-config\n slurp:\n src: \"{{ kube_config_dir }}/cinder_cloud_config\"\n register: cloud_config_secret\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cinder CSI Driver | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cinder-csi-driver, file: cinder-csi-driver.yml}\n - {name: cinder-csi-cloud-config-secret, file: cinder-csi-cloud-config-secret.yml}\n - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin-rbac.yml}\n - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin.yml}\n - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin-rbac.yml}\n - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin.yml}\n - {name: cinder-csi-poddisruptionbudget, file: cinder-csi-poddisruptionbudget.yml}\n register: cinder_csi_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cinder CSI Driver | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ cinder_csi_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2", "content": "# This YAML file contains secret objects,\n# which are necessary to run csi cinder plugin.\n\nkind: Secret\napiVersion: v1\nmetadata:\n name: cloud-config\n namespace: kube-system\ndata:\n cloud.conf: {{ cloud_config_secret.content }}\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2", "content": "[Global]\nauth-url=\"{{ cinder_auth_url }}\"\n{% if cinder_application_credential_id|length == 0 and cinder_application_credential_name|length == 0 %}\nusername=\"{{ cinder_username }}\"\npassword=\"{{ cinder_password }}\"\n{% endif %}\n{% if cinder_application_credential_id|length > 0 %}\napplication-credential-id={{ cinder_application_credential_id }}\n{% endif %}\n{% if cinder_application_credential_name|length > 0 %}\napplication-credential-name={{ cinder_application_credential_name }}\n{% endif %}\n{% if cinder_application_credential_secret|length > 0 %}\napplication-credential-secret={{ cinder_application_credential_secret }}\n{% endif %}\nregion=\"{{ cinder_region }}\"\n{% if cinder_tenant_id|length > 0 %}\ntenant-id=\"{{ cinder_tenant_id }}\"\n{% endif %}\n{% if cinder_tenant_name|length > 0 %}\ntenant-name=\"{{ cinder_tenant_name }}\"\n{% endif %}\n{% if cinder_domain_name|length > 0 %}\ndomain-name=\"{{ cinder_domain_name }}\"\n{% elif cinder_domain_id|length > 0 %}\ndomain-id =\"{{ cinder_domain_id }}\"\n{% endif %}\n{% if cinder_cacert|length > 0 %}\nca-file=\"{{ kube_config_dir }}/cinder-cacert.pem\"\n{% endif %}\n\n[BlockStorage]\n{% if cinder_blockstorage_version is defined %}\nbs-version={{ cinder_blockstorage_version }}\n{% endif %}\n{% if cinder_csi_ignore_volume_az is defined %}\nignore-volume-az={{ cinder_csi_ignore_volume_az | bool }}\n{% endif %}\n{% if node_volume_attach_limit is defined and node_volume_attach_limit != \"\" %}\nnode-volume-attach-limit=\"{{ node_volume_attach_limit }}\"\n{% endif %}\n{% if cinder_csi_rescan_on_resize is defined %}\nrescan-on-resize={{ cinder_csi_rescan_on_resize | bool }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2", "content": "# This YAML file contains RBAC API objects,\n# which are necessary to run csi controller plugin\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-cinder-controller-sa\n namespace: kube-system\n\n---\n# external attacher\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-attacher-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments/status\"]\n verbs: [\"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-attacher-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-cinder-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-attacher-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n# external Provisioner\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-provisioner-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-provisioner-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-cinder-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-provisioner-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n# external snapshotter\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-snapshotter-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"patch\", \"delete\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents/status\"]\n verbs: [\"update\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-snapshotter-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-cinder-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-snapshotter-role\n apiGroup: rbac.authorization.k8s.io\n---\n\n# External Resizer\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-resizer-role\nrules:\n # The following rule should be uncommented for plugins that require secrets\n # for provisioning.\n # - apiGroups: [\"\"]\n # resources: [\"secrets\"]\n # verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims/status\"]\n verbs: [\"patch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-resizer-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-cinder-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-resizer-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2", "content": "# This YAML file contains CSI Controller Plugin Sidecars\n# external-attacher, external-provisioner, external-snapshotter\n# external-resize, liveness-probe\n\n---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: csi-cinder-controllerplugin\n namespace: kube-system\nspec:\n replicas: {{ cinder_csi_controller_replicas }}\n selector:\n matchLabels:\n app: csi-cinder-controllerplugin\n template:\n metadata:\n labels:\n app: csi-cinder-controllerplugin\n spec:\n serviceAccount: csi-cinder-controller-sa\n containers:\n - name: csi-attacher\n image: {{ csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--timeout=3m\"\n{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}\n - --leader-election=true\n{% endif %}\n - \"--default-fstype=ext4\"\n{% for key, value in cinder_csi_attacher_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-provisioner\n image: {{ csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--timeout=3m\"\n - \"--default-fstype=ext4\"\n - \"--extra-create-metadata\"\n{% if cinder_topology is defined and cinder_topology %}\n - --feature-gates=Topology=true\n{% endif %}\n{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}\n - \"--leader-election=true\"\n{% endif %}\n{% for key, value in cinder_csi_provisioner_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-snapshotter\n image: {{ csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--timeout=3m\"\n - \"--extra-create-metadata\"\n{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}\n - --leader-election=true\n{% endif %}\n{% for key, value in cinder_csi_snapshotter_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - mountPath: /var/lib/csi/sockets/pluginproxy/\n name: socket-dir\n - name: csi-resizer\n image: {{ csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--timeout=3m\"\n - \"--handle-volume-inuse-error=false\"\n{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}\n - --leader-election=true\n{% endif %}\n{% for key, value in cinder_csi_resizer_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: liveness-probe\n image: {{ csi_livenessprobe_image_repo }}:{{ cinder_csi_livenessprobe_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n{% for key, value in cinder_liveness_probe_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n volumeMounts:\n - mountPath: /var/lib/csi/sockets/pluginproxy/\n name: socket-dir\n - name: cinder-csi-plugin\n image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - /bin/cinder-csi-plugin\n - \"--endpoint=$(CSI_ENDPOINT)\"\n - \"--cloud-config=$(CLOUD_CONFIG)\"\n - \"--cluster=$(CLUSTER_NAME)\"\n{% for key, value in cinder_csi_plugin_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n env:\n - name: CSI_ENDPOINT\n value: unix://csi/csi.sock\n - name: CLOUD_CONFIG\n value: /etc/config/cloud.conf\n - name: CLUSTER_NAME\n value: {{ cluster_name }}\n ports:\n - containerPort: 9808\n name: healthz\n protocol: TCP\n livenessProbe:\n failureThreshold: 5\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 10\n timeoutSeconds: 10\n periodSeconds: 60\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: secret-cinderplugin\n mountPath: /etc/config\n readOnly: true\n - name: ca-certs\n mountPath: /etc/ssl/certs\n readOnly: true\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n mountPath: {{ dir }}\n readOnly: true\n{% endfor %}\n{% endif %}\n{% if cinder_cacert is defined and cinder_cacert != \"\" %}\n - name: cinder-cacert\n mountPath: {{ kube_config_dir }}/cinder-cacert.pem\n readOnly: true\n{% endif %}\n volumes:\n - name: socket-dir\n emptyDir:\n - name: secret-cinderplugin\n secret:\n secretName: cloud-config\n - name: ca-certs\n hostPath:\n path: /etc/ssl/certs\n type: DirectoryOrCreate\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n hostPath:\n path: {{ dir }}\n type: DirectoryOrCreate\n{% endfor %}\n{% endif %}\n{% if cinder_cacert is defined and cinder_cacert != \"\" %}\n - name: cinder-cacert\n hostPath:\n path: {{ kube_config_dir }}/cinder-cacert.pem\n type: FileOrCreate\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2", "content": "apiVersion: storage.k8s.io/v1\nkind: CSIDriver\nmetadata:\n name: cinder.csi.openstack.org\nspec:\n attachRequired: true\n podInfoOnMount: true\n volumeLifecycleModes:\n - Persistent\n - Ephemeral\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2", "content": "# This YAML defines all API objects to create RBAC roles for csi node plugin.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-cinder-node-sa\n namespace: kube-system\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-nodeplugin-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-nodeplugin-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-cinder-node-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-nodeplugin-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2", "content": "# This YAML file contains driver-registrar & csi driver nodeplugin API objects,\n# which are necessary to run csi nodeplugin for cinder.\n\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: csi-cinder-nodeplugin\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n app: csi-cinder-nodeplugin\n template:\n metadata:\n labels:\n app: csi-cinder-nodeplugin\n spec:\n tolerations:\n - operator: Exists\n serviceAccountName: csi-cinder-node-sa\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n containers:\n - name: node-driver-registrar\n image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n - name: DRIVER_REG_SOCK_PATH\n value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: registration-dir\n mountPath: /registration\n - name: liveness-probe\n image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}\n args:\n - \"--csi-address=/csi/csi.sock\"\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: cinder-csi-plugin\n securityContext:\n privileged: true\n capabilities:\n add: [\"SYS_ADMIN\"]\n allowPrivilegeEscalation: true\n image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - /bin/cinder-csi-plugin\n - \"--endpoint=$(CSI_ENDPOINT)\"\n - \"--cloud-config=$(CLOUD_CONFIG)\"\n env:\n - name: CSI_ENDPOINT\n value: unix://csi/csi.sock\n - name: CLOUD_CONFIG\n value: /etc/config/cloud.conf\n ports:\n - containerPort: 9808\n name: healthz\n protocol: TCP\n livenessProbe:\n failureThreshold: 5\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 10\n timeoutSeconds: 3\n periodSeconds: 10\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: kubelet-dir\n mountPath: /var/lib/kubelet\n mountPropagation: \"Bidirectional\"\n - name: pods-probe-dir\n mountPath: /dev\n mountPropagation: \"HostToContainer\"\n - name: secret-cinderplugin\n mountPath: /etc/config\n readOnly: true\n - name: ca-certs\n mountPath: /etc/ssl/certs\n readOnly: true\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n mountPath: {{ dir }}\n readOnly: true\n{% endfor %}\n{% endif %}\n{% if cinder_cacert is defined and cinder_cacert != \"\" %}\n - name: cinder-cacert\n mountPath: {{ kube_config_dir }}/cinder-cacert.pem\n readOnly: true\n{% endif %}\n volumes:\n - name: socket-dir\n hostPath:\n path: /var/lib/kubelet/plugins/cinder.csi.openstack.org\n type: DirectoryOrCreate\n - name: registration-dir\n hostPath:\n path: /var/lib/kubelet/plugins_registry/\n type: Directory\n - name: kubelet-dir\n hostPath:\n path: /var/lib/kubelet\n type: Directory\n - name: pods-probe-dir\n hostPath:\n path: /dev\n type: Directory\n - name: secret-cinderplugin\n secret:\n secretName: cloud-config\n - name: ca-certs\n hostPath:\n path: /etc/ssl/certs\n type: DirectoryOrCreate\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n hostPath:\n path: {{ dir }}\n type: DirectoryOrCreate\n{% endfor %}\n{% endif %}\n{% if cinder_cacert is defined and cinder_cacert != \"\" %}\n - name: cinder-cacert\n hostPath:\n path: {{ kube_config_dir }}/cinder-cacert.pem\n type: FileOrCreate\n{% endif %}\n{% if cinder_tolerations %}\n tolerations:\n {{ cinder_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2", "content": "apiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: cinder-csi-pdb\n namespace: kube-system\nspec:\n{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}\n minAvailable: 1\n{% else %}\n minAvailable: 0\n{% endif %}\n selector:\n matchLabels:\n app: csi-cinder-controllerplugin\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml", "content": "---\n- name: CSI CRD | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: volumegroupsnapshotclasses, file: volumegroupsnapshotclasses.yml}\n - {name: volumegroupsnapshotcontents, file: volumegroupsnapshotcontents.yml}\n - {name: volumegroupsnapshots, file: volumegroupsnapshots.yml}\n - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml}\n - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml}\n - {name: volumesnapshots, file: volumesnapshots.yml}\n register: csi_crd_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: CSI CRD | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n wait: true\n with_items:\n - \"{{ csi_crd_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/templates/volumegroupsnapshotclasses.yml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: \"https://github.com/kubernetes-csi/external-snapshotter/pull/1150\"\n controller-gen.kubebuilder.io/version: v0.15.0\n name: volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io\nspec:\n group: groupsnapshot.storage.k8s.io\n names:\n kind: VolumeGroupSnapshotClass\n listKind: VolumeGroupSnapshotClassList\n plural: volumegroupsnapshotclasses\n shortNames:\n - vgsclass\n - vgsclasses\n singular: volumegroupsnapshotclass\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - jsonPath: .driver\n name: Driver\n type: string\n - description: Determines whether a VolumeGroupSnapshotContent created through\n the VolumeGroupSnapshotClass should be deleted when its bound VolumeGroupSnapshot\n is deleted.\n jsonPath: .deletionPolicy\n name: DeletionPolicy\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: |-\n VolumeGroupSnapshotClass specifies parameters that a underlying storage system\n uses when creating a volume group snapshot. A specific VolumeGroupSnapshotClass\n is used by specifying its name in a VolumeGroupSnapshot object.\n VolumeGroupSnapshotClasses are non-namespaced.\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n deletionPolicy:\n description: |-\n DeletionPolicy determines whether a VolumeGroupSnapshotContent created\n through the VolumeGroupSnapshotClass should be deleted when its bound\n VolumeGroupSnapshot is deleted.\n Supported values are \"Retain\" and \"Delete\".\n \"Retain\" means that the VolumeGroupSnapshotContent and its physical group\n snapshot on underlying storage system are kept.\n \"Delete\" means that the VolumeGroupSnapshotContent and its physical group\n snapshot on underlying storage system are deleted.\n Required.\n enum:\n - Delete\n - Retain\n type: string\n driver:\n description: |-\n Driver is the name of the storage driver expected to handle this VolumeGroupSnapshotClass.\n Required.\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n parameters:\n additionalProperties:\n type: string\n description: |-\n Parameters is a key-value map with storage driver specific parameters for\n creating group snapshots.\n These values are opaque to Kubernetes and are passed directly to the driver.\n type: object\n required:\n - deletionPolicy\n - driver\n type: object\n served: true\n storage: true\n subresources: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/templates/volumegroupsnapshotcontents.yml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: \"https://github.com/kubernetes-csi/external-snapshotter/pull/1150\"\n controller-gen.kubebuilder.io/version: v0.15.0\n name: volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io\nspec:\n group: groupsnapshot.storage.k8s.io\n names:\n kind: VolumeGroupSnapshotContent\n listKind: VolumeGroupSnapshotContentList\n plural: volumegroupsnapshotcontents\n shortNames:\n - vgsc\n - vgscs\n singular: volumegroupsnapshotcontent\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - description: Indicates if all the individual snapshots in the group are ready\n to be used to restore a group of volumes.\n jsonPath: .status.readyToUse\n name: ReadyToUse\n type: boolean\n - description: Determines whether this VolumeGroupSnapshotContent and its physical\n group snapshot on the underlying storage system should be deleted when its\n bound VolumeGroupSnapshot is deleted.\n jsonPath: .spec.deletionPolicy\n name: DeletionPolicy\n type: string\n - description: Name of the CSI driver used to create the physical group snapshot\n on the underlying storage system.\n jsonPath: .spec.driver\n name: Driver\n type: string\n - description: Name of the VolumeGroupSnapshotClass from which this group snapshot\n was (or will be) created.\n jsonPath: .spec.volumeGroupSnapshotClassName\n name: VolumeGroupSnapshotClass\n type: string\n - description: Namespace of the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent\n object is bound.\n jsonPath: .spec.volumeGroupSnapshotRef.namespace\n name: VolumeGroupSnapshotNamespace\n type: string\n - description: Name of the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent\n object is bound.\n jsonPath: .spec.volumeGroupSnapshotRef.name\n name: VolumeGroupSnapshot\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: |-\n VolumeGroupSnapshotContent represents the actual \"on-disk\" group snapshot object\n in the underlying storage system\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: |-\n Spec defines properties of a VolumeGroupSnapshotContent created by the underlying storage system.\n Required.\n properties:\n deletionPolicy:\n description: |-\n DeletionPolicy determines whether this VolumeGroupSnapshotContent and the\n physical group snapshot on the underlying storage system should be deleted\n when the bound VolumeGroupSnapshot is deleted.\n Supported values are \"Retain\" and \"Delete\".\n \"Retain\" means that the VolumeGroupSnapshotContent and its physical group\n snapshot on underlying storage system are kept.\n \"Delete\" means that the VolumeGroupSnapshotContent and its physical group\n snapshot on underlying storage system are deleted.\n For dynamically provisioned group snapshots, this field will automatically\n be filled in by the CSI snapshotter sidecar with the \"DeletionPolicy\" field\n defined in the corresponding VolumeGroupSnapshotClass.\n For pre-existing snapshots, users MUST specify this field when creating the\n VolumeGroupSnapshotContent object.\n Required.\n enum:\n - Delete\n - Retain\n type: string\n driver:\n description: |-\n Driver is the name of the CSI driver used to create the physical group snapshot on\n the underlying storage system.\n This MUST be the same as the name returned by the CSI GetPluginName() call for\n that driver.\n Required.\n type: string\n source:\n description: |-\n Source specifies whether the snapshot is (or should be) dynamically provisioned\n or already exists, and just requires a Kubernetes object representation.\n This field is immutable after creation.\n Required.\n properties:\n groupSnapshotHandles:\n description: |-\n GroupSnapshotHandles specifies the CSI \"group_snapshot_id\" of a pre-existing\n group snapshot and a list of CSI \"snapshot_id\" of pre-existing snapshots\n on the underlying storage system for which a Kubernetes object\n representation was (or should be) created.\n This field is immutable.\n properties:\n volumeGroupSnapshotHandle:\n description: |-\n VolumeGroupSnapshotHandle specifies the CSI \"group_snapshot_id\" of a pre-existing\n group snapshot on the underlying storage system for which a Kubernetes object\n representation was (or should be) created.\n This field is immutable.\n Required.\n type: string\n volumeSnapshotHandles:\n description: |-\n VolumeSnapshotHandles is a list of CSI \"snapshot_id\" of pre-existing\n snapshots on the underlying storage system for which Kubernetes objects\n representation were (or should be) created.\n This field is immutable.\n Required.\n items:\n type: string\n type: array\n required:\n - volumeGroupSnapshotHandle\n - volumeSnapshotHandles\n type: object\n x-kubernetes-validations:\n - message: groupSnapshotHandles is immutable\n rule: self == oldSelf\n volumeHandles:\n description: |-\n VolumeHandles is a list of volume handles on the backend to be snapshotted\n together. It is specified for dynamic provisioning of the VolumeGroupSnapshot.\n This field is immutable.\n items:\n type: string\n type: array\n x-kubernetes-validations:\n - message: volumeHandles is immutable\n rule: self == oldSelf\n type: object\n x-kubernetes-validations:\n - message: volumeHandles is required once set\n rule: '!has(oldSelf.volumeHandles) || has(self.volumeHandles)'\n - message: groupSnapshotHandles is required once set\n rule: '!has(oldSelf.groupSnapshotHandles) || has(self.groupSnapshotHandles)'\n - message: exactly one of volumeHandles and groupSnapshotHandles must\n be set\n rule: (has(self.volumeHandles) && !has(self.groupSnapshotHandles))\n || (!has(self.volumeHandles) && has(self.groupSnapshotHandles))\n volumeGroupSnapshotClassName:\n description: |-\n VolumeGroupSnapshotClassName is the name of the VolumeGroupSnapshotClass from\n which this group snapshot was (or will be) created.\n Note that after provisioning, the VolumeGroupSnapshotClass may be deleted or\n recreated with different set of values, and as such, should not be referenced\n post-snapshot creation.\n For dynamic provisioning, this field must be set.\n This field may be unset for pre-provisioned snapshots.\n type: string\n volumeGroupSnapshotRef:\n description: |-\n VolumeGroupSnapshotRef specifies the VolumeGroupSnapshot object to which this\n VolumeGroupSnapshotContent object is bound.\n VolumeGroupSnapshot.Spec.VolumeGroupSnapshotContentName field must reference to\n this VolumeGroupSnapshotContent's name for the bidirectional binding to be valid.\n For a pre-existing VolumeGroupSnapshotContent object, name and namespace of the\n VolumeGroupSnapshot object MUST be provided for binding to happen.\n This field is immutable after creation.\n Required.\n properties:\n apiVersion:\n description: API version of the referent.\n type: string\n fieldPath:\n description: |-\n If referring to a piece of an object instead of an entire object, this string\n should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].\n For example, if the object reference is to a container within a pod, this would take on a value like:\n \"spec.containers{name}\" (where \"name\" refers to the name of the container that triggered\n the event) or if no container name is specified \"spec.containers[2]\" (container with\n index 2 in this pod). This syntax is chosen only to have some well-defined way of\n referencing a part of an object.\n TODO: this design is not final and this field is subject to change in the future.\n type: string\n kind:\n description: |-\n Kind of the referent.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n name:\n description: |-\n Name of the referent.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n namespace:\n description: |-\n Namespace of the referent.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/\n type: string\n resourceVersion:\n description: |-\n Specific resourceVersion to which this reference is made, if any.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency\n type: string\n uid:\n description: |-\n UID of the referent.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids\n type: string\n type: object\n x-kubernetes-map-type: atomic\n x-kubernetes-validations:\n - message: both volumeGroupSnapshotRef.name and volumeGroupSnapshotRef.namespace\n must be set\n rule: has(self.name) && has(self.__namespace__)\n required:\n - deletionPolicy\n - driver\n - source\n - volumeGroupSnapshotRef\n type: object\n status:\n description: status represents the current information of a group snapshot.\n properties:\n creationTime:\n description: |-\n CreationTime is the timestamp when the point-in-time group snapshot is taken\n by the underlying storage system.\n If not specified, it indicates the creation time is unknown.\n If not specified, it means the readiness of a group snapshot is unknown.\n The format of this field is a Unix nanoseconds time encoded as an int64.\n On Unix, the command date +%s%N returns the current time in nanoseconds\n since 1970-01-01 00:00:00 UTC.\n This field is the source for the CreationTime field in VolumeGroupSnapshotStatus\n format: date-time\n type: string\n error:\n description: |-\n Error is the last observed error during group snapshot creation, if any.\n Upon success after retry, this error field will be cleared.\n properties:\n message:\n description: |-\n message is a string detailing the encountered error during snapshot\n creation if specified.\n NOTE: message may be logged, and it should not contain sensitive\n information.\n type: string\n time:\n description: time is the timestamp when the error was encountered.\n format: date-time\n type: string\n type: object\n readyToUse:\n description: |-\n ReadyToUse indicates if all the individual snapshots in the group are ready to be\n used to restore a group of volumes.\n ReadyToUse becomes true when ReadyToUse of all individual snapshots become true.\n type: boolean\n volumeGroupSnapshotHandle:\n description: |-\n VolumeGroupSnapshotHandle is a unique id returned by the CSI driver\n to identify the VolumeGroupSnapshot on the storage system.\n If a storage system does not provide such an id, the\n CSI driver can choose to return the VolumeGroupSnapshot name.\n type: string\n volumeSnapshotHandlePairList:\n description: |-\n VolumeSnapshotHandlePairList is a list of CSI \"volume_id\" and \"snapshot_id\"\n pair returned by the CSI driver to identify snapshots and their source volumes\n on the storage system.\n items:\n description: VolumeSnapshotHandlePair defines a pair of a source\n volume handle and a snapshot handle\n properties:\n snapshotHandle:\n description: |-\n SnapshotHandle is a unique id returned by the CSI driver to identify a volume\n snapshot on the storage system\n Required.\n type: string\n volumeHandle:\n description: |-\n VolumeHandle is a unique id returned by the CSI driver to identify a volume\n on the storage system\n Required.\n type: string\n required:\n - snapshotHandle\n - volumeHandle\n type: object\n type: array\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/templates/volumegroupsnapshots.yml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: \"https://github.com/kubernetes-csi/external-snapshotter/pull/1150\"\n controller-gen.kubebuilder.io/version: v0.15.0\n name: volumegroupsnapshots.groupsnapshot.storage.k8s.io\nspec:\n group: groupsnapshot.storage.k8s.io\n names:\n kind: VolumeGroupSnapshot\n listKind: VolumeGroupSnapshotList\n plural: volumegroupsnapshots\n shortNames:\n - vgs\n singular: volumegroupsnapshot\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - description: Indicates if all the individual snapshots in the group are ready\n to be used to restore a group of volumes.\n jsonPath: .status.readyToUse\n name: ReadyToUse\n type: boolean\n - description: The name of the VolumeGroupSnapshotClass requested by the VolumeGroupSnapshot.\n jsonPath: .spec.volumeGroupSnapshotClassName\n name: VolumeGroupSnapshotClass\n type: string\n - description: Name of the VolumeGroupSnapshotContent object to which the VolumeGroupSnapshot\n object intends to bind to. Please note that verification of binding actually\n requires checking both VolumeGroupSnapshot and VolumeGroupSnapshotContent\n to ensure both are pointing at each other. Binding MUST be verified prior\n to usage of this object.\n jsonPath: .status.boundVolumeGroupSnapshotContentName\n name: VolumeGroupSnapshotContent\n type: string\n - description: Timestamp when the point-in-time group snapshot was taken by the\n underlying storage system.\n jsonPath: .status.creationTime\n name: CreationTime\n type: date\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: |-\n VolumeGroupSnapshot is a user's request for creating either a point-in-time\n group snapshot or binding to a pre-existing group snapshot.\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: |-\n Spec defines the desired characteristics of a group snapshot requested by a user.\n Required.\n properties:\n source:\n description: |-\n Source specifies where a group snapshot will be created from.\n This field is immutable after creation.\n Required.\n properties:\n selector:\n description: |-\n Selector is a label query over persistent volume claims that are to be\n grouped together for snapshotting.\n This labelSelector will be used to match the label added to a PVC.\n If the label is added or removed to a volume after a group snapshot\n is created, the existing group snapshots won't be modified.\n Once a VolumeGroupSnapshotContent is created and the sidecar starts to process\n it, the volume list will not change with retries.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n x-kubernetes-validations:\n - message: selector is immutable\n rule: self == oldSelf\n volumeGroupSnapshotContentName:\n description: |-\n VolumeGroupSnapshotContentName specifies the name of a pre-existing VolumeGroupSnapshotContent\n object representing an existing volume group snapshot.\n This field should be set if the volume group snapshot already exists and\n only needs a representation in Kubernetes.\n This field is immutable.\n type: string\n x-kubernetes-validations:\n - message: volumeGroupSnapshotContentName is immutable\n rule: self == oldSelf\n type: object\n x-kubernetes-validations:\n - message: selector is required once set\n rule: '!has(oldSelf.selector) || has(self.selector)'\n - message: volumeGroupSnapshotContentName is required once set\n rule: '!has(oldSelf.volumeGroupSnapshotContentName) || has(self.volumeGroupSnapshotContentName)'\n - message: exactly one of selector and volumeGroupSnapshotContentName\n must be set\n rule: (has(self.selector) && !has(self.volumeGroupSnapshotContentName))\n || (!has(self.selector) && has(self.volumeGroupSnapshotContentName))\n volumeGroupSnapshotClassName:\n description: |-\n VolumeGroupSnapshotClassName is the name of the VolumeGroupSnapshotClass\n requested by the VolumeGroupSnapshot.\n VolumeGroupSnapshotClassName may be left nil to indicate that the default\n class will be used.\n Empty string is not allowed for this field.\n type: string\n x-kubernetes-validations:\n - message: volumeGroupSnapshotClassName must not be the empty string\n when set\n rule: size(self) > 0\n required:\n - source\n type: object\n status:\n description: |-\n Status represents the current information of a group snapshot.\n Consumers must verify binding between VolumeGroupSnapshot and\n VolumeGroupSnapshotContent objects is successful (by validating that both\n VolumeGroupSnapshot and VolumeGroupSnapshotContent point to each other) before\n using this object.\n properties:\n boundVolumeGroupSnapshotContentName:\n description: |-\n BoundVolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent\n object to which this VolumeGroupSnapshot object intends to bind to.\n If not specified, it indicates that the VolumeGroupSnapshot object has not\n been successfully bound to a VolumeGroupSnapshotContent object yet.\n NOTE: To avoid possible security issues, consumers must verify binding between\n VolumeGroupSnapshot and VolumeGroupSnapshotContent objects is successful\n (by validating that both VolumeGroupSnapshot and VolumeGroupSnapshotContent\n point at each other) before using this object.\n type: string\n creationTime:\n description: |-\n CreationTime is the timestamp when the point-in-time group snapshot is taken\n by the underlying storage system.\n If not specified, it may indicate that the creation time of the group snapshot\n is unknown.\n The format of this field is a Unix nanoseconds time encoded as an int64.\n On Unix, the command date +%s%N returns the current time in nanoseconds\n since 1970-01-01 00:00:00 UTC.\n This field is updated based on the CreationTime field in VolumeGroupSnapshotContentStatus\n format: date-time\n type: string\n error:\n description: |-\n Error is the last observed error during group snapshot creation, if any.\n This field could be helpful to upper level controllers (i.e., application\n controller) to decide whether they should continue on waiting for the group\n snapshot to be created based on the type of error reported.\n The snapshot controller will keep retrying when an error occurs during the\n group snapshot creation. Upon success, this error field will be cleared.\n properties:\n message:\n description: |-\n message is a string detailing the encountered error during snapshot\n creation if specified.\n NOTE: message may be logged, and it should not contain sensitive\n information.\n type: string\n time:\n description: time is the timestamp when the error was encountered.\n format: date-time\n type: string\n type: object\n readyToUse:\n description: |-\n ReadyToUse indicates if all the individual snapshots in the group are ready\n to be used to restore a group of volumes.\n ReadyToUse becomes true when ReadyToUse of all individual snapshots become true.\n If not specified, it means the readiness of a group snapshot is unknown.\n type: boolean\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: \"https://github.com/kubernetes-csi/external-snapshotter/pull/814\"\n controller-gen.kubebuilder.io/version: v0.15.0\n name: volumesnapshotclasses.snapshot.storage.k8s.io\nspec:\n group: snapshot.storage.k8s.io\n names:\n kind: VolumeSnapshotClass\n listKind: VolumeSnapshotClassList\n plural: volumesnapshotclasses\n shortNames:\n - vsclass\n - vsclasses\n singular: volumesnapshotclass\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - jsonPath: .driver\n name: Driver\n type: string\n - description: Determines whether a VolumeSnapshotContent created through the\n VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.\n jsonPath: .deletionPolicy\n name: DeletionPolicy\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1\n schema:\n openAPIV3Schema:\n description: |-\n VolumeSnapshotClass specifies parameters that a underlying storage system uses when\n creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its\n name in a VolumeSnapshot object.\n VolumeSnapshotClasses are non-namespaced\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n deletionPolicy:\n description: |-\n deletionPolicy determines whether a VolumeSnapshotContent created through\n the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.\n Supported values are \"Retain\" and \"Delete\".\n \"Retain\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept.\n \"Delete\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted.\n Required.\n enum:\n - Delete\n - Retain\n type: string\n driver:\n description: |-\n driver is the name of the storage driver that handles this VolumeSnapshotClass.\n Required.\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n parameters:\n additionalProperties:\n type: string\n description: |-\n parameters is a key-value map with storage driver specific parameters for creating snapshots.\n These values are opaque to Kubernetes.\n type: object\n required:\n - deletionPolicy\n - driver\n type: object\n served: true\n storage: true\n subresources: {}\n - additionalPrinterColumns:\n - jsonPath: .driver\n name: Driver\n type: string\n - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.\n jsonPath: .deletionPolicy\n name: DeletionPolicy\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1beta1\n # This indicates the v1beta1 version of the custom resource is deprecated.\n # API requests to this version receive a warning in the server response.\n deprecated: true\n # This overrides the default warning returned to clients making v1beta1 API requests.\n deprecationWarning: \"snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass\"\n schema:\n openAPIV3Schema:\n description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n deletionPolicy:\n description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are \"Retain\" and \"Delete\". \"Retain\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. \"Delete\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.\n enum:\n - Delete\n - Retain\n type: string\n driver:\n description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n parameters:\n additionalProperties:\n type: string\n description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.\n type: object\n required:\n - deletionPolicy\n - driver\n type: object\n served: false\n storage: false\n subresources: {}\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.15.0\n api-approved.kubernetes.io: \"https://github.com/kubernetes-csi/external-snapshotter/pull/955\"\n name: volumesnapshotcontents.snapshot.storage.k8s.io\nspec:\n group: snapshot.storage.k8s.io\n names:\n kind: VolumeSnapshotContent\n listKind: VolumeSnapshotContentList\n plural: volumesnapshotcontents\n shortNames:\n - vsc\n - vscs\n singular: volumesnapshotcontent\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - description: Indicates if the snapshot is ready to be used to restore a volume.\n jsonPath: .status.readyToUse\n name: ReadyToUse\n type: boolean\n - description: Represents the complete size of the snapshot in bytes\n jsonPath: .status.restoreSize\n name: RestoreSize\n type: integer\n - description: Determines whether this VolumeSnapshotContent and its physical\n snapshot on the underlying storage system should be deleted when its bound\n VolumeSnapshot is deleted.\n jsonPath: .spec.deletionPolicy\n name: DeletionPolicy\n type: string\n - description: Name of the CSI driver used to create the physical snapshot on\n the underlying storage system.\n jsonPath: .spec.driver\n name: Driver\n type: string\n - description: Name of the VolumeSnapshotClass to which this snapshot belongs.\n jsonPath: .spec.volumeSnapshotClassName\n name: VolumeSnapshotClass\n type: string\n - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent\n object is bound.\n jsonPath: .spec.volumeSnapshotRef.name\n name: VolumeSnapshot\n type: string\n - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent\n object is bound.\n jsonPath: .spec.volumeSnapshotRef.namespace\n name: VolumeSnapshotNamespace\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1\n schema:\n openAPIV3Schema:\n description: |-\n VolumeSnapshotContent represents the actual \"on-disk\" snapshot object in the\n underlying storage system\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: |-\n spec defines properties of a VolumeSnapshotContent created by the underlying storage system.\n Required.\n properties:\n deletionPolicy:\n description: |-\n deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on\n the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.\n Supported values are \"Retain\" and \"Delete\".\n \"Retain\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept.\n \"Delete\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted.\n For dynamically provisioned snapshots, this field will automatically be filled in by the\n CSI snapshotter sidecar with the \"DeletionPolicy\" field defined in the corresponding\n VolumeSnapshotClass.\n For pre-existing snapshots, users MUST specify this field when creating the\n VolumeSnapshotContent object.\n Required.\n enum:\n - Delete\n - Retain\n type: string\n driver:\n description: |-\n driver is the name of the CSI driver used to create the physical snapshot on\n the underlying storage system.\n This MUST be the same as the name returned by the CSI GetPluginName() call for\n that driver.\n Required.\n type: string\n source:\n description: |-\n source specifies whether the snapshot is (or should be) dynamically provisioned\n or already exists, and just requires a Kubernetes object representation.\n This field is immutable after creation.\n Required.\n properties:\n snapshotHandle:\n description: |-\n snapshotHandle specifies the CSI \"snapshot_id\" of a pre-existing snapshot on\n the underlying storage system for which a Kubernetes object representation\n was (or should be) created.\n This field is immutable.\n type: string\n x-kubernetes-validations:\n - message: snapshotHandle is immutable\n rule: self == oldSelf\n volumeHandle:\n description: |-\n volumeHandle specifies the CSI \"volume_id\" of the volume from which a snapshot\n should be dynamically taken from.\n This field is immutable.\n type: string\n x-kubernetes-validations:\n - message: volumeHandle is immutable\n rule: self == oldSelf\n type: object\n x-kubernetes-validations:\n - message: volumeHandle is required once set\n rule: '!has(oldSelf.volumeHandle) || has(self.volumeHandle)'\n - message: snapshotHandle is required once set\n rule: '!has(oldSelf.snapshotHandle) || has(self.snapshotHandle)'\n - message: exactly one of volumeHandle and snapshotHandle must be\n set\n rule: (has(self.volumeHandle) && !has(self.snapshotHandle)) || (!has(self.volumeHandle)\n && has(self.snapshotHandle))\n sourceVolumeMode:\n description: |-\n SourceVolumeMode is the mode of the volume whose snapshot is taken.\n Can be either “Filesystem” or “Block”.\n If not specified, it indicates the source volume's mode is unknown.\n This field is immutable.\n This field is an alpha field.\n type: string\n x-kubernetes-validations:\n - message: sourceVolumeMode is immutable\n rule: self == oldSelf\n volumeSnapshotClassName:\n description: |-\n name of the VolumeSnapshotClass from which this snapshot was (or will be)\n created.\n Note that after provisioning, the VolumeSnapshotClass may be deleted or\n recreated with different set of values, and as such, should not be referenced\n post-snapshot creation.\n type: string\n volumeSnapshotRef:\n description: |-\n volumeSnapshotRef specifies the VolumeSnapshot object to which this\n VolumeSnapshotContent object is bound.\n VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to\n this VolumeSnapshotContent's name for the bidirectional binding to be valid.\n For a pre-existing VolumeSnapshotContent object, name and namespace of the\n VolumeSnapshot object MUST be provided for binding to happen.\n This field is immutable after creation.\n Required.\n properties:\n apiVersion:\n description: API version of the referent.\n type: string\n fieldPath:\n description: |-\n If referring to a piece of an object instead of an entire object, this string\n should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].\n For example, if the object reference is to a container within a pod, this would take on a value like:\n \"spec.containers{name}\" (where \"name\" refers to the name of the container that triggered\n the event) or if no container name is specified \"spec.containers[2]\" (container with\n index 2 in this pod). This syntax is chosen only to have some well-defined way of\n referencing a part of an object.\n TODO: this design is not final and this field is subject to change in the future.\n type: string\n kind:\n description: |-\n Kind of the referent.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n name:\n description: |-\n Name of the referent.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n namespace:\n description: |-\n Namespace of the referent.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/\n type: string\n resourceVersion:\n description: |-\n Specific resourceVersion to which this reference is made, if any.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency\n type: string\n uid:\n description: |-\n UID of the referent.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids\n type: string\n type: object\n x-kubernetes-map-type: atomic\n x-kubernetes-validations:\n - message: both spec.volumeSnapshotRef.name and spec.volumeSnapshotRef.namespace\n must be set\n rule: has(self.name) && has(self.__namespace__)\n required:\n - deletionPolicy\n - driver\n - source\n - volumeSnapshotRef\n type: object\n x-kubernetes-validations:\n - message: sourceVolumeMode is required once set\n rule: '!has(oldSelf.sourceVolumeMode) || has(self.sourceVolumeMode)'\n status:\n description: status represents the current information of a snapshot.\n properties:\n creationTime:\n description: |-\n creationTime is the timestamp when the point-in-time snapshot is taken\n by the underlying storage system.\n In dynamic snapshot creation case, this field will be filled in by the\n CSI snapshotter sidecar with the \"creation_time\" value returned from CSI\n \"CreateSnapshot\" gRPC call.\n For a pre-existing snapshot, this field will be filled with the \"creation_time\"\n value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it.\n If not specified, it indicates the creation time is unknown.\n The format of this field is a Unix nanoseconds time encoded as an int64.\n On Unix, the command `date +%s%N` returns the current time in nanoseconds\n since 1970-01-01 00:00:00 UTC.\n format: int64\n type: integer\n error:\n description: |-\n error is the last observed error during snapshot creation, if any.\n Upon success after retry, this error field will be cleared.\n properties:\n message:\n description: |-\n message is a string detailing the encountered error during snapshot\n creation if specified.\n NOTE: message may be logged, and it should not contain sensitive\n information.\n type: string\n time:\n description: time is the timestamp when the error was encountered.\n format: date-time\n type: string\n type: object\n readyToUse:\n description: |-\n readyToUse indicates if a snapshot is ready to be used to restore a volume.\n In dynamic snapshot creation case, this field will be filled in by the\n CSI snapshotter sidecar with the \"ready_to_use\" value returned from CSI\n \"CreateSnapshot\" gRPC call.\n For a pre-existing snapshot, this field will be filled with the \"ready_to_use\"\n value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it,\n otherwise, this field will be set to \"True\".\n If not specified, it means the readiness of a snapshot is unknown.\n type: boolean\n restoreSize:\n description: |-\n restoreSize represents the complete size of the snapshot in bytes.\n In dynamic snapshot creation case, this field will be filled in by the\n CSI snapshotter sidecar with the \"size_bytes\" value returned from CSI\n \"CreateSnapshot\" gRPC call.\n For a pre-existing snapshot, this field will be filled with the \"size_bytes\"\n value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it.\n When restoring a volume from this snapshot, the size of the volume MUST NOT\n be smaller than the restoreSize if it is specified, otherwise the restoration will fail.\n If not specified, it indicates that the size is unknown.\n format: int64\n minimum: 0\n type: integer\n snapshotHandle:\n description: |-\n snapshotHandle is the CSI \"snapshot_id\" of a snapshot on the underlying storage system.\n If not specified, it indicates that dynamic snapshot creation has either failed\n or it is still in progress.\n type: string\n volumeGroupSnapshotHandle:\n description: |-\n VolumeGroupSnapshotHandle is the CSI \"group_snapshot_id\" of a group snapshot\n on the underlying storage system.\n type: string\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n - additionalPrinterColumns:\n - description: Indicates if the snapshot is ready to be used to restore a volume.\n jsonPath: .status.readyToUse\n name: ReadyToUse\n type: boolean\n - description: Represents the complete size of the snapshot in bytes\n jsonPath: .status.restoreSize\n name: RestoreSize\n type: integer\n - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.\n jsonPath: .spec.deletionPolicy\n name: DeletionPolicy\n type: string\n - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.\n jsonPath: .spec.driver\n name: Driver\n type: string\n - description: Name of the VolumeSnapshotClass to which this snapshot belongs.\n jsonPath: .spec.volumeSnapshotClassName\n name: VolumeSnapshotClass\n type: string\n - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.\n jsonPath: .spec.volumeSnapshotRef.name\n name: VolumeSnapshot\n type: string\n - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.\n jsonPath: .spec.volumeSnapshotRef.namespace\n name: VolumeSnapshotNamespace\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1beta1\n # This indicates the v1beta1 version of the custom resource is deprecated.\n # API requests to this version receive a warning in the server response.\n deprecated: true\n # This overrides the default warning returned to clients making v1beta1 API requests.\n deprecationWarning: \"snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent\"\n schema:\n openAPIV3Schema:\n description: VolumeSnapshotContent represents the actual \"on-disk\" snapshot object in the underlying storage system\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n spec:\n description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.\n properties:\n deletionPolicy:\n description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are \"Retain\" and \"Delete\". \"Retain\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. \"Delete\" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the \"DeletionPolicy\" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required.\n enum:\n - Delete\n - Retain\n type: string\n driver:\n description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.\n type: string\n source:\n description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.\n properties:\n snapshotHandle:\n description: snapshotHandle specifies the CSI \"snapshot_id\" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.\n type: string\n volumeHandle:\n description: volumeHandle specifies the CSI \"volume_id\" of the volume from which a snapshot should be dynamically taken from. This field is immutable.\n type: string\n type: object\n volumeSnapshotClassName:\n description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.\n type: string\n volumeSnapshotRef:\n description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.\n properties:\n apiVersion:\n description: API version of the referent.\n type: string\n fieldPath:\n description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: \"spec.containers{name}\" (where \"name\" refers to the name of the container that triggered the event) or if no container name is specified \"spec.containers[2]\" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'\n type: string\n kind:\n description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n name:\n description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'\n type: string\n namespace:\n description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'\n type: string\n resourceVersion:\n description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'\n type: string\n uid:\n description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'\n type: string\n type: object\n required:\n - deletionPolicy\n - driver\n - source\n - volumeSnapshotRef\n type: object\n status:\n description: status represents the current information of a snapshot.\n properties:\n creationTime:\n description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the \"creation_time\" value returned from CSI \"CreateSnapshot\" gRPC call. For a pre-existing snapshot, this field will be filled with the \"creation_time\" value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.\n format: int64\n type: integer\n error:\n description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.\n properties:\n message:\n description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'\n type: string\n time:\n description: time is the timestamp when the error was encountered.\n format: date-time\n type: string\n type: object\n readyToUse:\n description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the \"ready_to_use\" value returned from CSI \"CreateSnapshot\" gRPC call. For a pre-existing snapshot, this field will be filled with the \"ready_to_use\" value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it, otherwise, this field will be set to \"True\". If not specified, it means the readiness of a snapshot is unknown.\n type: boolean\n restoreSize:\n description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the \"size_bytes\" value returned from CSI \"CreateSnapshot\" gRPC call. For a pre-existing snapshot, this field will be filled with the \"size_bytes\" value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.\n format: int64\n minimum: 0\n type: integer\n snapshotHandle:\n description: snapshotHandle is the CSI \"snapshot_id\" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.\n type: string\n type: object\n required:\n - spec\n type: object\n served: false\n storage: false\n subresources:\n status: {}\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" }, { "path": "roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.15.0\n api-approved.kubernetes.io: \"https://github.com/kubernetes-csi/external-snapshotter/pull/814\"\n name: volumesnapshots.snapshot.storage.k8s.io\nspec:\n group: snapshot.storage.k8s.io\n names:\n kind: VolumeSnapshot\n listKind: VolumeSnapshotList\n plural: volumesnapshots\n shortNames:\n - vs\n singular: volumesnapshot\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - description: Indicates if the snapshot is ready to be used to restore a volume.\n jsonPath: .status.readyToUse\n name: ReadyToUse\n type: boolean\n - description: If a new snapshot needs to be created, this contains the name of\n the source PVC from which this snapshot was (or will be) created.\n jsonPath: .spec.source.persistentVolumeClaimName\n name: SourcePVC\n type: string\n - description: If a snapshot already exists, this contains the name of the existing\n VolumeSnapshotContent object representing the existing snapshot.\n jsonPath: .spec.source.volumeSnapshotContentName\n name: SourceSnapshotContent\n type: string\n - description: Represents the minimum size of volume required to rehydrate from\n this snapshot.\n jsonPath: .status.restoreSize\n name: RestoreSize\n type: string\n - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.\n jsonPath: .spec.volumeSnapshotClassName\n name: SnapshotClass\n type: string\n - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot\n object intends to bind to. Please note that verification of binding actually\n requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure\n both are pointing at each other. Binding MUST be verified prior to usage of\n this object.\n jsonPath: .status.boundVolumeSnapshotContentName\n name: SnapshotContent\n type: string\n - description: Timestamp when the point-in-time snapshot was taken by the underlying\n storage system.\n jsonPath: .status.creationTime\n name: CreationTime\n type: date\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1\n schema:\n openAPIV3Schema:\n description: |-\n VolumeSnapshot is a user's request for either creating a point-in-time\n snapshot of a persistent volume, or binding to a pre-existing snapshot.\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: |-\n spec defines the desired characteristics of a snapshot requested by a user.\n More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots\n Required.\n properties:\n source:\n description: |-\n source specifies where a snapshot will be created from.\n This field is immutable after creation.\n Required.\n properties:\n persistentVolumeClaimName:\n description: |-\n persistentVolumeClaimName specifies the name of the PersistentVolumeClaim\n object representing the volume from which a snapshot should be created.\n This PVC is assumed to be in the same namespace as the VolumeSnapshot\n object.\n This field should be set if the snapshot does not exists, and needs to be\n created.\n This field is immutable.\n type: string\n x-kubernetes-validations:\n - message: persistentVolumeClaimName is immutable\n rule: self == oldSelf\n volumeSnapshotContentName:\n description: |-\n volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent\n object representing an existing volume snapshot.\n This field should be set if the snapshot already exists and only needs a representation in Kubernetes.\n This field is immutable.\n type: string\n x-kubernetes-validations:\n - message: volumeSnapshotContentName is immutable\n rule: self == oldSelf\n type: object\n x-kubernetes-validations:\n - message: persistentVolumeClaimName is required once set\n rule: '!has(oldSelf.persistentVolumeClaimName) || has(self.persistentVolumeClaimName)'\n - message: volumeSnapshotContentName is required once set\n rule: '!has(oldSelf.volumeSnapshotContentName) || has(self.volumeSnapshotContentName)'\n - message: exactly one of volumeSnapshotContentName and persistentVolumeClaimName\n must be set\n rule: (has(self.volumeSnapshotContentName) && !has(self.persistentVolumeClaimName))\n || (!has(self.volumeSnapshotContentName) && has(self.persistentVolumeClaimName))\n volumeSnapshotClassName:\n description: |-\n VolumeSnapshotClassName is the name of the VolumeSnapshotClass\n requested by the VolumeSnapshot.\n VolumeSnapshotClassName may be left nil to indicate that the default\n SnapshotClass should be used.\n A given cluster may have multiple default Volume SnapshotClasses: one\n default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass,\n VolumeSnapshotSource will be checked to figure out what the associated\n CSI Driver is, and the default VolumeSnapshotClass associated with that\n CSI Driver will be used. If more than one VolumeSnapshotClass exist for\n a given CSI Driver and more than one have been marked as default,\n CreateSnapshot will fail and generate an event.\n Empty string is not allowed for this field.\n type: string\n x-kubernetes-validations:\n - message: volumeSnapshotClassName must not be the empty string when\n set\n rule: size(self) > 0\n required:\n - source\n type: object\n status:\n description: |-\n status represents the current information of a snapshot.\n Consumers must verify binding between VolumeSnapshot and\n VolumeSnapshotContent objects is successful (by validating that both\n VolumeSnapshot and VolumeSnapshotContent point at each other) before\n using this object.\n properties:\n boundVolumeSnapshotContentName:\n description: |-\n boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent\n object to which this VolumeSnapshot object intends to bind to.\n If not specified, it indicates that the VolumeSnapshot object has not been\n successfully bound to a VolumeSnapshotContent object yet.\n NOTE: To avoid possible security issues, consumers must verify binding between\n VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that\n both VolumeSnapshot and VolumeSnapshotContent point at each other) before using\n this object.\n type: string\n creationTime:\n description: |-\n creationTime is the timestamp when the point-in-time snapshot is taken\n by the underlying storage system.\n In dynamic snapshot creation case, this field will be filled in by the\n snapshot controller with the \"creation_time\" value returned from CSI\n \"CreateSnapshot\" gRPC call.\n For a pre-existing snapshot, this field will be filled with the \"creation_time\"\n value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it.\n If not specified, it may indicate that the creation time of the snapshot is unknown.\n format: date-time\n type: string\n error:\n description: |-\n error is the last observed error during snapshot creation, if any.\n This field could be helpful to upper level controllers(i.e., application controller)\n to decide whether they should continue on waiting for the snapshot to be created\n based on the type of error reported.\n The snapshot controller will keep retrying when an error occurs during the\n snapshot creation. Upon success, this error field will be cleared.\n properties:\n message:\n description: |-\n message is a string detailing the encountered error during snapshot\n creation if specified.\n NOTE: message may be logged, and it should not contain sensitive\n information.\n type: string\n time:\n description: time is the timestamp when the error was encountered.\n format: date-time\n type: string\n type: object\n readyToUse:\n description: |-\n readyToUse indicates if the snapshot is ready to be used to restore a volume.\n In dynamic snapshot creation case, this field will be filled in by the\n snapshot controller with the \"ready_to_use\" value returned from CSI\n \"CreateSnapshot\" gRPC call.\n For a pre-existing snapshot, this field will be filled with the \"ready_to_use\"\n value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it,\n otherwise, this field will be set to \"True\".\n If not specified, it means the readiness of a snapshot is unknown.\n type: boolean\n restoreSize:\n type: string\n description: |-\n restoreSize represents the minimum size of volume required to create a volume\n from this snapshot.\n In dynamic snapshot creation case, this field will be filled in by the\n snapshot controller with the \"size_bytes\" value returned from CSI\n \"CreateSnapshot\" gRPC call.\n For a pre-existing snapshot, this field will be filled with the \"size_bytes\"\n value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it.\n When restoring a volume from this snapshot, the size of the volume MUST NOT\n be smaller than the restoreSize if it is specified, otherwise the restoration will fail.\n If not specified, it indicates that the size is unknown.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n volumeGroupSnapshotName:\n description: |-\n VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this\n VolumeSnapshot is a part of.\n type: string\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n - additionalPrinterColumns:\n - description: Indicates if the snapshot is ready to be used to restore a volume.\n jsonPath: .status.readyToUse\n name: ReadyToUse\n type: boolean\n - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.\n jsonPath: .spec.source.persistentVolumeClaimName\n name: SourcePVC\n type: string\n - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.\n jsonPath: .spec.source.volumeSnapshotContentName\n name: SourceSnapshotContent\n type: string\n - description: Represents the minimum size of volume required to rehydrate from this snapshot.\n jsonPath: .status.restoreSize\n name: RestoreSize\n type: string\n - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.\n jsonPath: .spec.volumeSnapshotClassName\n name: SnapshotClass\n type: string\n - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.\n jsonPath: .status.boundVolumeSnapshotContentName\n name: SnapshotContent\n type: string\n - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.\n jsonPath: .status.creationTime\n name: CreationTime\n type: date\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1beta1\n # This indicates the v1beta1 version of the custom resource is deprecated.\n # API requests to this version receive a warning in the server response.\n deprecated: true\n # This overrides the default warning returned to clients making v1beta1 API requests.\n deprecationWarning: \"snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot\"\n schema:\n openAPIV3Schema:\n description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n spec:\n description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'\n properties:\n source:\n description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.\n properties:\n persistentVolumeClaimName:\n description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.\n type: string\n volumeSnapshotContentName:\n description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.\n type: string\n type: object\n volumeSnapshotClassName:\n description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'\n type: string\n required:\n - source\n type: object\n status:\n description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.\n properties:\n boundVolumeSnapshotContentName:\n description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'\n type: string\n creationTime:\n description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the \"creation_time\" value returned from CSI \"CreateSnapshot\" gRPC call. For a pre-existing snapshot, this field will be filled with the \"creation_time\" value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.\n format: date-time\n type: string\n error:\n description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared.\n properties:\n message:\n description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'\n type: string\n time:\n description: time is the timestamp when the error was encountered.\n format: date-time\n type: string\n type: object\n readyToUse:\n description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the \"ready_to_use\" value returned from CSI \"CreateSnapshot\" gRPC call. For a pre-existing snapshot, this field will be filled with the \"ready_to_use\" value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it, otherwise, this field will be set to \"True\". If not specified, it means the readiness of a snapshot is unknown.\n type: boolean\n restoreSize:\n type: string\n description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the \"size_bytes\" value returned from CSI \"CreateSnapshot\" gRPC call. For a pre-existing snapshot, this field will be filled with the \"size_bytes\" value returned from the CSI \"ListSnapshots\" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n type: object\n required:\n - spec\n type: object\n served: false\n storage: false\n subresources:\n status: {}\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml", "content": "---\ngcp_pd_csi_controller_replicas: 1\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml", "content": "---\n- name: GCP PD CSI Driver | Check if cloud-sa.json exists\n fail:\n msg: \"Credentials file cloud-sa.json is mandatory\"\n when: gcp_pd_csi_sa_cred_file is not defined or not gcp_pd_csi_sa_cred_file\n\n- name: GCP PD CSI Driver | Copy GCP credentials file\n copy:\n src: \"{{ gcp_pd_csi_sa_cred_file }}\"\n dest: \"{{ kube_config_dir }}/cloud-sa.json\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: GCP PD CSI Driver | Get base64 cloud-sa.json\n slurp:\n src: \"{{ kube_config_dir }}/cloud-sa.json\"\n register: gcp_cred_secret\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: GCP PD CSI Driver | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: gcp-pd-csi-cred-secret, file: gcp-pd-csi-cred-secret.yml}\n - {name: gcp-pd-csi-setup, file: gcp-pd-csi-setup.yml}\n - {name: gcp-pd-csi-controller, file: gcp-pd-csi-controller.yml}\n - {name: gcp-pd-csi-node, file: gcp-pd-csi-node.yml}\n - {name: gcp-pd-csi-sc-regional, file: gcp-pd-csi-sc-regional.yml}\n - {name: gcp-pd-csi-sc-zonal, file: gcp-pd-csi-sc-zonal.yml}\n register: gcp_pd_csi_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: GCP PD CSI Driver | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ gcp_pd_csi_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2", "content": "kind: Deployment\napiVersion: apps/v1\nmetadata:\n name: csi-gce-pd-controller\n namespace: kube-system\nspec:\n replicas: {{ gcp_pd_csi_controller_replicas }}\n selector:\n matchLabels:\n app: gcp-compute-persistent-disk-csi-driver\n template:\n metadata:\n labels:\n app: gcp-compute-persistent-disk-csi-driver\n spec:\n # Host network must be used for interaction with Workload Identity in GKE\n # since it replaces GCE Metadata Server with GKE Metadata Server. Remove\n # this requirement when issue is resolved and before any exposure of\n # metrics ports\n hostNetwork: true\n nodeSelector:\n kubernetes.io/os: linux\n serviceAccountName: csi-gce-pd-controller-sa\n priorityClassName: csi-gce-pd-controller\n containers:\n - name: csi-provisioner\n image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=/csi/csi.sock\"\n - \"--feature-gates=Topology=true\"\n - \"--http-endpoint=:22011\"\n - \"--leader-election-namespace=$(PDCSI_NAMESPACE)\"\n - \"--timeout=250s\"\n - \"--extra-create-metadata\"\n # - \"--run-controller-service=false\" # disable the controller service of the CSI driver\n # - \"--run-node-service=false\" # disable the node service of the CSI driver\n - \"--leader-election\"\n - \"--default-fstype=ext4\"\n - \"--controller-publish-readonly=true\"\n env:\n - name: PDCSI_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n ports:\n - containerPort: 22011\n name: http-endpoint\n protocol: TCP\n livenessProbe:\n failureThreshold: 1\n httpGet:\n path: /healthz/leader-election\n port: http-endpoint\n initialDelaySeconds: 10\n timeoutSeconds: 10\n periodSeconds: 20\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: csi-attacher\n image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=/csi/csi.sock\"\n - \"--http-endpoint=:22012\"\n - \"--leader-election\"\n - \"--leader-election-namespace=$(PDCSI_NAMESPACE)\"\n - \"--timeout=250s\"\n env:\n - name: PDCSI_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n ports:\n - containerPort: 22012\n name: http-endpoint\n protocol: TCP\n livenessProbe:\n failureThreshold: 1\n httpGet:\n path: /healthz/leader-election\n port: http-endpoint\n initialDelaySeconds: 10\n timeoutSeconds: 10\n periodSeconds: 20\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: csi-resizer\n image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=/csi/csi.sock\"\n - \"--http-endpoint=:22013\"\n - \"--leader-election\"\n - \"--leader-election-namespace=$(PDCSI_NAMESPACE)\"\n - \"--handle-volume-inuse-error=false\"\n env:\n - name: PDCSI_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n ports:\n - containerPort: 22013\n name: http-endpoint\n protocol: TCP\n livenessProbe:\n failureThreshold: 1\n httpGet:\n path: /healthz/leader-election\n port: http-endpoint\n initialDelaySeconds: 10\n timeoutSeconds: 10\n periodSeconds: 20\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: csi-snapshotter\n image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=/csi/csi.sock\"\n - \"--metrics-address=:22014\"\n - \"--leader-election\"\n - \"--leader-election-namespace=$(PDCSI_NAMESPACE)\"\n - \"--timeout=300s\"\n env:\n - name: PDCSI_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: gce-pd-driver\n # Don't change base image without changing pdImagePlaceholder in\n # test/k8s-integration/main.go\n image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }}\n args:\n - \"--v=5\"\n - \"--endpoint=unix:/csi/csi.sock\"\n env:\n - name: GOOGLE_APPLICATION_CREDENTIALS\n value: \"/etc/cloud-sa/cloud-sa.json\"\n volumeMounts:\n - name: socket-dir\n mountPath: /csi\n - name: cloud-sa-volume\n readOnly: true\n mountPath: \"/etc/cloud-sa\"\n volumes:\n - name: socket-dir\n emptyDir: {}\n - name: cloud-sa-volume\n secret:\n secretName: cloud-sa\n---\napiVersion: storage.k8s.io/v1\nkind: CSIDriver\nmetadata:\n name: pd.csi.storage.gke.io\nspec:\n attachRequired: true\n podInfoOnMount: false\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2", "content": "---\nkind: Secret\napiVersion: v1\nmetadata:\n name: cloud-sa\n namespace: kube-system\ndata:\n cloud-sa.json: {{ gcp_cred_secret.content }}\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2", "content": "kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: csi-gce-pd-node\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n app: gcp-compute-persistent-disk-csi-driver\n template:\n metadata:\n labels:\n app: gcp-compute-persistent-disk-csi-driver\n spec:\n # Host network must be used for interaction with Workload Identity in GKE\n # since it replaces GCE Metadata Server with GKE Metadata Server. Remove\n # this requirement when issue is resolved and before any exposure of\n # metrics ports.\n hostNetwork: true\n priorityClassName: csi-gce-pd-node\n serviceAccountName: csi-gce-pd-node-sa\n containers:\n - name: csi-driver-registrar\n image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=/csi/csi.sock\"\n - \"--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock\"\n lifecycle:\n preStop:\n exec:\n command: [\"/bin/sh\", \"-c\", \"rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock\"]\n env:\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n - name: registration-dir\n mountPath: /registration\n - name: gce-pd-driver\n securityContext:\n privileged: true\n # Don't change base image without changing pdImagePlaceholder in\n # test/k8s-integration/main.go\n image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }}\n args:\n - \"--v=5\"\n - \"--endpoint=unix:/csi/csi.sock\"\n - \"--run-controller-service=false\"\n volumeMounts:\n - name: kubelet-dir\n mountPath: /var/lib/kubelet\n mountPropagation: \"Bidirectional\"\n - name: plugin-dir\n mountPath: /csi\n - name: device-dir\n mountPath: /dev\n # The following mounts are required to trigger host udevadm from\n # container\n - name: udev-rules-etc\n mountPath: /etc/udev\n - name: udev-rules-lib\n mountPath: /lib/udev\n - name: udev-socket\n mountPath: /run/udev\n - name: sys\n mountPath: /sys\n nodeSelector:\n kubernetes.io/os: linux\n volumes:\n - name: registration-dir\n hostPath:\n path: /var/lib/kubelet/plugins_registry/\n type: Directory\n - name: kubelet-dir\n hostPath:\n path: /var/lib/kubelet\n type: Directory\n - name: plugin-dir\n hostPath:\n path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/\n type: DirectoryOrCreate\n - name: device-dir\n hostPath:\n path: /dev\n type: Directory\n # The following mounts are required to trigger host udevadm from\n # container\n - name: udev-rules-etc\n hostPath:\n path: /etc/udev\n type: Directory\n - name: udev-rules-lib\n hostPath:\n path: /lib/udev\n type: Directory\n - name: udev-socket\n hostPath:\n path: /run/udev\n type: Directory\n - name: sys\n hostPath:\n path: /sys\n type: Directory\n # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n # See \"special case\". This will tolerate everything. Node component should\n # be scheduled on all nodes.\n tolerations:\n - operator: Exists\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2", "content": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: csi-gce-pd-regional\nprovisioner: pd.csi.storage.gke.io\nparameters:\n type: pd-balanced\n replication-type: regional-pd\nvolumeBindingMode: WaitForFirstConsumer\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2", "content": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: csi-gce-pd-zonal\nprovisioner: pd.csi.storage.gke.io\nparameters:\n type: pd-balanced\nvolumeBindingMode: WaitForFirstConsumer\n" }, { "path": "roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2", "content": "##### Node Service Account, Roles, RoleBindings\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-gce-pd-node-sa\n namespace: kube-system\n\n---\n##### Controller Service Account, Roles, Rolebindings\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-gce-pd-controller-sa\n namespace: kube-system\n\n---\n# xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-provisioner-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"get\", \"list\"]\n # Access to volumeattachments is only needed when the CSI driver\n # has the PUBLISH_UNPUBLISH_VOLUME controller capability.\n # In that case, external-provisioner will watch volumeattachments\n # to determine when it is safe to delete a volume.\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-controller-provisioner-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-gce-pd-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-gce-pd-provisioner-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n# xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-attacher-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments/status\"]\n verbs: [\"patch\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-controller-attacher-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-gce-pd-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-gce-pd-attacher-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n\napiVersion: scheduling.k8s.io/v1\nkind: PriorityClass\nmetadata:\n name: csi-gce-pd-controller\nvalue: 900000000\nglobalDefault: false\ndescription: \"This priority class should be used for the GCE PD CSI driver controller deployment only.\"\n\n---\n\napiVersion: scheduling.k8s.io/v1\nkind: PriorityClass\nmetadata:\n name: csi-gce-pd-node\nvalue: 900001000\nglobalDefault: false\ndescription: \"This priority class should be used for the GCE PD CSI driver node deployment only.\"\n\n---\n\n# Resizer must be able to work with PVCs, PVs, SCs.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-resizer-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n # If handle-volume-inuse-error=true, the pod specific rbac is needed\n - apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-resizer-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-gce-pd-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-gce-pd-resizer-role\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: csi-gce-pd-controller\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: csi-gce-pd-node-deploy\nsubjects:\n- kind: ServiceAccount\n name: csi-gce-pd-controller-sa\n namespace: kube-system\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: csi-gce-pd-snapshotter-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n # Secrets resource omitted since GCE PD snapshots does not require them\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents/status\"]\n verbs: [\"update\", \"patch\"]\n---\n\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-controller-snapshotter-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-gce-pd-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-gce-pd-snapshotter-role\n apiGroup: rbac.authorization.k8s.io\n---\n\nkind: Role\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-leaderelection-role\n namespace: kube-system\n labels:\n k8s-app: gcp-compute-persistent-disk-csi-driver\nrules:\n- apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n\n---\n\nkind: RoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-gce-pd-controller-leaderelection-binding\n namespace: kube-system\n labels:\n k8s-app: gcp-compute-persistent-disk-csi-driver\nsubjects:\n- kind: ServiceAccount\n name: csi-gce-pd-controller-sa\n namespace: kube-system\nroleRef:\n kind: Role\n name: csi-gce-pd-leaderelection-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml", "content": "---\nupcloud_csi_controller_replicas: 1\nupcloud_csi_provisioner_image_tag: \"v3.1.0\"\nupcloud_csi_attacher_image_tag: \"v3.4.0\"\nupcloud_csi_resizer_image_tag: \"v1.4.0\"\nupcloud_csi_plugin_image_tag: \"v1.1.0\"\nupcloud_csi_node_image_tag: \"v2.5.0\"\nupcloud_username: \"{{ lookup('env', 'UPCLOUD_USERNAME') }}\"\nupcloud_password: \"{{ lookup('env', 'UPCLOUD_PASSWORD') }}\"\nupcloud_tolerations: []\nupcloud_csi_enable_volume_snapshot: false\nupcloud_csi_snapshot_controller_replicas: 2\nupcloud_csi_snapshotter_image_tag: \"v4.2.1\"\nupcloud_csi_snapshot_controller_image_tag: \"v4.2.1\"\nupcloud_csi_snapshot_validation_webhook_image_tag: \"v4.2.1\"\nupcloud_cacert: \"{{ lookup('env', 'OS_CACERT') }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml", "content": "---\n- name: UpCloud CSI Driver | Check if UPCLOUD_USERNAME exists\n fail:\n msg: \"UpCloud username is missing. Env UPCLOUD_USERNAME is mandatory\"\n when: upcloud_username is not defined or not upcloud_username\n\n- name: UpCloud CSI Driver | Check if UPCLOUD_PASSWORD exists\n fail:\n msg: \"UpCloud password is missing. Env UPCLOUD_PASSWORD is mandatory\"\n when:\n - upcloud_username is defined\n - upcloud_username | length > 0\n - upcloud_password is not defined or not upcloud_password\n\n- name: UpCloud CSI Driver | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: upcloud-csi-cred-secret, file: upcloud-csi-cred-secret.yml}\n - {name: upcloud-csi-setup, file: upcloud-csi-setup.yml}\n - {name: upcloud-csi-controller, file: upcloud-csi-controller.yml}\n - {name: upcloud-csi-node, file: upcloud-csi-node.yml}\n - {name: upcloud-csi-driver, file: upcloud-csi-driver.yml}\n register: upcloud_csi_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: UpCloud CSI Driver | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ upcloud_csi_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2", "content": "kind: StatefulSet\napiVersion: apps/v1\nmetadata:\n name: csi-upcloud-controller\n namespace: kube-system\nspec:\n serviceName: \"csi-upcloud\"\n replicas: {{ upcloud_csi_controller_replicas }}\n selector:\n matchLabels:\n app: csi-upcloud-controller\n template:\n metadata:\n labels:\n app: csi-upcloud-controller\n role: csi-upcloud\n spec:\n priorityClassName: system-cluster-critical\n serviceAccount: csi-upcloud-controller-sa\n containers:\n - name: csi-provisioner\n image: registry.k8s.io/sig-storage/csi-provisioner:{{ upcloud_csi_provisioner_image_tag }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--v=5\"\n - \"--timeout=600s\"\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n imagePullPolicy: \"Always\"\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-attacher\n image: registry.k8s.io/sig-storage/csi-attacher:{{ upcloud_csi_attacher_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--timeout=120s\"\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n imagePullPolicy: \"Always\"\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-resizer\n image: registry.k8s.io/sig-storage/csi-resizer:{{ upcloud_csi_resizer_image_tag }}\n args:\n - \"--v=5\"\n - \"--timeout=120s\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--handle-volume-inuse-error=true\"\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n imagePullPolicy: \"Always\"\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-snapshotter\n image: k8s.gcr.io/sig-storage/csi-snapshotter:{{ upcloud_csi_snapshotter_image_tag }}\n args:\n - \"--csi-address=$(ADDRESS)\"\n - \"--v=5\"\n - \"--timeout=600s\"\n - \"--leader-election=false\"\n env:\n - name: ADDRESS\n value: /var/lib/csi/sockets/pluginproxy/csi.sock\n imagePullPolicy: \"Always\"\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n - name: csi-upcloud-plugin\n image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }}\n args:\n - \"--endpoint=$(CSI_ENDPOINT)\"\n - \"--nodehost=$(NODE_ID)\"\n - \"--mode=monolith\"\n env:\n - name: CSI_ENDPOINT\n value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock\n - name: UPCLOUD_USERNAME\n valueFrom:\n secretKeyRef:\n name: upcloud\n key: username\n - name: UPCLOUD_PASSWORD\n valueFrom:\n secretKeyRef:\n name: upcloud\n key: password\n - name: NODE_ID\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n imagePullPolicy: \"Always\"\n volumeMounts:\n - name: socket-dir\n mountPath: /var/lib/csi/sockets/pluginproxy/\n imagePullSecrets:\n - name: upcloud\n volumes:\n - name: socket-dir\n emptyDir: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2", "content": "---\napiVersion: v1\nkind: Secret\nmetadata:\n name: upcloud\n namespace: kube-system\nstringData:\n username: {{ upcloud_username }}\n password: {{ upcloud_password }}\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2", "content": "apiVersion: storage.k8s.io/v1\nkind: CSIDriver\nmetadata:\n name: storage.csi.upcloud.com\nspec:\n attachRequired: true\n podInfoOnMount: true\n fsGroupPolicy: File\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2", "content": "kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: csi-upcloud-node\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n app: csi-upcloud-node\n template:\n metadata:\n labels:\n app: csi-upcloud-node\n role: csi-upcloud\n spec:\n priorityClassName: system-node-critical\n serviceAccount: csi-upcloud-node-sa\n hostNetwork: true\n containers:\n - name: csi-node-driver-registrar\n image: registry.k8s.io/sig-storage/csi-node-driver-registrar:{{ upcloud_csi_node_image_tag }}\n args:\n - \"--v=5\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n - name: DRIVER_REG_SOCK_PATH\n value: /var/lib/kubelet/plugins/storage.csi.upcloud.com/csi.sock\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi/\n - name: registration-dir\n mountPath: /registration/\n - name: csi-upcloud-plugin\n image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }}\n args:\n - \"--endpoint=$(CSI_ENDPOINT)\"\n - \"--nodehost=$(NODE_ID)\"\n - \"--mode=monolith\"\n env:\n - name: CSI_ENDPOINT\n value: unix:///csi/csi.sock\n - name: UPCLOUD_USERNAME\n valueFrom:\n secretKeyRef:\n name: upcloud\n key: username\n - name: UPCLOUD_PASSWORD\n valueFrom:\n secretKeyRef:\n name: upcloud\n key: password\n - name: NODE_ID\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n imagePullPolicy: \"Always\"\n securityContext:\n privileged: true\n capabilities:\n add: [\"SYS_ADMIN\"]\n allowPrivilegeEscalation: true\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n - name: pods-mount-dir\n mountPath: /var/lib/kubelet\n # needed so that any mounts setup inside this container are\n # propagated back to the host machine.\n mountPropagation: \"Bidirectional\"\n - name: device-dir\n mountPath: /dev\n imagePullSecrets:\n - name: upcloud\n volumes:\n - name: registration-dir\n hostPath:\n path: /var/lib/kubelet/plugins_registry/\n type: DirectoryOrCreate\n - name: plugin-dir\n hostPath:\n path: /var/lib/kubelet/plugins/storage.csi.upcloud.com\n type: DirectoryOrCreate\n - name: pods-mount-dir\n hostPath:\n path: /var/lib/kubelet\n type: Directory\n - name: device-dir\n hostPath:\n path: /dev\n{% if upcloud_tolerations %}\n tolerations:\n {{ upcloud_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2", "content": "kind: ServiceAccount\napiVersion: v1\nmetadata:\n name: csi-upcloud-controller-sa\n namespace: kube-system\n\n---\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: csi-upcloud-node-sa\n namespace: kube-system\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-node-driver-registrar-role\n namespace: kube-system\nrules:\n - apiGroups: [ \"\" ]\n resources: [ \"events\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\" ]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-node-driver-registrar-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-upcloud-node-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-upcloud-node-driver-registrar-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-provisioner-role\nrules:\n - apiGroups: [ \"\" ]\n resources: [ \"secrets\" ]\n verbs: [ \"get\", \"list\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"persistentvolumes\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"create\", \"delete\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"persistentvolumeclaims\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"update\" ]\n - apiGroups: [ \"storage.k8s.io\" ]\n resources: [ \"storageclasses\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n - apiGroups: [ \"storage.k8s.io\" ]\n resources: [ \"csinodes\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"events\" ]\n verbs: [ \"list\", \"watch\", \"create\", \"update\", \"patch\" ]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents/status\"]\n verbs: [\"update\"]\n - apiGroups: [ \"snapshot.storage.k8s.io\" ]\n resources: [ \"volumesnapshots\" ]\n verbs: [ \"get\", \"list\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"nodes\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-provisioner-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-upcloud-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-upcloud-provisioner-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n# Attacher must be able to work with PVs, nodes and VolumeAttachments\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-attacher-role\nrules:\n - apiGroups: [ \"\" ]\n resources: [ \"persistentvolumes\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"update\", \"patch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"nodes\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n - apiGroups: [ \"storage.k8s.io\" ]\n resources: [ \"csinodes\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n - apiGroups: [ \"storage.k8s.io\" ]\n resources: [ \"volumeattachments\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"update\", \"patch\" ]\n - apiGroups: [ \"storage.k8s.io\" ]\n resources: [ \"volumeattachments/status\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"update\", \"patch\" ]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-attacher-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-upcloud-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-upcloud-attacher-role\n apiGroup: rbac.authorization.k8s.io\n\n---\n# Provisioner must be able to work with endpoints and leases in current namespace\n# if (and only if) leadership election is enabled\nkind: Role\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n namespace: kube-system\n name: csi-upcloud-provisioner-cfg-role\nrules:\n- apiGroups: [\"\"]\n resources: [\"endpoints\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n- apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n\n---\nkind: RoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-provisioner-role-cfg-binding\n namespace: kube-system\nsubjects:\n - kind: ServiceAccount\n name: csi-upcloud-controller-sa\n namespace: kube-system\nroleRef:\n kind: Role\n name: csi-upcloud-provisioner-cfg-role\n apiGroup: rbac.authorization.k8s.io\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-resizer-role\nrules:\n - apiGroups: [ \"\" ]\n resources: [ \"persistentvolumes\" ]\n verbs: [ \"get\", \"list\", \"watch\", \"update\", \"patch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"persistentvolumeclaims\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"persistentvolumeclaims/status\" ]\n verbs: [ \"update\", \"patch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"events\" ]\n verbs: [ \"list\", \"watch\", \"create\", \"update\", \"patch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"pods\" ]\n verbs: [ \"watch\", \"list\" ]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-resizer-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-upcloud-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-upcloud-resizer-role\n apiGroup: rbac.authorization.k8s.io\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-snapshotter-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"get\", \"list\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: csi-upcloud-snapshotter-binding\nsubjects:\n - kind: ServiceAccount\n name: csi-upcloud-controller-sa\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: csi-upcloud-snapshotter-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml", "content": "---\nexternal_vsphere_vcenter_port: \"443\"\nexternal_vsphere_insecure: \"true\"\nexternal_vsphere_kubernetes_cluster_id: \"kubernetes-cluster-id\"\nexternal_vsphere_version: \"7.0u1\"\n\nvsphere_syncer_image_tag: \"v3.3.1\"\nvsphere_csi_attacher_image_tag: \"v4.3.0\"\nvsphere_csi_controller: \"v3.3.1\"\nvsphere_csi_liveness_probe_image_tag: \"v2.10.0\"\nvsphere_csi_provisioner_image_tag: \"v3.5.0\"\nvsphere_csi_snapshotter_image_tag: \"v6.2.2\"\nvsphere_csi_node_driver_registrar_image_tag: \"v2.8.0\"\nvsphere_csi_driver_image_tag: \"v3.3.1\"\nvsphere_csi_resizer_tag: \"v1.8.0\"\n\n# Set to kube-system for backward compatibility, should be change to vmware-system-csi on the long run\nvsphere_csi_namespace: \"kube-system\"\n\nvsphere_csi_controller_replicas: 1\n\ncsi_endpoint: '{% if external_vsphere_version >= \"7.0u1\" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}'\n\nvsphere_csi_aggressive_node_drain: false\nvsphere_csi_aggressive_node_unreachable_timeout: 300\nvsphere_csi_aggressive_node_not_ready_timeout: 300\n\nvsphere_csi_node_affinity: {}\n\n# https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/docs/book/features/volume_snapshot.md#how-to-enable-volume-snapshot--restore-feature-in-vsphere-csi-\n# according to the above link , we can controler the block-volume-snapshot parameter\nvsphere_csi_block_volume_snapshot: false\n\nexternal_vsphere_user: \"{{ lookup('env', 'VSPHERE_USER') }}\"\nexternal_vsphere_password: \"{{ lookup('env', 'VSPHERE_PASSWORD') }}\"\n\n# Controller resources\nvsphere_csi_snapshotter_resources: {}\nvsphere_csi_provisioner_resources: {}\nvsphere_syncer_resources: {}\nvsphere_csi_liveness_probe_controller_resources: {}\nvsphere_csi_resources: {}\nvsphere_csi_resizer_resources: {}\nvsphere_csi_attacher_resources: {}\n\n# DaemonSet node resources\nvsphere_csi_node_driver_registrar_resources: {}\nvsphere_csi_driver_resources: {}\nvsphere_csi_liveness_probe_ds_resources: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml", "content": "---\n- name: VSphere CSI Driver | Check vsphare credentials\n include_tasks: vsphere-credentials-check.yml\n\n- name: VSphere CSI Driver | Generate CSI cloud-config\n template:\n src: \"{{ item }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item }}\"\n mode: \"0640\"\n with_items:\n - vsphere-csi-cloud-config\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: VSphere CSI Driver | Generate Manifests\n template:\n src: \"{{ item }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item }}\"\n mode: \"0644\"\n with_items:\n - vsphere-csi-namespace.yml\n - vsphere-csi-driver.yml\n - vsphere-csi-controller-rbac.yml\n - vsphere-csi-node-rbac.yml\n - vsphere-csi-controller-config.yml\n - vsphere-csi-controller-deployment.yml\n - vsphere-csi-controller-service.yml\n - vsphere-csi-node.yml\n register: vsphere_csi_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: VSphere CSI Driver | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item }}\"\n state: \"latest\"\n with_items:\n - \"{{ vsphere_csi_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item }}\"\n\n- name: VSphere CSI Driver | Generate a CSI secret manifest\n command: \"{{ kubectl }} create secret generic vsphere-config-secret --from-file=csi-vsphere.conf={{ kube_config_dir }}/vsphere-csi-cloud-config -n {{ vsphere_csi_namespace }} --dry-run --save-config -o yaml\"\n register: vsphere_csi_secret_manifest\n when: inventory_hostname == groups['kube_control_plane'][0]\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n\n- name: VSphere CSI Driver | Apply a CSI secret manifest\n command:\n cmd: \"{{ kubectl }} apply -f -\"\n stdin: \"{{ vsphere_csi_secret_manifest.stdout }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n no_log: \"{{ not (unsafe_show_logs | bool) }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml", "content": "---\n- name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value\n fail:\n msg: \"external_vsphere_vcenter_ip is missing\"\n when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip\n\n- name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value\n fail:\n msg: \"external_vsphere_vcenter_port is missing\"\n when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port\n\n- name: External vSphere Cloud Provider | check external_vsphere_insecure value\n fail:\n msg: \"external_vsphere_insecure is missing\"\n when: external_vsphere_insecure is not defined or not external_vsphere_insecure\n\n- name: External vSphere Cloud Provider | check external_vsphere_user value\n fail:\n msg: \"external_vsphere_user is missing\"\n when: external_vsphere_user is not defined or not external_vsphere_user\n\n- name: External vSphere Cloud Provider | check external_vsphere_password value\n fail:\n msg: \"external_vsphere_password is missing\"\n when:\n - external_vsphere_password is not defined or not external_vsphere_password\n\n- name: External vSphere Cloud Provider | check external_vsphere_datacenter value\n fail:\n msg: \"external_vsphere_datacenter is missing\"\n when:\n - external_vsphere_datacenter is not defined or not external_vsphere_datacenter\n\n- name: External vSphere Cloud Provider | check external_vsphere_kubernetes_cluster_id value\n fail:\n msg: \"external_vsphere_kubernetes_cluster_id is missing\"\n when:\n - external_vsphere_kubernetes_cluster_id is not defined or not external_vsphere_kubernetes_cluster_id\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2", "content": "[Global]\ncluster-id = \"{{ external_vsphere_kubernetes_cluster_id }}\"\n\n[VirtualCenter \"{{ external_vsphere_vcenter_ip }}\"]\ninsecure-flag = \"{{ external_vsphere_insecure }}\"\nuser = \"{{ external_vsphere_user }}\"\npassword = \"{{ external_vsphere_password }}\"\nport = \"{{ external_vsphere_vcenter_port }}\"\ndatacenters = \"{{ external_vsphere_datacenter }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2", "content": "apiVersion: v1\ndata:\n{% if external_vsphere_version >= \"7.0\" %}\n \"csi-auth-check\": \"true\"\n{% else %}\n \"csi-auth-check\": \"false\"\n{% endif %}\n \"csi-auth-check\": \"true\"\n \"online-volume-extend\": \"true\"\n \"trigger-csi-fullsync\": \"false\"\n \"async-query-volume\": \"true\"\n \"block-volume-snapshot\": \"true\"\n \"csi-windows-support\": \"false\"\n \"list-volumes\": \"true\"\n \"pv-to-backingdiskobjectid-mapping\": \"false\"\n \"cnsmgr-suspend-create-volume\": \"true\"\n \"topology-preferential-datastores\": \"true\"\n \"max-pvscsi-targets-per-vm\": \"true\"\n \"multi-vcenter-csi-topology\": \"true\"\n \"csi-internal-generated-cluster-id\": \"true\"\n \"listview-tasks\": \"true\"\n \"improved-csi-idempotency\": \"true\"\n \"improved-volume-topology\": \"true\"\n \"use-csinode-id\": \"true\"\n \"list-volumes\": \"false\"\nkind: ConfigMap\nmetadata:\n name: internal-feature-states.csi.vsphere.vmware.com\n namespace: \"{{ vsphere_csi_namespace }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2", "content": "kind: Deployment\napiVersion: apps/v1\nmetadata:\n name: vsphere-csi-controller\n namespace: \"{{ vsphere_csi_namespace }}\"\nspec:\n replicas: {{ vsphere_csi_controller_replicas }}\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n maxSurge: 0\n selector:\n matchLabels:\n app: vsphere-csi-controller\n template:\n metadata:\n labels:\n app: vsphere-csi-controller\n role: vsphere-csi\n spec:\n priorityClassName: system-cluster-critical # Guarantees scheduling for critical system pods\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchExpressions:\n - key: \"app\"\n operator: In\n values:\n - vsphere-csi-controller\n topologyKey: \"kubernetes.io/hostname\"\n serviceAccountName: vsphere-csi-controller\n nodeSelector:\n node-role.kubernetes.io/control-plane: \"\"\n tolerations:\n - operator: \"Exists\"\n key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n{% if vsphere_csi_aggressive_node_drain %}\n # set below toleration if you need an aggressive pod eviction in case when\n # node becomes not-ready or unreachable. Default is 300 seconds if not specified.\n - key: node.kubernetes.io/not-ready\n operator: Exists\n effect: NoExecute\n tolerationSeconds: {{ vsphere_csi_aggressive_node_not_ready_timeout }}\n - key: node.kubernetes.io/unreachable\n operator: Exists\n effect: NoExecute\n tolerationSeconds: {{ vsphere_csi_aggressive_node_unreachable_timeout }}\n{% endif %}\n dnsPolicy: \"Default\"\n containers:\n - name: csi-attacher\n image: {{ kube_image_repo }}/sig-storage/csi-attacher:{{ vsphere_csi_attacher_image_tag }}\n args:\n - \"--v=4\"\n - \"--timeout=300s\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--leader-election\"\n - \"--leader-election-lease-duration=120s\"\n - \"--leader-election-renew-deadline=60s\"\n - \"--leader-election-retry-period=30s\"\n - \"--kube-api-qps=100\"\n - \"--kube-api-burst=100\"\n{% if vsphere_csi_attacher_resources | length > 0 %}\n resources:\n {{ vsphere_csi_attacher_resources | default({}) | to_nice_yaml | trim | indent(width=12) }}\n{% endif %}\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n{% if external_vsphere_version >= \"7.0\" %}\n - name: csi-resizer\n image: {{ kube_image_repo }}/sig-storage/csi-resizer:{{ vsphere_csi_resizer_tag }}\n args:\n - \"--v=4\"\n - \"--timeout=300s\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--handle-volume-inuse-error=false\"\n - \"--kube-api-qps=100\"\n - \"--kube-api-burst=100\"\n - \"--leader-election\"\n - \"--leader-election-lease-duration=120s\"\n - \"--leader-election-renew-deadline=60s\"\n - \"--leader-election-retry-period=30s\"\n{% if vsphere_csi_resizer_resources | length > 0 %}\n resources:\n {{ vsphere_csi_resizer_resources | default({}) | to_nice_yaml | trim | indent(width=12) }}\n{% endif %}\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n{% endif %}\n - name: vsphere-csi-controller\n image: {{ kube_image_repo }}/csi-vsphere/driver:{{ vsphere_csi_controller }}\n args:\n - \"--fss-name=internal-feature-states.csi.vsphere.vmware.com\"\n - \"--fss-namespace={{ vsphere_csi_namespace }}\"\n{% if vsphere_csi_resources | length > 0 %}\n resources:\n {{ vsphere_csi_resources | default({}) | to_nice_yaml | trim | indent(width=12) }}\n{% endif %}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n env:\n - name: CSI_ENDPOINT\n value: unix://{{ csi_endpoint }}/csi.sock\n - name: X_CSI_MODE\n value: \"controller\"\n - name: X_CSI_SPEC_DISABLE_LEN_CHECK\n value: \"true\"\n - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT\n value: 3m\n - name: VSPHERE_CSI_CONFIG\n value: \"/etc/cloud/csi-vsphere.conf\"\n - name: LOGGER_LEVEL\n value: \"PRODUCTION\" # Options: DEVELOPMENT, PRODUCTION\n{% if external_vsphere_version >= \"7.0u1\" %}\n - name: INCLUSTER_CLIENT_QPS\n value: \"100\"\n - name: INCLUSTER_CLIENT_BURST\n value: \"100\"\n{% endif %}\n volumeMounts:\n - mountPath: /etc/cloud\n name: vsphere-config-volume\n readOnly: true\n - mountPath: {{ csi_endpoint }}\n name: socket-dir\n securityContext:\n runAsNonRoot: true\n runAsUser: 65532\n runAsGroup: 65532\n ports:\n - name: healthz\n containerPort: 9808\n protocol: TCP\n - name: prometheus\n containerPort: 2112\n protocol: TCP\n livenessProbe:\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 30\n timeoutSeconds: 10\n periodSeconds: 180\n failureThreshold: 3\n - name: liveness-probe\n image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}\n args:\n - \"--v=4\"\n - \"--csi-address=$(ADDRESS)\"\n{% if vsphere_csi_liveness_probe_controller_resources | length > 0 %}\n resources:\n {{ vsphere_csi_liveness_probe_controller_resources | default({}) | to_nice_yaml | trim | indent(width=12) }}\n{% endif %}\n env:\n - name: ADDRESS\n value: {{ csi_endpoint }}/csi.sock\n volumeMounts:\n - name: socket-dir\n mountPath: {{ csi_endpoint }}\n - name: vsphere-syncer\n image: {{ kube_image_repo }}/csi-vsphere/syncer:{{ vsphere_syncer_image_tag }}\n args:\n - \"--leader-election\"\n - \"--leader-election-lease-duration=30s\"\n - \"--leader-election-renew-deadline=20s\"\n - \"--leader-election-retry-period=10s\"\n - \"--fss-name=internal-feature-states.csi.vsphere.vmware.com\"\n - \"--fss-namespace={{ vsphere_csi_namespace }}\"\n imagePullPolicy: {{ k8s_image_pull_policy }}\n securityContext:\n runAsNonRoot: true\n runAsUser: 65532\n runAsGroup: 65532\n ports:\n - containerPort: 2113\n name: prometheus\n protocol: TCP\n{% if vsphere_syncer_resources | length > 0 %}\n resources:\n {{ vsphere_syncer_resources | default({}) | to_nice_yaml | trim | indent(width=12) }}\n{% endif %}\n env:\n - name: FULL_SYNC_INTERVAL_MINUTES\n value: \"30\"\n - name: VSPHERE_CSI_CONFIG\n value: \"/etc/cloud/csi-vsphere.conf\"\n - name: LOGGER_LEVEL\n value: \"PRODUCTION\" # Options: DEVELOPMENT, PRODUCTION\n{% if external_vsphere_version >= \"7.0u1\" %}\n - name: INCLUSTER_CLIENT_QPS\n value: \"100\"\n - name: INCLUSTER_CLIENT_BURST\n value: \"100\"\n{% endif %}\n volumeMounts:\n - mountPath: /etc/cloud\n name: vsphere-config-volume\n readOnly: true\n - name: csi-provisioner\n image: {{ kube_image_repo }}/sig-storage/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}\n args:\n - \"--v=4\"\n - \"--timeout=300s\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--kube-api-qps=100\"\n - \"--kube-api-burst=100\"\n - \"--leader-election\"\n - \"--leader-election-lease-duration=120s\"\n - \"--leader-election-renew-deadline=60s\"\n - \"--leader-election-retry-period=30s\"\n - \"--default-fstype=ext4\"\n - \"--leader-election\"\n - \"--default-fstype=ext4\"\n # needed only for topology aware setup\n #- \"--feature-gates=Topology=true\"\n #- \"--strict-topology\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n{% if vsphere_csi_provisioner_resources | length > 0 %}\n resources:\n {{ vsphere_csi_provisioner_resources | default({}) | to_nice_yaml | trim | indent(width=12) }}\n{% endif %}\n - name: csi-snapshotter\n image: {{ kube_image_repo }}/sig-storage/csi-snapshotter:{{ vsphere_csi_snapshotter_image_tag }}\n args:\n - \"--v=4\"\n - \"--kube-api-qps=100\"\n - \"--kube-api-burst=100\"\n - \"--timeout=300s\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--leader-election\"\n - \"--leader-election-lease-duration=120s\"\n - \"--leader-election-renew-deadline=60s\"\n - \"--leader-election-retry-period=30s\"\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n volumeMounts:\n - mountPath: /csi\n name: socket-dir\n volumes:\n - name: vsphere-config-volume\n secret:\n secretName: vsphere-config-secret\n - name: socket-dir\n emptyDir: {}\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2", "content": "kind: ServiceAccount\napiVersion: v1\nmetadata:\n name: vsphere-csi-controller\n namespace: \"{{ vsphere_csi_namespace }}\"\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: vsphere-csi-controller-role\nrules:\n - apiGroups: [\"\"]\n resources: [\"nodes\", \"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"configmaps\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n{% if external_vsphere_version >= \"7.0\" %}\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims/status\"]\n{% if external_vsphere_version >= \"7.0u1\" %}\n verbs: [\"patch\"]\n{% else %}\n verbs: [\"update\", \"patch\"]\n{% endif %}\n{% endif %}\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"delete\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\",\"csinodes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\", \"update\"]\n - apiGroups: [\"cns.vmware.com\"]\n resources: [\"triggercsifullsyncs\"]\n verbs: [\"create\", \"get\", \"update\", \"watch\", \"list\"]\n - apiGroups: [\"cns.vmware.com\"]\n resources: [\"cnsvspherevolumemigrations\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\"]\n - apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"get\", \"create\", \"update\"]\n - apiGroups: [\"cns.vmware.com\"]\n resources: [\"cnsvolumeoperationrequests\"]\n verbs: [\"create\", \"get\", \"list\", \"update\", \"delete\"]\n - apiGroups: [ \"cns.vmware.com\" ]\n resources: [ \"csinodetopologies\" ]\n verbs: [\"get\", \"update\", \"watch\", \"list\"]\n - apiGroups: [\"storage.k8s.io\"]\n resources: [\"volumeattachments/status\"]\n verbs: [\"patch\"]\n - apiGroups: [ \"snapshot.storage.k8s.io\" ]\n resources: [ \"volumesnapshots\" ]\n verbs: [ \"get\", \"list\" ]\n - apiGroups: [ \"snapshot.storage.k8s.io\" ]\n resources: [ \"volumesnapshotclasses\" ]\n verbs: [ \"watch\", \"get\", \"list\" ]\n - apiGroups: [ \"snapshot.storage.k8s.io\" ]\n resources: [ \"volumesnapshotcontents\" ]\n verbs: [ \"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\", \"patch\" ]\n - apiGroups: [ \"snapshot.storage.k8s.io\" ]\n resources: [ \"volumesnapshotcontents/status\" ]\n verbs: [ \"update\", \"patch\" ]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: vsphere-csi-controller-binding\nsubjects:\n - kind: ServiceAccount\n name: vsphere-csi-controller\n namespace: \"{{ vsphere_csi_namespace }}\"\nroleRef:\n kind: ClusterRole\n name: vsphere-csi-controller-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: vsphere-csi-controller\n namespace: \"{{ vsphere_csi_namespace }}\"\n labels:\n app: vsphere-csi-controller\nspec:\n ports:\n - name: ctlr\n port: 2112\n targetPort: 2112\n protocol: TCP\n - name: syncer\n port: 2113\n targetPort: 2113\n protocol: TCP\n selector:\n app: vsphere-csi-controller\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2", "content": "apiVersion: storage.k8s.io/v1\nkind: CSIDriver\nmetadata:\n name: csi.vsphere.vmware.com\nspec:\n attachRequired: true\n podInfoOnMount: false\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2", "content": "apiVersion: v1\nkind: Namespace\nmetadata:\n name: \"{{ vsphere_csi_namespace }}\"\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2", "content": "---\nkind: ServiceAccount\napiVersion: v1\nmetadata:\n name: vsphere-csi-node\n namespace: \"{{ vsphere_csi_namespace }}\"\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: vsphere-csi-node-cluster-role\nrules:\n - apiGroups: [\"cns.vmware.com\"]\n resources: [\"csinodetopologies\"]\n verbs: [\"create\", \"watch\", \"get\", \"patch\" ]\n - apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\"]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: vsphere-csi-node-cluster-role-binding\nsubjects:\n - kind: ServiceAccount\n name: vsphere-csi-node\n namespace: \"{{ vsphere_csi_namespace }}\"\nroleRef:\n kind: ClusterRole\n name: vsphere-csi-node-cluster-role\n apiGroup: rbac.authorization.k8s.io\n---\nkind: Role\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: vsphere-csi-node-role\n namespace: \"{{ vsphere_csi_namespace }}\"\nrules:\n - apiGroups: [\"\"]\n resources: [\"configmaps\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\nkind: RoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: vsphere-csi-node-binding\n namespace: \"{{ vsphere_csi_namespace }}\"\nsubjects:\n - kind: ServiceAccount\n name: vsphere-csi-node\n namespace: \"{{ vsphere_csi_namespace }}\"\nroleRef:\n kind: Role\n name: vsphere-csi-node-role\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2", "content": "kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: vsphere-csi-node\n namespace: \"{{ vsphere_csi_namespace }}\"\nspec:\n selector:\n matchLabels:\n app: vsphere-csi-node\n updateStrategy:\n type: \"RollingUpdate\"\n rollingUpdate:\n maxUnavailable: 1\n template:\n metadata:\n labels:\n app: vsphere-csi-node\n role: vsphere-csi\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n{% if vsphere_csi_node_affinity %}\n affinity:\n {{ vsphere_csi_node_affinity | to_nice_yaml | indent(width=8) }}\n{% endif %}\n serviceAccountName: vsphere-csi-node\n hostNetwork: true\n dnsPolicy: \"ClusterFirstWithHostNet\"\n containers:\n - name: node-driver-registrar\n image: {{ kube_image_repo }}/sig-storage/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}\n{% if external_vsphere_version < \"7.0u1\" %}\n lifecycle:\n preStop:\n exec:\n command: [\"/bin/sh\", \"-c\", \"rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock\"]\n{% endif %}\n args:\n - \"--v=5\"\n - \"--csi-address=$(ADDRESS)\"\n - \"--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)\"\n{% if vsphere_csi_node_driver_registrar_resources | length > 0 %}\n resources:\n {{ vsphere_csi_node_driver_registrar_resources | default({}) | to_nice_yaml | trim | indent(width=10) }}\n{% endif %}\n env:\n - name: ADDRESS\n value: /csi/csi.sock\n - name: DRIVER_REG_SOCK_PATH\n value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n - name: registration-dir\n mountPath: /registration\n livenessProbe:\n exec:\n command:\n - /csi-node-driver-registrar\n - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock\n - --mode=kubelet-registration-probe\n initialDelaySeconds: 3\n - name: vsphere-csi-node\n image: {{ kube_image_repo }}/csi-vsphere/driver:{{ vsphere_csi_driver_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--fss-name=internal-feature-states.csi.vsphere.vmware.com\"\n - \"--fss-namespace={{ vsphere_csi_namespace }}\"\n imagePullPolicy: \"Always\"\n{% if vsphere_csi_driver_resources | length > 0 %}\n resources:\n {{ vsphere_csi_driver_resources | default({}) | to_nice_yaml | trim | indent(width=10) }}\n{% endif %}\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: CSI_ENDPOINT\n value: unix:///csi/csi.sock\n - name: MAX_VOLUMES_PER_NODE\n value: \"59\" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node.\n - name: X_CSI_MODE\n value: \"node\"\n - name: X_CSI_SPEC_REQ_VALIDATION\n value: \"false\"\n - name: X_CSI_DEBUG\n value: \"true\"\n - name: X_CSI_SPEC_DISABLE_LEN_CHECK\n value: \"true\"\n - name: LOGGER_LEVEL\n value: \"PRODUCTION\" # Options: DEVELOPMENT, PRODUCTION\n - name: GODEBUG\n value: x509sha1=1\n - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES\n value: \"1\"\n securityContext:\n privileged: true\n capabilities:\n add: [\"SYS_ADMIN\"]\n allowPrivilegeEscalation: true\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n - name: pods-mount-dir\n mountPath: /var/lib/kubelet\n # needed so that any mounts setup inside this container are\n # propagated back to the host machine.\n mountPropagation: \"Bidirectional\"\n - name: device-dir\n mountPath: /dev\n - name: blocks-dir\n mountPath: /sys/block\n - name: sys-devices-dir\n mountPath: /sys/devices\n ports:\n - containerPort: 9808\n name: healthz\n livenessProbe:\n httpGet:\n path: /healthz\n port: healthz\n initialDelaySeconds: 10\n timeoutSeconds: 5\n periodSeconds: 5\n failureThreshold: 3\n - name: liveness-probe\n image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}\n args:\n{% if external_vsphere_version >= \"7.0u1\" %}\n - \"--v=4\"\n{% endif %}\n - \"--csi-address=/csi/csi.sock\"\n{% if vsphere_csi_liveness_probe_ds_resources | length > 0 %}\n resources:\n {{ vsphere_csi_liveness_probe_ds_resources | default({}) | to_nice_yaml | trim | indent(width=10) }}\n{% endif %}\n volumeMounts:\n - name: plugin-dir\n mountPath: /csi\n volumes:\n - name: registration-dir\n hostPath:\n path: /var/lib/kubelet/plugins_registry\n type: Directory\n - name: plugin-dir\n hostPath:\n path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com\n type: DirectoryOrCreate\n - name: pods-mount-dir\n hostPath:\n path: /var/lib/kubelet\n type: Directory\n - name: device-dir\n hostPath:\n path: /dev\n - name: blocks-dir\n hostPath:\n path: /sys/block\n type: Directory\n - name: sys-devices-dir\n hostPath:\n path: /sys/devices\n type: Directory\n tolerations:\n - effect: NoExecute\n operator: Exists\n - effect: NoSchedule\n operator: Exists\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml", "content": "---\nexternal_hcloud_cloud:\n hcloud_api_token: \"\"\n token_secret_name: hcloud\n\n service_account_name: cloud-controller-manager\n\n controller_image_tag: \"latest\"\n ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset\n ## Format:\n ## external_hcloud_cloud.controller_extra_args:\n ## arg1: \"value1\"\n ## arg2: \"value2\"\n controller_extra_args: {}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml", "content": "---\n- name: External Hcloud Cloud Controller | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n with_items:\n - {name: external-hcloud-cloud-secret, file: external-hcloud-cloud-secret.yml}\n - {name: external-hcloud-cloud-service-account, file: external-hcloud-cloud-service-account.yml}\n - {name: external-hcloud-cloud-role-bindings, file: external-hcloud-cloud-role-bindings.yml}\n - {name: \"{{ 'external-hcloud-cloud-controller-manager-ds-with-networks' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds' }}\", file: \"{{ 'external-hcloud-cloud-controller-manager-ds-with-networks.yml' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds.yml' }}\"}\n\n register: external_hcloud_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-hcloud\n\n- name: External Hcloud Cloud Controller | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ external_hcloud_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n tags: external-hcloud\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2", "content": "---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: hcloud-cloud-controller-manager\n namespace: kube-system\n labels:\n k8s-app: hcloud-cloud-controller-manger\nspec:\n selector:\n matchLabels:\n app: hcloud-cloud-controller-manager\n template:\n metadata:\n labels:\n app: hcloud-cloud-controller-manager\n annotations:\n scheduler.alpha.kubernetes.io/critical-pod: ''\n spec:\n serviceAccountName: {{ external_hcloud_cloud.service_account_name }}\n dnsPolicy: Default\n tolerations:\n - key: \"node.cloudprovider.kubernetes.io/uninitialized\"\n value: \"true\"\n effect: \"NoSchedule\"\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: \"node-role.kubernetes.io/control-plane\"\n effect: NoSchedule\n operator: Exists\n - key: \"node.kubernetes.io/not-ready\"\n effect: \"NoSchedule\"\n hostNetwork: true\n containers:\n - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }}\n name: hcloud-cloud-controller-manager\n command:\n - \"/bin/hcloud-cloud-controller-manager\"\n - \"--cloud-provider=hcloud\"\n - \"--leader-elect=false\"\n - \"--allow-untagged-cloud\"\n - \"--allocate-node-cidrs=true\"\n - \"--cluster-cidr={{ kube_pods_subnet }}\"\n{% if external_hcloud_cloud.controller_extra_args is defined %}\n\n args:\n{% for key, value in external_hcloud_cloud.controller_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n{% endif %}\n resources:\n requests:\n cpu: 100m\n memory: 50Mi\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: HCLOUD_TOKEN\n valueFrom:\n secretKeyRef:\n name: {{ external_hcloud_cloud.token_secret_name }}\n key: token\n - name: HCLOUD_NETWORK\n valueFrom:\n secretKeyRef:\n name: {{ external_hcloud_cloud.token_secret_name }}\n key: network\n{% if external_hcloud_cloud.network_routes_enabled is defined %}\n - name: HCLOUD_NETWORK_ROUTES_ENABLED\n value: \"{{ external_hcloud_cloud.network_routes_enabled }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_location is defined %}\n - name: HCLOUD_LOAD_BALANCERS_LOCATION\n value: \"{{ external_hcloud_cloud.load_balancers_location }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_network_zone is defined %}\n - name: HCLOUD_LOAD_BALANCERS_NETWORK_ZONE\n value: \"{{ external_hcloud_cloud.load_balancers_network_zone }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_disable_private_ingress is defined %}\n - name: HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS\n value: \"{{ external_hcloud_cloud.load_balancers_disable_private_ingress }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_use_private_ip is defined %}\n - name: HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP\n value: \"{{ external_hcloud_cloud.load_balancers_use_private_ip }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_enabled is defined %}\n - name: HCLOUD_LOAD_BALANCERS_ENABLED\n value: \"{{ external_hcloud_cloud.load_balancers_enabled }}\"\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2", "content": "---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: hcloud-cloud-controller-manager\n namespace: kube-system\n labels:\n k8s-app: hcloud-cloud-controller-manger\nspec:\n selector:\n matchLabels:\n app: hcloud-cloud-controller-manager\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n app: hcloud-cloud-controller-manager\n annotations:\n scheduler.alpha.kubernetes.io/critical-pod: ''\n spec:\n serviceAccountName: {{ external_hcloud_cloud.service_account_name }}\n dnsPolicy: Default\n tolerations:\n - key: \"node.cloudprovider.kubernetes.io/uninitialized\"\n value: \"true\"\n effect: \"NoSchedule\"\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: \"node-role.kubernetes.io/control-plane\"\n effect: NoSchedule\n - key: \"node.kubernetes.io/not-ready\"\n effect: \"NoSchedule\"\n containers:\n - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }}\n name: hcloud-cloud-controller-manager\n command:\n - \"/bin/hcloud-cloud-controller-manager\"\n - \"--cloud-provider=hcloud\"\n - \"--leader-elect=false\"\n - \"--allow-untagged-cloud\"\n{% if external_hcloud_cloud.controller_extra_args is defined %}\n args:\n{% for key, value in external_hcloud_cloud.controller_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n{% endif %}\n resources:\n requests:\n cpu: 100m\n memory: 50Mi\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: HCLOUD_TOKEN\n valueFrom:\n secretKeyRef:\n name: {{ external_hcloud_cloud.token_secret_name }}\n key: token\n{% if external_hcloud_cloud.network_name is defined %}\n - name: HCLOUD_NETWORK\n valueFrom:\n secretKeyRef:\n name: {{ external_hcloud_cloud.token_secret_name }}\n key: network\n{% endif %}\n{% if external_hcloud_cloud.network_routes_enabled is defined %}\n - name: HCLOUD_NETWORK_ROUTES_ENABLED\n value: \"{{ external_hcloud_cloud.network_routes_enabled }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_location is defined %}\n - name: HCLOUD_LOAD_BALANCERS_LOCATION\n value: \"{{ external_hcloud_cloud.load_balancers_location }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_network_zone is defined %}\n - name: HCLOUD_LOAD_BALANCERS_NETWORK_ZONE\n value: \"{{ external_hcloud_cloud.load_balancers_network_zone }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_disable_private_ingress is defined %}\n - name: HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS\n value: \"{{ external_hcloud_cloud.load_balancers_disable_private_ingress }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_use_private_ip is defined %}\n - name: HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP\n value: \"{{ external_hcloud_cloud.load_balancers_use_private_ip }}\"\n{% endif %}\n{% if external_hcloud_cloud.load_balancers_enabled is defined %}\n - name: HCLOUD_LOAD_BALANCERS_ENABLED\n value: \"{{ external_hcloud_cloud.load_balancers_enabled }}\"\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2", "content": "---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:cloud-controller-manager\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n - kind: ServiceAccount\n name: {{ external_hcloud_cloud.service_account_name }}\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2", "content": "---\napiVersion: v1\nkind: Secret\nmetadata:\n name: \"{{ external_hcloud_cloud.token_secret_name }}\"\n namespace: kube-system\ndata:\n token: \"{{ external_hcloud_cloud.hcloud_api_token | b64encode }}\"\n{% if external_hcloud_cloud.with_networks or external_hcloud_cloud.network_name is defined %}\n{% if network_id is defined%}\n network: \"{{ network_id | b64encode }}\"\n{% else %}\n network: \"{{ external_hcloud_cloud.network_name | b64encode }}\"\n{% endif %}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: {{ external_hcloud_cloud.service_account_name }}\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml", "content": "---\n# The external cloud controller will need credentials to access\n# openstack apis. Per default these values will be\n# read from the environment.\nexternal_huaweicloud_auth_url: \"{{ lookup('env', 'OS_AUTH_URL') }}\"\nexternal_huaweicloud_access_key: \"{{ lookup('env', 'OS_ACCESS_KEY') }}\"\nexternal_huaweicloud_secret_key: \"{{ lookup('env', 'OS_SECRET_KEY') }}\"\nexternal_huaweicloud_region: \"{{ lookup('env', 'OS_REGION_NAME') }}\"\nexternal_huaweicloud_project_id: \"{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}\"\nexternal_huaweicloud_cloud: \"{{ lookup('env', 'OS_CLOUD') }}\"\n\n## A dictionary of extra arguments to add to the huawei cloud controller manager deployment\n## Format:\n## external_huawei_cloud_controller_extra_args:\n## arg1: \"value1\"\n## arg2: \"value2\"\nexternal_huawei_cloud_controller_extra_args: {}\nexternal_huawei_cloud_controller_image_repo: \"swr.ap-southeast-1.myhuaweicloud.com\"\nexternal_huawei_cloud_controller_image_tag: \"v0.26.8\"\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml", "content": "---\n- name: External Huawei Cloud Controller | check external_huaweicloud_auth_url value\n fail:\n msg: \"external_huaweicloud_auth_url is missing\"\n when: external_huaweicloud_auth_url is not defined or not external_huaweicloud_auth_url\n\n\n- name: External Huawei Cloud Controller | check external_huaweicloud_access_key value\n fail:\n msg: \"you must set external_huaweicloud_access_key\"\n when:\n - external_huaweicloud_access_key is not defined or not external_huaweicloud_access_key\n\n- name: External Huawei Cloud Controller | check external_huaweicloud_secret_key value\n fail:\n msg: \"external_huaweicloud_secret_key is missing\"\n when:\n - external_huaweicloud_access_key is defined\n - external_huaweicloud_access_key|length > 0\n - external_huaweicloud_secret_key is not defined or not external_huaweicloud_secret_key\n\n\n- name: External Huawei Cloud Controller | check external_huaweicloud_region value\n fail:\n msg: \"external_huaweicloud_region is missing\"\n when: external_huaweicloud_region is not defined or not external_huaweicloud_region\n\n\n- name: External Huawei Cloud Controller | check external_huaweicloud_project_id value\n fail:\n msg: \"one of external_huaweicloud_project_id must be specified\"\n when:\n - external_huaweicloud_project_id is not defined or not external_huaweicloud_project_id\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml", "content": "---\n- name: External Huawei Cloud Controller | Check Huawei credentials\n include_tasks: huaweicloud-credential-check.yml\n tags: external-huaweicloud\n\n- name: External huaweicloud Cloud Controller | Get base64 cacert\n slurp:\n src: \"{{ external_huaweicloud_cacert }}\"\n register: external_huaweicloud_cacert_b64\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - external_huaweicloud_cacert is defined\n - external_huaweicloud_cacert | length > 0\n tags: external-huaweicloud\n\n- name: External huaweicloud Cloud Controller | Get base64 cloud-config\n set_fact:\n external_huawei_cloud_config_secret: \"{{ lookup('template', 'external-huawei-cloud-config.j2') | b64encode }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-huaweicloud\n\n- name: External Huawei Cloud Controller | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n with_items:\n - {name: external-huawei-cloud-config-secret, file: external-huawei-cloud-config-secret.yml}\n - {name: external-huawei-cloud-controller-manager-roles, file: external-huawei-cloud-controller-manager-roles.yml}\n - {name: external-huawei-cloud-controller-manager-role-bindings, file: external-huawei-cloud-controller-manager-role-bindings.yml}\n - {name: external-huawei-cloud-controller-manager-ds, file: external-huawei-cloud-controller-manager-ds.yml}\n register: external_huaweicloud_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-huaweicloud\n\n- name: External Huawei Cloud Controller | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ external_huaweicloud_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n tags: external-huaweicloud\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2", "content": "# This YAML file contains secret objects,\n# which are necessary to run external huaweicloud cloud controller.\n\nkind: Secret\napiVersion: v1\nmetadata:\n name: external-huawei-cloud-config\n namespace: kube-system\ndata:\n cloud-config: {{ external_huawei_cloud_config_secret }}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2", "content": "[Global]\nauth-url=\"{{ external_huaweicloud_auth_url }}\"\n{% if external_huaweicloud_access_key is defined and external_huaweicloud_access_key != \"\" %}\naccess-key={{ external_huaweicloud_access_key }}\n{% endif %}\n{% if external_huaweicloud_secret_key is defined and external_huaweicloud_secret_key != \"\" %}\nsecret-key={{ external_huaweicloud_secret_key }}\n{% endif %}\nregion=\"{{ external_huaweicloud_region }}\"\n{% if external_huaweicloud_project_id is defined and external_huaweicloud_project_id != \"\" %}\nproject-id=\"{{ external_huaweicloud_project_id }}\"\n{% endif %}\n{% if external_huaweicloud_cloud is defined and external_huaweicloud_cloud != \"\" %}\ncloud=\"{{ external_huaweicloud_cloud }}\"\n{% endif %}\n\n[VPC]\n{% if external_huaweicloud_lbaas_subnet_id is defined %}\nsubnet-id={{ external_huaweicloud_lbaas_subnet_id }}\n{% endif %}\n{% if external_huaweicloud_lbaas_network_id is defined %}\nid={{ external_huaweicloud_lbaas_network_id }}\n{% endif %}\n{% if external_huaweicloud_security_group_id is defined %}\nsecurity-group-id={{ external_huaweicloud_security_group_id }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2", "content": "kind: Namespace\napiVersion: v1\nmetadata:\n name: huawei-cloud-provider\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cloud-controller-manager\n namespace: kube-system\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: huawei-cloud-controller-manager\n namespace: kube-system\n labels:\n k8s-app: huawei-cloud-controller-manager\nspec:\n selector:\n matchLabels:\n k8s-app: huawei-cloud-controller-manager\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n k8s-app: huawei-cloud-controller-manager\n spec:\n nodeSelector:\n node-role.kubernetes.io/control-plane: \"\"\n securityContext:\n runAsUser: 1001\n tolerations:\n - key: node.cloudprovider.kubernetes.io/uninitialized\n value: \"true\"\n effect: NoSchedule\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n serviceAccountName: cloud-controller-manager\n containers:\n - name: huawei-cloud-controller-manager\n image: {{ external_huawei_cloud_controller_image_repo }}/k8s-cloudprovider/huawei-cloud-controller-manager:{{ external_huawei_cloud_controller_image_tag }}\n args:\n - /bin/huawei-cloud-controller-manager\n - --v=1\n - --cloud-config=$(CLOUD_CONFIG)\n - --cloud-provider=huaweicloud\n - --use-service-account-credentials=true\n - --node-status-update-frequency=5s\n - --node-monitor-period=5s\n - --leader-elect-lease-duration=30s\n - --leader-elect-renew-deadline=20s\n - --leader-elect-retry-period=2s\n{% for key, value in external_huawei_cloud_controller_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n volumeMounts:\n - mountPath: /etc/kubernetes\n name: k8s-certs\n readOnly: true\n - mountPath: /etc/ssl/certs\n name: ca-certs\n readOnly: true\n - mountPath: /etc/config\n name: cloud-config-volume\n readOnly: true\n{% if kubelet_flexvolumes_plugins_dir is defined %}\n - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec\n name: flexvolume-dir\n{% endif %}\n resources:\n requests:\n cpu: 200m\n env:\n - name: CLOUD_CONFIG\n value: /etc/config/cloud-config\n hostNetwork: true\n volumes:\n{% if kubelet_flexvolumes_plugins_dir is defined %}\n - name: flexvolume-dir\n hostPath:\n path: \"{{ kubelet_flexvolumes_plugins_dir }}\"\n type: DirectoryOrCreate\n{% endif %}\n - name: k8s-certs\n hostPath:\n path: /etc/kubernetes\n type: DirectoryOrCreate\n - name: ca-certs\n hostPath:\n path: /etc/ssl/certs\n type: DirectoryOrCreate\n - name: cloud-config-volume\n secret:\n secretName: external-huawei-cloud-config\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2", "content": "kind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: system:cloud-controller-manager\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:cloud-controller-manager\nsubjects:\n - kind: ServiceAccount\n name: cloud-controller-manager\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: system:cloud-controller-manager\nrules:\n - resources:\n - tokenreviews\n verbs:\n - get\n - list\n - watch\n - create\n - update\n - patch\n apiGroups:\n - authentication.k8s.io\n - resources:\n - configmaps\n - endpoints\n - pods\n - services\n - secrets\n - serviceaccounts\n - serviceaccounts/token\n verbs:\n - get\n - list\n - watch\n - create\n - update\n - patch\n apiGroups:\n - ''\n - resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n - delete\n - patch\n - update\n apiGroups:\n - ''\n - resources:\n - services/status\n - pods/status\n verbs:\n - update\n - patch\n apiGroups:\n - ''\n - resources:\n - nodes/status\n verbs:\n - patch\n - update\n apiGroups:\n - ''\n - resources:\n - events\n - endpoints\n verbs:\n - create\n - patch\n - update\n apiGroups:\n - ''\n - resources:\n - leases\n verbs:\n - get\n - update\n - create\n - delete\n apiGroups:\n - coordination.k8s.io\n - resources:\n - customresourcedefinitions\n verbs:\n - get\n - update\n - create\n - delete\n apiGroups:\n - apiextensions.k8s.io\n - resources:\n - ingresses\n verbs:\n - get\n - list\n - watch\n - update\n - create\n - patch\n - delete\n apiGroups:\n - networking.k8s.io\n - resources:\n - ingresses/status\n verbs:\n - update\n - patch\n apiGroups:\n - networking.k8s.io\n - resources:\n - endpointslices\n verbs:\n - get\n - list\n - watch\n apiGroups:\n - discovery.k8s.io\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/external_cloud_controller/openstack\n when:\n - cloud_provider == \"external\"\n - external_cloud_provider == \"openstack\"\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - external-cloud-controller\n - external-openstack\n - role: kubernetes-apps/external_cloud_controller/vsphere\n when:\n - cloud_provider == \"external\"\n - external_cloud_provider == \"vsphere\"\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - external-cloud-controller\n - external-vsphere\n - role: kubernetes-apps/external_cloud_controller/hcloud\n when:\n - cloud_provider == \"external\"\n - external_cloud_provider == \"hcloud\"\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - external-cloud-controller\n - external-hcloud\n - role: kubernetes-apps/external_cloud_controller/huaweicloud\n when:\n - cloud_provider == \"external\"\n - external_cloud_provider == \"huaweicloud\"\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - external-cloud-controller\n - external-huaweicloud\n - role: kubernetes-apps/external_cloud_controller/oci\n when:\n - cloud_provider == \"external\"\n - external_cloud_provider == \"oci\"\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - external-cloud-controller\n - external-oci\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml", "content": "---\n## External Oracle Cloud Controller Manager\n## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml\nexternal_oracle_auth_region: \"\"\nexternal_oracle_auth_tenancy: \"\"\nexternal_oracle_auth_user: \"\"\nexternal_oracle_auth_key: \"\"\nexternal_oracle_auth_passphrase: \"\"\nexternal_oracle_auth_fingerprint: \"\"\nexternal_oracle_auth_use_instance_principals: false\n\nexternal_oracle_compartment: \"\"\nexternal_oracle_vcn: \"\"\nexternal_oracle_load_balancer_subnet1: \"\"\nexternal_oracle_load_balancer_subnet2: \"\"\nexternal_oracle_load_balancer_security_list_management_mode: All\nexternal_oracle_load_balancer_security_lists: {}\n\nexternal_oracle_ratelimiter_qps_read: 20.0\nexternal_oracle_ratelimiter_bucket_read: 5\nexternal_oracle_ratelimiter_qps_write: 20.0\nexternal_oracle_ratelimiter_bucket_write: 5\n\nexternal_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci\nexternal_oracle_cloud_controller_image_tag: \"v1.29.0\"\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml", "content": "---\n- name: \"External OCI Cloud Controller Manager | Check credentials\"\n ansible.builtin.assert:\n that:\n - external_oracle_auth_key | length > 0\n - external_oracle_auth_region | length > 0\n - external_oracle_auth_tenancy | length > 0\n - external_oracle_auth_user | length > 0\n - external_oracle_auth_fingerprint | length > 0\n when: not external_oracle_auth_use_instance_principals\n\n- name: \"External OCI Cloud Controller Manager | Check settings\"\n ansible.builtin.assert:\n that:\n - external_oracle_compartment | length > 0\n - external_oracle_vcn | length > 0\n - external_oracle_load_balancer_subnet1 | length > 0\n - external_oracle_load_balancer_subnet2 | length > 0\n - external_oracle_load_balancer_security_list_management_mode in [\"All\", \"Frontend\", \"None\"]\n\n- name: External OCI Cloud Controller Manager | Get base64 cloud-config\n set_fact:\n external_oracle_cloud_config_secret: \"{{ lookup('template', 'external-oci-cloud-config.j2') | b64encode }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-oci\n\n- name: External OCI Cloud Controller Manager | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n with_items:\n - {name: external-oci-cloud-config-secret, file: external-oci-cloud-config-secret.yml}\n - {name: external-oci-cloud-controller-manager-rbac, file: external-oci-cloud-controller-manager-rbac.yml}\n - {name: external-oci-cloud-controller-manager, file: external-oci-cloud-controller-manager.yml}\n register: external_oracle_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-oci\n\n- name: External OCI Cloud Controller Manager | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ external_oracle_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n tags: external-oci\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2", "content": "# This YAML file contains secret objects,\n# which are necessary to run external oci cloud controller.\n\nkind: Secret\napiVersion: v1\nmetadata:\n name: oci-cloud-controller-manager\n namespace: kube-system\ndata:\n cloud-provider.yaml: {{ external_oracle_cloud_config_secret }}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2", "content": "{% if external_oracle_auth_use_instance_principals %}\nuseInstancePrincipals: true\n{% endif %}\n\nauth:\n{% if external_oracle_auth_use_instance_principals %}\n useInstancePrincipals: true\n{% else %}\n useInstancePrincipals: false\n region: {{ external_oracle_auth_region }}\n tenancy: {{ external_oracle_auth_tenancy }}\n user: {{ external_oracle_auth_user }}\n key: |\n {{ external_oracle_auth_key }}\n {% if external_oracle_auth_passphrase is defined %}\n # Omit if there is not a password for the key\n passphrase: {{ external_oracle_auth_passphrase }}\n {% endif %}\n fingerprint: {{ external_oracle_auth_fingerprint }}\n{% endif %}\n\ncompartment: {{ external_oracle_compartment }}\n\nvcn: {{ external_oracle_vcn }}\n\nloadBalancer:\n subnet1: {{ external_oracle_load_balancer_subnet1 }}\n subnet2: {{ external_oracle_load_balancer_subnet2 }}\n\n securityListManagementMode: {{ external_oracle_load_balancer_security_list_management_mode }}\n\n{% if external_oracle_security_lists is defined and external_oracle_security_lists | length > 0 %}\n # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off.\n securityLists:\n {% for subnet_ocid, list_ocid in external_oracle_load_balancer_security_lists.items() %}\n {{ subnet_ocid }}: {{ list_ocid }}\n {% endfor %}\n{% endif %}\n\n# Optional rate limit controls for accessing OCI API\nrateLimiter:\n rateLimitQPSRead: {{ external_oracle_ratelimiter_qps_read }}\n rateLimitBucketRead: {{ external_oracle_ratelimiter_bucket_read }}\n rateLimitQPSWrite: {{ external_oracle_ratelimiter_qps_write }}\n rateLimitBucketWrite: {{ external_oracle_ratelimiter_bucket_write }}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cloud-controller-manager\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: system:cloud-controller-manager\n labels:\n kubernetes.io/cluster-service: \"true\"\nrules:\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - '*'\n\n- apiGroups:\n - \"\"\n resources:\n - nodes/status\n verbs:\n - patch\n\n- apiGroups:\n - \"\"\n resources:\n - services\n verbs:\n - list\n - watch\n - patch\n - get\n\n- apiGroups:\n - \"\"\n resources:\n - services/status\n verbs:\n - patch\n - get\n - update\n\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n resourceNames:\n - \"extension-apiserver-authentication\"\n verbs:\n - get\n\n- apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - list\n - watch\n - create\n - patch\n - update\n\n# For leader election\n- apiGroups:\n - \"\"\n resources:\n - endpoints\n verbs:\n - create\n\n- apiGroups:\n - \"\"\n resources:\n - endpoints\n resourceNames:\n - \"cloud-controller-manager\"\n verbs:\n - get\n - list\n - watch\n - update\n\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n verbs:\n - create\n\n- apiGroups:\n - \"coordination.k8s.io\"\n resources:\n - leases\n verbs:\n - get\n - create\n - update\n - delete\n - patch\n - watch\n\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n resourceNames:\n - \"cloud-controller-manager\"\n verbs:\n - get\n - update\n\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n resourceNames:\n - \"extension-apiserver-authentication\"\n verbs:\n - get\n - list\n - watch\n\n- apiGroups:\n - \"\"\n resources:\n - serviceaccounts\n verbs:\n - create\n - list\n - get\n - watch\n- apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - get\n - list\n\n# For the PVL\n- apiGroups:\n - \"\"\n resources:\n - persistentvolumes\n verbs:\n - list\n - watch\n - patch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: oci-cloud-controller-manager\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:cloud-controller-manager\nsubjects:\n- kind: ServiceAccount\n name: cloud-controller-manager\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2", "content": "---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: oci-cloud-controller-manager\n namespace: kube-system\n labels:\n k8s-app: oci-cloud-controller-manager\nspec:\n selector:\n matchLabels:\n component: oci-cloud-controller-manager\n tier: control-plane\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n component: oci-cloud-controller-manager\n tier: control-plane\n spec:\n serviceAccountName: cloud-controller-manager\n hostNetwork: true\n nodeSelector:\n node-role.kubernetes.io/control-plane: \"\"\n tolerations:\n - key: node.cloudprovider.kubernetes.io/uninitialized\n value: \"true\"\n effect: NoSchedule\n - key: node-role.kubernetes.io/control-plane\n operator: Exists\n effect: NoSchedule\n - key: node-role.kubernetes.io/master\n operator: Exists\n effect: NoSchedule\n volumes:\n - name: cfg\n secret:\n secretName: oci-cloud-controller-manager\n - name: kubernetes\n hostPath:\n path: /etc/kubernetes\n containers:\n - name: oci-cloud-controller-manager\n image: {{ external_oracle_cloud_controller_image_repo }}:{{ external_oracle_cloud_controller_image_tag }}\n command: [\"/usr/local/bin/oci-cloud-controller-manager\"]\n args:\n - --cloud-config=/etc/oci/cloud-provider.yaml\n - --cloud-provider=oci\n - --leader-elect-resource-lock=leases\n - --concurrent-service-syncs=3\n - --v=2\n volumeMounts:\n - name: cfg\n mountPath: /etc/oci\n readOnly: true\n - name: kubernetes\n mountPath: /etc/kubernetes\n readOnly: true\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml", "content": "---\n# The external cloud controller will need credentials to access\n# openstack apis. Per default these values will be\n# read from the environment.\nexternal_openstack_auth_url: \"{{ lookup('env', 'OS_AUTH_URL') }}\"\nexternal_openstack_username: \"{{ lookup('env', 'OS_USERNAME') }}\"\nexternal_openstack_password: \"{{ lookup('env', 'OS_PASSWORD') }}\"\nexternal_openstack_application_credential_id: \"{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}\"\nexternal_openstack_application_credential_name: \"{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}\"\nexternal_openstack_application_credential_secret: \"{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}\"\nexternal_openstack_region: \"{{ lookup('env', 'OS_REGION_NAME') }}\"\nexternal_openstack_tenant_id: \"{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}\"\nexternal_openstack_tenant_name: \"{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}\"\nexternal_openstack_domain_name: \"{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}\"\nexternal_openstack_domain_id: \"{{ lookup('env', 'OS_USER_DOMAIN_ID') }}\"\nexternal_openstack_cacert: \"{{ lookup('env', 'OS_CACERT') }}\"\n\n## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset\n## Format:\n## external_openstack_cloud_controller_extra_args:\n## arg1: \"value1\"\n## arg2: \"value2\"\nexternal_openstack_cloud_controller_extra_args: {}\nexternal_openstack_cloud_controller_image_tag: \"v1.35.0\"\nexternal_openstack_cloud_controller_bind_address: 127.0.0.1\nexternal_openstack_cloud_controller_dns_policy: ClusterFirst\n\nexternal_openstack_lbaas_member_subnet_id: \"{{ external_openstack_lbaas_subnet_id }}\"\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml", "content": "---\n- name: External OpenStack Cloud Controller | Check OpenStack credentials\n include_tasks: openstack-credential-check.yml\n tags: external-openstack\n\n- name: External OpenStack Cloud Controller | Get base64 cacert\n slurp:\n src: \"{{ external_openstack_cacert }}\"\n register: external_openstack_cacert_b64\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - external_openstack_cacert is defined\n - external_openstack_cacert | length > 0\n tags: external-openstack\n\n- name: External OpenStack Cloud Controller | Get base64 cloud-config\n set_fact:\n external_openstack_cloud_config_secret: \"{{ lookup('template', 'external-openstack-cloud-config.j2') | b64encode }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-openstack\n\n- name: External OpenStack Cloud Controller | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n group: \"{{ kube_cert_group }}\"\n mode: \"0640\"\n with_items:\n - {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml}\n - {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml}\n - {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml}\n - {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml}\n register: external_openstack_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: external-openstack\n\n- name: External OpenStack Cloud Controller | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ external_openstack_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n tags: external-openstack\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml", "content": "---\n- name: External OpenStack Cloud Controller | check external_openstack_auth_url value\n fail:\n msg: \"external_openstack_auth_url is missing\"\n when: external_openstack_auth_url is not defined or not external_openstack_auth_url\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_username or external_openstack_application_credential_name value\n fail:\n msg: \"you must either set external_openstack_username or external_openstack_application_credential_name\"\n when:\n - external_openstack_username is not defined or not external_openstack_username\n - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_application_credential_id value\n fail:\n msg: \"external_openstack_application_credential_id is missing\"\n when:\n - external_openstack_application_credential_name is defined\n - external_openstack_application_credential_name | length > 0\n - external_openstack_application_credential_id is not defined or not external_openstack_application_credential_id\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_application_credential_secret value\n fail:\n msg: \"external_openstack_application_credential_secret is missing\"\n when:\n - external_openstack_application_credential_name is defined\n - external_openstack_application_credential_name | length > 0\n - external_openstack_application_credential_secret is not defined or not external_openstack_application_credential_secret\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_password value\n fail:\n msg: \"external_openstack_password is missing\"\n when:\n - external_openstack_username is defined\n - external_openstack_username | length > 0\n - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name\n - external_openstack_application_credential_secret is not defined or not external_openstack_application_credential_secret\n - external_openstack_password is not defined or not external_openstack_password\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_region value\n fail:\n msg: \"external_openstack_region is missing\"\n when: external_openstack_region is not defined or not external_openstack_region\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_tenant_id value\n fail:\n msg: \"one of external_openstack_tenant_id or external_openstack_tenant_name must be specified\"\n when:\n - external_openstack_tenant_id is not defined or not external_openstack_tenant_id\n - external_openstack_tenant_name is not defined or not external_openstack_tenant_name\n - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name\n\n\n- name: External OpenStack Cloud Controller | check external_openstack_domain_id value\n fail:\n msg: \"one of external_openstack_domain_id or external_openstack_domain_name must be specified\"\n when:\n - external_openstack_domain_id is not defined or not external_openstack_domain_id\n - external_openstack_domain_name is not defined or not external_openstack_domain_name\n - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2", "content": "# This YAML file contains secret objects,\n# which are necessary to run external openstack cloud controller.\n\nkind: Secret\napiVersion: v1\nmetadata:\n name: external-openstack-cloud-config\n namespace: kube-system\ndata:\n cloud.conf: {{ external_openstack_cloud_config_secret }}\n{% if external_openstack_cacert_b64.content is defined %}\n ca.cert: {{ external_openstack_cacert_b64.content }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2", "content": "[Global]\nauth-url=\"{{ external_openstack_auth_url }}\"\n{% if external_openstack_application_credential_id == \"\" and external_openstack_application_credential_name == \"\" %}\nusername=\"{{ external_openstack_username }}\"\npassword=\"{{ external_openstack_password }}\"\n{% endif %}\n{% if external_openstack_application_credential_id is defined and external_openstack_application_credential_id != \"\" %}\napplication-credential-id={{ external_openstack_application_credential_id }}\n{% endif %}\n{% if external_openstack_application_credential_name is defined and external_openstack_application_credential_name != \"\" %}\napplication-credential-name={{ external_openstack_application_credential_name }}\n{% endif %}\n{% if external_openstack_application_credential_secret is defined and external_openstack_application_credential_secret != \"\" %}\napplication-credential-secret={{ external_openstack_application_credential_secret }}\n{% endif %}\nregion=\"{{ external_openstack_region }}\"\n{% if external_openstack_tenant_id is defined and external_openstack_tenant_id != \"\" %}\ntenant-id=\"{{ external_openstack_tenant_id }}\"\n{% endif %}\n{% if external_openstack_tenant_name is defined and external_openstack_tenant_name != \"\" %}\ntenant-name=\"{{ external_openstack_tenant_name }}\"\n{% endif %}\n{% if external_openstack_domain_name is defined and external_openstack_domain_name != \"\" %}\ndomain-name=\"{{ external_openstack_domain_name }}\"\n{% elif external_openstack_domain_id is defined and external_openstack_domain_id != \"\" %}\ndomain-id =\"{{ external_openstack_domain_id }}\"\n{% endif %}\n{% if external_openstack_cacert is defined and external_openstack_cacert != \"\" %}\nca-file=\"{{ kube_config_dir }}/external-openstack-cacert.pem\"\n{% endif %}\n\n[LoadBalancer]\nenabled={{ external_openstack_lbaas_enabled | string | lower }}\n{% if external_openstack_lbaas_floating_network_id is defined %}\nfloating-network-id={{ external_openstack_lbaas_floating_network_id }}\n{% endif %}\n{% if external_openstack_lbaas_floating_subnet_id is defined %}\nfloating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }}\n{% endif %}\n{% if external_openstack_lbaas_method is defined %}\nlb-method={{ external_openstack_lbaas_method }}\n{% endif %}\n{% if external_openstack_lbaas_provider is defined %}\nlb-provider={{ external_openstack_lbaas_provider }}\n{% endif %}\n{% if external_openstack_lbaas_subnet_id is defined %}\nsubnet-id={{ external_openstack_lbaas_subnet_id }}\nmember-subnet-id={{ external_openstack_lbaas_member_subnet_id }}\n{% endif %}\n{% if external_openstack_lbaas_network_id is defined %}\nnetwork-id={{ external_openstack_lbaas_network_id }}\n{% endif %}\n{% if external_openstack_lbaas_manage_security_groups is defined %}\nmanage-security-groups={{ external_openstack_lbaas_manage_security_groups }}\n{% endif %}\n{% if external_openstack_lbaas_create_monitor is defined %}\ncreate-monitor={{ external_openstack_lbaas_create_monitor }}\n{% endif %}\n{% if external_openstack_lbaas_monitor_delay is defined %}\nmonitor-delay={{ external_openstack_lbaas_monitor_delay }}\n{% endif %}\n{% if external_openstack_lbaas_monitor_max_retries is defined %}\nmonitor-max-retries={{ external_openstack_lbaas_monitor_max_retries }}\n{% endif %}\n{% if external_openstack_lbaas_monitor_timeout is defined %}\nmonitor-timeout={{ external_openstack_lbaas_monitor_timeout }}\n{% endif %}\n{% if external_openstack_lbaas_internal_lb is defined %}\ninternal-lb={{ external_openstack_lbaas_internal_lb }}\n{% endif %}\n{% if external_openstack_enable_ingress_hostname is defined %}\nenable-ingress-hostname={{ external_openstack_enable_ingress_hostname | string | lower }}\n{% endif %}\n{% if external_openstack_ingress_hostname_suffix is defined %}\ningress-hostname-suffix={{ external_openstack_ingress_hostname_suffix | string | lower }}\n{% endif %}\n{% if external_openstack_max_shared_lb is defined %}\nmax-shared-lb={{ external_openstack_max_shared_lb }}\n{% endif %}\n\n[Networking]\nipv6-support-disabled={{ external_openstack_network_ipv6_disabled | string | lower }}\n{% for network_name in external_openstack_network_internal_networks %}\ninternal-network-name=\"{{ network_name }}\"\n{% endfor %}\n{% for network_name in external_openstack_network_public_networks %}\npublic-network-name=\"{{ network_name }}\"\n{% endfor %}\n\n[Metadata]\n{% if external_openstack_metadata_search_order is defined %}\nsearch-order=\"{{ external_openstack_metadata_search_order }}\"\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cloud-controller-manager\n namespace: kube-system\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: openstack-cloud-controller-manager\n namespace: kube-system\n labels:\n k8s-app: openstack-cloud-controller-manager\nspec:\n selector:\n matchLabels:\n k8s-app: openstack-cloud-controller-manager\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n k8s-app: openstack-cloud-controller-manager\n spec:\n nodeSelector:\n node-role.kubernetes.io/control-plane: \"\"\n securityContext:\n runAsUser: 999\n tolerations:\n - key: node.cloudprovider.kubernetes.io/uninitialized\n value: \"true\"\n effect: NoSchedule\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n serviceAccountName: cloud-controller-manager\n containers:\n - name: openstack-cloud-controller-manager\n image: {{ external_openstack_cloud_controller_image_repo }}:{{ external_openstack_cloud_controller_image_tag }}\n args:\n - /bin/openstack-cloud-controller-manager\n - --v=1\n - --cloud-config=$(CLOUD_CONFIG)\n - --cloud-provider=openstack\n - --cluster-name={{ cluster_name }}\n - --use-service-account-credentials=true\n - --bind-address={{ external_openstack_cloud_controller_bind_address }}\n{% for key, value in external_openstack_cloud_controller_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n volumeMounts:\n - mountPath: /etc/kubernetes/pki\n name: k8s-certs\n readOnly: true\n - mountPath: /etc/ssl/certs\n name: ca-certs\n readOnly: true\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n mountPath: {{ dir }}\n readOnly: true\n{% endfor %}\n{% endif %}\n - mountPath: /etc/config/cloud.conf\n name: cloud-config-volume\n readOnly: true\n subPath: cloud.conf\n - mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem\n name: cloud-config-volume\n readOnly: true\n subPath: ca.cert\n{% if kubelet_flexvolumes_plugins_dir is defined %}\n - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec\n name: flexvolume-dir\n{% endif %}\n resources:\n requests:\n cpu: 200m\n env:\n - name: CLOUD_CONFIG\n value: /etc/config/cloud.conf\n hostNetwork: true\n{% if external_openstack_cloud_controller_dns_policy is defined %}\n dnsPolicy: {{ external_openstack_cloud_controller_dns_policy }}\n{% endif %}\n volumes:\n{% if kubelet_flexvolumes_plugins_dir is defined %}\n - name: flexvolume-dir\n hostPath:\n path: \"{{ kubelet_flexvolumes_plugins_dir }}\"\n type: DirectoryOrCreate\n{% endif %}\n - name: k8s-certs\n hostPath:\n path: /etc/kubernetes/pki\n type: DirectoryOrCreate\n - name: ca-certs\n hostPath:\n path: /etc/ssl/certs\n type: DirectoryOrCreate\n{% if ssl_ca_dirs | length %}\n{% for dir in ssl_ca_dirs %}\n - name: {{ dir | regex_replace('^/(.*)$', '\\\\1' ) | regex_replace('/', '-') }}\n hostPath:\n path: {{ dir }}\n type: DirectoryOrCreate\n{% endfor %}\n{% endif %}\n - name: cloud-config-volume\n secret:\n secretName: external-openstack-cloud-config\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2", "content": "apiVersion: v1\nitems:\n- apiVersion: rbac.authorization.k8s.io/v1\n kind: ClusterRoleBinding\n metadata:\n name: system:cloud-controller-manager\n roleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:cloud-controller-manager\n subjects:\n - kind: ServiceAccount\n name: cloud-controller-manager\n namespace: kube-system\nkind: List\nmetadata: {}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2", "content": "apiVersion: v1\nitems:\n- apiVersion: rbac.authorization.k8s.io/v1\n kind: ClusterRole\n metadata:\n name: system:cloud-controller-manager\n rules:\n - apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs:\n - get\n - create\n - update\n - apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n - update\n - apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - '*'\n - apiGroups:\n - \"\"\n resources:\n - nodes/status\n verbs:\n - patch\n - apiGroups:\n - \"\"\n resources:\n - services\n verbs:\n - list\n - patch\n - update\n - watch\n - apiGroups:\n - \"\"\n resources:\n - services/status\n verbs:\n - patch\n - apiGroups:\n - \"\"\n resources:\n - serviceaccounts/token\n verbs:\n - create\n - apiGroups:\n - \"\"\n resources:\n - serviceaccounts\n verbs:\n - create\n - get\n - apiGroups:\n - \"\"\n resources:\n - persistentvolumes\n verbs:\n - '*'\n - apiGroups:\n - \"\"\n resources:\n - endpoints\n verbs:\n - create\n - get\n - list\n - watch\n - update\n - apiGroups:\n - \"\"\n resources:\n - configmaps\n verbs:\n - get\n - list\n - watch\n - apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - list\n - get\n - watch\n - apiGroups:\n - authentication.k8s.io\n resources:\n - tokenreviews\n verbs:\n - create\n - apiGroups:\n - authorization.k8s.io\n resources:\n - subjectaccessreviews\n verbs:\n - create\nkind: List\nmetadata: {}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml", "content": "---\nexternal_vsphere_vcenter_port: \"443\"\nexternal_vsphere_insecure: \"true\"\n\n## A dictionary of extra arguments to add to the vsphere cloud controller manager daemonset\n## Format:\n## external_vsphere_cloud_controller_extra_args:\n## arg1: \"value1\"\n## arg2: \"value2\"\nexternal_vsphere_cloud_controller_extra_args: {}\nexternal_vsphere_cloud_controller_image_tag: \"v1.31.0\"\n\nexternal_vsphere_user: \"{{ lookup('env', 'VSPHERE_USER') }}\"\nexternal_vsphere_password: \"{{ lookup('env', 'VSPHERE_PASSWORD') }}\"\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml", "content": "---\n- name: External vSphere Cloud Controller | Check vsphere credentials\n include_tasks: vsphere-credentials-check.yml\n\n- name: External vSphere Cloud Controller | Generate CPI cloud-config\n template:\n src: \"{{ item }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item }}\"\n mode: \"0640\"\n with_items:\n - external-vsphere-cpi-cloud-config\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: External vSphere Cloud Controller | Generate Manifests\n template:\n src: \"{{ item }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item }}\"\n mode: \"0644\"\n with_items:\n - external-vsphere-cpi-cloud-config-secret.yml\n - external-vsphere-cloud-controller-manager-roles.yml\n - external-vsphere-cloud-controller-manager-role-bindings.yml\n - external-vsphere-cloud-controller-manager-ds.yml\n register: external_vsphere_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: External vSphere Cloud Provider Interface | Create a CPI configMap manifest\n command: \"{{ bin_dir }}/kubectl create configmap cloud-config --from-file=vsphere.conf={{ kube_config_dir }}/external-vsphere-cpi-cloud-config -n kube-system --dry-run --save-config -o yaml\"\n register: external_vsphere_configmap_manifest\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: External vSphere Cloud Provider Interface | Apply a CPI configMap manifest\n command:\n cmd: \"{{ bin_dir }}/kubectl apply -f -\"\n stdin: \"{{ external_vsphere_configmap_manifest.stdout }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: External vSphere Cloud Controller | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item }}\"\n state: \"latest\"\n with_items:\n - \"{{ external_vsphere_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item }}\"\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/vsphere-credentials-check.yml", "content": "---\n- name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value\n fail:\n msg: \"external_vsphere_vcenter_ip is missing\"\n when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip\n\n- name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value\n fail:\n msg: \"external_vsphere_vcenter_port is missing\"\n when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port\n\n- name: External vSphere Cloud Provider | check external_vsphere_insecure value\n fail:\n msg: \"external_vsphere_insecure is missing\"\n when: external_vsphere_insecure is not defined or not external_vsphere_insecure\n\n- name: External vSphere Cloud Provider | check external_vsphere_user value\n fail:\n msg: \"external_vsphere_user is missing\"\n when: external_vsphere_user is not defined or not external_vsphere_user\n\n- name: External vSphere Cloud Provider | check external_vsphere_password value\n fail:\n msg: \"external_vsphere_password is missing\"\n when:\n - external_vsphere_password is not defined or not external_vsphere_password\n\n- name: External vSphere Cloud Provider | check external_vsphere_datacenter value\n fail:\n msg: \"external_vsphere_datacenter is missing\"\n when:\n - external_vsphere_datacenter is not defined or not external_vsphere_datacenter\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cloud-controller-manager\n namespace: kube-system\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: vsphere-cloud-controller-manager\n namespace: kube-system\n labels:\n k8s-app: vsphere-cloud-controller-manager\nspec:\n selector:\n matchLabels:\n k8s-app: vsphere-cloud-controller-manager\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n k8s-app: vsphere-cloud-controller-manager\n spec:\n nodeSelector:\n node-role.kubernetes.io/control-plane: \"\"\n securityContext:\n runAsUser: 0\n tolerations:\n - key: node.cloudprovider.kubernetes.io/uninitialized\n value: \"true\"\n effect: NoSchedule\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n serviceAccountName: cloud-controller-manager\n containers:\n - name: vsphere-cloud-controller-manager\n image: {{ kube_image_repo }}/cloud-pv-vsphere/cloud-provider-vsphere:{{ external_vsphere_cloud_controller_image_tag }}\n args:\n - --v=2\n - --cloud-provider=vsphere\n - --cloud-config=/etc/cloud/vsphere.conf\n{% for key, value in external_vsphere_cloud_controller_extra_args.items() %}\n - \"{{ '--' + key + '=' + value }}\"\n{% endfor %}\n volumeMounts:\n - mountPath: /etc/cloud\n name: vsphere-config-volume\n readOnly: true\n resources:\n requests:\n cpu: 200m\n hostNetwork: true\n volumes:\n - name: vsphere-config-volume\n configMap:\n name: cloud-config\n---\napiVersion: v1\nkind: Service\nmetadata:\n labels:\n component: cloud-controller-manager\n name: vsphere-cloud-controller-manager\n namespace: kube-system\nspec:\n type: NodePort\n ports:\n - port: 43001\n protocol: TCP\n targetPort: 43001\n selector:\n component: cloud-controller-manager\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-role-bindings.yml.j2", "content": "apiVersion: v1\nitems:\n- apiVersion: rbac.authorization.k8s.io/v1\n kind: RoleBinding\n metadata:\n name: servicecatalog.k8s.io:apiserver-authentication-reader\n namespace: kube-system\n roleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: extension-apiserver-authentication-reader\n subjects:\n - apiGroup: \"\"\n kind: ServiceAccount\n name: cloud-controller-manager\n namespace: kube-system\n - apiGroup: \"\"\n kind: User\n name: cloud-controller-manager\n- apiVersion: rbac.authorization.k8s.io/v1\n kind: ClusterRoleBinding\n metadata:\n name: system:cloud-controller-manager\n roleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:cloud-controller-manager\n subjects:\n - kind: ServiceAccount\n name: cloud-controller-manager\n namespace: kube-system\n - kind: User\n name: cloud-controller-manager\nkind: List\nmetadata: {}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-roles.yml.j2", "content": "apiVersion: v1\nitems:\n- apiVersion: rbac.authorization.k8s.io/v1\n kind: ClusterRole\n metadata:\n name: system:cloud-controller-manager\n rules:\n - apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n - update\n - apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - '*'\n - apiGroups:\n - \"\"\n resources:\n - nodes/status\n verbs:\n - patch\n - apiGroups:\n - \"\"\n resources:\n - services\n verbs:\n - list\n - patch\n - update\n - watch\n - apiGroups:\n - \"\"\n resources:\n - services/status\n verbs:\n - patch\n - apiGroups:\n - \"\"\n resources:\n - serviceaccounts\n verbs:\n - create\n - get\n - list\n - watch\n - update\n - apiGroups:\n - \"\"\n resources:\n - persistentvolumes\n verbs:\n - get\n - list\n - update\n - watch\n - apiGroups:\n - \"\"\n resources:\n - endpoints\n verbs:\n - create\n - get\n - list\n - watch\n - update\n - apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - get\n - list\n - watch\n - apiGroups:\n - \"coordination.k8s.io\"\n resources:\n - leases\n verbs:\n - get\n - list\n - watch\n - create\n - update\nkind: List\nmetadata: {}\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config-secret.yml.j2", "content": "# This YAML file contains secret objects,\n# which are necessary to run external vsphere cloud controller.\n\napiVersion: v1\nkind: Secret\nmetadata:\n name: cpi-global-secret\n namespace: kube-system\nstringData:\n {{ external_vsphere_vcenter_ip }}.username: \"{{ external_vsphere_user }}\"\n {{ external_vsphere_vcenter_ip }}.password: \"{{ external_vsphere_password }}\"\n" }, { "path": "roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config.j2", "content": "[Global]\nport = \"{{ external_vsphere_vcenter_port }}\"\ninsecure-flag = \"{{ external_vsphere_insecure }}\"\nsecret-name = \"cpi-global-secret\"\nsecret-namespace = \"kube-system\"\n\n[VirtualCenter \"{{ external_vsphere_vcenter_ip }}\"]\ndatacenters = \"{{ external_vsphere_datacenter }}\"\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml", "content": "---\nlocal_path_provisioner_enabled: false\nlocal_path_provisioner_namespace: \"local-path-storage\"\nlocal_path_provisioner_storage_class: \"local-path\"\nlocal_path_provisioner_reclaim_policy: Delete\nlocal_path_provisioner_claim_root: /opt/local-path-provisioner/\nlocal_path_provisioner_is_default_storageclass: \"true\"\nlocal_path_provisioner_debug: false\nlocal_path_provisioner_helper_image_repo: \"busybox\"\nlocal_path_provisioner_helper_image_tag: \"latest\"\nlocal_path_provisioner_resources: {}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml", "content": "---\n- name: Local Path Provisioner | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/local_path_provisioner\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Local Path Provisioner | Create claim root dir\n file:\n path: \"{{ local_path_provisioner_claim_root }}\"\n state: directory\n mode: \"0755\"\n\n- name: Local Path Provisioner | Render Template\n set_fact:\n local_path_provisioner_templates:\n - { name: local-path-storage-ns, file: local-path-storage-ns.yml, type: ns }\n - { name: local-path-storage-sa, file: local-path-storage-sa.yml, type: sa }\n - { name: local-path-storage-cr, file: local-path-storage-cr.yml, type: cr }\n - { name: local-path-storage-clusterrolebinding, file: local-path-storage-clusterrolebinding.yml, type: clusterrolebinding }\n - { name: local-path-storage-cm, file: local-path-storage-cm.yml, type: cm }\n - { name: local-path-storage-deployment, file: local-path-storage-deployment.yml, type: deployment }\n - { name: local-path-storage-sc, file: local-path-storage-sc.yml, type: sc }\n\n- name: Local Path Provisioner | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ local_path_provisioner_templates }}\"\n register: local_path_provisioner_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Local Path Provisioner | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"{{ local_path_provisioner_namespace }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ local_path_provisioner_manifests.results }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: local-path-provisioner-bind\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: local-path-provisioner-role\nsubjects:\n - kind: ServiceAccount\n name: local-path-provisioner-service-account\n namespace: {{ local_path_provisioner_namespace }}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2", "content": "---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: local-path-config\n namespace: {{ local_path_provisioner_namespace }}\ndata:\n config.json: |-\n {\n \"nodePathMap\":[\n {\n \"node\":\"DEFAULT_PATH_FOR_NON_LISTED_NODES\",\n \"paths\":[\"{{ local_path_provisioner_claim_root }}\"]\n }\n ]\n }\n setup: |-\n #!/bin/sh\n set -eu\n mkdir -m 0777 -p \"$VOL_DIR\"\n teardown: |-\n #!/bin/sh\n set -eu\n rm -rf \"$VOL_DIR\"\n helperPod.yaml: |-\n apiVersion: v1\n kind: Pod\n metadata:\n name: helper-pod\n spec:\n containers:\n - name: helper-pod\n image: \"{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}\"\n imagePullPolicy: IfNotPresent\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: local-path-provisioner-role\nrules:\n - apiGroups: [ \"\" ]\n resources: [ \"nodes\", \"persistentvolumeclaims\", \"configmaps\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"endpoints\", \"persistentvolumes\", \"pods\" ]\n verbs: [ \"*\" ]\n - apiGroups: [ \"\" ]\n resources: [ \"events\" ]\n verbs: [ \"create\", \"patch\" ]\n - apiGroups: [ \"storage.k8s.io\" ]\n resources: [ \"storageclasses\" ]\n verbs: [ \"get\", \"list\", \"watch\" ]\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2", "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: local-path-provisioner\n namespace: {{ local_path_provisioner_namespace }}\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: local-path-provisioner\n template:\n metadata:\n labels:\n app: local-path-provisioner\n spec:\n serviceAccountName: local-path-provisioner-service-account\n containers:\n - name: local-path-provisioner\n image: {{ local_path_provisioner_image_repo }}:{{ local_path_provisioner_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command:\n - local-path-provisioner\n - start\n - --config\n - /etc/config/config.json\n{% if local_path_provisioner_debug | default(false) %}\n - --debug\n{% endif %}\n volumeMounts:\n - name: config-volume\n mountPath: /etc/config/\n env:\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n{% if local_path_provisioner_resources %}\n resources:\n {{ local_path_provisioner_resources | to_nice_yaml | indent(10) | trim }}\n{% endif %}\n volumes:\n - name: config-volume\n configMap:\n name: local-path-config\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ local_path_provisioner_namespace }}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: local-path-provisioner-service-account\n namespace: {{ local_path_provisioner_namespace }}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2", "content": "---\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: {{ local_path_provisioner_storage_class }}\n annotations:\n storageclass.kubernetes.io/is-default-class: \"{{ local_path_provisioner_is_default_storageclass }}\"\nprovisioner: rancher.io/local-path\nvolumeBindingMode: WaitForFirstConsumer\nreclaimPolicy: {{ local_path_provisioner_reclaim_policy }}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml", "content": "---\nlocal_volume_provisioner_namespace: \"kube-system\"\n# List of node labels to be copied to the PVs created by the provisioner\nlocal_volume_provisioner_nodelabels: []\n# - kubernetes.io/hostname\n# - topology.kubernetes.io/region\n# - topology.kubernetes.io/zone\nlocal_volume_provisioner_tolerations: []\nlocal_volume_provisioner_use_node_name_only: false\n# Leverages Ansible's string to Python datatype casting. Otherwise the dict_key isn't substituted.\n# see https://github.com/ansible/ansible/issues/17324\nlocal_volume_provisioner_storage_classes: |\n {\n \"{{ local_volume_provisioner_storage_class | default('local-storage') }}\": {\n \"host_dir\": \"{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}\",\n \"mount_dir\": \"{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}\",\n \"volume_mode\": \"Filesystem\",\n \"fs_type\": \"ext4\"\n }\n }\nlocal_volume_provisioner_log_level: 2\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml", "content": "---\n# include to workaround mitogen issue\n# https://github.com/dw/mitogen/issues/663\n\n- name: \"Local Volume Provisioner | Ensure base dir {{ delegate_host_base_dir.1 }} is created on {{ delegate_host_base_dir.0 }}\"\n file:\n path: \"{{ local_volume_provisioner_storage_classes[delegate_host_base_dir.1].host_dir }}\"\n state: directory\n owner: root\n group: root\n mode: \"{{ local_volume_provisioner_directory_mode }}\"\n delegate_to: \"{{ delegate_host_base_dir.0 }}\"\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml", "content": "---\n\n- name: Local Volume Provisioner | Ensure base dir is created on all hosts\n include_tasks: basedirs.yml\n loop_control:\n loop_var: delegate_host_base_dir\n loop: \"{{ groups['k8s_cluster'] | product(local_volume_provisioner_storage_classes.keys()) | list }}\"\n\n- name: Local Volume Provisioner | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/local_volume_provisioner\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n\n- name: Local Volume Provisioner | Templates list\n set_fact:\n local_volume_provisioner_templates:\n - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns }\n - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa }\n - { name: local-volume-provisioner-clusterrole, file: local-volume-provisioner-clusterrole.yml, type: clusterrole }\n - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type: clusterrolebinding }\n - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type: cm }\n - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type: ds }\n - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type: sc }\n\n- name: Local Volume Provisioner | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ local_volume_provisioner_templates }}\"\n register: local_volume_provisioner_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Local Volume Provisioner | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"{{ local_volume_provisioner_namespace }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ local_volume_provisioner_manifests.results }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: local-volume-provisioner-node-clusterrole\n namespace: {{ local_volume_provisioner_namespace }}\nrules:\n- apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\"]\n- apiGroups: [\"storage.k8s.io\"]\n resources: [\"storageclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"watch\"]\n- apiGroups: [\"\", \"events.k8s.io\"]\n resources: [\"events\"]\n verbs: [\"create\", \"update\", \"patch\"]\n- apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\"]\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: local-volume-provisioner-system-node\n namespace: {{ local_volume_provisioner_namespace }}\nsubjects:\n- kind: ServiceAccount\n name: local-volume-provisioner\n namespace: {{ local_volume_provisioner_namespace }}\nroleRef:\n kind: ClusterRole\n name: local-volume-provisioner-node-clusterrole\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2", "content": "# Macro to convert camelCase dictionary keys to snake_case keys\n{% macro convert_keys(mydict) -%}\n {% for key in mydict.keys() | list -%}\n {% set key_split = key.split('_') -%}\n {% set new_key = key_split[0] + key_split[1:] | map('capitalize') | join -%}\n {% set value = mydict.pop(key) -%}\n {{ mydict.__setitem__(new_key, value) -}}\n {{ convert_keys(value) if value is mapping else None -}}\n {% endfor -%}\n{% endmacro -%}\n\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: local-volume-provisioner\n namespace: {{ local_volume_provisioner_namespace }}\ndata:\n{% if local_volume_provisioner_nodelabels | length > 0 %}\n nodeLabelsForPV: |\n{% for nodelabel in local_volume_provisioner_nodelabels %}\n - {{ nodelabel }}\n{% endfor %}\n{% endif %}\n{% if local_volume_provisioner_use_node_name_only %}\n useNodeNameOnly: \"true\"\n{% endif %}\n storageClassMap: |\n{% for class_name, storage_class in local_volume_provisioner_storage_classes.items() %}\n {{ class_name }}:\n {{- convert_keys(storage_class) }}\n {{ storage_class | to_nice_yaml(indent=2) | indent(6) }}\n{%- endfor %}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2", "content": "---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: local-volume-provisioner\n namespace: {{ local_volume_provisioner_namespace }}\n labels:\n k8s-app: local-volume-provisioner\n version: {{ local_volume_provisioner_image_tag }}\nspec:\n selector:\n matchLabels:\n k8s-app: local-volume-provisioner\n version: {{ local_volume_provisioner_image_tag }}\n template:\n metadata:\n labels:\n k8s-app: local-volume-provisioner\n version: {{ local_volume_provisioner_image_tag }}\n spec:\n priorityClassName: {% if local_volume_provisioner_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}\n serviceAccountName: local-volume-provisioner\n nodeSelector:\n kubernetes.io/os: linux\n{% if local_volume_provisioner_tolerations %}\n tolerations:\n {{ local_volume_provisioner_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{% endif %}\n containers:\n - name: provisioner\n image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"-v={{ local_volume_provisioner_log_level }}\"\n securityContext:\n privileged: true\n env:\n - name: MY_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: MY_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n volumeMounts:\n - name: local-volume-provisioner\n mountPath: /etc/provisioner/config\n readOnly: true\n - mountPath: /dev\n name: provisioner-dev\n{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %}\n - name: local-volume-provisioner-hostpath-{{ class_name }}\n mountPath: {{ class_config.mount_dir }}\n mountPropagation: \"HostToContainer\"\n{% endfor %}\n volumes:\n - name: local-volume-provisioner\n configMap:\n name: local-volume-provisioner\n - name: provisioner-dev\n hostPath:\n path: /dev\n{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %}\n - name: local-volume-provisioner-hostpath-{{ class_name }}\n hostPath:\n path: {{ class_config.host_dir }}\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ local_volume_provisioner_namespace }}\n labels:\n name: {{ local_volume_provisioner_namespace }}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: local-volume-provisioner\n namespace: {{ local_volume_provisioner_namespace }}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2", "content": "{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %}\n---\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: {{ class_name }}\nprovisioner: kubernetes.io/no-provisioner\nvolumeBindingMode: WaitForFirstConsumer\n{% if class_config.reclaim_policy is defined %}\nreclaimPolicy: {{ class_config.reclaim_policy }}\n{% endif %}\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/external_provisioner/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/external_provisioner/local_volume_provisioner\n when:\n - local_volume_provisioner_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - apps\n - local-volume-provisioner\n - external-provisioner\n\n - role: kubernetes-apps/external_provisioner/local_path_provisioner\n when: local_path_provisioner_enabled\n tags:\n - apps\n - local-path-provisioner\n - external-provisioner\n" }, { "path": "roles/kubernetes-apps/helm/.gitkeep", "content": "" }, { "path": "roles/kubernetes-apps/helm/defaults/main.yml", "content": "---\nhelm_enabled: false\n" }, { "path": "roles/kubernetes-apps/helm/tasks/main.yml", "content": "---\n- name: Helm | Gather os specific variables\n include_vars: \"{{ item }}\"\n with_first_found:\n - files:\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}.yml\"\n - \"{{ ansible_os_family | lower }}.yml\"\n - defaults.yml\n paths:\n - ../vars\n skip: true\n\n- name: Helm | Install PyYaml\n package:\n name: \"{{ pyyaml_package }}\"\n state: present\n when: pyyaml_package is defined\n\n- name: Helm | Install PyYaml [flatcar]\n include_tasks: pyyaml-flatcar.yml\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n\n- name: Helm | Download helm\n include_tasks: \"../../../download/tasks/download_file.yml\"\n vars:\n download: \"{{ download_defaults | combine(downloads.helm) }}\"\n\n- name: Helm | Copy helm binary from download dir\n copy:\n src: \"{{ local_release_dir }}/helm-{{ helm_version }}/linux-{{ image_arch }}/helm\"\n dest: \"{{ bin_dir }}/helm\"\n mode: \"0755\"\n remote_src: true\n\n- name: Helm | Get helm completion\n command: \"{{ bin_dir }}/helm completion bash\"\n changed_when: false\n register: helm_completion\n check_mode: false\n\n- name: Helm | Install helm completion\n copy:\n dest: /etc/bash_completion.d/helm.sh\n content: \"{{ helm_completion.stdout }}\"\n mode: \"0755\"\n become: true\n" }, { "path": "roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml", "content": "---\n- name: Get installed pip version\n command: \"{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip --version\"\n register: pip_version_output\n ignore_errors: true\n changed_when: false\n\n- name: Get installed PyYAML version\n command: \"{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip show PyYAML\"\n register: pyyaml_version_output\n ignore_errors: true\n changed_when: false\n\n- name: Install pip\n command: \"{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m ensurepip --upgrade\"\n when: (pyyaml_version_output is failed) and (pip_version_output is failed)\n\n- name: Install PyYAML\n ansible.builtin.pip:\n name:\n - PyYAML\n when: (pyyaml_version_output is failed)\n" }, { "path": "roles/kubernetes-apps/helm/vars/amazon.yml", "content": "---\npyyaml_package: PyYAML\n" }, { "path": "roles/kubernetes-apps/helm/vars/centos-7.yml", "content": "---\npyyaml_package: PyYAML\n" }, { "path": "roles/kubernetes-apps/helm/vars/centos.yml", "content": "---\npyyaml_package: python3-pyyaml\n" }, { "path": "roles/kubernetes-apps/helm/vars/debian.yml", "content": "---\npyyaml_package: python3-yaml\n" }, { "path": "roles/kubernetes-apps/helm/vars/fedora.yml", "content": "---\npyyaml_package: python3-pyyaml\n" }, { "path": "roles/kubernetes-apps/helm/vars/redhat-7.yml", "content": "---\npyyaml_package: PyYAML\n" }, { "path": "roles/kubernetes-apps/helm/vars/redhat.yml", "content": "---\npyyaml_package: python3-pyyaml\n" }, { "path": "roles/kubernetes-apps/helm/vars/suse.yml", "content": "---\npyyaml_package: python3-PyYAML\n" }, { "path": "roles/kubernetes-apps/helm/vars/ubuntu.yml", "content": "---\npyyaml_package: python3-yaml\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/defaults/main.yml", "content": "---\nalb_ingress_controller_namespace: kube-system\nalb_ingress_aws_region: \"us-east-1\"\n\n# Enables logging on all outbound requests sent to the AWS API.\n# If logging is desired, set to true.\nalb_ingress_aws_debug: \"false\"\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml", "content": "---\n\n- name: ALB Ingress Controller | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/alb_ingress\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n\n- name: ALB Ingress Controller | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/alb_ingress/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - { name: alb-ingress-clusterrole, file: alb-ingress-clusterrole.yml, type: clusterrole }\n - { name: alb-ingress-clusterrolebinding, file: alb-ingress-clusterrolebinding.yml, type: clusterrolebinding }\n - { name: alb-ingress-ns, file: alb-ingress-ns.yml, type: ns }\n - { name: alb-ingress-sa, file: alb-ingress-sa.yml, type: sa }\n - { name: alb-ingress-deploy, file: alb-ingress-deploy.yml, type: deploy }\n register: alb_ingress_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: ALB Ingress Controller | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"{{ alb_ingress_controller_namespace }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/alb_ingress/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ alb_ingress_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrole.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: alb-ingress\n namespace: {{ alb_ingress_controller_namespace }}\nrules:\n - apiGroups: [\"\", \"extensions\"]\n resources: [\"configmaps\", \"endpoints\", \"nodes\", \"pods\", \"secrets\", \"events\", \"ingresses\", \"ingresses/status\", \"services\"]\n verbs: [\"list\", \"create\", \"get\", \"update\", \"watch\", \"patch\"]\n - apiGroups: [\"\", \"extensions\"]\n resources: [\"nodes\", \"pods\", \"secrets\", \"services\", \"namespaces\"]\n verbs: [\"get\", \"list\", \"watch\"]\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrolebinding.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: alb-ingress\n namespace: {{ alb_ingress_controller_namespace }}\nsubjects:\n - kind: ServiceAccount\n name: alb-ingress\n namespace: {{ alb_ingress_controller_namespace }}\nroleRef:\n kind: ClusterRole\n name: alb-ingress\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-deploy.yml.j2", "content": "# Application Load Balancer (ALB) Ingress Controller Deployment Manifest.\n# This manifest details sensible defaults for deploying an ALB Ingress Controller.\n# GitHub: https://github.com/coreos/alb-ingress-controller\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: alb-ingress-controller\n labels:\n k8s-app: alb-ingress-controller\n # Namespace the ALB Ingress Controller should run in. Does not impact which\n # namespaces it's able to resolve ingress resource for. For limiting ingress\n # namespace scope, see --watch-namespace.\n namespace: {{ alb_ingress_controller_namespace }}\nspec:\n replicas: 1\n selector:\n matchLabels:\n k8s-app: alb-ingress-controller\n strategy:\n rollingUpdate:\n maxSurge: 1\n maxUnavailable: 1\n type: RollingUpdate\n template:\n metadata:\n creationTimestamp: null\n labels:\n k8s-app: alb-ingress-controller\n spec:\n containers:\n - args:\n # Limit the namespace where this ALB Ingress Controller deployment will\n # resolve ingress resources. If left commented, all namespaces are used.\n #- --watch-namespace=your-k8s-namespace\n\n # Setting the ingress-class flag below will ensure that only ingress resources with the\n # annotation kubernetes.io/ingress.class: \"alb\" are respected by the controller. You may\n # choose any class you'd like for this controller to respect.\n - --ingress-class=alb\n # Name of your cluster. Used when naming resources created\n # by the ALB Ingress Controller, providing distinction between\n # clusters.\n - --cluster-name={{ cluster_name }}\n\n # Enables logging on all outbound requests sent to the AWS API.\n # If logging is desired, set to true.\n # - ---aws-api-debug\n{% if alb_ingress_aws_debug %}\n - --aws-api-debug\n{% endif %}\n # Maximum number of times to retry the aws calls.\n # defaults to 10.\n # - --aws-max-retries=10\n\n # AWS region this ingress controller will operate in.\n # If unspecified, it will be discovered from ec2metadata.\n # List of regions: http://docs.aws.amazon.com/general/latest/gr/rande.html#vpc_region\n{% if alb_ingress_aws_region is defined %}\n - --aws-region={{ alb_ingress_aws_region }}\n{% endif %}\n\n image: \"{{ alb_ingress_image_repo }}:{{ alb_ingress_image_tag }}\"\n imagePullPolicy: {{ k8s_image_pull_policy }}\n name: server\n resources: {}\n terminationMessagePath: /dev/termination-log\n dnsPolicy: ClusterFirst\n restartPolicy: Always\n securityContext: {}\n terminationGracePeriodSeconds: 30\n{% if rbac_enabled %}\n serviceAccountName: alb-ingress\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-ns.yml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ alb_ingress_controller_namespace }}\n labels:\n name: {{ alb_ingress_controller_namespace }}\n" }, { "path": "roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-sa.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: alb-ingress\n namespace: {{ alb_ingress_controller_namespace }}\n" }, { "path": "roles/kubernetes-apps/ingress_controller/cert_manager/defaults/main.yml", "content": "---\ncert_manager_namespace: \"cert-manager\"\ncert_manager_user: 1001\ncert_manager_tolerations: []\ncert_manager_affinity: {}\ncert_manager_nodeselector: {}\ncert_manager_dns_policy: \"ClusterFirst\"\ncert_manager_dns_config: {}\ncert_manager_controller_extra_args: []\n\n## Allow http_proxy, https_proxy and no_proxy environment variables\n## Details https://github.com/kubernetes-sigs/kubespray/blob/master/docs/proxy.md\ncert_manager_http_proxy: \"{{ http_proxy | default('') }}\"\ncert_manager_https_proxy: \"{{ https_proxy | default('') }}\"\ncert_manager_no_proxy: \"{{ no_proxy | default('') }}\"\n\n## Change leader election namespace when deploying on GKE Autopilot that forbid the changes on kube-system namespace.\n## See https://github.com/jetstack/cert-manager/issues/3717\ncert_manager_leader_election_namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml", "content": "---\n\n- name: Cert Manager | Remove legacy addon dir and manifests\n file:\n path: \"{{ kube_config_dir }}/addons/cert_manager\"\n state: absent\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - upgrade\n\n- name: Cert Manager | Remove legacy namespace\n command: >\n {{ kubectl }} delete namespace {{ cert_manager_namespace }}\n ignore_errors: true # noqa ignore-errors\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - upgrade\n\n- name: Cert Manager | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/cert_manager\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cert Manager | Templates list\n set_fact:\n cert_manager_templates:\n - { name: cert-manager, file: cert-manager.yml, type: all }\n - { name: cert-manager.crds, file: cert-manager.crds.yml, type: crd }\n\n- name: Cert Manager | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/cert_manager/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ cert_manager_templates }}\"\n register: cert_manager_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cert Manager | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/cert_manager/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ cert_manager_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2", "content": "# Copyright 2022 The cert-manager Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Source: cert-manager/deploy/crds/crd-clusterissuers.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: clusterissuers.cert-manager.io\n labels:\n app: 'cert-manager'\n app.kubernetes.io/name: 'cert-manager'\n app.kubernetes.io/instance: \"cert-manager\"\n # Generated labels\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n group: cert-manager.io\n names:\n kind: ClusterIssuer\n listKind: ClusterIssuerList\n plural: clusterissuers\n singular: clusterissuer\n categories:\n - cert-manager\n scope: Cluster\n versions:\n - name: v1\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].status\n name: Ready\n type: string\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].message\n name: Status\n priority: 1\n type: string\n - jsonPath: .metadata.creationTimestamp\n description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n name: Age\n type: date\n schema:\n openAPIV3Schema:\n description: |-\n A ClusterIssuer represents a certificate issuing authority which can be\n referenced as part of `issuerRef` fields.\n It is similar to an Issuer, however it is cluster-scoped and therefore can\n be referenced by resources that exist in *any* namespace, not just the same\n namespace as the referent.\n type: object\n required:\n - spec\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: Desired state of the ClusterIssuer resource.\n type: object\n properties:\n acme:\n description: |-\n ACME configures this issuer to communicate with a RFC8555 (ACME) server\n to obtain signed x509 certificates.\n type: object\n required:\n - privateKeySecretRef\n - server\n properties:\n caBundle:\n description: |-\n Base64-encoded bundle of PEM CAs which can be used to validate the certificate\n chain presented by the ACME server.\n Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various\n kinds of security vulnerabilities.\n If CABundle and SkipTLSVerify are unset, the system certificate bundle inside\n the container is used to validate the TLS connection.\n type: string\n format: byte\n disableAccountKeyGeneration:\n description: |-\n Enables or disables generating a new ACME account key.\n If true, the Issuer resource will *not* request a new account but will expect\n the account key to be supplied via an existing secret.\n If false, the cert-manager system will generate a new ACME account key\n for the Issuer.\n Defaults to false.\n type: boolean\n email:\n description: |-\n Email is the email address to be associated with the ACME account.\n This field is optional, but it is strongly recommended to be set.\n It will be used to contact you in case of issues with your account or\n certificates, including expiry notification emails.\n This field may be updated after the account is initially registered.\n type: string\n enableDurationFeature:\n description: |-\n Enables requesting a Not After date on certificates that matches the\n duration of the certificate. This is not supported by all ACME servers\n like Let's Encrypt. If set to true when the ACME server does not support\n it, it will create an error on the Order.\n Defaults to false.\n type: boolean\n externalAccountBinding:\n description: |-\n ExternalAccountBinding is a reference to a CA external account of the ACME\n server.\n If set, upon registration cert-manager will attempt to associate the given\n external account credentials with the registered ACME account.\n type: object\n required:\n - keyID\n - keySecretRef\n properties:\n keyAlgorithm:\n description: |-\n Deprecated: keyAlgorithm field exists for historical compatibility\n reasons and should not be used. The algorithm is now hardcoded to HS256\n in golang/x/crypto/acme.\n type: string\n enum:\n - HS256\n - HS384\n - HS512\n keyID:\n description: keyID is the ID of the CA key that the External Account is bound to.\n type: string\n keySecretRef:\n description: |-\n keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes\n Secret which holds the symmetric MAC key of the External Account Binding.\n The `key` is the index string that is paired with the key data in the\n Secret and should not be confused with the key data itself, or indeed with\n the External Account Binding keyID above.\n The secret key stored in the Secret **must** be un-padded, base64 URL\n encoded data.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n preferredChain:\n description: |-\n PreferredChain is the chain to use if the ACME server outputs multiple.\n PreferredChain is no guarantee that this one gets delivered by the ACME\n endpoint.\n For example, for Let's Encrypt's DST crosssign you would use:\n \"DST Root CA X3\" or \"ISRG Root X1\" for the newer Let's Encrypt root CA.\n This value picks the first certificate bundle in the combined set of\n ACME default and alternative chains that has a root-most certificate with\n this value as its issuer's commonname.\n type: string\n maxLength: 64\n privateKeySecretRef:\n description: |-\n PrivateKey is the name of a Kubernetes Secret resource that will be used to\n store the automatically generated ACME account private key.\n Optionally, a `key` may be specified to select a specific entry within\n the named Secret resource.\n If `key` is not specified, a default of `tls.key` will be used.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n server:\n description: |-\n Server is the URL used to access the ACME server's 'directory' endpoint.\n For example, for Let's Encrypt's staging endpoint, you would use:\n \"https://acme-staging-v02.api.letsencrypt.org/directory\".\n Only ACME v2 endpoints (i.e. RFC 8555) are supported.\n type: string\n skipTLSVerify:\n description: |-\n INSECURE: Enables or disables validation of the ACME server TLS certificate.\n If true, requests to the ACME server will not have the TLS certificate chain\n validated.\n Mutually exclusive with CABundle; prefer using CABundle to prevent various\n kinds of security vulnerabilities.\n Only enable this option in development environments.\n If CABundle and SkipTLSVerify are unset, the system certificate bundle inside\n the container is used to validate the TLS connection.\n Defaults to false.\n type: boolean\n solvers:\n description: |-\n Solvers is a list of challenge solvers that will be used to solve\n ACME challenges for the matching domains.\n Solver configurations must be provided in order to obtain certificates\n from an ACME server.\n For more information, see: https://cert-manager.io/docs/configuration/acme/\n type: array\n items:\n description: |-\n An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of.\n A selector may be provided to use different solving strategies for different DNS names.\n Only one of HTTP01 or DNS01 must be provided.\n type: object\n properties:\n dns01:\n description: |-\n Configures cert-manager to attempt to complete authorizations by\n performing the DNS01 challenge flow.\n type: object\n properties:\n acmeDNS:\n description: |-\n Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage\n DNS01 challenge records.\n type: object\n required:\n - accountSecretRef\n - host\n properties:\n accountSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n host:\n type: string\n akamai:\n description: Use the Akamai DNS zone management API to manage DNS01 challenge records.\n type: object\n required:\n - accessTokenSecretRef\n - clientSecretSecretRef\n - clientTokenSecretRef\n - serviceConsumerDomain\n properties:\n accessTokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientSecretSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientTokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n serviceConsumerDomain:\n type: string\n azureDNS:\n description: Use the Microsoft Azure DNS API to manage DNS01 challenge records.\n type: object\n required:\n - resourceGroupName\n - subscriptionID\n properties:\n clientID:\n description: |-\n Auth: Azure Service Principal:\n The ClientID of the Azure Service Principal used to authenticate with Azure DNS.\n If set, ClientSecret and TenantID must also be set.\n type: string\n clientSecretSecretRef:\n description: |-\n Auth: Azure Service Principal:\n A reference to a Secret containing the password associated with the Service Principal.\n If set, ClientID and TenantID must also be set.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n environment:\n description: name of the Azure environment (default AzurePublicCloud)\n type: string\n enum:\n - AzurePublicCloud\n - AzureChinaCloud\n - AzureGermanCloud\n - AzureUSGovernmentCloud\n hostedZoneName:\n description: name of the DNS zone that should be used\n type: string\n managedIdentity:\n description: |-\n Auth: Azure Workload Identity or Azure Managed Service Identity:\n Settings to enable Azure Workload Identity or Azure Managed Service Identity\n If set, ClientID, ClientSecret and TenantID must not be set.\n type: object\n properties:\n clientID:\n description: client ID of the managed identity, can not be used at the same time as resourceID\n type: string\n resourceID:\n description: |-\n resource ID of the managed identity, can not be used at the same time as clientID\n Cannot be used for Azure Managed Service Identity\n type: string\n resourceGroupName:\n description: resource group the DNS zone is located in\n type: string\n subscriptionID:\n description: ID of the Azure subscription\n type: string\n tenantID:\n description: |-\n Auth: Azure Service Principal:\n The TenantID of the Azure Service Principal used to authenticate with Azure DNS.\n If set, ClientID and ClientSecret must also be set.\n type: string\n cloudDNS:\n description: Use the Google Cloud DNS API to manage DNS01 challenge records.\n type: object\n required:\n - project\n properties:\n hostedZoneName:\n description: |-\n HostedZoneName is an optional field that tells cert-manager in which\n Cloud DNS zone the challenge record has to be created.\n If left empty cert-manager will automatically choose a zone.\n type: string\n project:\n type: string\n serviceAccountSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n cloudflare:\n description: Use the Cloudflare API to manage DNS01 challenge records.\n type: object\n properties:\n apiKeySecretRef:\n description: |-\n API key to use to authenticate with Cloudflare.\n Note: using an API token to authenticate is now the recommended method\n as it allows greater control of permissions.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n apiTokenSecretRef:\n description: API token used to authenticate with Cloudflare.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n email:\n description: Email of the account, only required when using API key based authentication.\n type: string\n cnameStrategy:\n description: |-\n CNAMEStrategy configures how the DNS01 provider should handle CNAME\n records when found in DNS zones.\n type: string\n enum:\n - None\n - Follow\n digitalocean:\n description: Use the DigitalOcean DNS API to manage DNS01 challenge records.\n type: object\n required:\n - tokenSecretRef\n properties:\n tokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n rfc2136:\n description: |-\n Use RFC2136 (\"Dynamic Updates in the Domain Name System\") (https://datatracker.ietf.org/doc/rfc2136/)\n to manage DNS01 challenge records.\n type: object\n required:\n - nameserver\n properties:\n nameserver:\n description: |-\n The IP address or hostname of an authoritative DNS server supporting\n RFC2136 in the form host:port. If the host is an IPv6 address it must be\n enclosed in square brackets (e.g [2001:db8::1]) ; port is optional.\n This field is required.\n type: string\n tsigAlgorithm:\n description: |-\n The TSIG Algorithm configured in the DNS supporting RFC2136. Used only\n when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined.\n Supported values are (case-insensitive): ``HMACMD5`` (default),\n ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.\n type: string\n tsigKeyName:\n description: |-\n The TSIG Key name configured in the DNS.\n If ``tsigSecretSecretRef`` is defined, this field is required.\n type: string\n tsigSecretSecretRef:\n description: |-\n The name of the secret containing the TSIG value.\n If ``tsigKeyName`` is defined, this field is required.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n route53:\n description: Use the AWS Route53 API to manage DNS01 challenge records.\n type: object\n required:\n - region\n properties:\n accessKeyID:\n description: |-\n The AccessKeyID is used for authentication.\n Cannot be set when SecretAccessKeyID is set.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: string\n accessKeyIDSecretRef:\n description: |-\n The SecretAccessKey is used for authentication. If set, pull the AWS\n access key ID from a key within a Kubernetes Secret.\n Cannot be set when AccessKeyID is set.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n auth:\n description: Auth configures how cert-manager authenticates.\n type: object\n required:\n - kubernetes\n properties:\n kubernetes:\n description: |-\n Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity\n by passing a bound ServiceAccount token.\n type: object\n required:\n - serviceAccountRef\n properties:\n serviceAccountRef:\n description: |-\n A reference to a service account that will be used to request a bound\n token (also known as \"projected token\"). To use this field, you must\n configure an RBAC rule to let cert-manager request a token.\n type: object\n required:\n - name\n properties:\n audiences:\n description: |-\n TokenAudiences is an optional list of audiences to include in the\n token passed to AWS. The default token consisting of the issuer's namespace\n and name is always included.\n If unset the audience defaults to `sts.amazonaws.com`.\n type: array\n items:\n type: string\n name:\n description: Name of the ServiceAccount used to request a token.\n type: string\n hostedZoneID:\n description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call.\n type: string\n region:\n description: Always set the region when using AccessKeyID and SecretAccessKey\n type: string\n role:\n description: |-\n Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey\n or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata\n type: string\n secretAccessKeySecretRef:\n description: |-\n The SecretAccessKey is used for authentication.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n webhook:\n description: |-\n Configure an external webhook based DNS01 challenge solver to manage\n DNS01 challenge records.\n type: object\n required:\n - groupName\n - solverName\n properties:\n config:\n description: |-\n Additional configuration that should be passed to the webhook apiserver\n when challenges are processed.\n This can contain arbitrary JSON data.\n Secret values should not be specified in this stanza.\n If secret values are needed (e.g. credentials for a DNS service), you\n should use a SecretKeySelector to reference a Secret resource.\n For details on the schema of this field, consult the webhook provider\n implementation's documentation.\n x-kubernetes-preserve-unknown-fields: true\n groupName:\n description: |-\n The API group name that should be used when POSTing ChallengePayload\n resources to the webhook apiserver.\n This should be the same as the GroupName specified in the webhook\n provider implementation.\n type: string\n solverName:\n description: |-\n The name of the solver to use, as defined in the webhook provider\n implementation.\n This will typically be the name of the provider, e.g. 'cloudflare'.\n type: string\n http01:\n description: |-\n Configures cert-manager to attempt to complete authorizations by\n performing the HTTP01 challenge flow.\n It is not possible to obtain certificates for wildcard domain names\n (e.g. `*.example.com`) using the HTTP01 challenge mechanism.\n type: object\n properties:\n gatewayHTTPRoute:\n description: |-\n The Gateway API is a sig-network community API that models service networking\n in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will\n create HTTPRoutes with the specified labels in the same namespace as the challenge.\n This solver is experimental, and fields / behaviour may change in the future.\n type: object\n properties:\n labels:\n description: |-\n Custom labels that will be applied to HTTPRoutes created by cert-manager\n while solving HTTP-01 challenges.\n type: object\n additionalProperties:\n type: string\n parentRefs:\n description: |-\n When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute.\n cert-manager needs to know which parentRefs should be used when creating\n the HTTPRoute. Usually, the parentRef references a Gateway. See:\n https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways\n type: array\n items:\n description: |-\n ParentReference identifies an API object (usually a Gateway) that can be considered\n a parent of this resource (usually a route). There are two kinds of parent resources\n with \"Core\" support:\n\n\n * Gateway (Gateway conformance profile)\n * Service (Mesh conformance profile, ClusterIP Services only)\n\n\n This API may be extended in the future to support additional kinds of parent\n resources.\n\n\n The API object must be valid in the cluster; the Group and Kind must\n be registered in the cluster for this reference to be valid.\n type: object\n required:\n - name\n properties:\n group:\n description: |-\n Group is the group of the referent.\n When unspecified, \"gateway.networking.k8s.io\" is inferred.\n To set the core API group (such as for a \"Service\" kind referent),\n Group must be explicitly set to \"\" (empty string).\n\n\n Support: Core\n type: string\n default: gateway.networking.k8s.io\n maxLength: 253\n pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\n kind:\n description: |-\n Kind is kind of the referent.\n\n\n There are two kinds of parent resources with \"Core\" support:\n\n\n * Gateway (Gateway conformance profile)\n * Service (Mesh conformance profile, ClusterIP Services only)\n\n\n Support for other resources is Implementation-Specific.\n type: string\n default: Gateway\n maxLength: 63\n minLength: 1\n pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$\n name:\n description: |-\n Name is the name of the referent.\n\n\n Support: Core\n type: string\n maxLength: 253\n minLength: 1\n namespace:\n description: |-\n Namespace is the namespace of the referent. When unspecified, this refers\n to the local namespace of the Route.\n\n\n Note that there are specific rules for ParentRefs which cross namespace\n boundaries. Cross-namespace references are only valid if they are explicitly\n allowed by something in the namespace they are referring to. For example:\n Gateway has the AllowedRoutes field, and ReferenceGrant provides a\n generic way to enable any other kind of cross-namespace reference.\n\n\n \n ParentRefs from a Route to a Service in the same namespace are \"producer\"\n routes, which apply default routing rules to inbound connections from\n any namespace to the Service.\n\n\n ParentRefs from a Route to a Service in a different namespace are\n \"consumer\" routes, and these routing rules are only applied to outbound\n connections originating from the same namespace as the Route, for which\n the intended destination of the connections are a Service targeted as a\n ParentRef of the Route.\n \n\n\n Support: Core\n type: string\n maxLength: 63\n minLength: 1\n pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$\n port:\n description: |-\n Port is the network port this Route targets. It can be interpreted\n differently based on the type of parent resource.\n\n\n When the parent resource is a Gateway, this targets all listeners\n listening on the specified port that also support this kind of Route(and\n select this Route). It's not recommended to set `Port` unless the\n networking behaviors specified in a Route must apply to a specific port\n as opposed to a listener(s) whose port(s) may be changed. When both Port\n and SectionName are specified, the name and port of the selected listener\n must match both specified values.\n\n\n \n When the parent resource is a Service, this targets a specific port in the\n Service spec. When both Port (experimental) and SectionName are specified,\n the name and port of the selected port must match both specified values.\n \n\n\n Implementations MAY choose to support other parent resources.\n Implementations supporting other types of parent resources MUST clearly\n document how/if Port is interpreted.\n\n\n For the purpose of status, an attachment is considered successful as\n long as the parent resource accepts it partially. For example, Gateway\n listeners can restrict which Routes can attach to them by Route kind,\n namespace, or hostname. If 1 of 2 Gateway listeners accept attachment\n from the referencing Route, the Route MUST be considered successfully\n attached. If no Gateway listeners accept attachment from this Route,\n the Route MUST be considered detached from the Gateway.\n\n\n Support: Extended\n type: integer\n format: int32\n maximum: 65535\n minimum: 1\n sectionName:\n description: |-\n SectionName is the name of a section within the target resource. In the\n following resources, SectionName is interpreted as the following:\n\n\n * Gateway: Listener name. When both Port (experimental) and SectionName\n are specified, the name and port of the selected listener must match\n both specified values.\n * Service: Port name. When both Port (experimental) and SectionName\n are specified, the name and port of the selected listener must match\n both specified values.\n\n\n Implementations MAY choose to support attaching Routes to other resources.\n If that is the case, they MUST clearly document how SectionName is\n interpreted.\n\n\n When unspecified (empty string), this will reference the entire resource.\n For the purpose of status, an attachment is considered successful if at\n least one section in the parent resource accepts it. For example, Gateway\n listeners can restrict which Routes can attach to them by Route kind,\n namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from\n the referencing Route, the Route MUST be considered successfully\n attached. If no Gateway listeners accept attachment from this Route, the\n Route MUST be considered detached from the Gateway.\n\n\n Support: Core\n type: string\n maxLength: 253\n minLength: 1\n pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\n serviceType:\n description: |-\n Optional service type for Kubernetes solver service. Supported values\n are NodePort or ClusterIP. If unset, defaults to NodePort.\n type: string\n ingress:\n description: |-\n The ingress based HTTP01 challenge solver will solve challenges by\n creating or modifying Ingress resources in order to route requests for\n '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are\n provisioned by cert-manager for each Challenge to be completed.\n type: object\n properties:\n class:\n description: |-\n This field configures the annotation `kubernetes.io/ingress.class` when\n creating Ingress resources to solve ACME challenges that use this\n challenge solver. Only one of `class`, `name` or `ingressClassName` may\n be specified.\n type: string\n ingressClassName:\n description: |-\n This field configures the field `ingressClassName` on the created Ingress\n resources used to solve ACME challenges that use this challenge solver.\n This is the recommended way of configuring the ingress class. Only one of\n `class`, `name` or `ingressClassName` may be specified.\n type: string\n ingressTemplate:\n description: |-\n Optional ingress template used to configure the ACME challenge solver\n ingress used for HTTP01 challenges.\n type: object\n properties:\n metadata:\n description: |-\n ObjectMeta overrides for the ingress used to solve HTTP01 challenges.\n Only the 'labels' and 'annotations' fields may be set.\n If labels or annotations overlap with in-built values, the values here\n will override the in-built values.\n type: object\n properties:\n annotations:\n description: Annotations that should be added to the created ACME HTTP01 solver ingress.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels that should be added to the created ACME HTTP01 solver ingress.\n type: object\n additionalProperties:\n type: string\n name:\n description: |-\n The name of the ingress resource that should have ACME challenge solving\n routes inserted into it in order to solve HTTP01 challenges.\n This is typically used in conjunction with ingress controllers like\n ingress-gce, which maintains a 1:1 mapping between external IPs and\n ingress resources. Only one of `class`, `name` or `ingressClassName` may\n be specified.\n type: string\n podTemplate:\n description: |-\n Optional pod template used to configure the ACME challenge solver pods\n used for HTTP01 challenges.\n type: object\n properties:\n metadata:\n description: |-\n ObjectMeta overrides for the pod used to solve HTTP01 challenges.\n Only the 'labels' and 'annotations' fields may be set.\n If labels or annotations overlap with in-built values, the values here\n will override the in-built values.\n type: object\n properties:\n annotations:\n description: Annotations that should be added to the create ACME HTTP01 solver pods.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels that should be added to the created ACME HTTP01 solver pods.\n type: object\n additionalProperties:\n type: string\n spec:\n description: |-\n PodSpec defines overrides for the HTTP01 challenge solver pod.\n Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields.\n All other fields will be ignored.\n type: object\n properties:\n affinity:\n description: If specified, the pod's scheduling constraints\n type: object\n properties:\n nodeAffinity:\n description: Describes node affinity scheduling rules for the pod.\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node matches the corresponding matchExpressions; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: |-\n An empty preferred scheduling term matches all objects with implicit weight 0\n (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n type: object\n required:\n - preference\n - weight\n properties:\n preference:\n description: A node selector term, associated with the corresponding weight.\n type: object\n properties:\n matchExpressions:\n description: A list of node selector requirements by node's labels.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector requirements by node's fields.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n weight:\n description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to an update), the system\n may or may not try to eventually evict the pod from its node.\n type: object\n required:\n - nodeSelectorTerms\n properties:\n nodeSelectorTerms:\n description: Required. A list of node selector terms. The terms are ORed.\n type: array\n items:\n description: |-\n A null or empty node selector term matches no objects. The requirements of\n them are ANDed.\n The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n type: object\n properties:\n matchExpressions:\n description: A list of node selector requirements by node's labels.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector requirements by node's fields.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n podAffinity:\n description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n type: object\n required:\n - podAffinityTerm\n - weight\n properties:\n podAffinityTerm:\n description: Required. A pod affinity term, associated with the corresponding weight.\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n type: array\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n x-kubernetes-list-type: atomic\n podAntiAffinity:\n description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the anti-affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling anti-affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n type: object\n required:\n - podAffinityTerm\n - weight\n properties:\n podAffinityTerm:\n description: Required. A pod affinity term, associated with the corresponding weight.\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the anti-affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the anti-affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n type: array\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n x-kubernetes-list-type: atomic\n imagePullSecrets:\n description: If specified, the pod's imagePullSecrets\n type: array\n items:\n description: |-\n LocalObjectReference contains enough information to let you locate the\n referenced object inside the same namespace.\n type: object\n properties:\n name:\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n TODO: Add other useful fields. apiVersion, kind, uid?\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.\n type: string\n default: \"\"\n x-kubernetes-map-type: atomic\n nodeSelector:\n description: |-\n NodeSelector is a selector which must be true for the pod to fit on a node.\n Selector which must match a node's labels for the pod to be scheduled on that node.\n More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/\n type: object\n additionalProperties:\n type: string\n priorityClassName:\n description: If specified, the pod's priorityClassName.\n type: string\n serviceAccountName:\n description: If specified, the pod's service account\n type: string\n tolerations:\n description: If specified, the pod's tolerations.\n type: array\n items:\n description: |-\n The pod this Toleration is attached to tolerates any taint that matches\n the triple using the matching operator .\n type: object\n properties:\n effect:\n description: |-\n Effect indicates the taint effect to match. Empty means match all taint effects.\n When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n type: string\n key:\n description: |-\n Key is the taint key that the toleration applies to. Empty means match all taint keys.\n If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n type: string\n operator:\n description: |-\n Operator represents a key's relationship to the value.\n Valid operators are Exists and Equal. Defaults to Equal.\n Exists is equivalent to wildcard for value, so that a pod can\n tolerate all taints of a particular category.\n type: string\n tolerationSeconds:\n description: |-\n TolerationSeconds represents the period of time the toleration (which must be\n of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,\n it is not set, which means tolerate the taint forever (do not evict). Zero and\n negative values will be treated as 0 (evict immediately) by the system.\n type: integer\n format: int64\n value:\n description: |-\n Value is the taint value the toleration matches to.\n If the operator is Exists, the value should be empty, otherwise just a regular string.\n type: string\n serviceType:\n description: |-\n Optional service type for Kubernetes solver service. Supported values\n are NodePort or ClusterIP. If unset, defaults to NodePort.\n type: string\n selector:\n description: |-\n Selector selects a set of DNSNames on the Certificate resource that\n should be solved using this challenge solver.\n If not specified, the solver will be treated as the 'default' solver\n with the lowest priority, i.e. if any other solver has a more specific\n match, it will be used instead.\n type: object\n properties:\n dnsNames:\n description: |-\n List of DNSNames that this solver will be used to solve.\n If specified and a match is found, a dnsNames selector will take\n precedence over a dnsZones selector.\n If multiple solvers match with the same dnsNames value, the solver\n with the most matching labels in matchLabels will be selected.\n If neither has more matches, the solver defined earlier in the list\n will be selected.\n type: array\n items:\n type: string\n dnsZones:\n description: |-\n List of DNSZones that this solver will be used to solve.\n The most specific DNS zone match specified here will take precedence\n over other DNS zone matches, so a solver specifying sys.example.com\n will be selected over one specifying example.com for the domain\n www.sys.example.com.\n If multiple solvers match with the same dnsZones value, the solver\n with the most matching labels in matchLabels will be selected.\n If neither has more matches, the solver defined earlier in the list\n will be selected.\n type: array\n items:\n type: string\n matchLabels:\n description: |-\n A label selector that is used to refine the set of certificate's that\n this challenge solver will apply to.\n type: object\n additionalProperties:\n type: string\n ca:\n description: |-\n CA configures this issuer to sign certificates using a signing CA keypair\n stored in a Secret resource.\n This is used to build internal PKIs that are managed by cert-manager.\n type: object\n required:\n - secretName\n properties:\n crlDistributionPoints:\n description: |-\n The CRL distribution points is an X.509 v3 certificate extension which identifies\n the location of the CRL from which the revocation of this certificate can be checked.\n If not set, certificates will be issued without distribution points set.\n type: array\n items:\n type: string\n issuingCertificateURLs:\n description: |-\n IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates\n it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details.\n As an example, such a URL might be \"http://ca.domain.com/ca.crt\".\n type: array\n items:\n type: string\n ocspServers:\n description: |-\n The OCSP server list is an X.509 v3 extension that defines a list of\n URLs of OCSP responders. The OCSP responders can be queried for the\n revocation status of an issued certificate. If not set, the\n certificate will be issued with no OCSP servers set. For example, an\n OCSP server URL could be \"http://ocsp.int-x3.letsencrypt.org\".\n type: array\n items:\n type: string\n secretName:\n description: |-\n SecretName is the name of the secret used to sign Certificates issued\n by this Issuer.\n type: string\n selfSigned:\n description: |-\n SelfSigned configures this issuer to 'self sign' certificates using the\n private key used to create the CertificateRequest object.\n type: object\n properties:\n crlDistributionPoints:\n description: |-\n The CRL distribution points is an X.509 v3 certificate extension which identifies\n the location of the CRL from which the revocation of this certificate can be checked.\n If not set certificate will be issued without CDP. Values are strings.\n type: array\n items:\n type: string\n vault:\n description: |-\n Vault configures this issuer to sign certificates using a HashiCorp Vault\n PKI backend.\n type: object\n required:\n - auth\n - path\n - server\n properties:\n auth:\n description: Auth configures how cert-manager authenticates with the Vault server.\n type: object\n properties:\n appRole:\n description: |-\n AppRole authenticates with Vault using the App Role auth mechanism,\n with the role and secret stored in a Kubernetes Secret resource.\n type: object\n required:\n - path\n - roleId\n - secretRef\n properties:\n path:\n description: |-\n Path where the App Role authentication backend is mounted in Vault, e.g:\n \"approle\"\n type: string\n roleId:\n description: |-\n RoleID configured in the App Role authentication backend when setting\n up the authentication backend in Vault.\n type: string\n secretRef:\n description: |-\n Reference to a key in a Secret that contains the App Role secret used\n to authenticate with Vault.\n The `key` field must be specified and denotes which entry within the Secret\n resource is used as the app role secret.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n kubernetes:\n description: |-\n Kubernetes authenticates with Vault by passing the ServiceAccount\n token stored in the named Secret resource to the Vault server.\n type: object\n required:\n - role\n properties:\n mountPath:\n description: |-\n The Vault mountPath here is the mount path to use when authenticating with\n Vault. For example, setting a value to `/v1/auth/foo`, will use the path\n `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the\n default value \"/v1/auth/kubernetes\" will be used.\n type: string\n role:\n description: |-\n A required field containing the Vault Role to assume. A Role binds a\n Kubernetes ServiceAccount with a set of Vault policies.\n type: string\n secretRef:\n description: |-\n The required Secret field containing a Kubernetes ServiceAccount JWT used\n for authenticating with Vault. Use of 'ambient credentials' is not\n supported.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n serviceAccountRef:\n description: |-\n A reference to a service account that will be used to request a bound\n token (also known as \"projected token\"). Compared to using \"secretRef\",\n using this field means that you don't rely on statically bound tokens. To\n use this field, you must configure an RBAC rule to let cert-manager\n request a token.\n type: object\n required:\n - name\n properties:\n audiences:\n description: |-\n TokenAudiences is an optional list of extra audiences to include in the token passed to Vault. The default token\n consisting of the issuer's namespace and name is always included.\n type: array\n items:\n type: string\n name:\n description: Name of the ServiceAccount used to request a token.\n type: string\n tokenSecretRef:\n description: TokenSecretRef authenticates with Vault by presenting a token.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n caBundle:\n description: |-\n Base64-encoded bundle of PEM CAs which will be used to validate the certificate\n chain presented by Vault. Only used if using HTTPS to connect to Vault and\n ignored for HTTP connections.\n Mutually exclusive with CABundleSecretRef.\n If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in\n the cert-manager controller container is used to validate the TLS connection.\n type: string\n format: byte\n caBundleSecretRef:\n description: |-\n Reference to a Secret containing a bundle of PEM-encoded CAs to use when\n verifying the certificate chain presented by Vault when using HTTPS.\n Mutually exclusive with CABundle.\n If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in\n the cert-manager controller container is used to validate the TLS connection.\n If no key for the Secret is specified, cert-manager will default to 'ca.crt'.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientCertSecretRef:\n description: |-\n Reference to a Secret containing a PEM-encoded Client Certificate to use when the\n Vault server requires mTLS.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientKeySecretRef:\n description: |-\n Reference to a Secret containing a PEM-encoded Client Private Key to use when the\n Vault server requires mTLS.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n namespace:\n description: |-\n Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: \"ns1\"\n More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces\n type: string\n path:\n description: |-\n Path is the mount path of the Vault PKI backend's `sign` endpoint, e.g:\n \"my_pki_mount/sign/my-role-name\".\n type: string\n server:\n description: 'Server is the connection address for the Vault server, e.g: \"https://vault.example.com:8200\".'\n type: string\n venafi:\n description: |-\n Venafi configures this issuer to sign certificates using a Venafi TPP\n or Venafi Cloud policy zone.\n type: object\n required:\n - zone\n properties:\n cloud:\n description: |-\n Cloud specifies the Venafi cloud configuration settings.\n Only one of TPP or Cloud may be specified.\n type: object\n required:\n - apiTokenSecretRef\n properties:\n apiTokenSecretRef:\n description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n url:\n description: |-\n URL is the base URL for Venafi Cloud.\n Defaults to \"https://api.venafi.cloud/v1\".\n type: string\n tpp:\n description: |-\n TPP specifies Trust Protection Platform configuration settings.\n Only one of TPP or Cloud may be specified.\n type: object\n required:\n - credentialsRef\n - url\n properties:\n caBundle:\n description: |-\n Base64-encoded bundle of PEM CAs which will be used to validate the certificate\n chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP.\n If undefined, the certificate bundle in the cert-manager controller container\n is used to validate the chain.\n type: string\n format: byte\n credentialsRef:\n description: |-\n CredentialsRef is a reference to a Secret containing the username and\n password for the TPP server.\n The secret must contain two keys, 'username' and 'password'.\n type: object\n required:\n - name\n properties:\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n url:\n description: |-\n URL is the base URL for the vedsdk endpoint of the Venafi TPP instance,\n for example: \"https://tpp.example.com/vedsdk\".\n type: string\n zone:\n description: |-\n Zone is the Venafi Policy Zone to use for this issuer.\n All requests made to the Venafi platform will be restricted by the named\n zone policy.\n This field is required.\n type: string\n status:\n description: Status of the ClusterIssuer. This is set and managed automatically.\n type: object\n properties:\n acme:\n description: |-\n ACME specific status options.\n This field should only be set if the Issuer is configured to use an ACME\n server to issue certificates.\n type: object\n properties:\n lastPrivateKeyHash:\n description: |-\n LastPrivateKeyHash is a hash of the private key associated with the latest\n registered ACME account, in order to track changes made to registered account\n associated with the Issuer\n type: string\n lastRegisteredEmail:\n description: |-\n LastRegisteredEmail is the email associated with the latest registered\n ACME account, in order to track changes made to registered account\n associated with the Issuer\n type: string\n uri:\n description: |-\n URI is the unique account identifier, which can also be used to retrieve\n account details from the CA\n type: string\n conditions:\n description: |-\n List of status conditions to indicate the status of a CertificateRequest.\n Known condition types are `Ready`.\n type: array\n items:\n description: IssuerCondition contains condition information for an Issuer.\n type: object\n required:\n - status\n - type\n properties:\n lastTransitionTime:\n description: |-\n LastTransitionTime is the timestamp corresponding to the last status\n change of this condition.\n type: string\n format: date-time\n message:\n description: |-\n Message is a human readable description of the details of the last\n transition, complementing reason.\n type: string\n observedGeneration:\n description: |-\n If set, this represents the .metadata.generation that the condition was\n set based upon.\n For instance, if .metadata.generation is currently 12, but the\n .status.condition[x].observedGeneration is 9, the condition is out of date\n with respect to the current state of the Issuer.\n type: integer\n format: int64\n reason:\n description: |-\n Reason is a brief machine readable explanation for the condition's last\n transition.\n type: string\n status:\n description: Status of the condition, one of (`True`, `False`, `Unknown`).\n type: string\n enum:\n - \"True\"\n - \"False\"\n - Unknown\n type:\n description: Type of the condition, known values are (`Ready`).\n type: string\n x-kubernetes-list-map-keys:\n - type\n x-kubernetes-list-type: map\n served: true\n storage: true\n---\n# Source: cert-manager/deploy/crds/crd-challenges.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: challenges.acme.cert-manager.io\n labels:\n app: 'cert-manager'\n app.kubernetes.io/name: 'cert-manager'\n app.kubernetes.io/instance: 'cert-manager'\n # Generated labels\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n group: acme.cert-manager.io\n names:\n kind: Challenge\n listKind: ChallengeList\n plural: challenges\n singular: challenge\n categories:\n - cert-manager\n - cert-manager-acme\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - jsonPath: .status.state\n name: State\n type: string\n - jsonPath: .spec.dnsName\n name: Domain\n type: string\n - jsonPath: .status.reason\n name: Reason\n priority: 1\n type: string\n - description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1\n schema:\n openAPIV3Schema:\n description: Challenge is a type to represent a Challenge request with an ACME server\n type: object\n required:\n - metadata\n - spec\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n type: object\n required:\n - authorizationURL\n - dnsName\n - issuerRef\n - key\n - solver\n - token\n - type\n - url\n properties:\n authorizationURL:\n description: |-\n The URL to the ACME Authorization resource that this\n challenge is a part of.\n type: string\n dnsName:\n description: |-\n dnsName is the identifier that this challenge is for, e.g. example.com.\n If the requested DNSName is a 'wildcard', this field MUST be set to the\n non-wildcard domain, e.g. for `*.example.com`, it must be `example.com`.\n type: string\n issuerRef:\n description: |-\n References a properly configured ACME-type Issuer which should\n be used to create this Challenge.\n If the Issuer does not exist, processing will be retried.\n If the Issuer is not an 'ACME' Issuer, an error will be returned and the\n Challenge will be marked as failed.\n type: object\n required:\n - name\n properties:\n group:\n description: Group of the resource being referred to.\n type: string\n kind:\n description: Kind of the resource being referred to.\n type: string\n name:\n description: Name of the resource being referred to.\n type: string\n key:\n description: |-\n The ACME challenge key for this challenge\n For HTTP01 challenges, this is the value that must be responded with to\n complete the HTTP01 challenge in the format:\n `.`.\n For DNS01 challenges, this is the base64 encoded SHA256 sum of the\n `.`\n text that must be set as the TXT record content.\n type: string\n solver:\n description: |-\n Contains the domain solving configuration that should be used to\n solve this challenge resource.\n type: object\n properties:\n dns01:\n description: |-\n Configures cert-manager to attempt to complete authorizations by\n performing the DNS01 challenge flow.\n type: object\n properties:\n acmeDNS:\n description: |-\n Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage\n DNS01 challenge records.\n type: object\n required:\n - accountSecretRef\n - host\n properties:\n accountSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n host:\n type: string\n akamai:\n description: Use the Akamai DNS zone management API to manage DNS01 challenge records.\n type: object\n required:\n - accessTokenSecretRef\n - clientSecretSecretRef\n - clientTokenSecretRef\n - serviceConsumerDomain\n properties:\n accessTokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientSecretSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientTokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n serviceConsumerDomain:\n type: string\n azureDNS:\n description: Use the Microsoft Azure DNS API to manage DNS01 challenge records.\n type: object\n required:\n - resourceGroupName\n - subscriptionID\n properties:\n clientID:\n description: |-\n Auth: Azure Service Principal:\n The ClientID of the Azure Service Principal used to authenticate with Azure DNS.\n If set, ClientSecret and TenantID must also be set.\n type: string\n clientSecretSecretRef:\n description: |-\n Auth: Azure Service Principal:\n A reference to a Secret containing the password associated with the Service Principal.\n If set, ClientID and TenantID must also be set.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n environment:\n description: name of the Azure environment (default AzurePublicCloud)\n type: string\n enum:\n - AzurePublicCloud\n - AzureChinaCloud\n - AzureGermanCloud\n - AzureUSGovernmentCloud\n hostedZoneName:\n description: name of the DNS zone that should be used\n type: string\n managedIdentity:\n description: |-\n Auth: Azure Workload Identity or Azure Managed Service Identity:\n Settings to enable Azure Workload Identity or Azure Managed Service Identity\n If set, ClientID, ClientSecret and TenantID must not be set.\n type: object\n properties:\n clientID:\n description: client ID of the managed identity, can not be used at the same time as resourceID\n type: string\n resourceID:\n description: |-\n resource ID of the managed identity, can not be used at the same time as clientID\n Cannot be used for Azure Managed Service Identity\n type: string\n resourceGroupName:\n description: resource group the DNS zone is located in\n type: string\n subscriptionID:\n description: ID of the Azure subscription\n type: string\n tenantID:\n description: |-\n Auth: Azure Service Principal:\n The TenantID of the Azure Service Principal used to authenticate with Azure DNS.\n If set, ClientID and ClientSecret must also be set.\n type: string\n cloudDNS:\n description: Use the Google Cloud DNS API to manage DNS01 challenge records.\n type: object\n required:\n - project\n properties:\n hostedZoneName:\n description: |-\n HostedZoneName is an optional field that tells cert-manager in which\n Cloud DNS zone the challenge record has to be created.\n If left empty cert-manager will automatically choose a zone.\n type: string\n project:\n type: string\n serviceAccountSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n cloudflare:\n description: Use the Cloudflare API to manage DNS01 challenge records.\n type: object\n properties:\n apiKeySecretRef:\n description: |-\n API key to use to authenticate with Cloudflare.\n Note: using an API token to authenticate is now the recommended method\n as it allows greater control of permissions.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n apiTokenSecretRef:\n description: API token used to authenticate with Cloudflare.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n email:\n description: Email of the account, only required when using API key based authentication.\n type: string\n cnameStrategy:\n description: |-\n CNAMEStrategy configures how the DNS01 provider should handle CNAME\n records when found in DNS zones.\n type: string\n enum:\n - None\n - Follow\n digitalocean:\n description: Use the DigitalOcean DNS API to manage DNS01 challenge records.\n type: object\n required:\n - tokenSecretRef\n properties:\n tokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n rfc2136:\n description: |-\n Use RFC2136 (\"Dynamic Updates in the Domain Name System\") (https://datatracker.ietf.org/doc/rfc2136/)\n to manage DNS01 challenge records.\n type: object\n required:\n - nameserver\n properties:\n nameserver:\n description: |-\n The IP address or hostname of an authoritative DNS server supporting\n RFC2136 in the form host:port. If the host is an IPv6 address it must be\n enclosed in square brackets (e.g [2001:db8::1]) ; port is optional.\n This field is required.\n type: string\n tsigAlgorithm:\n description: |-\n The TSIG Algorithm configured in the DNS supporting RFC2136. Used only\n when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined.\n Supported values are (case-insensitive): ``HMACMD5`` (default),\n ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.\n type: string\n tsigKeyName:\n description: |-\n The TSIG Key name configured in the DNS.\n If ``tsigSecretSecretRef`` is defined, this field is required.\n type: string\n tsigSecretSecretRef:\n description: |-\n The name of the secret containing the TSIG value.\n If ``tsigKeyName`` is defined, this field is required.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n route53:\n description: Use the AWS Route53 API to manage DNS01 challenge records.\n type: object\n required:\n - region\n properties:\n accessKeyID:\n description: |-\n The AccessKeyID is used for authentication.\n Cannot be set when SecretAccessKeyID is set.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: string\n accessKeyIDSecretRef:\n description: |-\n The SecretAccessKey is used for authentication. If set, pull the AWS\n access key ID from a key within a Kubernetes Secret.\n Cannot be set when AccessKeyID is set.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n auth:\n description: Auth configures how cert-manager authenticates.\n type: object\n required:\n - kubernetes\n properties:\n kubernetes:\n description: |-\n Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity\n by passing a bound ServiceAccount token.\n type: object\n required:\n - serviceAccountRef\n properties:\n serviceAccountRef:\n description: |-\n A reference to a service account that will be used to request a bound\n token (also known as \"projected token\"). To use this field, you must\n configure an RBAC rule to let cert-manager request a token.\n type: object\n required:\n - name\n properties:\n audiences:\n description: |-\n TokenAudiences is an optional list of audiences to include in the\n token passed to AWS. The default token consisting of the issuer's namespace\n and name is always included.\n If unset the audience defaults to `sts.amazonaws.com`.\n type: array\n items:\n type: string\n name:\n description: Name of the ServiceAccount used to request a token.\n type: string\n hostedZoneID:\n description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call.\n type: string\n region:\n description: Always set the region when using AccessKeyID and SecretAccessKey\n type: string\n role:\n description: |-\n Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey\n or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata\n type: string\n secretAccessKeySecretRef:\n description: |-\n The SecretAccessKey is used for authentication.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n webhook:\n description: |-\n Configure an external webhook based DNS01 challenge solver to manage\n DNS01 challenge records.\n type: object\n required:\n - groupName\n - solverName\n properties:\n config:\n description: |-\n Additional configuration that should be passed to the webhook apiserver\n when challenges are processed.\n This can contain arbitrary JSON data.\n Secret values should not be specified in this stanza.\n If secret values are needed (e.g. credentials for a DNS service), you\n should use a SecretKeySelector to reference a Secret resource.\n For details on the schema of this field, consult the webhook provider\n implementation's documentation.\n x-kubernetes-preserve-unknown-fields: true\n groupName:\n description: |-\n The API group name that should be used when POSTing ChallengePayload\n resources to the webhook apiserver.\n This should be the same as the GroupName specified in the webhook\n provider implementation.\n type: string\n solverName:\n description: |-\n The name of the solver to use, as defined in the webhook provider\n implementation.\n This will typically be the name of the provider, e.g. 'cloudflare'.\n type: string\n http01:\n description: |-\n Configures cert-manager to attempt to complete authorizations by\n performing the HTTP01 challenge flow.\n It is not possible to obtain certificates for wildcard domain names\n (e.g. `*.example.com`) using the HTTP01 challenge mechanism.\n type: object\n properties:\n gatewayHTTPRoute:\n description: |-\n The Gateway API is a sig-network community API that models service networking\n in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will\n create HTTPRoutes with the specified labels in the same namespace as the challenge.\n This solver is experimental, and fields / behaviour may change in the future.\n type: object\n properties:\n labels:\n description: |-\n Custom labels that will be applied to HTTPRoutes created by cert-manager\n while solving HTTP-01 challenges.\n type: object\n additionalProperties:\n type: string\n parentRefs:\n description: |-\n When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute.\n cert-manager needs to know which parentRefs should be used when creating\n the HTTPRoute. Usually, the parentRef references a Gateway. See:\n https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways\n type: array\n items:\n description: |-\n ParentReference identifies an API object (usually a Gateway) that can be considered\n a parent of this resource (usually a route). There are two kinds of parent resources\n with \"Core\" support:\n\n\n * Gateway (Gateway conformance profile)\n * Service (Mesh conformance profile, ClusterIP Services only)\n\n\n This API may be extended in the future to support additional kinds of parent\n resources.\n\n\n The API object must be valid in the cluster; the Group and Kind must\n be registered in the cluster for this reference to be valid.\n type: object\n required:\n - name\n properties:\n group:\n description: |-\n Group is the group of the referent.\n When unspecified, \"gateway.networking.k8s.io\" is inferred.\n To set the core API group (such as for a \"Service\" kind referent),\n Group must be explicitly set to \"\" (empty string).\n\n\n Support: Core\n type: string\n default: gateway.networking.k8s.io\n maxLength: 253\n pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\n kind:\n description: |-\n Kind is kind of the referent.\n\n\n There are two kinds of parent resources with \"Core\" support:\n\n\n * Gateway (Gateway conformance profile)\n * Service (Mesh conformance profile, ClusterIP Services only)\n\n\n Support for other resources is Implementation-Specific.\n type: string\n default: Gateway\n maxLength: 63\n minLength: 1\n pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$\n name:\n description: |-\n Name is the name of the referent.\n\n\n Support: Core\n type: string\n maxLength: 253\n minLength: 1\n namespace:\n description: |-\n Namespace is the namespace of the referent. When unspecified, this refers\n to the local namespace of the Route.\n\n\n Note that there are specific rules for ParentRefs which cross namespace\n boundaries. Cross-namespace references are only valid if they are explicitly\n allowed by something in the namespace they are referring to. For example:\n Gateway has the AllowedRoutes field, and ReferenceGrant provides a\n generic way to enable any other kind of cross-namespace reference.\n\n\n \n ParentRefs from a Route to a Service in the same namespace are \"producer\"\n routes, which apply default routing rules to inbound connections from\n any namespace to the Service.\n\n\n ParentRefs from a Route to a Service in a different namespace are\n \"consumer\" routes, and these routing rules are only applied to outbound\n connections originating from the same namespace as the Route, for which\n the intended destination of the connections are a Service targeted as a\n ParentRef of the Route.\n \n\n\n Support: Core\n type: string\n maxLength: 63\n minLength: 1\n pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$\n port:\n description: |-\n Port is the network port this Route targets. It can be interpreted\n differently based on the type of parent resource.\n\n\n When the parent resource is a Gateway, this targets all listeners\n listening on the specified port that also support this kind of Route(and\n select this Route). It's not recommended to set `Port` unless the\n networking behaviors specified in a Route must apply to a specific port\n as opposed to a listener(s) whose port(s) may be changed. When both Port\n and SectionName are specified, the name and port of the selected listener\n must match both specified values.\n\n\n \n When the parent resource is a Service, this targets a specific port in the\n Service spec. When both Port (experimental) and SectionName are specified,\n the name and port of the selected port must match both specified values.\n \n\n\n Implementations MAY choose to support other parent resources.\n Implementations supporting other types of parent resources MUST clearly\n document how/if Port is interpreted.\n\n\n For the purpose of status, an attachment is considered successful as\n long as the parent resource accepts it partially. For example, Gateway\n listeners can restrict which Routes can attach to them by Route kind,\n namespace, or hostname. If 1 of 2 Gateway listeners accept attachment\n from the referencing Route, the Route MUST be considered successfully\n attached. If no Gateway listeners accept attachment from this Route,\n the Route MUST be considered detached from the Gateway.\n\n\n Support: Extended\n type: integer\n format: int32\n maximum: 65535\n minimum: 1\n sectionName:\n description: |-\n SectionName is the name of a section within the target resource. In the\n following resources, SectionName is interpreted as the following:\n\n\n * Gateway: Listener name. When both Port (experimental) and SectionName\n are specified, the name and port of the selected listener must match\n both specified values.\n * Service: Port name. When both Port (experimental) and SectionName\n are specified, the name and port of the selected listener must match\n both specified values.\n\n\n Implementations MAY choose to support attaching Routes to other resources.\n If that is the case, they MUST clearly document how SectionName is\n interpreted.\n\n\n When unspecified (empty string), this will reference the entire resource.\n For the purpose of status, an attachment is considered successful if at\n least one section in the parent resource accepts it. For example, Gateway\n listeners can restrict which Routes can attach to them by Route kind,\n namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from\n the referencing Route, the Route MUST be considered successfully\n attached. If no Gateway listeners accept attachment from this Route, the\n Route MUST be considered detached from the Gateway.\n\n\n Support: Core\n type: string\n maxLength: 253\n minLength: 1\n pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\n serviceType:\n description: |-\n Optional service type for Kubernetes solver service. Supported values\n are NodePort or ClusterIP. If unset, defaults to NodePort.\n type: string\n ingress:\n description: |-\n The ingress based HTTP01 challenge solver will solve challenges by\n creating or modifying Ingress resources in order to route requests for\n '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are\n provisioned by cert-manager for each Challenge to be completed.\n type: object\n properties:\n class:\n description: |-\n This field configures the annotation `kubernetes.io/ingress.class` when\n creating Ingress resources to solve ACME challenges that use this\n challenge solver. Only one of `class`, `name` or `ingressClassName` may\n be specified.\n type: string\n ingressClassName:\n description: |-\n This field configures the field `ingressClassName` on the created Ingress\n resources used to solve ACME challenges that use this challenge solver.\n This is the recommended way of configuring the ingress class. Only one of\n `class`, `name` or `ingressClassName` may be specified.\n type: string\n ingressTemplate:\n description: |-\n Optional ingress template used to configure the ACME challenge solver\n ingress used for HTTP01 challenges.\n type: object\n properties:\n metadata:\n description: |-\n ObjectMeta overrides for the ingress used to solve HTTP01 challenges.\n Only the 'labels' and 'annotations' fields may be set.\n If labels or annotations overlap with in-built values, the values here\n will override the in-built values.\n type: object\n properties:\n annotations:\n description: Annotations that should be added to the created ACME HTTP01 solver ingress.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels that should be added to the created ACME HTTP01 solver ingress.\n type: object\n additionalProperties:\n type: string\n name:\n description: |-\n The name of the ingress resource that should have ACME challenge solving\n routes inserted into it in order to solve HTTP01 challenges.\n This is typically used in conjunction with ingress controllers like\n ingress-gce, which maintains a 1:1 mapping between external IPs and\n ingress resources. Only one of `class`, `name` or `ingressClassName` may\n be specified.\n type: string\n podTemplate:\n description: |-\n Optional pod template used to configure the ACME challenge solver pods\n used for HTTP01 challenges.\n type: object\n properties:\n metadata:\n description: |-\n ObjectMeta overrides for the pod used to solve HTTP01 challenges.\n Only the 'labels' and 'annotations' fields may be set.\n If labels or annotations overlap with in-built values, the values here\n will override the in-built values.\n type: object\n properties:\n annotations:\n description: Annotations that should be added to the create ACME HTTP01 solver pods.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels that should be added to the created ACME HTTP01 solver pods.\n type: object\n additionalProperties:\n type: string\n spec:\n description: |-\n PodSpec defines overrides for the HTTP01 challenge solver pod.\n Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields.\n All other fields will be ignored.\n type: object\n properties:\n affinity:\n description: If specified, the pod's scheduling constraints\n type: object\n properties:\n nodeAffinity:\n description: Describes node affinity scheduling rules for the pod.\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node matches the corresponding matchExpressions; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: |-\n An empty preferred scheduling term matches all objects with implicit weight 0\n (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n type: object\n required:\n - preference\n - weight\n properties:\n preference:\n description: A node selector term, associated with the corresponding weight.\n type: object\n properties:\n matchExpressions:\n description: A list of node selector requirements by node's labels.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector requirements by node's fields.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n weight:\n description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to an update), the system\n may or may not try to eventually evict the pod from its node.\n type: object\n required:\n - nodeSelectorTerms\n properties:\n nodeSelectorTerms:\n description: Required. A list of node selector terms. The terms are ORed.\n type: array\n items:\n description: |-\n A null or empty node selector term matches no objects. The requirements of\n them are ANDed.\n The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n type: object\n properties:\n matchExpressions:\n description: A list of node selector requirements by node's labels.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector requirements by node's fields.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n podAffinity:\n description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n type: object\n required:\n - podAffinityTerm\n - weight\n properties:\n podAffinityTerm:\n description: Required. A pod affinity term, associated with the corresponding weight.\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n type: array\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n x-kubernetes-list-type: atomic\n podAntiAffinity:\n description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the anti-affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling anti-affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n type: object\n required:\n - podAffinityTerm\n - weight\n properties:\n podAffinityTerm:\n description: Required. A pod affinity term, associated with the corresponding weight.\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the anti-affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the anti-affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n type: array\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n x-kubernetes-list-type: atomic\n imagePullSecrets:\n description: If specified, the pod's imagePullSecrets\n type: array\n items:\n description: |-\n LocalObjectReference contains enough information to let you locate the\n referenced object inside the same namespace.\n type: object\n properties:\n name:\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n TODO: Add other useful fields. apiVersion, kind, uid?\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.\n type: string\n default: \"\"\n x-kubernetes-map-type: atomic\n nodeSelector:\n description: |-\n NodeSelector is a selector which must be true for the pod to fit on a node.\n Selector which must match a node's labels for the pod to be scheduled on that node.\n More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/\n type: object\n additionalProperties:\n type: string\n priorityClassName:\n description: If specified, the pod's priorityClassName.\n type: string\n serviceAccountName:\n description: If specified, the pod's service account\n type: string\n tolerations:\n description: If specified, the pod's tolerations.\n type: array\n items:\n description: |-\n The pod this Toleration is attached to tolerates any taint that matches\n the triple using the matching operator .\n type: object\n properties:\n effect:\n description: |-\n Effect indicates the taint effect to match. Empty means match all taint effects.\n When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n type: string\n key:\n description: |-\n Key is the taint key that the toleration applies to. Empty means match all taint keys.\n If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n type: string\n operator:\n description: |-\n Operator represents a key's relationship to the value.\n Valid operators are Exists and Equal. Defaults to Equal.\n Exists is equivalent to wildcard for value, so that a pod can\n tolerate all taints of a particular category.\n type: string\n tolerationSeconds:\n description: |-\n TolerationSeconds represents the period of time the toleration (which must be\n of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,\n it is not set, which means tolerate the taint forever (do not evict). Zero and\n negative values will be treated as 0 (evict immediately) by the system.\n type: integer\n format: int64\n value:\n description: |-\n Value is the taint value the toleration matches to.\n If the operator is Exists, the value should be empty, otherwise just a regular string.\n type: string\n serviceType:\n description: |-\n Optional service type for Kubernetes solver service. Supported values\n are NodePort or ClusterIP. If unset, defaults to NodePort.\n type: string\n selector:\n description: |-\n Selector selects a set of DNSNames on the Certificate resource that\n should be solved using this challenge solver.\n If not specified, the solver will be treated as the 'default' solver\n with the lowest priority, i.e. if any other solver has a more specific\n match, it will be used instead.\n type: object\n properties:\n dnsNames:\n description: |-\n List of DNSNames that this solver will be used to solve.\n If specified and a match is found, a dnsNames selector will take\n precedence over a dnsZones selector.\n If multiple solvers match with the same dnsNames value, the solver\n with the most matching labels in matchLabels will be selected.\n If neither has more matches, the solver defined earlier in the list\n will be selected.\n type: array\n items:\n type: string\n dnsZones:\n description: |-\n List of DNSZones that this solver will be used to solve.\n The most specific DNS zone match specified here will take precedence\n over other DNS zone matches, so a solver specifying sys.example.com\n will be selected over one specifying example.com for the domain\n www.sys.example.com.\n If multiple solvers match with the same dnsZones value, the solver\n with the most matching labels in matchLabels will be selected.\n If neither has more matches, the solver defined earlier in the list\n will be selected.\n type: array\n items:\n type: string\n matchLabels:\n description: |-\n A label selector that is used to refine the set of certificate's that\n this challenge solver will apply to.\n type: object\n additionalProperties:\n type: string\n token:\n description: |-\n The ACME challenge token for this challenge.\n This is the raw value returned from the ACME server.\n type: string\n type:\n description: |-\n The type of ACME challenge this resource represents.\n One of \"HTTP-01\" or \"DNS-01\".\n type: string\n enum:\n - HTTP-01\n - DNS-01\n url:\n description: |-\n The URL of the ACME Challenge resource for this challenge.\n This can be used to lookup details about the status of this challenge.\n type: string\n wildcard:\n description: |-\n wildcard will be true if this challenge is for a wildcard identifier,\n for example '*.example.com'.\n type: boolean\n status:\n type: object\n properties:\n presented:\n description: |-\n presented will be set to true if the challenge values for this challenge\n are currently 'presented'.\n This *does not* imply the self check is passing. Only that the values\n have been 'submitted' for the appropriate challenge mechanism (i.e. the\n DNS01 TXT record has been presented, or the HTTP01 configuration has been\n configured).\n type: boolean\n processing:\n description: |-\n Used to denote whether this challenge should be processed or not.\n This field will only be set to true by the 'scheduling' component.\n It will only be set to false by the 'challenges' controller, after the\n challenge has reached a final state or timed out.\n If this field is set to false, the challenge controller will not take\n any more action.\n type: boolean\n reason:\n description: |-\n Contains human readable information on why the Challenge is in the\n current state.\n type: string\n state:\n description: |-\n Contains the current 'state' of the challenge.\n If not set, the state of the challenge is unknown.\n type: string\n enum:\n - valid\n - ready\n - pending\n - processing\n - invalid\n - expired\n - errored\n served: true\n storage: true\n subresources:\n status: {}\n---\n# Source: cert-manager/deploy/crds/crd-certificaterequests.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: certificaterequests.cert-manager.io\n labels:\n app: 'cert-manager'\n app.kubernetes.io/name: 'cert-manager'\n app.kubernetes.io/instance: 'cert-manager'\n # Generated labels\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n group: cert-manager.io\n names:\n kind: CertificateRequest\n listKind: CertificateRequestList\n plural: certificaterequests\n shortNames:\n - cr\n - crs\n singular: certificaterequest\n categories:\n - cert-manager\n scope: Namespaced\n versions:\n - name: v1\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.conditions[?(@.type==\"Approved\")].status\n name: Approved\n type: string\n - jsonPath: .status.conditions[?(@.type==\"Denied\")].status\n name: Denied\n type: string\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].status\n name: Ready\n type: string\n - jsonPath: .spec.issuerRef.name\n name: Issuer\n type: string\n - jsonPath: .spec.username\n name: Requestor\n type: string\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].message\n name: Status\n priority: 1\n type: string\n - jsonPath: .metadata.creationTimestamp\n description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n name: Age\n type: date\n schema:\n openAPIV3Schema:\n description: |-\n A CertificateRequest is used to request a signed certificate from one of the\n configured issuers.\n\n\n All fields within the CertificateRequest's `spec` are immutable after creation.\n A CertificateRequest will either succeed or fail, as denoted by its `Ready` status\n condition and its `status.failureTime` field.\n\n\n A CertificateRequest is a one-shot resource, meaning it represents a single\n point in time request for a certificate and cannot be re-used.\n type: object\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: |-\n Specification of the desired state of the CertificateRequest resource.\n https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\n type: object\n required:\n - issuerRef\n - request\n properties:\n duration:\n description: |-\n Requested 'duration' (i.e. lifetime) of the Certificate. Note that the\n issuer may choose to ignore the requested duration, just like any other\n requested attribute.\n type: string\n extra:\n description: |-\n Extra contains extra attributes of the user that created the CertificateRequest.\n Populated by the cert-manager webhook on creation and immutable.\n type: object\n additionalProperties:\n type: array\n items:\n type: string\n groups:\n description: |-\n Groups contains group membership of the user that created the CertificateRequest.\n Populated by the cert-manager webhook on creation and immutable.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n isCA:\n description: |-\n Requested basic constraints isCA value. Note that the issuer may choose\n to ignore the requested isCA value, just like any other requested attribute.\n\n\n NOTE: If the CSR in the `Request` field has a BasicConstraints extension,\n it must have the same isCA value as specified here.\n\n\n If true, this will automatically add the `cert sign` usage to the list\n of requested `usages`.\n type: boolean\n issuerRef:\n description: |-\n Reference to the issuer responsible for issuing the certificate.\n If the issuer is namespace-scoped, it must be in the same namespace\n as the Certificate. If the issuer is cluster-scoped, it can be used\n from any namespace.\n\n\n The `name` field of the reference must always be specified.\n type: object\n required:\n - name\n properties:\n group:\n description: Group of the resource being referred to.\n type: string\n kind:\n description: Kind of the resource being referred to.\n type: string\n name:\n description: Name of the resource being referred to.\n type: string\n request:\n description: |-\n The PEM-encoded X.509 certificate signing request to be submitted to the\n issuer for signing.\n\n\n If the CSR has a BasicConstraints extension, its isCA attribute must\n match the `isCA` value of this CertificateRequest.\n If the CSR has a KeyUsage extension, its key usages must match the\n key usages in the `usages` field of this CertificateRequest.\n If the CSR has a ExtKeyUsage extension, its extended key usages\n must match the extended key usages in the `usages` field of this\n CertificateRequest.\n type: string\n format: byte\n uid:\n description: |-\n UID contains the uid of the user that created the CertificateRequest.\n Populated by the cert-manager webhook on creation and immutable.\n type: string\n usages:\n description: |-\n Requested key usages and extended key usages.\n\n\n NOTE: If the CSR in the `Request` field has uses the KeyUsage or\n ExtKeyUsage extension, these extensions must have the same values\n as specified here without any additional values.\n\n\n If unset, defaults to `digital signature` and `key encipherment`.\n type: array\n items:\n description: |-\n KeyUsage specifies valid usage contexts for keys.\n See:\n https://tools.ietf.org/html/rfc5280#section-4.2.1.3\n https://tools.ietf.org/html/rfc5280#section-4.2.1.12\n\n\n Valid KeyUsage values are as follows:\n \"signing\",\n \"digital signature\",\n \"content commitment\",\n \"key encipherment\",\n \"key agreement\",\n \"data encipherment\",\n \"cert sign\",\n \"crl sign\",\n \"encipher only\",\n \"decipher only\",\n \"any\",\n \"server auth\",\n \"client auth\",\n \"code signing\",\n \"email protection\",\n \"s/mime\",\n \"ipsec end system\",\n \"ipsec tunnel\",\n \"ipsec user\",\n \"timestamping\",\n \"ocsp signing\",\n \"microsoft sgc\",\n \"netscape sgc\"\n type: string\n enum:\n - signing\n - digital signature\n - content commitment\n - key encipherment\n - key agreement\n - data encipherment\n - cert sign\n - crl sign\n - encipher only\n - decipher only\n - any\n - server auth\n - client auth\n - code signing\n - email protection\n - s/mime\n - ipsec end system\n - ipsec tunnel\n - ipsec user\n - timestamping\n - ocsp signing\n - microsoft sgc\n - netscape sgc\n username:\n description: |-\n Username contains the name of the user that created the CertificateRequest.\n Populated by the cert-manager webhook on creation and immutable.\n type: string\n status:\n description: |-\n Status of the CertificateRequest.\n This is set and managed automatically.\n Read-only.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\n type: object\n properties:\n ca:\n description: |-\n The PEM encoded X.509 certificate of the signer, also known as the CA\n (Certificate Authority).\n This is set on a best-effort basis by different issuers.\n If not set, the CA is assumed to be unknown/not available.\n type: string\n format: byte\n certificate:\n description: |-\n The PEM encoded X.509 certificate resulting from the certificate\n signing request.\n If not set, the CertificateRequest has either not been completed or has\n failed. More information on failure can be found by checking the\n `conditions` field.\n type: string\n format: byte\n conditions:\n description: |-\n List of status conditions to indicate the status of a CertificateRequest.\n Known condition types are `Ready`, `InvalidRequest`, `Approved` and `Denied`.\n type: array\n items:\n description: CertificateRequestCondition contains condition information for a CertificateRequest.\n type: object\n required:\n - status\n - type\n properties:\n lastTransitionTime:\n description: |-\n LastTransitionTime is the timestamp corresponding to the last status\n change of this condition.\n type: string\n format: date-time\n message:\n description: |-\n Message is a human readable description of the details of the last\n transition, complementing reason.\n type: string\n reason:\n description: |-\n Reason is a brief machine readable explanation for the condition's last\n transition.\n type: string\n status:\n description: Status of the condition, one of (`True`, `False`, `Unknown`).\n type: string\n enum:\n - \"True\"\n - \"False\"\n - Unknown\n type:\n description: |-\n Type of the condition, known values are (`Ready`, `InvalidRequest`,\n `Approved`, `Denied`).\n type: string\n x-kubernetes-list-map-keys:\n - type\n x-kubernetes-list-type: map\n failureTime:\n description: |-\n FailureTime stores the time that this CertificateRequest failed. This is\n used to influence garbage collection and back-off.\n type: string\n format: date-time\n served: true\n storage: true\n---\n# Source: cert-manager/deploy/crds/crd-issuers.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: issuers.cert-manager.io\n labels:\n app: 'cert-manager'\n app.kubernetes.io/name: 'cert-manager'\n app.kubernetes.io/instance: \"cert-manager\"\n # Generated labels\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n group: cert-manager.io\n names:\n kind: Issuer\n listKind: IssuerList\n plural: issuers\n singular: issuer\n categories:\n - cert-manager\n scope: Namespaced\n versions:\n - name: v1\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].status\n name: Ready\n type: string\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].message\n name: Status\n priority: 1\n type: string\n - jsonPath: .metadata.creationTimestamp\n description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n name: Age\n type: date\n schema:\n openAPIV3Schema:\n description: |-\n An Issuer represents a certificate issuing authority which can be\n referenced as part of `issuerRef` fields.\n It is scoped to a single namespace and can therefore only be referenced by\n resources within the same namespace.\n type: object\n required:\n - spec\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: Desired state of the Issuer resource.\n type: object\n properties:\n acme:\n description: |-\n ACME configures this issuer to communicate with a RFC8555 (ACME) server\n to obtain signed x509 certificates.\n type: object\n required:\n - privateKeySecretRef\n - server\n properties:\n caBundle:\n description: |-\n Base64-encoded bundle of PEM CAs which can be used to validate the certificate\n chain presented by the ACME server.\n Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various\n kinds of security vulnerabilities.\n If CABundle and SkipTLSVerify are unset, the system certificate bundle inside\n the container is used to validate the TLS connection.\n type: string\n format: byte\n disableAccountKeyGeneration:\n description: |-\n Enables or disables generating a new ACME account key.\n If true, the Issuer resource will *not* request a new account but will expect\n the account key to be supplied via an existing secret.\n If false, the cert-manager system will generate a new ACME account key\n for the Issuer.\n Defaults to false.\n type: boolean\n email:\n description: |-\n Email is the email address to be associated with the ACME account.\n This field is optional, but it is strongly recommended to be set.\n It will be used to contact you in case of issues with your account or\n certificates, including expiry notification emails.\n This field may be updated after the account is initially registered.\n type: string\n enableDurationFeature:\n description: |-\n Enables requesting a Not After date on certificates that matches the\n duration of the certificate. This is not supported by all ACME servers\n like Let's Encrypt. If set to true when the ACME server does not support\n it, it will create an error on the Order.\n Defaults to false.\n type: boolean\n externalAccountBinding:\n description: |-\n ExternalAccountBinding is a reference to a CA external account of the ACME\n server.\n If set, upon registration cert-manager will attempt to associate the given\n external account credentials with the registered ACME account.\n type: object\n required:\n - keyID\n - keySecretRef\n properties:\n keyAlgorithm:\n description: |-\n Deprecated: keyAlgorithm field exists for historical compatibility\n reasons and should not be used. The algorithm is now hardcoded to HS256\n in golang/x/crypto/acme.\n type: string\n enum:\n - HS256\n - HS384\n - HS512\n keyID:\n description: keyID is the ID of the CA key that the External Account is bound to.\n type: string\n keySecretRef:\n description: |-\n keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes\n Secret which holds the symmetric MAC key of the External Account Binding.\n The `key` is the index string that is paired with the key data in the\n Secret and should not be confused with the key data itself, or indeed with\n the External Account Binding keyID above.\n The secret key stored in the Secret **must** be un-padded, base64 URL\n encoded data.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n preferredChain:\n description: |-\n PreferredChain is the chain to use if the ACME server outputs multiple.\n PreferredChain is no guarantee that this one gets delivered by the ACME\n endpoint.\n For example, for Let's Encrypt's DST crosssign you would use:\n \"DST Root CA X3\" or \"ISRG Root X1\" for the newer Let's Encrypt root CA.\n This value picks the first certificate bundle in the combined set of\n ACME default and alternative chains that has a root-most certificate with\n this value as its issuer's commonname.\n type: string\n maxLength: 64\n privateKeySecretRef:\n description: |-\n PrivateKey is the name of a Kubernetes Secret resource that will be used to\n store the automatically generated ACME account private key.\n Optionally, a `key` may be specified to select a specific entry within\n the named Secret resource.\n If `key` is not specified, a default of `tls.key` will be used.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n server:\n description: |-\n Server is the URL used to access the ACME server's 'directory' endpoint.\n For example, for Let's Encrypt's staging endpoint, you would use:\n \"https://acme-staging-v02.api.letsencrypt.org/directory\".\n Only ACME v2 endpoints (i.e. RFC 8555) are supported.\n type: string\n skipTLSVerify:\n description: |-\n INSECURE: Enables or disables validation of the ACME server TLS certificate.\n If true, requests to the ACME server will not have the TLS certificate chain\n validated.\n Mutually exclusive with CABundle; prefer using CABundle to prevent various\n kinds of security vulnerabilities.\n Only enable this option in development environments.\n If CABundle and SkipTLSVerify are unset, the system certificate bundle inside\n the container is used to validate the TLS connection.\n Defaults to false.\n type: boolean\n solvers:\n description: |-\n Solvers is a list of challenge solvers that will be used to solve\n ACME challenges for the matching domains.\n Solver configurations must be provided in order to obtain certificates\n from an ACME server.\n For more information, see: https://cert-manager.io/docs/configuration/acme/\n type: array\n items:\n description: |-\n An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of.\n A selector may be provided to use different solving strategies for different DNS names.\n Only one of HTTP01 or DNS01 must be provided.\n type: object\n properties:\n dns01:\n description: |-\n Configures cert-manager to attempt to complete authorizations by\n performing the DNS01 challenge flow.\n type: object\n properties:\n acmeDNS:\n description: |-\n Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage\n DNS01 challenge records.\n type: object\n required:\n - accountSecretRef\n - host\n properties:\n accountSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n host:\n type: string\n akamai:\n description: Use the Akamai DNS zone management API to manage DNS01 challenge records.\n type: object\n required:\n - accessTokenSecretRef\n - clientSecretSecretRef\n - clientTokenSecretRef\n - serviceConsumerDomain\n properties:\n accessTokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientSecretSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientTokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n serviceConsumerDomain:\n type: string\n azureDNS:\n description: Use the Microsoft Azure DNS API to manage DNS01 challenge records.\n type: object\n required:\n - resourceGroupName\n - subscriptionID\n properties:\n clientID:\n description: |-\n Auth: Azure Service Principal:\n The ClientID of the Azure Service Principal used to authenticate with Azure DNS.\n If set, ClientSecret and TenantID must also be set.\n type: string\n clientSecretSecretRef:\n description: |-\n Auth: Azure Service Principal:\n A reference to a Secret containing the password associated with the Service Principal.\n If set, ClientID and TenantID must also be set.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n environment:\n description: name of the Azure environment (default AzurePublicCloud)\n type: string\n enum:\n - AzurePublicCloud\n - AzureChinaCloud\n - AzureGermanCloud\n - AzureUSGovernmentCloud\n hostedZoneName:\n description: name of the DNS zone that should be used\n type: string\n managedIdentity:\n description: |-\n Auth: Azure Workload Identity or Azure Managed Service Identity:\n Settings to enable Azure Workload Identity or Azure Managed Service Identity\n If set, ClientID, ClientSecret and TenantID must not be set.\n type: object\n properties:\n clientID:\n description: client ID of the managed identity, can not be used at the same time as resourceID\n type: string\n resourceID:\n description: |-\n resource ID of the managed identity, can not be used at the same time as clientID\n Cannot be used for Azure Managed Service Identity\n type: string\n resourceGroupName:\n description: resource group the DNS zone is located in\n type: string\n subscriptionID:\n description: ID of the Azure subscription\n type: string\n tenantID:\n description: |-\n Auth: Azure Service Principal:\n The TenantID of the Azure Service Principal used to authenticate with Azure DNS.\n If set, ClientID and ClientSecret must also be set.\n type: string\n cloudDNS:\n description: Use the Google Cloud DNS API to manage DNS01 challenge records.\n type: object\n required:\n - project\n properties:\n hostedZoneName:\n description: |-\n HostedZoneName is an optional field that tells cert-manager in which\n Cloud DNS zone the challenge record has to be created.\n If left empty cert-manager will automatically choose a zone.\n type: string\n project:\n type: string\n serviceAccountSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n cloudflare:\n description: Use the Cloudflare API to manage DNS01 challenge records.\n type: object\n properties:\n apiKeySecretRef:\n description: |-\n API key to use to authenticate with Cloudflare.\n Note: using an API token to authenticate is now the recommended method\n as it allows greater control of permissions.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n apiTokenSecretRef:\n description: API token used to authenticate with Cloudflare.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n email:\n description: Email of the account, only required when using API key based authentication.\n type: string\n cnameStrategy:\n description: |-\n CNAMEStrategy configures how the DNS01 provider should handle CNAME\n records when found in DNS zones.\n type: string\n enum:\n - None\n - Follow\n digitalocean:\n description: Use the DigitalOcean DNS API to manage DNS01 challenge records.\n type: object\n required:\n - tokenSecretRef\n properties:\n tokenSecretRef:\n description: |-\n A reference to a specific 'key' within a Secret resource.\n In some instances, `key` is a required field.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n rfc2136:\n description: |-\n Use RFC2136 (\"Dynamic Updates in the Domain Name System\") (https://datatracker.ietf.org/doc/rfc2136/)\n to manage DNS01 challenge records.\n type: object\n required:\n - nameserver\n properties:\n nameserver:\n description: |-\n The IP address or hostname of an authoritative DNS server supporting\n RFC2136 in the form host:port. If the host is an IPv6 address it must be\n enclosed in square brackets (e.g [2001:db8::1]) ; port is optional.\n This field is required.\n type: string\n tsigAlgorithm:\n description: |-\n The TSIG Algorithm configured in the DNS supporting RFC2136. Used only\n when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined.\n Supported values are (case-insensitive): ``HMACMD5`` (default),\n ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.\n type: string\n tsigKeyName:\n description: |-\n The TSIG Key name configured in the DNS.\n If ``tsigSecretSecretRef`` is defined, this field is required.\n type: string\n tsigSecretSecretRef:\n description: |-\n The name of the secret containing the TSIG value.\n If ``tsigKeyName`` is defined, this field is required.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n route53:\n description: Use the AWS Route53 API to manage DNS01 challenge records.\n type: object\n required:\n - region\n properties:\n accessKeyID:\n description: |-\n The AccessKeyID is used for authentication.\n Cannot be set when SecretAccessKeyID is set.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: string\n accessKeyIDSecretRef:\n description: |-\n The SecretAccessKey is used for authentication. If set, pull the AWS\n access key ID from a key within a Kubernetes Secret.\n Cannot be set when AccessKeyID is set.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n auth:\n description: Auth configures how cert-manager authenticates.\n type: object\n required:\n - kubernetes\n properties:\n kubernetes:\n description: |-\n Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity\n by passing a bound ServiceAccount token.\n type: object\n required:\n - serviceAccountRef\n properties:\n serviceAccountRef:\n description: |-\n A reference to a service account that will be used to request a bound\n token (also known as \"projected token\"). To use this field, you must\n configure an RBAC rule to let cert-manager request a token.\n type: object\n required:\n - name\n properties:\n audiences:\n description: |-\n TokenAudiences is an optional list of audiences to include in the\n token passed to AWS. The default token consisting of the issuer's namespace\n and name is always included.\n If unset the audience defaults to `sts.amazonaws.com`.\n type: array\n items:\n type: string\n name:\n description: Name of the ServiceAccount used to request a token.\n type: string\n hostedZoneID:\n description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call.\n type: string\n region:\n description: Always set the region when using AccessKeyID and SecretAccessKey\n type: string\n role:\n description: |-\n Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey\n or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata\n type: string\n secretAccessKeySecretRef:\n description: |-\n The SecretAccessKey is used for authentication.\n If neither the Access Key nor Key ID are set, we fall-back to using env\n vars, shared credentials file or AWS Instance metadata,\n see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n webhook:\n description: |-\n Configure an external webhook based DNS01 challenge solver to manage\n DNS01 challenge records.\n type: object\n required:\n - groupName\n - solverName\n properties:\n config:\n description: |-\n Additional configuration that should be passed to the webhook apiserver\n when challenges are processed.\n This can contain arbitrary JSON data.\n Secret values should not be specified in this stanza.\n If secret values are needed (e.g. credentials for a DNS service), you\n should use a SecretKeySelector to reference a Secret resource.\n For details on the schema of this field, consult the webhook provider\n implementation's documentation.\n x-kubernetes-preserve-unknown-fields: true\n groupName:\n description: |-\n The API group name that should be used when POSTing ChallengePayload\n resources to the webhook apiserver.\n This should be the same as the GroupName specified in the webhook\n provider implementation.\n type: string\n solverName:\n description: |-\n The name of the solver to use, as defined in the webhook provider\n implementation.\n This will typically be the name of the provider, e.g. 'cloudflare'.\n type: string\n http01:\n description: |-\n Configures cert-manager to attempt to complete authorizations by\n performing the HTTP01 challenge flow.\n It is not possible to obtain certificates for wildcard domain names\n (e.g. `*.example.com`) using the HTTP01 challenge mechanism.\n type: object\n properties:\n gatewayHTTPRoute:\n description: |-\n The Gateway API is a sig-network community API that models service networking\n in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will\n create HTTPRoutes with the specified labels in the same namespace as the challenge.\n This solver is experimental, and fields / behaviour may change in the future.\n type: object\n properties:\n labels:\n description: |-\n Custom labels that will be applied to HTTPRoutes created by cert-manager\n while solving HTTP-01 challenges.\n type: object\n additionalProperties:\n type: string\n parentRefs:\n description: |-\n When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute.\n cert-manager needs to know which parentRefs should be used when creating\n the HTTPRoute. Usually, the parentRef references a Gateway. See:\n https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways\n type: array\n items:\n description: |-\n ParentReference identifies an API object (usually a Gateway) that can be considered\n a parent of this resource (usually a route). There are two kinds of parent resources\n with \"Core\" support:\n\n\n * Gateway (Gateway conformance profile)\n * Service (Mesh conformance profile, ClusterIP Services only)\n\n\n This API may be extended in the future to support additional kinds of parent\n resources.\n\n\n The API object must be valid in the cluster; the Group and Kind must\n be registered in the cluster for this reference to be valid.\n type: object\n required:\n - name\n properties:\n group:\n description: |-\n Group is the group of the referent.\n When unspecified, \"gateway.networking.k8s.io\" is inferred.\n To set the core API group (such as for a \"Service\" kind referent),\n Group must be explicitly set to \"\" (empty string).\n\n\n Support: Core\n type: string\n default: gateway.networking.k8s.io\n maxLength: 253\n pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\n kind:\n description: |-\n Kind is kind of the referent.\n\n\n There are two kinds of parent resources with \"Core\" support:\n\n\n * Gateway (Gateway conformance profile)\n * Service (Mesh conformance profile, ClusterIP Services only)\n\n\n Support for other resources is Implementation-Specific.\n type: string\n default: Gateway\n maxLength: 63\n minLength: 1\n pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$\n name:\n description: |-\n Name is the name of the referent.\n\n\n Support: Core\n type: string\n maxLength: 253\n minLength: 1\n namespace:\n description: |-\n Namespace is the namespace of the referent. When unspecified, this refers\n to the local namespace of the Route.\n\n\n Note that there are specific rules for ParentRefs which cross namespace\n boundaries. Cross-namespace references are only valid if they are explicitly\n allowed by something in the namespace they are referring to. For example:\n Gateway has the AllowedRoutes field, and ReferenceGrant provides a\n generic way to enable any other kind of cross-namespace reference.\n\n\n \n ParentRefs from a Route to a Service in the same namespace are \"producer\"\n routes, which apply default routing rules to inbound connections from\n any namespace to the Service.\n\n\n ParentRefs from a Route to a Service in a different namespace are\n \"consumer\" routes, and these routing rules are only applied to outbound\n connections originating from the same namespace as the Route, for which\n the intended destination of the connections are a Service targeted as a\n ParentRef of the Route.\n \n\n\n Support: Core\n type: string\n maxLength: 63\n minLength: 1\n pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$\n port:\n description: |-\n Port is the network port this Route targets. It can be interpreted\n differently based on the type of parent resource.\n\n\n When the parent resource is a Gateway, this targets all listeners\n listening on the specified port that also support this kind of Route(and\n select this Route). It's not recommended to set `Port` unless the\n networking behaviors specified in a Route must apply to a specific port\n as opposed to a listener(s) whose port(s) may be changed. When both Port\n and SectionName are specified, the name and port of the selected listener\n must match both specified values.\n\n\n \n When the parent resource is a Service, this targets a specific port in the\n Service spec. When both Port (experimental) and SectionName are specified,\n the name and port of the selected port must match both specified values.\n \n\n\n Implementations MAY choose to support other parent resources.\n Implementations supporting other types of parent resources MUST clearly\n document how/if Port is interpreted.\n\n\n For the purpose of status, an attachment is considered successful as\n long as the parent resource accepts it partially. For example, Gateway\n listeners can restrict which Routes can attach to them by Route kind,\n namespace, or hostname. If 1 of 2 Gateway listeners accept attachment\n from the referencing Route, the Route MUST be considered successfully\n attached. If no Gateway listeners accept attachment from this Route,\n the Route MUST be considered detached from the Gateway.\n\n\n Support: Extended\n type: integer\n format: int32\n maximum: 65535\n minimum: 1\n sectionName:\n description: |-\n SectionName is the name of a section within the target resource. In the\n following resources, SectionName is interpreted as the following:\n\n\n * Gateway: Listener name. When both Port (experimental) and SectionName\n are specified, the name and port of the selected listener must match\n both specified values.\n * Service: Port name. When both Port (experimental) and SectionName\n are specified, the name and port of the selected listener must match\n both specified values.\n\n\n Implementations MAY choose to support attaching Routes to other resources.\n If that is the case, they MUST clearly document how SectionName is\n interpreted.\n\n\n When unspecified (empty string), this will reference the entire resource.\n For the purpose of status, an attachment is considered successful if at\n least one section in the parent resource accepts it. For example, Gateway\n listeners can restrict which Routes can attach to them by Route kind,\n namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from\n the referencing Route, the Route MUST be considered successfully\n attached. If no Gateway listeners accept attachment from this Route, the\n Route MUST be considered detached from the Gateway.\n\n\n Support: Core\n type: string\n maxLength: 253\n minLength: 1\n pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$\n serviceType:\n description: |-\n Optional service type for Kubernetes solver service. Supported values\n are NodePort or ClusterIP. If unset, defaults to NodePort.\n type: string\n ingress:\n description: |-\n The ingress based HTTP01 challenge solver will solve challenges by\n creating or modifying Ingress resources in order to route requests for\n '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are\n provisioned by cert-manager for each Challenge to be completed.\n type: object\n properties:\n class:\n description: |-\n This field configures the annotation `kubernetes.io/ingress.class` when\n creating Ingress resources to solve ACME challenges that use this\n challenge solver. Only one of `class`, `name` or `ingressClassName` may\n be specified.\n type: string\n ingressClassName:\n description: |-\n This field configures the field `ingressClassName` on the created Ingress\n resources used to solve ACME challenges that use this challenge solver.\n This is the recommended way of configuring the ingress class. Only one of\n `class`, `name` or `ingressClassName` may be specified.\n type: string\n ingressTemplate:\n description: |-\n Optional ingress template used to configure the ACME challenge solver\n ingress used for HTTP01 challenges.\n type: object\n properties:\n metadata:\n description: |-\n ObjectMeta overrides for the ingress used to solve HTTP01 challenges.\n Only the 'labels' and 'annotations' fields may be set.\n If labels or annotations overlap with in-built values, the values here\n will override the in-built values.\n type: object\n properties:\n annotations:\n description: Annotations that should be added to the created ACME HTTP01 solver ingress.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels that should be added to the created ACME HTTP01 solver ingress.\n type: object\n additionalProperties:\n type: string\n name:\n description: |-\n The name of the ingress resource that should have ACME challenge solving\n routes inserted into it in order to solve HTTP01 challenges.\n This is typically used in conjunction with ingress controllers like\n ingress-gce, which maintains a 1:1 mapping between external IPs and\n ingress resources. Only one of `class`, `name` or `ingressClassName` may\n be specified.\n type: string\n podTemplate:\n description: |-\n Optional pod template used to configure the ACME challenge solver pods\n used for HTTP01 challenges.\n type: object\n properties:\n metadata:\n description: |-\n ObjectMeta overrides for the pod used to solve HTTP01 challenges.\n Only the 'labels' and 'annotations' fields may be set.\n If labels or annotations overlap with in-built values, the values here\n will override the in-built values.\n type: object\n properties:\n annotations:\n description: Annotations that should be added to the create ACME HTTP01 solver pods.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels that should be added to the created ACME HTTP01 solver pods.\n type: object\n additionalProperties:\n type: string\n spec:\n description: |-\n PodSpec defines overrides for the HTTP01 challenge solver pod.\n Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields.\n All other fields will be ignored.\n type: object\n properties:\n affinity:\n description: If specified, the pod's scheduling constraints\n type: object\n properties:\n nodeAffinity:\n description: Describes node affinity scheduling rules for the pod.\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node matches the corresponding matchExpressions; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: |-\n An empty preferred scheduling term matches all objects with implicit weight 0\n (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n type: object\n required:\n - preference\n - weight\n properties:\n preference:\n description: A node selector term, associated with the corresponding weight.\n type: object\n properties:\n matchExpressions:\n description: A list of node selector requirements by node's labels.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector requirements by node's fields.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n weight:\n description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to an update), the system\n may or may not try to eventually evict the pod from its node.\n type: object\n required:\n - nodeSelectorTerms\n properties:\n nodeSelectorTerms:\n description: Required. A list of node selector terms. The terms are ORed.\n type: array\n items:\n description: |-\n A null or empty node selector term matches no objects. The requirements of\n them are ANDed.\n The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n type: object\n properties:\n matchExpressions:\n description: A list of node selector requirements by node's labels.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector requirements by node's fields.\n type: array\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: The label key that the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n x-kubernetes-list-type: atomic\n x-kubernetes-map-type: atomic\n podAffinity:\n description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n type: object\n required:\n - podAffinityTerm\n - weight\n properties:\n podAffinityTerm:\n description: Required. A pod affinity term, associated with the corresponding weight.\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n type: array\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n x-kubernetes-list-type: atomic\n podAntiAffinity:\n description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\n type: object\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the anti-affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling anti-affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n type: array\n items:\n description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n type: object\n required:\n - podAffinityTerm\n - weight\n properties:\n podAffinityTerm:\n description: Required. A pod affinity term, associated with the corresponding weight.\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n type: integer\n format: int32\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the anti-affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the anti-affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n type: array\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n type: object\n required:\n - topologyKey\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n type: object\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n type: array\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n type: object\n required:\n - key\n - operator\n properties:\n key:\n description: key is the label key that the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n x-kubernetes-list-type: atomic\n matchLabels:\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n additionalProperties:\n type: string\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n type: array\n items:\n type: string\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n x-kubernetes-list-type: atomic\n imagePullSecrets:\n description: If specified, the pod's imagePullSecrets\n type: array\n items:\n description: |-\n LocalObjectReference contains enough information to let you locate the\n referenced object inside the same namespace.\n type: object\n properties:\n name:\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n TODO: Add other useful fields. apiVersion, kind, uid?\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.\n type: string\n default: \"\"\n x-kubernetes-map-type: atomic\n nodeSelector:\n description: |-\n NodeSelector is a selector which must be true for the pod to fit on a node.\n Selector which must match a node's labels for the pod to be scheduled on that node.\n More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/\n type: object\n additionalProperties:\n type: string\n priorityClassName:\n description: If specified, the pod's priorityClassName.\n type: string\n serviceAccountName:\n description: If specified, the pod's service account\n type: string\n tolerations:\n description: If specified, the pod's tolerations.\n type: array\n items:\n description: |-\n The pod this Toleration is attached to tolerates any taint that matches\n the triple using the matching operator .\n type: object\n properties:\n effect:\n description: |-\n Effect indicates the taint effect to match. Empty means match all taint effects.\n When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n type: string\n key:\n description: |-\n Key is the taint key that the toleration applies to. Empty means match all taint keys.\n If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n type: string\n operator:\n description: |-\n Operator represents a key's relationship to the value.\n Valid operators are Exists and Equal. Defaults to Equal.\n Exists is equivalent to wildcard for value, so that a pod can\n tolerate all taints of a particular category.\n type: string\n tolerationSeconds:\n description: |-\n TolerationSeconds represents the period of time the toleration (which must be\n of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,\n it is not set, which means tolerate the taint forever (do not evict). Zero and\n negative values will be treated as 0 (evict immediately) by the system.\n type: integer\n format: int64\n value:\n description: |-\n Value is the taint value the toleration matches to.\n If the operator is Exists, the value should be empty, otherwise just a regular string.\n type: string\n serviceType:\n description: |-\n Optional service type for Kubernetes solver service. Supported values\n are NodePort or ClusterIP. If unset, defaults to NodePort.\n type: string\n selector:\n description: |-\n Selector selects a set of DNSNames on the Certificate resource that\n should be solved using this challenge solver.\n If not specified, the solver will be treated as the 'default' solver\n with the lowest priority, i.e. if any other solver has a more specific\n match, it will be used instead.\n type: object\n properties:\n dnsNames:\n description: |-\n List of DNSNames that this solver will be used to solve.\n If specified and a match is found, a dnsNames selector will take\n precedence over a dnsZones selector.\n If multiple solvers match with the same dnsNames value, the solver\n with the most matching labels in matchLabels will be selected.\n If neither has more matches, the solver defined earlier in the list\n will be selected.\n type: array\n items:\n type: string\n dnsZones:\n description: |-\n List of DNSZones that this solver will be used to solve.\n The most specific DNS zone match specified here will take precedence\n over other DNS zone matches, so a solver specifying sys.example.com\n will be selected over one specifying example.com for the domain\n www.sys.example.com.\n If multiple solvers match with the same dnsZones value, the solver\n with the most matching labels in matchLabels will be selected.\n If neither has more matches, the solver defined earlier in the list\n will be selected.\n type: array\n items:\n type: string\n matchLabels:\n description: |-\n A label selector that is used to refine the set of certificate's that\n this challenge solver will apply to.\n type: object\n additionalProperties:\n type: string\n ca:\n description: |-\n CA configures this issuer to sign certificates using a signing CA keypair\n stored in a Secret resource.\n This is used to build internal PKIs that are managed by cert-manager.\n type: object\n required:\n - secretName\n properties:\n crlDistributionPoints:\n description: |-\n The CRL distribution points is an X.509 v3 certificate extension which identifies\n the location of the CRL from which the revocation of this certificate can be checked.\n If not set, certificates will be issued without distribution points set.\n type: array\n items:\n type: string\n issuingCertificateURLs:\n description: |-\n IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates\n it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details.\n As an example, such a URL might be \"http://ca.domain.com/ca.crt\".\n type: array\n items:\n type: string\n ocspServers:\n description: |-\n The OCSP server list is an X.509 v3 extension that defines a list of\n URLs of OCSP responders. The OCSP responders can be queried for the\n revocation status of an issued certificate. If not set, the\n certificate will be issued with no OCSP servers set. For example, an\n OCSP server URL could be \"http://ocsp.int-x3.letsencrypt.org\".\n type: array\n items:\n type: string\n secretName:\n description: |-\n SecretName is the name of the secret used to sign Certificates issued\n by this Issuer.\n type: string\n selfSigned:\n description: |-\n SelfSigned configures this issuer to 'self sign' certificates using the\n private key used to create the CertificateRequest object.\n type: object\n properties:\n crlDistributionPoints:\n description: |-\n The CRL distribution points is an X.509 v3 certificate extension which identifies\n the location of the CRL from which the revocation of this certificate can be checked.\n If not set certificate will be issued without CDP. Values are strings.\n type: array\n items:\n type: string\n vault:\n description: |-\n Vault configures this issuer to sign certificates using a HashiCorp Vault\n PKI backend.\n type: object\n required:\n - auth\n - path\n - server\n properties:\n auth:\n description: Auth configures how cert-manager authenticates with the Vault server.\n type: object\n properties:\n appRole:\n description: |-\n AppRole authenticates with Vault using the App Role auth mechanism,\n with the role and secret stored in a Kubernetes Secret resource.\n type: object\n required:\n - path\n - roleId\n - secretRef\n properties:\n path:\n description: |-\n Path where the App Role authentication backend is mounted in Vault, e.g:\n \"approle\"\n type: string\n roleId:\n description: |-\n RoleID configured in the App Role authentication backend when setting\n up the authentication backend in Vault.\n type: string\n secretRef:\n description: |-\n Reference to a key in a Secret that contains the App Role secret used\n to authenticate with Vault.\n The `key` field must be specified and denotes which entry within the Secret\n resource is used as the app role secret.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n kubernetes:\n description: |-\n Kubernetes authenticates with Vault by passing the ServiceAccount\n token stored in the named Secret resource to the Vault server.\n type: object\n required:\n - role\n properties:\n mountPath:\n description: |-\n The Vault mountPath here is the mount path to use when authenticating with\n Vault. For example, setting a value to `/v1/auth/foo`, will use the path\n `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the\n default value \"/v1/auth/kubernetes\" will be used.\n type: string\n role:\n description: |-\n A required field containing the Vault Role to assume. A Role binds a\n Kubernetes ServiceAccount with a set of Vault policies.\n type: string\n secretRef:\n description: |-\n The required Secret field containing a Kubernetes ServiceAccount JWT used\n for authenticating with Vault. Use of 'ambient credentials' is not\n supported.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n serviceAccountRef:\n description: |-\n A reference to a service account that will be used to request a bound\n token (also known as \"projected token\"). Compared to using \"secretRef\",\n using this field means that you don't rely on statically bound tokens. To\n use this field, you must configure an RBAC rule to let cert-manager\n request a token.\n type: object\n required:\n - name\n properties:\n audiences:\n description: |-\n TokenAudiences is an optional list of extra audiences to include in the token passed to Vault. The default token\n consisting of the issuer's namespace and name is always included.\n type: array\n items:\n type: string\n name:\n description: Name of the ServiceAccount used to request a token.\n type: string\n tokenSecretRef:\n description: TokenSecretRef authenticates with Vault by presenting a token.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n caBundle:\n description: |-\n Base64-encoded bundle of PEM CAs which will be used to validate the certificate\n chain presented by Vault. Only used if using HTTPS to connect to Vault and\n ignored for HTTP connections.\n Mutually exclusive with CABundleSecretRef.\n If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in\n the cert-manager controller container is used to validate the TLS connection.\n type: string\n format: byte\n caBundleSecretRef:\n description: |-\n Reference to a Secret containing a bundle of PEM-encoded CAs to use when\n verifying the certificate chain presented by Vault when using HTTPS.\n Mutually exclusive with CABundle.\n If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in\n the cert-manager controller container is used to validate the TLS connection.\n If no key for the Secret is specified, cert-manager will default to 'ca.crt'.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientCertSecretRef:\n description: |-\n Reference to a Secret containing a PEM-encoded Client Certificate to use when the\n Vault server requires mTLS.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n clientKeySecretRef:\n description: |-\n Reference to a Secret containing a PEM-encoded Client Private Key to use when the\n Vault server requires mTLS.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n namespace:\n description: |-\n Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: \"ns1\"\n More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces\n type: string\n path:\n description: |-\n Path is the mount path of the Vault PKI backend's `sign` endpoint, e.g:\n \"my_pki_mount/sign/my-role-name\".\n type: string\n server:\n description: 'Server is the connection address for the Vault server, e.g: \"https://vault.example.com:8200\".'\n type: string\n venafi:\n description: |-\n Venafi configures this issuer to sign certificates using a Venafi TPP\n or Venafi Cloud policy zone.\n type: object\n required:\n - zone\n properties:\n cloud:\n description: |-\n Cloud specifies the Venafi cloud configuration settings.\n Only one of TPP or Cloud may be specified.\n type: object\n required:\n - apiTokenSecretRef\n properties:\n apiTokenSecretRef:\n description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n url:\n description: |-\n URL is the base URL for Venafi Cloud.\n Defaults to \"https://api.venafi.cloud/v1\".\n type: string\n tpp:\n description: |-\n TPP specifies Trust Protection Platform configuration settings.\n Only one of TPP or Cloud may be specified.\n type: object\n required:\n - credentialsRef\n - url\n properties:\n caBundle:\n description: |-\n Base64-encoded bundle of PEM CAs which will be used to validate the certificate\n chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP.\n If undefined, the certificate bundle in the cert-manager controller container\n is used to validate the chain.\n type: string\n format: byte\n credentialsRef:\n description: |-\n CredentialsRef is a reference to a Secret containing the username and\n password for the TPP server.\n The secret must contain two keys, 'username' and 'password'.\n type: object\n required:\n - name\n properties:\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n url:\n description: |-\n URL is the base URL for the vedsdk endpoint of the Venafi TPP instance,\n for example: \"https://tpp.example.com/vedsdk\".\n type: string\n zone:\n description: |-\n Zone is the Venafi Policy Zone to use for this issuer.\n All requests made to the Venafi platform will be restricted by the named\n zone policy.\n This field is required.\n type: string\n status:\n description: Status of the Issuer. This is set and managed automatically.\n type: object\n properties:\n acme:\n description: |-\n ACME specific status options.\n This field should only be set if the Issuer is configured to use an ACME\n server to issue certificates.\n type: object\n properties:\n lastPrivateKeyHash:\n description: |-\n LastPrivateKeyHash is a hash of the private key associated with the latest\n registered ACME account, in order to track changes made to registered account\n associated with the Issuer\n type: string\n lastRegisteredEmail:\n description: |-\n LastRegisteredEmail is the email associated with the latest registered\n ACME account, in order to track changes made to registered account\n associated with the Issuer\n type: string\n uri:\n description: |-\n URI is the unique account identifier, which can also be used to retrieve\n account details from the CA\n type: string\n conditions:\n description: |-\n List of status conditions to indicate the status of a CertificateRequest.\n Known condition types are `Ready`.\n type: array\n items:\n description: IssuerCondition contains condition information for an Issuer.\n type: object\n required:\n - status\n - type\n properties:\n lastTransitionTime:\n description: |-\n LastTransitionTime is the timestamp corresponding to the last status\n change of this condition.\n type: string\n format: date-time\n message:\n description: |-\n Message is a human readable description of the details of the last\n transition, complementing reason.\n type: string\n observedGeneration:\n description: |-\n If set, this represents the .metadata.generation that the condition was\n set based upon.\n For instance, if .metadata.generation is currently 12, but the\n .status.condition[x].observedGeneration is 9, the condition is out of date\n with respect to the current state of the Issuer.\n type: integer\n format: int64\n reason:\n description: |-\n Reason is a brief machine readable explanation for the condition's last\n transition.\n type: string\n status:\n description: Status of the condition, one of (`True`, `False`, `Unknown`).\n type: string\n enum:\n - \"True\"\n - \"False\"\n - Unknown\n type:\n description: Type of the condition, known values are (`Ready`).\n type: string\n x-kubernetes-list-map-keys:\n - type\n x-kubernetes-list-type: map\n served: true\n storage: true\n---\n# Source: cert-manager/deploy/crds/crd-certificates.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: certificates.cert-manager.io\n labels:\n app: 'cert-manager'\n app.kubernetes.io/name: 'cert-manager'\n app.kubernetes.io/instance: 'cert-manager'\n # Generated labels\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n group: cert-manager.io\n names:\n kind: Certificate\n listKind: CertificateList\n plural: certificates\n shortNames:\n - cert\n - certs\n singular: certificate\n categories:\n - cert-manager\n scope: Namespaced\n versions:\n - name: v1\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].status\n name: Ready\n type: string\n - jsonPath: .spec.secretName\n name: Secret\n type: string\n - jsonPath: .spec.issuerRef.name\n name: Issuer\n priority: 1\n type: string\n - jsonPath: .status.conditions[?(@.type==\"Ready\")].message\n name: Status\n priority: 1\n type: string\n - jsonPath: .metadata.creationTimestamp\n description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n name: Age\n type: date\n schema:\n openAPIV3Schema:\n description: |-\n A Certificate resource should be created to ensure an up to date and signed\n X.509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`.\n\n\n The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`).\n type: object\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: |-\n Specification of the desired state of the Certificate resource.\n https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\n type: object\n required:\n - issuerRef\n - secretName\n properties:\n additionalOutputFormats:\n description: |-\n Defines extra output formats of the private key and signed certificate chain\n to be written to this Certificate's target Secret.\n\n\n This is a Beta Feature enabled by default. It can be disabled with the\n `--feature-gates=AdditionalCertificateOutputFormats=false` option set on both\n the controller and webhook components.\n type: array\n items:\n description: |-\n CertificateAdditionalOutputFormat defines an additional output format of a\n Certificate resource. These contain supplementary data formats of the signed\n certificate chain and paired private key.\n type: object\n required:\n - type\n properties:\n type:\n description: |-\n Type is the name of the format type that should be written to the\n Certificate's target Secret.\n type: string\n enum:\n - DER\n - CombinedPEM\n commonName:\n description: |-\n Requested common name X509 certificate subject attribute.\n More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6\n NOTE: TLS clients will ignore this value when any subject alternative name is\n set (see https://tools.ietf.org/html/rfc6125#section-6.4.4).\n\n\n Should have a length of 64 characters or fewer to avoid generating invalid CSRs.\n Cannot be set if the `literalSubject` field is set.\n type: string\n dnsNames:\n description: Requested DNS subject alternative names.\n type: array\n items:\n type: string\n duration:\n description: |-\n Requested 'duration' (i.e. lifetime) of the Certificate. Note that the\n issuer may choose to ignore the requested duration, just like any other\n requested attribute.\n\n\n If unset, this defaults to 90 days.\n Minimum accepted duration is 1 hour.\n Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration.\n type: string\n emailAddresses:\n description: Requested email subject alternative names.\n type: array\n items:\n type: string\n encodeUsagesInRequest:\n description: |-\n Whether the KeyUsage and ExtKeyUsage extensions should be set in the encoded CSR.\n\n\n This option defaults to true, and should only be disabled if the target\n issuer does not support CSRs with these X509 KeyUsage/ ExtKeyUsage extensions.\n type: boolean\n ipAddresses:\n description: Requested IP address subject alternative names.\n type: array\n items:\n type: string\n isCA:\n description: |-\n Requested basic constraints isCA value.\n The isCA value is used to set the `isCA` field on the created CertificateRequest\n resources. Note that the issuer may choose to ignore the requested isCA value, just\n like any other requested attribute.\n\n\n If true, this will automatically add the `cert sign` usage to the list\n of requested `usages`.\n type: boolean\n issuerRef:\n description: |-\n Reference to the issuer responsible for issuing the certificate.\n If the issuer is namespace-scoped, it must be in the same namespace\n as the Certificate. If the issuer is cluster-scoped, it can be used\n from any namespace.\n\n\n The `name` field of the reference must always be specified.\n type: object\n required:\n - name\n properties:\n group:\n description: Group of the resource being referred to.\n type: string\n kind:\n description: Kind of the resource being referred to.\n type: string\n name:\n description: Name of the resource being referred to.\n type: string\n keystores:\n description: Additional keystore output formats to be stored in the Certificate's Secret.\n type: object\n properties:\n jks:\n description: |-\n JKS configures options for storing a JKS keystore in the\n `spec.secretName` Secret resource.\n type: object\n required:\n - create\n - passwordSecretRef\n properties:\n alias:\n description: |-\n Alias specifies the alias of the key in the keystore, required by the JKS format.\n If not provided, the default alias `certificate` will be used.\n type: string\n create:\n description: |-\n Create enables JKS keystore creation for the Certificate.\n If true, a file named `keystore.jks` will be created in the target\n Secret resource, encrypted using the password stored in\n `passwordSecretRef`.\n The keystore file will be updated immediately.\n If the issuer provided a CA certificate, a file named `truststore.jks`\n will also be created in the target Secret resource, encrypted using the\n password stored in `passwordSecretRef`\n containing the issuing Certificate Authority\n type: boolean\n passwordSecretRef:\n description: |-\n PasswordSecretRef is a reference to a key in a Secret resource\n containing the password used to encrypt the JKS keystore.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n pkcs12:\n description: |-\n PKCS12 configures options for storing a PKCS12 keystore in the\n `spec.secretName` Secret resource.\n type: object\n required:\n - create\n - passwordSecretRef\n properties:\n create:\n description: |-\n Create enables PKCS12 keystore creation for the Certificate.\n If true, a file named `keystore.p12` will be created in the target\n Secret resource, encrypted using the password stored in\n `passwordSecretRef`.\n The keystore file will be updated immediately.\n If the issuer provided a CA certificate, a file named `truststore.p12` will\n also be created in the target Secret resource, encrypted using the\n password stored in `passwordSecretRef` containing the issuing Certificate\n Authority\n type: boolean\n passwordSecretRef:\n description: |-\n PasswordSecretRef is a reference to a key in a Secret resource\n containing the password used to encrypt the PKCS12 keystore.\n type: object\n required:\n - name\n properties:\n key:\n description: |-\n The key of the entry in the Secret resource's `data` field to be used.\n Some instances of this field may be defaulted, in others it may be\n required.\n type: string\n name:\n description: |-\n Name of the resource being referred to.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n profile:\n description: |-\n Profile specifies the key and certificate encryption algorithms and the HMAC algorithm\n used to create the PKCS12 keystore. Default value is `LegacyRC2` for backward compatibility.\n\n\n If provided, allowed values are:\n `LegacyRC2`: Deprecated. Not supported by default in OpenSSL 3 or Java 20.\n `LegacyDES`: Less secure algorithm. Use this option for maximal compatibility.\n `Modern2023`: Secure algorithm. Use this option in case you have to always use secure algorithms\n (eg. because of company policy). Please note that the security of the algorithm is not that important\n in reality, because the unencrypted certificate and private key are also stored in the Secret.\n type: string\n enum:\n - LegacyRC2\n - LegacyDES\n - Modern2023\n literalSubject:\n description: |-\n Requested X.509 certificate subject, represented using the LDAP \"String\n Representation of a Distinguished Name\" [1].\n Important: the LDAP string format also specifies the order of the attributes\n in the subject, this is important when issuing certs for LDAP authentication.\n Example: `CN=foo,DC=corp,DC=example,DC=com`\n More info [1]: https://datatracker.ietf.org/doc/html/rfc4514\n More info: https://github.com/cert-manager/cert-manager/issues/3203\n More info: https://github.com/cert-manager/cert-manager/issues/4424\n\n\n Cannot be set if the `subject` or `commonName` field is set.\n type: string\n nameConstraints:\n description: |-\n x.509 certificate NameConstraint extension which MUST NOT be used in a non-CA certificate.\n More Info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.10\n\n\n This is an Alpha Feature and is only enabled with the\n `--feature-gates=NameConstraints=true` option set on both\n the controller and webhook components.\n type: object\n properties:\n critical:\n description: if true then the name constraints are marked critical.\n type: boolean\n excluded:\n description: |-\n Excluded contains the constraints which must be disallowed. Any name matching a\n restriction in the excluded field is invalid regardless\n of information appearing in the permitted\n type: object\n properties:\n dnsDomains:\n description: DNSDomains is a list of DNS domains that are permitted or excluded.\n type: array\n items:\n type: string\n emailAddresses:\n description: EmailAddresses is a list of Email Addresses that are permitted or excluded.\n type: array\n items:\n type: string\n ipRanges:\n description: |-\n IPRanges is a list of IP Ranges that are permitted or excluded.\n This should be a valid CIDR notation.\n type: array\n items:\n type: string\n uriDomains:\n description: URIDomains is a list of URI domains that are permitted or excluded.\n type: array\n items:\n type: string\n permitted:\n description: Permitted contains the constraints in which the names must be located.\n type: object\n properties:\n dnsDomains:\n description: DNSDomains is a list of DNS domains that are permitted or excluded.\n type: array\n items:\n type: string\n emailAddresses:\n description: EmailAddresses is a list of Email Addresses that are permitted or excluded.\n type: array\n items:\n type: string\n ipRanges:\n description: |-\n IPRanges is a list of IP Ranges that are permitted or excluded.\n This should be a valid CIDR notation.\n type: array\n items:\n type: string\n uriDomains:\n description: URIDomains is a list of URI domains that are permitted or excluded.\n type: array\n items:\n type: string\n otherNames:\n description: |-\n `otherNames` is an escape hatch for SAN that allows any type. We currently restrict the support to string like otherNames, cf RFC 5280 p 37\n Any UTF8 String valued otherName can be passed with by setting the keys oid: x.x.x.x and UTF8Value: somevalue for `otherName`.\n Most commonly this would be UPN set with oid: 1.3.6.1.4.1.311.20.2.3\n You should ensure that any OID passed is valid for the UTF8String type as we do not explicitly validate this.\n type: array\n items:\n type: object\n properties:\n oid:\n description: |-\n OID is the object identifier for the otherName SAN.\n The object identifier must be expressed as a dotted string, for\n example, \"1.2.840.113556.1.4.221\".\n type: string\n utf8Value:\n description: |-\n utf8Value is the string value of the otherName SAN.\n The utf8Value accepts any valid UTF8 string to set as value for the otherName SAN.\n type: string\n privateKey:\n description: |-\n Private key options. These include the key algorithm and size, the used\n encoding and the rotation policy.\n type: object\n properties:\n algorithm:\n description: |-\n Algorithm is the private key algorithm of the corresponding private key\n for this certificate.\n\n\n If provided, allowed values are either `RSA`, `ECDSA` or `Ed25519`.\n If `algorithm` is specified and `size` is not provided,\n key size of 2048 will be used for `RSA` key algorithm and\n key size of 256 will be used for `ECDSA` key algorithm.\n key size is ignored when using the `Ed25519` key algorithm.\n type: string\n enum:\n - RSA\n - ECDSA\n - Ed25519\n encoding:\n description: |-\n The private key cryptography standards (PKCS) encoding for this\n certificate's private key to be encoded in.\n\n\n If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1\n and PKCS#8, respectively.\n Defaults to `PKCS1` if not specified.\n type: string\n enum:\n - PKCS1\n - PKCS8\n rotationPolicy:\n description: |-\n RotationPolicy controls how private keys should be regenerated when a\n re-issuance is being processed.\n\n\n If set to `Never`, a private key will only be generated if one does not\n already exist in the target `spec.secretName`. If one does exists but it\n does not have the correct algorithm or size, a warning will be raised\n to await user intervention.\n If set to `Always`, a private key matching the specified requirements\n will be generated whenever a re-issuance occurs.\n Default is `Never` for backward compatibility.\n type: string\n enum:\n - Never\n - Always\n size:\n description: |-\n Size is the key bit size of the corresponding private key for this certificate.\n\n\n If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`,\n and will default to `2048` if not specified.\n If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`,\n and will default to `256` if not specified.\n If `algorithm` is set to `Ed25519`, Size is ignored.\n No other values are allowed.\n type: integer\n renewBefore:\n description: |-\n How long before the currently issued certificate's expiry cert-manager should\n renew the certificate. For example, if a certificate is valid for 60 minutes,\n and `renewBefore=10m`, cert-manager will begin to attempt to renew the certificate\n 50 minutes after it was issued (i.e. when there are 10 minutes remaining until\n the certificate is no longer valid).\n\n\n NOTE: The actual lifetime of the issued certificate is used to determine the\n renewal time. If an issuer returns a certificate with a different lifetime than\n the one requested, cert-manager will use the lifetime of the issued certificate.\n\n\n If unset, this defaults to 1/3 of the issued certificate's lifetime.\n Minimum accepted value is 5 minutes.\n Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration.\n type: string\n revisionHistoryLimit:\n description: |-\n The maximum number of CertificateRequest revisions that are maintained in\n the Certificate's history. Each revision represents a single `CertificateRequest`\n created by this Certificate, either when it was created, renewed, or Spec\n was changed. Revisions will be removed by oldest first if the number of\n revisions exceeds this number.\n\n\n If set, revisionHistoryLimit must be a value of `1` or greater.\n If unset (`nil`), revisions will not be garbage collected.\n Default value is `nil`.\n type: integer\n format: int32\n secretName:\n description: |-\n Name of the Secret resource that will be automatically created and\n managed by this Certificate resource. It will be populated with a\n private key and certificate, signed by the denoted issuer. The Secret\n resource lives in the same namespace as the Certificate resource.\n type: string\n secretTemplate:\n description: |-\n Defines annotations and labels to be copied to the Certificate's Secret.\n Labels and annotations on the Secret will be changed as they appear on the\n SecretTemplate when added or removed. SecretTemplate annotations are added\n in conjunction with, and cannot overwrite, the base set of annotations\n cert-manager sets on the Certificate's Secret.\n type: object\n properties:\n annotations:\n description: Annotations is a key value map to be copied to the target Kubernetes Secret.\n type: object\n additionalProperties:\n type: string\n labels:\n description: Labels is a key value map to be copied to the target Kubernetes Secret.\n type: object\n additionalProperties:\n type: string\n subject:\n description: |-\n Requested set of X509 certificate subject attributes.\n More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6\n\n\n The common name attribute is specified separately in the `commonName` field.\n Cannot be set if the `literalSubject` field is set.\n type: object\n properties:\n countries:\n description: Countries to be used on the Certificate.\n type: array\n items:\n type: string\n localities:\n description: Cities to be used on the Certificate.\n type: array\n items:\n type: string\n organizationalUnits:\n description: Organizational Units to be used on the Certificate.\n type: array\n items:\n type: string\n organizations:\n description: Organizations to be used on the Certificate.\n type: array\n items:\n type: string\n postalCodes:\n description: Postal codes to be used on the Certificate.\n type: array\n items:\n type: string\n provinces:\n description: State/Provinces to be used on the Certificate.\n type: array\n items:\n type: string\n serialNumber:\n description: Serial number to be used on the Certificate.\n type: string\n streetAddresses:\n description: Street addresses to be used on the Certificate.\n type: array\n items:\n type: string\n uris:\n description: Requested URI subject alternative names.\n type: array\n items:\n type: string\n usages:\n description: |-\n Requested key usages and extended key usages.\n These usages are used to set the `usages` field on the created CertificateRequest\n resources. If `encodeUsagesInRequest` is unset or set to `true`, the usages\n will additionally be encoded in the `request` field which contains the CSR blob.\n\n\n If unset, defaults to `digital signature` and `key encipherment`.\n type: array\n items:\n description: |-\n KeyUsage specifies valid usage contexts for keys.\n See:\n https://tools.ietf.org/html/rfc5280#section-4.2.1.3\n https://tools.ietf.org/html/rfc5280#section-4.2.1.12\n\n\n Valid KeyUsage values are as follows:\n \"signing\",\n \"digital signature\",\n \"content commitment\",\n \"key encipherment\",\n \"key agreement\",\n \"data encipherment\",\n \"cert sign\",\n \"crl sign\",\n \"encipher only\",\n \"decipher only\",\n \"any\",\n \"server auth\",\n \"client auth\",\n \"code signing\",\n \"email protection\",\n \"s/mime\",\n \"ipsec end system\",\n \"ipsec tunnel\",\n \"ipsec user\",\n \"timestamping\",\n \"ocsp signing\",\n \"microsoft sgc\",\n \"netscape sgc\"\n type: string\n enum:\n - signing\n - digital signature\n - content commitment\n - key encipherment\n - key agreement\n - data encipherment\n - cert sign\n - crl sign\n - encipher only\n - decipher only\n - any\n - server auth\n - client auth\n - code signing\n - email protection\n - s/mime\n - ipsec end system\n - ipsec tunnel\n - ipsec user\n - timestamping\n - ocsp signing\n - microsoft sgc\n - netscape sgc\n status:\n description: |-\n Status of the Certificate.\n This is set and managed automatically.\n Read-only.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\n type: object\n properties:\n conditions:\n description: |-\n List of status conditions to indicate the status of certificates.\n Known condition types are `Ready` and `Issuing`.\n type: array\n items:\n description: CertificateCondition contains condition information for an Certificate.\n type: object\n required:\n - status\n - type\n properties:\n lastTransitionTime:\n description: |-\n LastTransitionTime is the timestamp corresponding to the last status\n change of this condition.\n type: string\n format: date-time\n message:\n description: |-\n Message is a human readable description of the details of the last\n transition, complementing reason.\n type: string\n observedGeneration:\n description: |-\n If set, this represents the .metadata.generation that the condition was\n set based upon.\n For instance, if .metadata.generation is currently 12, but the\n .status.condition[x].observedGeneration is 9, the condition is out of date\n with respect to the current state of the Certificate.\n type: integer\n format: int64\n reason:\n description: |-\n Reason is a brief machine readable explanation for the condition's last\n transition.\n type: string\n status:\n description: Status of the condition, one of (`True`, `False`, `Unknown`).\n type: string\n enum:\n - \"True\"\n - \"False\"\n - Unknown\n type:\n description: Type of the condition, known values are (`Ready`, `Issuing`).\n type: string\n x-kubernetes-list-map-keys:\n - type\n x-kubernetes-list-type: map\n failedIssuanceAttempts:\n description: |-\n The number of continuous failed issuance attempts up till now. This\n field gets removed (if set) on a successful issuance and gets set to\n 1 if unset and an issuance has failed. If an issuance has failed, the\n delay till the next issuance will be calculated using formula\n time.Hour * 2 ^ (failedIssuanceAttempts - 1).\n type: integer\n lastFailureTime:\n description: |-\n LastFailureTime is set only if the lastest issuance for this\n Certificate failed and contains the time of the failure. If an\n issuance has failed, the delay till the next issuance will be\n calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts -\n 1). If the latest issuance has succeeded this field will be unset.\n type: string\n format: date-time\n nextPrivateKeySecretName:\n description: |-\n The name of the Secret resource containing the private key to be used\n for the next certificate iteration.\n The keymanager controller will automatically set this field if the\n `Issuing` condition is set to `True`.\n It will automatically unset this field when the Issuing condition is\n not set or False.\n type: string\n notAfter:\n description: |-\n The expiration time of the certificate stored in the secret named\n by this resource in `spec.secretName`.\n type: string\n format: date-time\n notBefore:\n description: |-\n The time after which the certificate stored in the secret named\n by this resource in `spec.secretName` is valid.\n type: string\n format: date-time\n renewalTime:\n description: |-\n RenewalTime is the time at which the certificate will be next\n renewed.\n If not set, no upcoming renewal is scheduled.\n type: string\n format: date-time\n revision:\n description: |-\n The current 'revision' of the certificate as issued.\n\n\n When a CertificateRequest resource is created, it will have the\n `cert-manager.io/certificate-revision` set to one greater than the\n current value of this field.\n\n\n Upon issuance, this field will be set to the value of the annotation\n on the CertificateRequest resource used to issue the certificate.\n\n\n Persisting the value on the CertificateRequest resource allows the\n certificates controller to know whether a request is part of an old\n issuance or if it is part of the ongoing revision's issuance by\n checking if the revision value in the annotation is greater than this\n field.\n type: integer\n served: true\n storage: true\n---\n# Source: cert-manager/deploy/crds/crd-orders.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: orders.acme.cert-manager.io\n labels:\n app: 'cert-manager'\n app.kubernetes.io/name: 'cert-manager'\n app.kubernetes.io/instance: 'cert-manager'\n # Generated labels\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n group: acme.cert-manager.io\n names:\n kind: Order\n listKind: OrderList\n plural: orders\n singular: order\n categories:\n - cert-manager\n - cert-manager-acme\n scope: Namespaced\n versions:\n - name: v1\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.state\n name: State\n type: string\n - jsonPath: .spec.issuerRef.name\n name: Issuer\n priority: 1\n type: string\n - jsonPath: .status.reason\n name: Reason\n priority: 1\n type: string\n - jsonPath: .metadata.creationTimestamp\n description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n name: Age\n type: date\n schema:\n openAPIV3Schema:\n description: Order is a type to represent an Order with an ACME server\n type: object\n required:\n - metadata\n - spec\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n type: object\n required:\n - issuerRef\n - request\n properties:\n commonName:\n description: |-\n CommonName is the common name as specified on the DER encoded CSR.\n If specified, this value must also be present in `dnsNames` or `ipAddresses`.\n This field must match the corresponding field on the DER encoded CSR.\n type: string\n dnsNames:\n description: |-\n DNSNames is a list of DNS names that should be included as part of the Order\n validation process.\n This field must match the corresponding field on the DER encoded CSR.\n type: array\n items:\n type: string\n duration:\n description: |-\n Duration is the duration for the not after date for the requested certificate.\n this is set on order creation as pe the ACME spec.\n type: string\n ipAddresses:\n description: |-\n IPAddresses is a list of IP addresses that should be included as part of the Order\n validation process.\n This field must match the corresponding field on the DER encoded CSR.\n type: array\n items:\n type: string\n issuerRef:\n description: |-\n IssuerRef references a properly configured ACME-type Issuer which should\n be used to create this Order.\n If the Issuer does not exist, processing will be retried.\n If the Issuer is not an 'ACME' Issuer, an error will be returned and the\n Order will be marked as failed.\n type: object\n required:\n - name\n properties:\n group:\n description: Group of the resource being referred to.\n type: string\n kind:\n description: Kind of the resource being referred to.\n type: string\n name:\n description: Name of the resource being referred to.\n type: string\n request:\n description: |-\n Certificate signing request bytes in DER encoding.\n This will be used when finalizing the order.\n This field must be set on the order.\n type: string\n format: byte\n status:\n type: object\n properties:\n authorizations:\n description: |-\n Authorizations contains data returned from the ACME server on what\n authorizations must be completed in order to validate the DNS names\n specified on the Order.\n type: array\n items:\n description: |-\n ACMEAuthorization contains data returned from the ACME server on an\n authorization that must be completed in order validate a DNS name on an ACME\n Order resource.\n type: object\n required:\n - url\n properties:\n challenges:\n description: |-\n Challenges specifies the challenge types offered by the ACME server.\n One of these challenge types will be selected when validating the DNS\n name and an appropriate Challenge resource will be created to perform\n the ACME challenge process.\n type: array\n items:\n description: |-\n Challenge specifies a challenge offered by the ACME server for an Order.\n An appropriate Challenge resource can be created to perform the ACME\n challenge process.\n type: object\n required:\n - token\n - type\n - url\n properties:\n token:\n description: |-\n Token is the token that must be presented for this challenge.\n This is used to compute the 'key' that must also be presented.\n type: string\n type:\n description: |-\n Type is the type of challenge being offered, e.g. 'http-01', 'dns-01',\n 'tls-sni-01', etc.\n This is the raw value retrieved from the ACME server.\n Only 'http-01' and 'dns-01' are supported by cert-manager, other values\n will be ignored.\n type: string\n url:\n description: |-\n URL is the URL of this challenge. It can be used to retrieve additional\n metadata about the Challenge from the ACME server.\n type: string\n identifier:\n description: Identifier is the DNS name to be validated as part of this authorization\n type: string\n initialState:\n description: |-\n InitialState is the initial state of the ACME authorization when first\n fetched from the ACME server.\n If an Authorization is already 'valid', the Order controller will not\n create a Challenge resource for the authorization. This will occur when\n working with an ACME server that enables 'authz reuse' (such as Let's\n Encrypt's production endpoint).\n If not set and 'identifier' is set, the state is assumed to be pending\n and a Challenge will be created.\n type: string\n enum:\n - valid\n - ready\n - pending\n - processing\n - invalid\n - expired\n - errored\n url:\n description: URL is the URL of the Authorization that must be completed\n type: string\n wildcard:\n description: |-\n Wildcard will be true if this authorization is for a wildcard DNS name.\n If this is true, the identifier will be the *non-wildcard* version of\n the DNS name.\n For example, if '*.example.com' is the DNS name being validated, this\n field will be 'true' and the 'identifier' field will be 'example.com'.\n type: boolean\n certificate:\n description: |-\n Certificate is a copy of the PEM encoded certificate for this Order.\n This field will be populated after the order has been successfully\n finalized with the ACME server, and the order has transitioned to the\n 'valid' state.\n type: string\n format: byte\n failureTime:\n description: |-\n FailureTime stores the time that this order failed.\n This is used to influence garbage collection and back-off.\n type: string\n format: date-time\n finalizeURL:\n description: |-\n FinalizeURL of the Order.\n This is used to obtain certificates for this order once it has been completed.\n type: string\n reason:\n description: |-\n Reason optionally provides more information about a why the order is in\n the current state.\n type: string\n state:\n description: |-\n State contains the current state of this Order resource.\n States 'success' and 'expired' are 'final'\n type: string\n enum:\n - valid\n - ready\n - pending\n - processing\n - invalid\n - expired\n - errored\n url:\n description: |-\n URL of the Order.\n This will initially be empty when the resource is first created.\n The Order controller will populate this field when the Order is first processed.\n This field will be immutable after it is initially set.\n type: string\n served: true\n storage: true\n" }, { "path": "roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2", "content": "# Copyright 2022 The cert-manager Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ cert_manager_namespace }}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nautomountServiceAccountToken: true\nmetadata:\n name: cert-manager-cainjector\n namespace: {{ cert_manager_namespace }}\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nautomountServiceAccountToken: true\nmetadata:\n name: cert-manager\n namespace: {{ cert_manager_namespace }}\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nautomountServiceAccountToken: true\nmetadata:\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/controller-config.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cert-manager\n namespace: {{ cert_manager_namespace }}\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\ndata:\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-config.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\ndata:\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-cainjector\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"get\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"admissionregistration.k8s.io\"]\n resources: [\"validatingwebhookconfigurations\", \"mutatingwebhookconfigurations\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"apiregistration.k8s.io\"]\n resources: [\"apiservices\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"apiextensions.k8s.io\"]\n resources: [\"customresourcedefinitions\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# Issuer controller role\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-issuers\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"issuers\", \"issuers/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"issuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# ClusterIssuer controller role\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-clusterissuers\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"clusterissuers\", \"clusterissuers/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"clusterissuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"delete\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# Certificates controller role\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-certificates\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\", \"certificates/status\", \"certificaterequests\", \"certificaterequests/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\", \"certificaterequests\", \"clusterissuers\", \"issuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n # We require these rules to support users with the OwnerReferencesPermissionEnforcement\n # admission controller enabled:\n # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates/finalizers\", \"certificaterequests/finalizers\"]\n verbs: [\"update\"]\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"orders\"]\n verbs: [\"create\", \"delete\", \"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"update\", \"delete\", \"patch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# Orders controller role\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-orders\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"orders\", \"orders/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"orders\", \"challenges\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"clusterissuers\", \"issuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"challenges\"]\n verbs: [\"create\", \"delete\"]\n # We require these rules to support users with the OwnerReferencesPermissionEnforcement\n # admission controller enabled:\n # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"orders/finalizers\"]\n verbs: [\"update\"]\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# Challenges controller role\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-challenges\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n # Use to update challenge resource status\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"challenges\", \"challenges/status\"]\n verbs: [\"update\", \"patch\"]\n # Used to watch challenge resources\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"challenges\"]\n verbs: [\"get\", \"list\", \"watch\"]\n # Used to watch challenges, issuer and clusterissuer resources\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"issuers\", \"clusterissuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n # Need to be able to retrieve ACME account private key to complete challenges\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n # Used to create events\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\"]\n # HTTP01 rules\n - apiGroups: [\"\"]\n resources: [\"pods\", \"services\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\"]\n - apiGroups: [\"networking.k8s.io\"]\n resources: [\"ingresses\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\", \"update\"]\n - apiGroups: [ \"gateway.networking.k8s.io\" ]\n resources: [ \"httproutes\" ]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\", \"update\"]\n # We require the ability to specify a custom hostname when we are creating\n # new ingress resources.\n # See: https://github.com/openshift/origin/blob/21f191775636f9acadb44fa42beeb4f75b255532/pkg/route/apiserver/admission/ingress_admission.go#L84-L148\n - apiGroups: [\"route.openshift.io\"]\n resources: [\"routes/custom-host\"]\n verbs: [\"create\"]\n # We require these rules to support users with the OwnerReferencesPermissionEnforcement\n # admission controller enabled:\n # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"challenges/finalizers\"]\n verbs: [\"update\"]\n # DNS01 rules (duplicated above)\n - apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# ingress-shim controller role\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-ingress-shim\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\", \"certificaterequests\"]\n verbs: [\"create\", \"update\", \"delete\"]\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\", \"certificaterequests\", \"issuers\", \"clusterissuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"networking.k8s.io\"]\n resources: [\"ingresses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n # We require these rules to support users with the OwnerReferencesPermissionEnforcement\n # admission controller enabled:\n # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement\n - apiGroups: [\"networking.k8s.io\"]\n resources: [\"ingresses/finalizers\"]\n verbs: [\"update\"]\n - apiGroups: [\"gateway.networking.k8s.io\"]\n resources: [\"gateways\", \"httproutes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"gateway.networking.k8s.io\"]\n resources: [\"gateways/finalizers\", \"httproutes/finalizers\"]\n verbs: [\"update\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-cluster-view\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n rbac.authorization.k8s.io/aggregate-to-cluster-reader: \"true\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"clusterissuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-view\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n rbac.authorization.k8s.io/aggregate-to-view: \"true\"\n rbac.authorization.k8s.io/aggregate-to-edit: \"true\"\n rbac.authorization.k8s.io/aggregate-to-admin: \"true\"\n rbac.authorization.k8s.io/aggregate-to-cluster-reader: \"true\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\", \"certificaterequests\", \"issuers\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"challenges\", \"orders\"]\n verbs: [\"get\", \"list\", \"watch\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-edit\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n rbac.authorization.k8s.io/aggregate-to-edit: \"true\"\n rbac.authorization.k8s.io/aggregate-to-admin: \"true\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates\", \"certificaterequests\", \"issuers\"]\n verbs: [\"create\", \"delete\", \"deletecollection\", \"patch\", \"update\"]\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"certificates/status\"]\n verbs: [\"update\"]\n - apiGroups: [\"acme.cert-manager.io\"]\n resources: [\"challenges\", \"orders\"]\n verbs: [\"create\", \"delete\", \"deletecollection\", \"patch\", \"update\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# Permission to approve CertificateRequests referencing cert-manager.io Issuers and ClusterIssuers\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-approve:cert-manager-io\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cert-manager\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"cert-manager.io\"]\n resources: [\"signers\"]\n verbs: [\"approve\"]\n resourceNames:\n - \"issuers.cert-manager.io/*\"\n - \"clusterissuers.cert-manager.io/*\"\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# Permission to:\n# - Update and sign CertificatSigningeRequests referencing cert-manager.io Issuers and ClusterIssuers\n# - Perform SubjectAccessReviews to test whether users are able to reference Namespaced Issuers\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-controller-certificatesigningrequests\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cert-manager\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"certificates.k8s.io\"]\n resources: [\"certificatesigningrequests\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"certificates.k8s.io\"]\n resources: [\"certificatesigningrequests/status\"]\n verbs: [\"update\", \"patch\"]\n - apiGroups: [\"certificates.k8s.io\"]\n resources: [\"signers\"]\n resourceNames: [\"issuers.cert-manager.io/*\", \"clusterissuers.cert-manager.io/*\"]\n verbs: [\"sign\"]\n - apiGroups: [\"authorization.k8s.io\"]\n resources: [\"subjectaccessreviews\"]\n verbs: [\"create\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cert-manager-webhook:subjectaccessreviews\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n- apiGroups: [\"authorization.k8s.io\"]\n resources: [\"subjectaccessreviews\"]\n verbs: [\"create\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-cainjector\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-cainjector\nsubjects:\n - name: cert-manager-cainjector\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-issuers\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-issuers\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-clusterissuers\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-clusterissuers\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-certificates\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-certificates\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-orders\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-orders\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-challenges\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-challenges\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-ingress-shim\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-ingress-shim\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-approve:cert-manager-io\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cert-manager\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-approve:cert-manager-io\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-controller-certificatesigningrequests\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cert-manager\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-controller-certificatesigningrequests\nsubjects:\n - name: cert-manager\n namespace: {{ cert_manager_namespace }}\n kind: ServiceAccount\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cert-manager-webhook:subjectaccessreviews\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cert-manager-webhook:subjectaccessreviews\nsubjects:\n- apiGroup: \"\"\n kind: ServiceAccount\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml\n# leader election rules\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: cert-manager-cainjector:leaderelection\n namespace: {{ cert_manager_leader_election_namespace }}\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n # Used for leader election by the controller\n # cert-manager-cainjector-leader-election is used by the CertificateBased injector controller\n # see cmd/cainjector/start.go#L113\n # cert-manager-cainjector-leader-election-core is used by the SecretBased injector controller\n # see cmd/cainjector/start.go#L137\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n resourceNames: [\"cert-manager-cainjector-leader-election\", \"cert-manager-cainjector-leader-election-core\"]\n verbs: [\"get\", \"update\", \"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"create\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: cert-manager:leaderelection\n namespace: {{ cert_manager_leader_election_namespace }}\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n resourceNames: [\"cert-manager-controller\"]\n verbs: [\"get\", \"update\", \"patch\"]\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"create\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: cert-manager-webhook:dynamic-serving\n namespace: {{ cert_manager_namespace }}\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nrules:\n- apiGroups: [\"\"]\n resources: [\"secrets\"]\n resourceNames:\n - 'cert-manager-webhook-ca'\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n# It's not possible to grant CREATE permission on a single resourceName.\n- apiGroups: [\"\"]\n resources: [\"secrets\"]\n verbs: [\"create\"]\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml\n# grant cert-manager permission to manage the leaderelection configmap in the\n# leader election namespace\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: cert-manager-cainjector:leaderelection\n namespace: {{ cert_manager_leader_election_namespace }}\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: cert-manager-cainjector:leaderelection\nsubjects:\n - kind: ServiceAccount\n name: cert-manager-cainjector\n namespace: {{ cert_manager_namespace }}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml\n# grant cert-manager permission to manage the leaderelection configmap in the\n# leader election namespace\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: cert-manager:leaderelection\n namespace: {{ cert_manager_leader_election_namespace }}\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: cert-manager:leaderelection\nsubjects:\n - apiGroup: \"\"\n kind: ServiceAccount\n name: cert-manager\n namespace: {{ cert_manager_namespace }}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: cert-manager-webhook:dynamic-serving\n namespace: {{ cert_manager_namespace }}\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: cert-manager-webhook:dynamic-serving\nsubjects:\n- apiGroup: \"\"\n kind: ServiceAccount\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: cert-manager\n namespace: {{ cert_manager_namespace }}\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n type: ClusterIP\n ports:\n - protocol: TCP\n port: 9402\n name: tcp-prometheus-servicemonitor\n targetPort: 9402\n selector:\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n type: ClusterIP\n ports:\n - name: https\n port: 443\n protocol: TCP\n targetPort: \"https\"\n selector:\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-deployment.yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: cert-manager-cainjector\n namespace: {{ cert_manager_namespace }}\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n replicas: 1\n selector:\n matchLabels:\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n template:\n metadata:\n labels:\n app: cainjector\n app.kubernetes.io/name: cainjector\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"cainjector\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n spec:\n serviceAccountName: cert-manager-cainjector\n enableServiceLinks: false\n securityContext:\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n containers:\n - name: cert-manager-cainjector\n image: \"{{ cert_manager_cainjector_image_repo }}:{{ cert_manager_cainjector_image_tag }}\"\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --v=2\n - --leader-election-namespace={{ cert_manager_leader_election_namespace }}\n env:\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n{% if cert_manager_http_proxy is defined and cert_manager_http_proxy != \"\" %}\n - name: HTTP_PROXY\n value: \"{{ cert_manager_http_proxy }}\"\n{% endif %}\n{% if cert_manager_https_proxy is defined and cert_manager_https_proxy != \"\" %}\n - name: HTTPS_PROXY\n value: \"{{ cert_manager_https_proxy }}\"\n{% endif %}\n{% if cert_manager_no_proxy is defined and cert_manager_no_proxy != \"\" %}\n - name: NO_PROXY\n value: \"{{ cert_manager_no_proxy }}\"\n{% endif %}\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n readOnlyRootFilesystem: true\n{% if cert_manager_tolerations %}\n tolerations:\n {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{% endif %}\n{% if cert_manager_nodeselector %}\n nodeSelector:\n {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }}\n{% endif %}\n{% if cert_manager_affinity %}\n affinity:\n {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }}\n{% endif %}\n---\n{% if cert_manager_trusted_internal_ca is defined %}\napiVersion: v1\ndata:\n internal-ca.pem: |\n {{ cert_manager_trusted_internal_ca | indent(width=4, first=False) }}\nkind: ConfigMap\nmetadata:\n name: ca-internal-truststore\n namespace: {{ cert_manager_namespace }}\n---\n{% endif %}\n# Source: cert-manager/deploy/charts/cert-manager/templates/deployment.yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: cert-manager\n namespace: {{ cert_manager_namespace }}\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n replicas: 1\n selector:\n matchLabels:\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n template:\n metadata:\n labels:\n app: cert-manager\n app.kubernetes.io/name: cert-manager\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"controller\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n annotations:\n prometheus.io/path: \"/metrics\"\n prometheus.io/scrape: 'true'\n prometheus.io/port: '9402'\n spec:\n serviceAccountName: cert-manager\n enableServiceLinks: false\n securityContext:\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n containers:\n - name: cert-manager-controller\n image: \"{{ cert_manager_controller_image_repo }}:{{ cert_manager_controller_image_tag }}\"\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --v=2\n - --cluster-resource-namespace=$(POD_NAMESPACE)\n - --leader-election-namespace={{ cert_manager_leader_election_namespace }}\n{% for extra_arg in cert_manager_controller_extra_args %}\n - {{ extra_arg }}\n{% endfor %}\n ports:\n - containerPort: 9402\n name: http-metrics\n protocol: TCP\n - containerPort: 9403\n name: http-healthz\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n readOnlyRootFilesystem: true\n env:\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n{% if cert_manager_http_proxy is defined and cert_manager_http_proxy != \"\" %}\n - name: HTTP_PROXY\n value: \"{{ cert_manager_http_proxy }}\"\n{% endif %}\n{% if cert_manager_https_proxy is defined and cert_manager_https_proxy != \"\" %}\n - name: HTTPS_PROXY\n value: \"{{ cert_manager_https_proxy }}\"\n{% endif %}\n{% if cert_manager_no_proxy is defined and cert_manager_no_proxy != \"\" %}\n - name: NO_PROXY\n value: \"{{ cert_manager_no_proxy }}\"\n{% endif %}\n livenessProbe:\n httpGet:\n port: http-healthz\n path: /livez\n scheme: HTTP\n initialDelaySeconds: 10\n periodSeconds: 10\n timeoutSeconds: 15\n successThreshold: 1\n failureThreshold: 8\n{% if cert_manager_trusted_internal_ca is defined %}\n volumeMounts:\n - mountPath: /etc/ssl/certs/internal-ca.pem\n name: ca-internal-truststore\n subPath: internal-ca.pem\n volumes:\n - configMap:\n defaultMode: 420\n name: ca-internal-truststore\n name: ca-internal-truststore\n{% endif %}\n{% if cert_manager_tolerations %}\n tolerations:\n {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{% endif %}\n{% if cert_manager_nodeselector %}\n nodeSelector:\n {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }}\n{% endif %}\n{% if cert_manager_affinity %}\n affinity:\n {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }}\n{% endif %}\n{% if cert_manager_dns_policy %}\n dnsPolicy: {{ cert_manager_dns_policy }}\n{% endif %}\n{% if cert_manager_dns_config %}\n dnsConfig:\n {{ cert_manager_dns_config | to_nice_yaml | indent(width=8) }}\n{% endif %}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-deployment.yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\nspec:\n replicas: 1\n selector:\n matchLabels:\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n template:\n metadata:\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n spec:\n serviceAccountName: cert-manager-webhook\n enableServiceLinks: false\n securityContext:\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n containers:\n - name: cert-manager-webhook\n image: \"{{ cert_manager_webhook_image_repo }}:{{ cert_manager_webhook_image_tag }}\"\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --v=2\n - --secure-port=10250\n - --dynamic-serving-ca-secret-namespace=$(POD_NAMESPACE)\n - --dynamic-serving-ca-secret-name=cert-manager-webhook-ca\n - --dynamic-serving-dns-names=cert-manager-webhook\n - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE)\n - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE).svc\n ports:\n - name: https\n protocol: TCP\n containerPort: 10250\n - name: healthcheck\n protocol: TCP\n containerPort: 6080\n livenessProbe:\n httpGet:\n path: /livez\n port: 6080\n scheme: HTTP\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n successThreshold: 1\n failureThreshold: 3\n readinessProbe:\n httpGet:\n path: /healthz\n port: 6080\n scheme: HTTP\n initialDelaySeconds: 5\n periodSeconds: 5\n timeoutSeconds: 1\n successThreshold: 1\n failureThreshold: 3\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n readOnlyRootFilesystem: true\n env:\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n{% if cert_manager_http_proxy is defined and cert_manager_http_proxy != \"\" %}\n - name: HTTP_PROXY\n value: \"{{ cert_manager_http_proxy }}\"\n{% endif %}\n{% if cert_manager_https_proxy is defined and cert_manager_https_proxy != \"\" %}\n - name: HTTPS_PROXY\n value: \"{{ cert_manager_https_proxy }}\"\n{% endif %}\n{% if cert_manager_no_proxy is defined and cert_manager_no_proxy != \"\" %}\n - name: NO_PROXY\n value: \"{{ cert_manager_no_proxy }}\"\n{% endif %}\n{% if cert_manager_tolerations %}\n tolerations:\n {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{% endif %}\n{% if cert_manager_nodeselector %}\n nodeSelector:\n {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }}\n{% endif %}\n{% if cert_manager_affinity %}\n affinity:\n {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }}\n{% endif %}\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-mutating-webhook.yaml\napiVersion: admissionregistration.k8s.io/v1\nkind: MutatingWebhookConfiguration\nmetadata:\n name: cert-manager-webhook\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n annotations:\n cert-manager.io/inject-ca-from-secret: \"{{ cert_manager_namespace }}/cert-manager-webhook-ca\"\nwebhooks:\n - name: webhook.cert-manager.io\n rules:\n - apiGroups:\n - \"cert-manager.io\"\n apiVersions:\n - \"v1\"\n operations:\n - CREATE\n resources:\n - \"certificaterequests\"\n admissionReviewVersions: [\"v1\"]\n # This webhook only accepts v1 cert-manager resources.\n # Equivalent matchPolicy ensures that non-v1 resource requests are sent to\n # this webhook (after the resources have been converted to v1).\n matchPolicy: Equivalent\n timeoutSeconds: 30\n failurePolicy: Fail\n # Only include 'sideEffects' field in Kubernetes 1.12+\n sideEffects: None\n clientConfig:\n service:\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n path: /mutate\n---\n# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-validating-webhook.yaml\napiVersion: admissionregistration.k8s.io/v1\nkind: ValidatingWebhookConfiguration\nmetadata:\n name: cert-manager-webhook\n labels:\n app: webhook\n app.kubernetes.io/name: webhook\n app.kubernetes.io/instance: cert-manager\n app.kubernetes.io/component: \"webhook\"\n app.kubernetes.io/version: \"{{ cert_manager_version }}\"\n annotations:\n cert-manager.io/inject-ca-from-secret: \"{{ cert_manager_namespace }}/cert-manager-webhook-ca\"\nwebhooks:\n - name: webhook.cert-manager.io\n namespaceSelector:\n matchExpressions:\n - key: cert-manager.io/disable-validation\n operator: NotIn\n values:\n - \"true\"\n rules:\n - apiGroups:\n - \"cert-manager.io\"\n - \"acme.cert-manager.io\"\n apiVersions:\n - \"v1\"\n operations:\n - CREATE\n - UPDATE\n resources:\n - \"*/*\"\n admissionReviewVersions: [\"v1\"]\n # This webhook only accepts v1 cert-manager resources.\n # Equivalent matchPolicy ensures that non-v1 resource requests are sent to\n # this webhook (after the resources have been converted to v1).\n matchPolicy: Equivalent\n timeoutSeconds: 30\n failurePolicy: Fail\n sideEffects: None\n clientConfig:\n service:\n name: cert-manager-webhook\n namespace: {{ cert_manager_namespace }}\n path: /validate\n" }, { "path": "roles/kubernetes-apps/ingress_controller/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/ingress_controller/cert_manager\n when: cert_manager_enabled\n tags:\n - apps\n - ingress-controller\n - cert-manager\n\n - role: kubernetes-apps/ingress_controller/alb_ingress_controller\n when: ingress_alb_enabled\n tags:\n - apps\n - ingress-controller\n - ingress_alb\n" }, { "path": "roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml", "content": "---\nkubelet_csr_approver_enabled: \"{{ kubelet_rotate_server_certificates }}\"\nkubelet_csr_approver_namespace: kube-system\n\nkubelet_csr_approver_repository_name: kubelet-csr-approver\nkubelet_csr_approver_repository_url: https://postfinance.github.io/kubelet-csr-approver\nkubelet_csr_approver_chart_ref: \"{{ kubelet_csr_approver_repository_name }}/kubelet-csr-approver\"\nkubelet_csr_approver_chart_version: 1.1.0\n\n# Fill values override here\n# See upstream https://github.com/postfinance/kubelet-csr-approver\nkubelet_csr_approver_values: {}\n" }, { "path": "roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml", "content": "---\ndependencies:\n - role: helm-apps\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - kubelet_csr_approver_enabled\n environment:\n http_proxy: \"{{ http_proxy | default('') }}\"\n https_proxy: \"{{ https_proxy | default('') }}\"\n release_common_opts: {}\n releases:\n - name: kubelet-csr-approver\n namespace: \"{{ kubelet_csr_approver_namespace }}\"\n chart_ref: \"{{ kubelet_csr_approver_chart_ref }}\"\n chart_version: \"{{ kubelet_csr_approver_chart_version }}\"\n wait: \"{{ kube_network_plugin != 'cni' }}\"\n atomic: \"{{ kube_network_plugin != 'cni' }}\"\n values: \"{{ kubelet_csr_approver_values }}\"\n repositories:\n - name: \"{{ kubelet_csr_approver_repository_name }}\"\n url: \"{{ kubelet_csr_approver_repository_url }}\"\n" }, { "path": "roles/kubernetes-apps/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/utils\n\n - role: kubernetes-apps/ansible\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n - role: kubernetes-apps/helm\n when:\n - helm_enabled\n tags:\n - helm\n\n - role: kubernetes-apps/registry\n when:\n - registry_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - registry\n\n - role: kubernetes-apps/metrics_server\n when:\n - metrics_server_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - metrics_server\n\n - role: kubernetes-apps/csi_driver/csi_crd\n when:\n - cinder_csi_enabled or csi_snapshot_controller_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - csi-driver\n\n - role: kubernetes-apps/csi_driver/cinder\n when:\n - cinder_csi_enabled\n tags:\n - cinder-csi-driver\n - csi-driver\n\n - role: kubernetes-apps/csi_driver/aws_ebs\n when:\n - aws_ebs_csi_enabled\n tags:\n - aws-ebs-csi-driver\n - csi-driver\n\n - role: kubernetes-apps/csi_driver/azuredisk\n when:\n - azure_csi_enabled\n tags:\n - azure-csi-driver\n - csi-driver\n\n - role: kubernetes-apps/csi_driver/gcp_pd\n when:\n - gcp_pd_csi_enabled\n tags:\n - gcp-pd-csi-driver\n - csi-driver\n\n - role: kubernetes-apps/csi_driver/upcloud\n when:\n - upcloud_csi_enabled\n tags:\n - upcloud-csi-driver\n - csi-driver\n\n - role: kubernetes-apps/csi_driver/vsphere\n when:\n - vsphere_csi_enabled\n tags:\n - vsphere-csi-driver\n - csi-driver\n\n - role: kubernetes-apps/persistent_volumes\n when:\n - persistent_volumes_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - persistent_volumes\n\n - role: kubernetes-apps/snapshots\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - snapshots\n - csi-driver\n\n - role: kubernetes-apps/container_runtimes\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - container-runtimes\n\n - role: kubernetes-apps/container_engine_accelerator\n when: nvidia_accelerator_enabled\n tags:\n - container_engine_accelerator\n\n - role: kubernetes-apps/kubelet-csr-approver\n when:\n - kubelet_csr_approver_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - kubelet-csr-approver\n\n - role: kubernetes-apps/metallb\n when:\n - metallb_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - metallb\n\n - role: kubernetes-apps/argocd\n when:\n - argocd_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - argocd\n\n - role: kubernetes-apps/scheduler_plugins\n when:\n - scheduler_plugins_enabled\n - kube_major_version is version('1.29', '<')\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - scheduler_plugins\n\n - role: kubernetes-apps/node_feature_discovery\n when:\n - node_feature_discovery_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - node_feature_discovery\n" }, { "path": "roles/kubernetes-apps/metallb/defaults/main.yml", "content": "---\nmetallb_enabled: false\nmetallb_log_level: info\nmetallb_namespace: \"metallb-system\"\nmetallb_port: \"7472\"\nmetallb_memberlist_port: \"7946\"\nmetallb_speaker_enabled: \"{{ metallb_enabled }}\"\nmetallb_speaker_nodeselector:\n kubernetes.io/os: \"linux\"\nmetallb_controller_nodeselector:\n kubernetes.io/os: \"linux\"\nmetallb_speaker_tolerations:\n - effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n operator: Exists\nmetallb_controller_tolerations: []\nmetallb_loadbalancer_class: \"\"\n" }, { "path": "roles/kubernetes-apps/metallb/tasks/main.yml", "content": "---\n- name: Kubernetes Apps | Check cluster settings for MetalLB\n fail:\n msg: \"MetalLB require kube_proxy_strict_arp = true, see https://github.com/danderson/metallb/issues/153#issuecomment-518651132\"\n when:\n - \"kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp\"\n\n- name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore\n fail:\n msg: \"'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead\"\n when:\n - matallb_auto_assign is defined\n\n- name: Kubernetes Apps | Lay Down MetalLB\n become: true\n template:\n src: \"metallb.yaml.j2\"\n dest: \"{{ kube_config_dir }}/metallb.yaml\"\n mode: \"0644\"\n register: metallb_rendering\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Apps | Install and configure MetalLB\n kube:\n name: \"MetalLB\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/metallb.yaml\"\n state: \"{{ metallb_rendering.changed | ternary('latest', 'present') }}\"\n wait: true\n become: true\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Apps | Wait for MetalLB controller to be running\n command: \"{{ bin_dir }}/kubectl rollout status -n {{ metallb_namespace }} deployment -l app=metallb,component=controller --timeout=2m\"\n become: true\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: MetalLB | Address pools\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - metallb_config.address_pools is defined\n block:\n - name: MetalLB | Layout address pools template\n ansible.builtin.template:\n src: pools.yaml.j2\n dest: \"{{ kube_config_dir }}/pools.yaml\"\n mode: \"0644\"\n register: pools_rendering\n\n - name: MetalLB | Create address pools configuration\n kube:\n name: \"MetalLB\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/pools.yaml\"\n state: \"{{ pools_rendering.changed | ternary('latest', 'present') }}\"\n become: true\n\n- name: MetalLB | Layer2\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - metallb_config.layer2 is defined\n block:\n - name: MetalLB | Layout layer2 template\n ansible.builtin.template:\n src: layer2.yaml.j2\n dest: \"{{ kube_config_dir }}/layer2.yaml\"\n mode: \"0644\"\n register: layer2_rendering\n\n - name: MetalLB | Create layer2 configuration\n kube:\n name: \"MetalLB\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/layer2.yaml\"\n state: \"{{ layer2_rendering.changed | ternary('latest', 'present') }}\"\n become: true\n\n- name: MetalLB | Layer3\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - metallb_config.layer3 is defined\n block:\n - name: MetalLB | Layout layer3 template\n ansible.builtin.template:\n src: layer3.yaml.j2\n dest: \"{{ kube_config_dir }}/layer3.yaml\"\n mode: \"0644\"\n register: layer3_rendering\n\n - name: MetalLB | Create layer3 configuration\n kube:\n name: \"MetalLB\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/layer3.yaml\"\n state: \"{{ layer3_rendering.changed | ternary('latest', 'present') }}\"\n become: true\n\n\n- name: Kubernetes Apps | Delete MetalLB ConfigMap\n kube:\n name: config\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: ConfigMap\n namespace: \"{{ metallb_namespace }}\"\n state: absent\n" }, { "path": "roles/kubernetes-apps/metallb/templates/layer2.yaml.j2", "content": "#jinja2: trim_blocks: True, lstrip_blocks: True\n# yamllint disable-file\n---\n\n# Create layer2 configuration\n{% for entry in metallb_config.layer2 %}\n\n---\n# L2 Configuration\napiVersion: metallb.io/v1beta1\nkind: L2Advertisement\nmetadata:\n name: \"{{ entry }}\"\n namespace: \"{{ metallb_namespace }}\"\nspec:\n ipAddressPools:\n - \"{{ entry }}\"\n\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/metallb/templates/layer3.yaml.j2", "content": "#jinja2: trim_blocks: True, lstrip_blocks: True\n# yamllint disable-file\n---\n# Create layer3 configuration\n{% if metallb_config.layer3.communities is defined %}\n{% for community_name, community in metallb_config.layer3.communities.items() %}\n---\napiVersion: metallb.io/v1beta1\nkind: Community\nmetadata:\n name: \"{{ community_name }}\"\n namespace: \"{{ metallb_namespace }}\"\nspec:\n communities:\n - name: \"{{ community_name }}\"\n value: \"{{ community }}\"\n{% endfor %}\n{% endif %}\n---\napiVersion: metallb.io/v1beta1\nkind: Community\nmetadata:\n name: well-known\n namespace: \"{{ metallb_namespace }}\"\nspec:\n communities:\n - name: no-export\n value: 65535:65281\n - name: no-advertise\n value: 65535:65282\n - name: local-as\n value: 65535:65283\n - name: nopeer\n value: 65535:65284\n\n# BGPAdvertisement is used to advertise address pools to the BGP peer. Specific pools can be listed to be advertised.\n# Local BGP Advertisement specifies that the IP specified in the address pool will be used as remote source address for traffic entering your cluster from the remote peer.\n# When using this option, be sure to use a subnet and routable IP for your address pool.\n# This is good: 10.0.0.10/24. This is also good: 10.0.0.129/25. This is bad: 10.0.0.0/24. This is also bad: 10.0.0.128/25.\n# In this example, 10.0.0.10 will be used as the remote source address.\n# This is also bad: 10.0.0.10-10.0.0.25. Remember: you are working with aggregationLength, which specifies a subnet, not an IP range!\n# The no-advertise community is set on the local advertisement to prevent this route from being published to the BGP peer.\n# Your aggregationLength ideally is the same size as your address pool.\n\n{% for peer_name, peer in metallb_config.layer3.metallb_peers.items() %}\n\n{% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %}\n\n---\napiVersion: metallb.io/v1beta1\nkind: BGPAdvertisement\nmetadata:\n name: \"{{ peer_name }}-local\"\n namespace: \"{{ metallb_namespace }}\"\nspec:\n aggregationLength: 32\n aggregationLengthV6: 128\n communities:\n - no-advertise\n localpref: \"{{ peer.localpref | default(\"100\") }}\"\n ipAddressPools:\n {% for address_pool in peer.address_pool %}\n - \"{{ address_pool }}\"\n {% endfor %}\n{% endif %}\n\n# External BGP Advertisement. The IP range specied in the address pool is advertised to the BGP peer.\n---\napiVersion: metallb.io/v1beta1\nkind: BGPAdvertisement\nmetadata:\n name: \"{{ peer_name }}-external\"\n namespace: \"{{ metallb_namespace }}\"\nspec:\n {% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %}\n aggregationLength: {{ peer.aggregation_length }}\n {% endif %}\n ipAddressPools:\n {% for address_pool in peer.address_pool %}\n - \"{{ address_pool }}\"\n {% endfor %}\n {% if peer.communities is defined %}\n {% for community in peer.communities %}\n communities:\n - \"{{ community }}\"\n {% endfor %}\n {% endif %}\n\n\n# Configuration for the BGP peer.\n---\napiVersion: metallb.io/v1beta2\nkind: BGPPeer\nmetadata:\n name: \"{{ peer_name }}\"\n namespace: \"{{ metallb_namespace }}\"\nspec:\n myASN: {{ peer.my_asn }}\n peerASN: {{ peer.peer_asn }}\n peerAddress: {{ peer.peer_address }}\n {% if peer.peer_port is defined %}\n peerPort: {{ peer.peer_port }}\n {% else %}\n peerPort: {{ metallb_config.layer3.defaults.peer_port }}\n {% endif -%}\n\n {% if peer.password is defined %}\n password: \"{{ peer.password }}\"\n {% endif -%}\n\n {% if peer.router_id is defined %}\n routerID: \"{{ peer.router_id }}\"\n {% endif -%}\n\n {% if peer.hold_time is defined %}\n holdTime: {{ peer.hold_time }}\n {% elif metallb_config.layer3.defaults.hold_time is defined %}\n holdTime: {{ metallb_config.layer3.defaults.hold_time }}\n {% endif -%}\n\n {% if peer.multihop is defined %}\n ebgpMultiHop: {{ peer.multihop }}\n {% endif -%}\n\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/metallb/templates/metallb.yaml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n labels:\n pod-security.kubernetes.io/audit: privileged\n pod-security.kubernetes.io/enforce: privileged\n pod-security.kubernetes.io/warn: privileged\n name: {{ metallb_namespace }}\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n name: addresspools.metallb.io\nspec:\n conversion:\n strategy: Webhook\n webhook:\n clientConfig:\n caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /convert\n conversionReviewVersions:\n - v1alpha1\n - v1beta1\n group: metallb.io\n names:\n kind: AddressPool\n listKind: AddressPoolList\n plural: addresspools\n singular: addresspool\n scope: Namespaced\n versions:\n - deprecated: true\n deprecationWarning: metallb.io v1alpha1 AddressPool is deprecated\n name: v1alpha1\n schema:\n openAPIV3Schema:\n description: AddressPool is the Schema for the addresspools API.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: AddressPoolSpec defines the desired state of AddressPool.\n properties:\n addresses:\n description: A list of IP address ranges over which MetalLB has authority.\n You can list multiple ranges in a single pool, they will all share\n the same settings. Each range can be either a CIDR prefix, or an\n explicit start-end range of IPs.\n items:\n type: string\n type: array\n autoAssign:\n default: true\n description: AutoAssign flag used to prevent MetallB from automatic\n allocation for a pool.\n type: boolean\n bgpAdvertisements:\n description: When an IP is allocated from this pool, how should it\n be translated into BGP announcements?\n items:\n properties:\n aggregationLength:\n default: 32\n description: The aggregation-length advertisement option lets\n you “roll up” the /32s into a larger prefix.\n format: int32\n minimum: 1\n type: integer\n aggregationLengthV6:\n default: 128\n description: Optional, defaults to 128 (i.e. no aggregation)\n if not specified.\n format: int32\n type: integer\n communities:\n description: BGP communities\n items:\n type: string\n type: array\n localPref:\n description: BGP LOCAL_PREF attribute which is used by BGP best\n path algorithm, Path with higher localpref is preferred over\n one with lower localpref.\n format: int32\n type: integer\n type: object\n type: array\n protocol:\n description: Protocol can be used to select how the announcement is\n done.\n enum:\n - layer2\n - bgp\n type: string\n required:\n - addresses\n - protocol\n type: object\n status:\n description: AddressPoolStatus defines the observed state of AddressPool.\n type: object\n required:\n - spec\n type: object\n served: true\n storage: false\n subresources:\n status: {}\n - deprecated: true\n deprecationWarning: metallb.io v1beta1 AddressPool is deprecated, consider using\n IPAddressPool\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: AddressPool represents a pool of IP addresses that can be allocated\n to LoadBalancer services. AddressPool is deprecated and being replaced by\n IPAddressPool.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: AddressPoolSpec defines the desired state of AddressPool.\n properties:\n addresses:\n description: A list of IP address ranges over which MetalLB has authority.\n You can list multiple ranges in a single pool, they will all share\n the same settings. Each range can be either a CIDR prefix, or an\n explicit start-end range of IPs.\n items:\n type: string\n type: array\n autoAssign:\n default: true\n description: AutoAssign flag used to prevent MetallB from automatic\n allocation for a pool.\n type: boolean\n bgpAdvertisements:\n description: Drives how an IP allocated from this pool should translated\n into BGP announcements.\n items:\n properties:\n aggregationLength:\n default: 32\n description: The aggregation-length advertisement option lets\n you “roll up” the /32s into a larger prefix.\n format: int32\n minimum: 1\n type: integer\n aggregationLengthV6:\n default: 128\n description: Optional, defaults to 128 (i.e. no aggregation)\n if not specified.\n format: int32\n type: integer\n communities:\n description: BGP communities to be associated with the given\n advertisement.\n items:\n type: string\n type: array\n localPref:\n description: BGP LOCAL_PREF attribute which is used by BGP best\n path algorithm, Path with higher localpref is preferred over\n one with lower localpref.\n format: int32\n type: integer\n type: object\n type: array\n protocol:\n description: Protocol can be used to select how the announcement is\n done.\n enum:\n - layer2\n - bgp\n type: string\n required:\n - addresses\n - protocol\n type: object\n status:\n description: AddressPoolStatus defines the observed state of AddressPool.\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: bfdprofiles.metallb.io\nspec:\n group: metallb.io\n names:\n kind: BFDProfile\n listKind: BFDProfileList\n plural: bfdprofiles\n singular: bfdprofile\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.passiveMode\n name: Passive Mode\n type: boolean\n - jsonPath: .spec.transmitInterval\n name: Transmit Interval\n type: integer\n - jsonPath: .spec.receiveInterval\n name: Receive Interval\n type: integer\n - jsonPath: .spec.detectMultiplier\n name: Multiplier\n type: integer\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: BFDProfile represents the settings of the bfd session that can\n be optionally associated with a BGP session.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BFDProfileSpec defines the desired state of BFDProfile.\n properties:\n detectMultiplier:\n description: Configures the detection multiplier to determine packet\n loss. The remote transmission interval will be multiplied by this\n value to determine the connection loss detection timer.\n format: int32\n maximum: 255\n minimum: 2\n type: integer\n echoInterval:\n description: Configures the minimal echo receive transmission interval\n that this system is capable of handling in milliseconds. Defaults\n to 50ms\n format: int32\n maximum: 60000\n minimum: 10\n type: integer\n echoMode:\n description: Enables or disables the echo transmission mode. This\n mode is disabled by default, and not supported on multi hops setups.\n type: boolean\n minimumTtl:\n description: 'For multi hop sessions only: configure the minimum expected\n TTL for an incoming BFD control packet.'\n format: int32\n maximum: 254\n minimum: 1\n type: integer\n passiveMode:\n description: 'Mark session as passive: a passive session will not\n attempt to start the connection and will wait for control packets\n from peer before it begins replying.'\n type: boolean\n receiveInterval:\n description: The minimum interval that this system is capable of receiving\n control packets in milliseconds. Defaults to 300ms.\n format: int32\n maximum: 60000\n minimum: 10\n type: integer\n transmitInterval:\n description: The minimum transmission interval (less jitter) that\n this system wants to use to send BFD control packets in milliseconds.\n Defaults to 300ms\n format: int32\n maximum: 60000\n minimum: 10\n type: integer\n type: object\n status:\n description: BFDProfileStatus defines the observed state of BFDProfile.\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: bgpadvertisements.metallb.io\nspec:\n group: metallb.io\n names:\n kind: BGPAdvertisement\n listKind: BGPAdvertisementList\n plural: bgpadvertisements\n singular: bgpadvertisement\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.ipAddressPools\n name: IPAddressPools\n type: string\n - jsonPath: .spec.ipAddressPoolSelectors\n name: IPAddressPool Selectors\n type: string\n - jsonPath: .spec.peers\n name: Peers\n type: string\n - jsonPath: .spec.nodeSelectors\n name: Node Selectors\n priority: 10\n type: string\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: BGPAdvertisement allows to advertise the IPs coming from the\n selected IPAddressPools via BGP, setting the parameters of the BGP Advertisement.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPAdvertisementSpec defines the desired state of BGPAdvertisement.\n properties:\n aggregationLength:\n default: 32\n description: The aggregation-length advertisement option lets you\n “roll up” the /32s into a larger prefix. Defaults to 32. Works for\n IPv4 addresses.\n format: int32\n minimum: 1\n type: integer\n aggregationLengthV6:\n default: 128\n description: The aggregation-length advertisement option lets you\n “roll up” the /128s into a larger prefix. Defaults to 128. Works\n for IPv6 addresses.\n format: int32\n type: integer\n communities:\n description: The BGP communities to be associated with the announcement.\n Each item can be a community of the form 1234:1234 or the name of\n an alias defined in the Community CRD.\n items:\n type: string\n type: array\n ipAddressPoolSelectors:\n description: A selector for the IPAddressPools which would get advertised\n via this advertisement. If no IPAddressPool is selected by this\n or by the list, the advertisement is applied to all the IPAddressPools.\n items:\n description: A label selector is a label query over a set of resources.\n The result of matchLabels and matchExpressions are ANDed. An empty\n label selector matches all objects. A null label selector matches\n no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements.\n The requirements are ANDed.\n items:\n description: A label selector requirement is a selector that\n contains values, a key, and an operator that relates the\n key and values.\n properties:\n key:\n description: key is the label key that the selector applies\n to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn, Exists\n and DoesNotExist.\n type: string\n values:\n description: values is an array of string values. If the\n operator is In or NotIn, the values array must be non-empty.\n If the operator is Exists or DoesNotExist, the values\n array must be empty. This array is replaced during a\n strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs. A single\n {key,value} in the matchLabels map is equivalent to an element\n of matchExpressions, whose key field is \"key\", the operator\n is \"In\", and the values array contains only \"value\". The requirements\n are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n ipAddressPools:\n description: The list of IPAddressPools to advertise via this advertisement,\n selected by name.\n items:\n type: string\n type: array\n localPref:\n description: The BGP LOCAL_PREF attribute which is used by BGP best\n path algorithm, Path with higher localpref is preferred over one\n with lower localpref.\n format: int32\n type: integer\n nodeSelectors:\n description: NodeSelectors allows to limit the nodes to announce as\n next hops for the LoadBalancer IP. When empty, all the nodes having are\n announced as next hops.\n items:\n description: A label selector is a label query over a set of resources.\n The result of matchLabels and matchExpressions are ANDed. An empty\n label selector matches all objects. A null label selector matches\n no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements.\n The requirements are ANDed.\n items:\n description: A label selector requirement is a selector that\n contains values, a key, and an operator that relates the\n key and values.\n properties:\n key:\n description: key is the label key that the selector applies\n to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn, Exists\n and DoesNotExist.\n type: string\n values:\n description: values is an array of string values. If the\n operator is In or NotIn, the values array must be non-empty.\n If the operator is Exists or DoesNotExist, the values\n array must be empty. This array is replaced during a\n strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs. A single\n {key,value} in the matchLabels map is equivalent to an element\n of matchExpressions, whose key field is \"key\", the operator\n is \"In\", and the values array contains only \"value\". The requirements\n are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n peers:\n description: Peers limits the bgppeer to advertise the ips of the\n selected pools to. When empty, the loadbalancer IP is announced\n to all the BGPPeers configured.\n items:\n type: string\n type: array\n type: object\n status:\n description: BGPAdvertisementStatus defines the observed state of BGPAdvertisement.\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n name: bgppeers.metallb.io\nspec:\n conversion:\n strategy: Webhook\n webhook:\n clientConfig:\n caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /convert\n conversionReviewVersions:\n - v1beta1\n - v1beta2\n group: metallb.io\n names:\n kind: BGPPeer\n listKind: BGPPeerList\n plural: bgppeers\n singular: bgppeer\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.peerAddress\n name: Address\n type: string\n - jsonPath: .spec.peerASN\n name: ASN\n type: string\n - jsonPath: .spec.bfdProfile\n name: BFD Profile\n type: string\n - jsonPath: .spec.ebgpMultiHop\n name: Multi Hops\n type: string\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: BGPPeer is the Schema for the peers API.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPPeerSpec defines the desired state of Peer.\n properties:\n bfdProfile:\n type: string\n ebgpMultiHop:\n description: EBGP peer is multi-hops away\n type: boolean\n holdTime:\n description: Requested BGP hold time, per RFC4271.\n type: string\n keepaliveTime:\n description: Requested BGP keepalive time, per RFC4271.\n type: string\n myASN:\n description: AS number to use for the local end of the session.\n format: int32\n maximum: 4294967295\n minimum: 0\n type: integer\n nodeSelectors:\n description: Only connect to this peer on nodes that match one of\n these selectors.\n items:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n minItems: 1\n type: array\n required:\n - key\n - operator\n - values\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n password:\n description: Authentication password for routers enforcing TCP MD5\n authenticated sessions\n type: string\n peerASN:\n description: AS number to expect from the remote end of the session.\n format: int32\n maximum: 4294967295\n minimum: 0\n type: integer\n peerAddress:\n description: Address to dial when establishing the session.\n type: string\n peerPort:\n description: Port to dial when establishing the session.\n maximum: 16384\n minimum: 0\n type: integer\n routerID:\n description: BGP router ID to advertise to the peer\n type: string\n sourceAddress:\n description: Source address to use when establishing the session.\n type: string\n required:\n - myASN\n - peerASN\n - peerAddress\n type: object\n status:\n description: BGPPeerStatus defines the observed state of Peer.\n type: object\n type: object\n served: true\n storage: false\n subresources:\n status: {}\n - additionalPrinterColumns:\n - jsonPath: .spec.peerAddress\n name: Address\n type: string\n - jsonPath: .spec.peerASN\n name: ASN\n type: string\n - jsonPath: .spec.bfdProfile\n name: BFD Profile\n type: string\n - jsonPath: .spec.ebgpMultiHop\n name: Multi Hops\n type: string\n name: v1beta2\n schema:\n openAPIV3Schema:\n description: BGPPeer is the Schema for the peers API.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPPeerSpec defines the desired state of Peer.\n properties:\n bfdProfile:\n description: The name of the BFD Profile to be used for the BFD session\n associated to the BGP session. If not set, the BFD session won't\n be set up.\n type: string\n ebgpMultiHop:\n description: To set if the BGPPeer is multi-hops away. Needed for\n FRR mode only.\n type: boolean\n holdTime:\n description: Requested BGP hold time, per RFC4271.\n type: string\n keepaliveTime:\n description: Requested BGP keepalive time, per RFC4271.\n type: string\n myASN:\n description: AS number to use for the local end of the session.\n format: int32\n maximum: 4294967295\n minimum: 0\n type: integer\n nodeSelectors:\n description: Only connect to this peer on nodes that match one of\n these selectors.\n items:\n description: A label selector is a label query over a set of resources.\n The result of matchLabels and matchExpressions are ANDed. An empty\n label selector matches all objects. A null label selector matches\n no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements.\n The requirements are ANDed.\n items:\n description: A label selector requirement is a selector that\n contains values, a key, and an operator that relates the\n key and values.\n properties:\n key:\n description: key is the label key that the selector applies\n to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn, Exists\n and DoesNotExist.\n type: string\n values:\n description: values is an array of string values. If the\n operator is In or NotIn, the values array must be non-empty.\n If the operator is Exists or DoesNotExist, the values\n array must be empty. This array is replaced during a\n strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs. A single\n {key,value} in the matchLabels map is equivalent to an element\n of matchExpressions, whose key field is \"key\", the operator\n is \"In\", and the values array contains only \"value\". The requirements\n are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n password:\n description: Authentication password for routers enforcing TCP MD5\n authenticated sessions\n type: string\n passwordSecret:\n description: passwordSecret is name of the authentication secret for\n BGP Peer. the secret must be of type \"kubernetes.io/basic-auth\",\n and created in the same namespace as the MetalLB deployment. The\n password is stored in the secret as the key \"password\".\n properties:\n name:\n description: name is unique within a namespace to reference a\n secret resource.\n type: string\n namespace:\n description: namespace defines the space within which the secret\n name must be unique.\n type: string\n type: object\n x-kubernetes-map-type: atomic\n peerASN:\n description: AS number to expect from the remote end of the session.\n format: int32\n maximum: 4294967295\n minimum: 0\n type: integer\n peerAddress:\n description: Address to dial when establishing the session.\n type: string\n peerPort:\n default: 179\n description: Port to dial when establishing the session.\n maximum: 16384\n minimum: 0\n type: integer\n routerID:\n description: BGP router ID to advertise to the peer\n type: string\n sourceAddress:\n description: Source address to use when establishing the session.\n type: string\n vrf:\n description: To set if we want to peer with the BGPPeer using an interface\n belonging to a host vrf\n type: string\n required:\n - myASN\n - peerASN\n - peerAddress\n type: object\n status:\n description: BGPPeerStatus defines the observed state of Peer.\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: communities.metallb.io\nspec:\n group: metallb.io\n names:\n kind: Community\n listKind: CommunityList\n plural: communities\n singular: community\n scope: Namespaced\n versions:\n - name: v1beta1\n schema:\n openAPIV3Schema:\n description: Community is a collection of aliases for communities. Users can\n define named aliases to be used in the BGPPeer CRD.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: CommunitySpec defines the desired state of Community.\n properties:\n communities:\n items:\n properties:\n name:\n description: The name of the alias for the community.\n type: string\n value:\n description: The BGP community value corresponding to the given\n name.\n type: string\n type: object\n type: array\n type: object\n status:\n description: CommunityStatus defines the observed state of Community.\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: ipaddresspools.metallb.io\nspec:\n group: metallb.io\n names:\n kind: IPAddressPool\n listKind: IPAddressPoolList\n plural: ipaddresspools\n singular: ipaddresspool\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.autoAssign\n name: Auto Assign\n type: boolean\n - jsonPath: .spec.avoidBuggyIPs\n name: Avoid Buggy IPs\n type: boolean\n - jsonPath: .spec.addresses\n name: Addresses\n type: string\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: IPAddressPool represents a pool of IP addresses that can be allocated\n to LoadBalancer services.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAddressPoolSpec defines the desired state of IPAddressPool.\n properties:\n addresses:\n description: A list of IP address ranges over which MetalLB has authority.\n You can list multiple ranges in a single pool, they will all share\n the same settings. Each range can be either a CIDR prefix, or an\n explicit start-end range of IPs.\n items:\n type: string\n type: array\n autoAssign:\n default: true\n description: AutoAssign flag used to prevent MetallB from automatic\n allocation for a pool.\n type: boolean\n avoidBuggyIPs:\n default: false\n description: AvoidBuggyIPs prevents addresses ending with .0 and .255\n to be used by a pool.\n type: boolean\n serviceAllocation:\n description: AllocateTo makes ip pool allocation to specific namespace\n and/or service. The controller will use the pool with lowest value\n of priority in case of multiple matches. A pool with no priority\n set will be used only if the pools with priority can't be used.\n If multiple matching IPAddressPools are available it will check\n for the availability of IPs sorting the matching IPAddressPools\n by priority, starting from the highest to the lowest. If multiple\n IPAddressPools have the same priority, choice will be random.\n properties:\n namespaceSelectors:\n description: NamespaceSelectors list of label selectors to select\n namespace(s) for ip pool, an alternative to using namespace\n list.\n items:\n description: A label selector is a label query over a set of\n resources. The result of matchLabels and matchExpressions\n are ANDed. An empty label selector matches all objects. A\n null label selector matches no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: A label selector requirement is a selector\n that contains values, a key, and an operator that relates\n the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn,\n Exists and DoesNotExist.\n type: string\n values:\n description: values is an array of string values.\n If the operator is In or NotIn, the values array\n must be non-empty. If the operator is Exists or\n DoesNotExist, the values array must be empty. This\n array is replaced during a strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs.\n A single {key,value} in the matchLabels map is equivalent\n to an element of matchExpressions, whose key field is\n \"key\", the operator is \"In\", and the values array contains\n only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n namespaces:\n description: Namespaces list of namespace(s) on which ip pool\n can be attached.\n items:\n type: string\n type: array\n priority:\n description: Priority priority given for ip pool while ip allocation\n on a service.\n type: integer\n serviceSelectors:\n description: ServiceSelectors list of label selector to select\n service(s) for which ip pool can be used for ip allocation.\n items:\n description: A label selector is a label query over a set of\n resources. The result of matchLabels and matchExpressions\n are ANDed. An empty label selector matches all objects. A\n null label selector matches no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: A label selector requirement is a selector\n that contains values, a key, and an operator that relates\n the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn,\n Exists and DoesNotExist.\n type: string\n values:\n description: values is an array of string values.\n If the operator is In or NotIn, the values array\n must be non-empty. If the operator is Exists or\n DoesNotExist, the values array must be empty. This\n array is replaced during a strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs.\n A single {key,value} in the matchLabels map is equivalent\n to an element of matchExpressions, whose key field is\n \"key\", the operator is \"In\", and the values array contains\n only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n type: object\n required:\n - addresses\n type: object\n status:\n description: IPAddressPoolStatus defines the observed state of IPAddressPool.\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: l2advertisements.metallb.io\nspec:\n group: metallb.io\n names:\n kind: L2Advertisement\n listKind: L2AdvertisementList\n plural: l2advertisements\n singular: l2advertisement\n scope: Namespaced\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.ipAddressPools\n name: IPAddressPools\n type: string\n - jsonPath: .spec.ipAddressPoolSelectors\n name: IPAddressPool Selectors\n type: string\n - jsonPath: .spec.interfaces\n name: Interfaces\n type: string\n - jsonPath: .spec.nodeSelectors\n name: Node Selectors\n priority: 10\n type: string\n name: v1beta1\n schema:\n openAPIV3Schema:\n description: L2Advertisement allows to advertise the LoadBalancer IPs provided\n by the selected pools via L2.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: L2AdvertisementSpec defines the desired state of L2Advertisement.\n properties:\n interfaces:\n description: A list of interfaces to announce from. The LB IP will\n be announced only from these interfaces. If the field is not set,\n we advertise from all the interfaces on the host.\n items:\n type: string\n type: array\n ipAddressPoolSelectors:\n description: A selector for the IPAddressPools which would get advertised\n via this advertisement. If no IPAddressPool is selected by this\n or by the list, the advertisement is applied to all the IPAddressPools.\n items:\n description: A label selector is a label query over a set of resources.\n The result of matchLabels and matchExpressions are ANDed. An empty\n label selector matches all objects. A null label selector matches\n no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements.\n The requirements are ANDed.\n items:\n description: A label selector requirement is a selector that\n contains values, a key, and an operator that relates the\n key and values.\n properties:\n key:\n description: key is the label key that the selector applies\n to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn, Exists\n and DoesNotExist.\n type: string\n values:\n description: values is an array of string values. If the\n operator is In or NotIn, the values array must be non-empty.\n If the operator is Exists or DoesNotExist, the values\n array must be empty. This array is replaced during a\n strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs. A single\n {key,value} in the matchLabels map is equivalent to an element\n of matchExpressions, whose key field is \"key\", the operator\n is \"In\", and the values array contains only \"value\". The requirements\n are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n ipAddressPools:\n description: The list of IPAddressPools to advertise via this advertisement,\n selected by name.\n items:\n type: string\n type: array\n nodeSelectors:\n description: NodeSelectors allows to limit the nodes to announce as\n next hops for the LoadBalancer IP. When empty, all the nodes having are\n announced as next hops.\n items:\n description: A label selector is a label query over a set of resources.\n The result of matchLabels and matchExpressions are ANDed. An empty\n label selector matches all objects. A null label selector matches\n no objects.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector requirements.\n The requirements are ANDed.\n items:\n description: A label selector requirement is a selector that\n contains values, a key, and an operator that relates the\n key and values.\n properties:\n key:\n description: key is the label key that the selector applies\n to.\n type: string\n operator:\n description: operator represents a key's relationship\n to a set of values. Valid operators are In, NotIn, Exists\n and DoesNotExist.\n type: string\n values:\n description: values is an array of string values. If the\n operator is In or NotIn, the values array must be non-empty.\n If the operator is Exists or DoesNotExist, the values\n array must be empty. This array is replaced during a\n strategic merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: matchLabels is a map of {key,value} pairs. A single\n {key,value} in the matchLabels map is equivalent to an element\n of matchExpressions, whose key field is \"key\", the operator\n is \"In\", and the values array contains only \"value\". The requirements\n are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n type: array\n type: object\n status:\n description: L2AdvertisementStatus defines the observed state of L2Advertisement.\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n labels:\n app: metallb\n pod-security.kubernetes.io/audit: privileged\n pod-security.kubernetes.io/enforce: privileged\n pod-security.kubernetes.io/warn: privileged\n name: controller\n namespace: \"{{ metallb_namespace }}\"\n\n{% if metallb_speaker_enabled %}\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n labels:\n app: metallb\n name: speaker\n namespace: \"{{ metallb_namespace }}\"\n{% endif %}\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n labels:\n app: metallb\n name: controller\n namespace: \"{{ metallb_namespace }}\"\nrules:\n- apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - create\n - delete\n - get\n - list\n - patch\n - update\n - watch\n- apiGroups:\n - \"\"\n resourceNames:\n - memberlist\n resources:\n - secrets\n verbs:\n - list\n- apiGroups:\n - apps\n resourceNames:\n - controller\n resources:\n - deployments\n verbs:\n - get\n- apiGroups:\n - metallb.io\n resources:\n - bgppeers\n verbs:\n - get\n - list\n- apiGroups:\n - metallb.io\n resources:\n - addresspools\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - bfdprofiles\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - ipaddresspools\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - bgpadvertisements\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - l2advertisements\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - communities\n verbs:\n - get\n - list\n - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n labels:\n app: metallb\n name: pod-lister\n namespace: \"{{ metallb_namespace }}\"\nrules:\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - list\n- apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - addresspools\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - bfdprofiles\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - bgppeers\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - l2advertisements\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - bgpadvertisements\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - ipaddresspools\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - metallb.io\n resources:\n - communities\n verbs:\n - get\n - list\n - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n app: metallb\n name: {{ metallb_namespace }}:controller\nrules:\n- apiGroups:\n - \"\"\n resources:\n - services\n - namespaces\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - services/status\n verbs:\n - update\n- apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n- apiGroups:\n - admissionregistration.k8s.io\n resourceNames:\n - metallb-webhook-configuration\n resources:\n - validatingwebhookconfigurations\n - mutatingwebhookconfigurations\n verbs:\n - create\n - delete\n - get\n - list\n - patch\n - update\n - watch\n- apiGroups:\n - admissionregistration.k8s.io\n resources:\n - validatingwebhookconfigurations\n - mutatingwebhookconfigurations\n verbs:\n - list\n - watch\n- apiGroups:\n - apiextensions.k8s.io\n resourceNames:\n - addresspools.metallb.io\n - bfdprofiles.metallb.io\n - bgpadvertisements.metallb.io\n - bgppeers.metallb.io\n - ipaddresspools.metallb.io\n - l2advertisements.metallb.io\n - communities.metallb.io\n resources:\n - customresourcedefinitions\n verbs:\n - create\n - delete\n - get\n - list\n - patch\n - update\n - watch\n- apiGroups:\n - apiextensions.k8s.io\n resources:\n - customresourcedefinitions\n verbs:\n - list\n - watch\n---\n{% if metallb_speaker_enabled %}\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n app: metallb\n name: {{ metallb_namespace }}:speaker\nrules:\n- apiGroups:\n - \"\"\n resources:\n - services\n - endpoints\n - nodes\n - namespaces\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n{% endif %}\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n labels:\n app: metallb\n name: controller\n namespace: \"{{ metallb_namespace }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: controller\nsubjects:\n- kind: ServiceAccount\n name: controller\n namespace: \"{{ metallb_namespace }}\"\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n labels:\n app: metallb\n name: pod-lister\n namespace: \"{{ metallb_namespace }}\"\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: pod-lister\nsubjects:\n- kind: ServiceAccount\n name: speaker\n namespace: \"{{ metallb_namespace }}\"\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n labels:\n app: metallb\n name: {{ metallb_namespace }}:controller\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: {{ metallb_namespace }}:controller\nsubjects:\n- kind: ServiceAccount\n name: controller\n namespace: \"{{ metallb_namespace }}\"\n\n{% if metallb_speaker_enabled %}\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n labels:\n app: metallb\n name: {{ metallb_namespace }}:speaker\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: {{ metallb_namespace }}:speaker\nsubjects:\n- kind: ServiceAccount\n name: speaker\n namespace: \"{{ metallb_namespace }}\"\n{% endif %}\n\n---\napiVersion: v1\nkind: Secret\nmetadata:\n name: webhook-server-cert\n namespace: \"{{ metallb_namespace }}\"\n\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\nspec:\n ports:\n - port: 443\n targetPort: 9443\n selector:\n component: controller\n\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n labels:\n app: metallb\n component: controller\n name: controller\n namespace: \"{{ metallb_namespace }}\"\nspec:\n revisionHistoryLimit: 3\n selector:\n matchLabels:\n app: metallb\n component: controller\n template:\n metadata:\n annotations:\n prometheus.io/port: '{{ metallb_port }}'\n prometheus.io/scrape: 'true'\n labels:\n app: metallb\n component: controller\n spec:\n priorityClassName: system-cluster-critical\n containers:\n - args:\n - --port={{ metallb_port }}\n - --log-level={{ metallb_log_level }}\n{% if metallb_loadbalancer_class != \"\" %}\n - --lb-class={{ metallb_loadbalancer_class }}\n{% endif %}\n env:\n - name: METALLB_ML_SECRET_NAME\n value: memberlist\n - name: METALLB_DEPLOYMENT\n value: controller\n image: \"{{ metallb_controller_image_repo }}:{{ metallb_image_tag }}\"\n livenessProbe:\n failureThreshold: 3\n httpGet:\n path: /metrics\n port: monitoring\n initialDelaySeconds: 10\n periodSeconds: 10\n successThreshold: 1\n timeoutSeconds: 1\n name: controller\n ports:\n - containerPort: {{ metallb_port }}\n name: monitoring\n - containerPort: 9443\n name: webhook-server\n protocol: TCP\n readinessProbe:\n failureThreshold: 3\n httpGet:\n path: /metrics\n port: monitoring\n initialDelaySeconds: 10\n periodSeconds: 10\n successThreshold: 1\n timeoutSeconds: 1\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - mountPath: /tmp/k8s-webhook-server/serving-certs\n name: cert\n readOnly: true\n{% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %}\n tolerations:\n {{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }}\n{%- endif %}\n nodeSelector:\n {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}}\n {% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %}\n {{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}}\n {%- endif %}\n securityContext:\n fsGroup: 65534\n runAsNonRoot: true\n runAsUser: 65534\n serviceAccountName: controller\n terminationGracePeriodSeconds: 0\n volumes:\n - name: cert\n secret:\n defaultMode: 420\n secretName: webhook-server-cert\n\n---\n{% if metallb_speaker_enabled %}\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n labels:\n app: metallb\n component: speaker\n name: speaker\n namespace: \"{{ metallb_namespace }}\"\nspec:\n selector:\n matchLabels:\n app: metallb\n component: speaker\n template:\n metadata:\n annotations:\n prometheus.io/port: '{{ metallb_port }}'\n prometheus.io/scrape: 'true'\n labels:\n app: metallb\n component: speaker\n spec:\n containers:\n - args:\n - --port={{ metallb_port }}\n - --log-level={{ metallb_log_level }}\n{% if metallb_loadbalancer_class != \"\" %}\n - --lb-class={{ metallb_loadbalancer_class }}\n{% endif %}\n env:\n - name: METALLB_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: METALLB_HOST\n valueFrom:\n fieldRef:\n fieldPath: status.hostIP\n - name: METALLB_ML_BIND_ADDR\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: METALLB_ML_LABELS\n value: app=metallb,component=speaker\n - name: METALLB_ML_SECRET_KEY\n valueFrom:\n secretKeyRef:\n key: secretkey\n name: memberlist\n image: \"{{ metallb_speaker_image_repo }}:{{ metallb_image_tag }}\"\n livenessProbe:\n failureThreshold: 3\n httpGet:\n path: /metrics\n port: monitoring\n initialDelaySeconds: 10\n periodSeconds: 10\n successThreshold: 1\n timeoutSeconds: 1\n name: speaker\n ports:\n - containerPort: {{ metallb_port }}\n name: monitoring\n - containerPort: {{ metallb_memberlist_port }}\n name: memberlist-tcp\n - containerPort: {{ metallb_memberlist_port }}\n name: memberlist-udp\n protocol: UDP\n readinessProbe:\n failureThreshold: 3\n httpGet:\n path: /metrics\n port: monitoring\n initialDelaySeconds: 10\n periodSeconds: 10\n successThreshold: 1\n timeoutSeconds: 1\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_RAW\n drop:\n - ALL\n readOnlyRootFilesystem: true\n hostNetwork: true\n nodeSelector:\n {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}}\n {% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %}\n {{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}}\n {%- endif %}\n\n serviceAccountName: speaker\n terminationGracePeriodSeconds: 2\n tolerations:\n {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}\n {% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %}\n {{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}\n {% endif %}\n{% endif %}\n\n---\napiVersion: admissionregistration.k8s.io/v1\nkind: ValidatingWebhookConfiguration\nmetadata:\n creationTimestamp: null\n name: metallb-webhook-configuration\nwebhooks:\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta2-bgppeer\n failurePolicy: Fail\n name: bgppeersvalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta2\n operations:\n - CREATE\n - UPDATE\n resources:\n - bgppeers\n sideEffects: None\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta1-addresspool\n failurePolicy: Fail\n name: addresspoolvalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta1\n operations:\n - CREATE\n - UPDATE\n resources:\n - addresspools\n sideEffects: None\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta1-bfdprofile\n failurePolicy: Fail\n name: bfdprofilevalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta1\n operations:\n - CREATE\n - DELETE\n resources:\n - bfdprofiles\n sideEffects: None\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta1-bgpadvertisement\n failurePolicy: Fail\n name: bgpadvertisementvalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta1\n operations:\n - CREATE\n - UPDATE\n resources:\n - bgpadvertisements\n sideEffects: None\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta1-community\n failurePolicy: Fail\n name: communityvalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta1\n operations:\n - CREATE\n - UPDATE\n resources:\n - communities\n sideEffects: None\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta1-ipaddresspool\n failurePolicy: Fail\n name: ipaddresspoolvalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta1\n operations:\n - CREATE\n - UPDATE\n resources:\n - ipaddresspools\n sideEffects: None\n- admissionReviewVersions:\n - v1\n clientConfig:\n service:\n name: webhook-service\n namespace: \"{{ metallb_namespace }}\"\n path: /validate-metallb-io-v1beta1-l2advertisement\n failurePolicy: Fail\n name: l2advertisementvalidationwebhook.metallb.io\n rules:\n - apiGroups:\n - metallb.io\n apiVersions:\n - v1beta1\n operations:\n - CREATE\n - UPDATE\n resources:\n - l2advertisements\n sideEffects: None\n" }, { "path": "roles/kubernetes-apps/metallb/templates/pools.yaml.j2", "content": "#jinja2: trim_blocks: True, lstrip_blocks: True\n# yamllint disable-file\n---\n\n# Create all pools\n{% for pool_name, pool in metallb_config.address_pools.items() %}\n\n---\napiVersion: metallb.io/v1beta1\nkind: IPAddressPool\nmetadata:\n namespace: \"{{ metallb_namespace }}\"\n name: \"{{ pool_name }}\"\nspec:\n addresses:\n{% for ip_range in pool.ip_range %}\n - \"{{ ip_range }}\"\n{% endfor %}\n autoAssign: {{ pool.auto_assign | default(true) }}\n avoidBuggyIPs: {{ pool.avoid_buggy_ips | default(false) }}\n\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/metrics_server/defaults/main.yml", "content": "---\nmetrics_server_container_port: 10250\nmetrics_server_kubelet_insecure_tls: true\nmetrics_server_kubelet_preferred_address_types: \"InternalIP,ExternalIP,Hostname\"\nmetrics_server_metric_resolution: 15s\nmetrics_server_limits_cpu: 100m\nmetrics_server_limits_memory: 200Mi\nmetrics_server_requests_cpu: 100m\nmetrics_server_requests_memory: 200Mi\nmetrics_server_host_network: false\nmetrics_server_replicas: 1\nmetrics_server_extra_tolerations: []\nmetrics_server_extra_affinity: {}\nmetrics_server_nodeselector: {}\n" }, { "path": "roles/kubernetes-apps/metrics_server/tasks/main.yml", "content": "---\n- name: Metrics Server | Delete addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/metrics_server\"\n state: absent\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - upgrade\n\n- name: Metrics Server | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/metrics_server\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Metrics Server | Templates list\n set_fact:\n metrics_server_templates:\n - { name: auth-delegator, file: auth-delegator.yaml, type: clusterrolebinding }\n - { name: auth-reader, file: auth-reader.yaml, type: rolebinding }\n - { name: metrics-server-sa, file: metrics-server-sa.yaml, type: sa }\n - { name: metrics-server-deployment, file: metrics-server-deployment.yaml, type: deploy }\n - { name: metrics-server-service, file: metrics-server-service.yaml, type: service }\n - { name: metrics-apiservice, file: metrics-apiservice.yaml, type: service }\n - { name: resource-reader-clusterrolebinding, file: resource-reader-clusterrolebinding.yaml, type: clusterrolebinding }\n - { name: resource-reader, file: resource-reader.yaml, type: clusterrole }\n\n- name: Metrics Server | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/metrics_server/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ metrics_server_templates }}\"\n register: metrics_server_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Metrics Server | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/metrics_server/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ metrics_server_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: metrics-server:system:auth-delegator\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:auth-delegator\nsubjects:\n- kind: ServiceAccount\n name: metrics-server\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: metrics-server-auth-reader\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: extension-apiserver-authentication-reader\nsubjects:\n- kind: ServiceAccount\n name: metrics-server\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2", "content": "apiVersion: apiregistration.k8s.io/v1\nkind: APIService\nmetadata:\n name: v1beta1.metrics.k8s.io\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n service:\n name: metrics-server\n namespace: kube-system\n group: metrics.k8s.io\n version: v1beta1\n insecureSkipTLSVerify: {{ metrics_server_kubelet_insecure_tls }}\n groupPriorityMinimum: 100\n versionPriority: 100\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2", "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: metrics-server\n namespace: kube-system\n labels:\n app.kubernetes.io/name: metrics-server\n addonmanager.kubernetes.io/mode: Reconcile\n version: {{ metrics_server_version }}\nspec:\n replicas: {{ metrics_server_replicas }}\n selector:\n matchLabels:\n app.kubernetes.io/name: metrics-server\n version: {{ metrics_server_version }}\n strategy:\n rollingUpdate:\n maxUnavailable: 0\n template:\n metadata:\n name: metrics-server\n labels:\n app.kubernetes.io/name: metrics-server\n version: {{ metrics_server_version }}\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: metrics-server\n hostNetwork: {{ metrics_server_host_network | default(false) }}\n containers:\n - name: metrics-server\n image: {{ metrics_server_image_repo }}:{{ metrics_server_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --cert-dir=/tmp\n - --secure-port={{ metrics_server_container_port }}\n{% if metrics_server_kubelet_preferred_address_types %}\n - --kubelet-preferred-address-types={{ metrics_server_kubelet_preferred_address_types }}\n{% endif %}\n - --kubelet-use-node-status-port\n{% if metrics_server_kubelet_insecure_tls %}\n - --kubelet-insecure-tls=true\n{% endif %}\n - --metric-resolution={{ metrics_server_metric_resolution }}\n ports:\n - containerPort: {{ metrics_server_container_port }}\n name: https\n protocol: TCP\n volumeMounts:\n - name: tmp\n mountPath: /tmp\n livenessProbe:\n httpGet:\n path: /livez\n port: https\n scheme: HTTPS\n periodSeconds: 10\n failureThreshold: 3\n initialDelaySeconds: 40\n readinessProbe:\n httpGet:\n path: /readyz\n port: https\n scheme: HTTPS\n periodSeconds: 10\n failureThreshold: 3\n initialDelaySeconds: 40\n securityContext:\n readOnlyRootFilesystem: true\n runAsNonRoot: true\n runAsUser: 1000\n allowPrivilegeEscalation: false\n seccompProfile:\n type: RuntimeDefault\n capabilities:\n drop:\n - ALL\n resources:\n limits:\n cpu: {{ metrics_server_limits_cpu }}\n memory: {{ metrics_server_limits_memory }}\n requests:\n cpu: {{ metrics_server_requests_cpu }}\n memory: {{ metrics_server_requests_memory }}\n volumes:\n - name: tmp\n emptyDir: {}\n tolerations:\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n{% if metrics_server_extra_tolerations %}\n {{ metrics_server_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}\n{%- endif %}\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - weight: 100\n podAffinityTerm:\n labelSelector:\n matchExpressions:\n - key: app.kubernetes.io/name\n operator: In\n values:\n - metrics-server\n topologyKey: kubernetes.io/hostname\n namespaces:\n - kube-system\n{% if metrics_server_extra_affinity %}\n {{ metrics_server_extra_affinity | to_nice_yaml(indent=2) | indent(8) }}\n{%- endif %}\n{% if metrics_server_nodeselector %}\n nodeSelector:\n {{ metrics_server_nodeselector | to_nice_yaml(indent=2) | indent(8) }}\n{%- endif %}\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: metrics-server\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: metrics-server\n namespace: kube-system\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\n app.kubernetes.io/name: \"metrics-server\"\nspec:\n type: ClusterIP\n selector:\n app.kubernetes.io/name: metrics-server\n ports:\n - name: https\n port: 443\n protocol: TCP\n targetPort: https\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: system:metrics-server\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:metrics-server\nsubjects:\n- kind: ServiceAccount\n name: metrics-server\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: system:metrics-server\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nrules:\n- apiGroups:\n - \"\"\n resources:\n - nodes/metrics\n verbs:\n - get\n- apiGroups:\n - \"\"\n resources:\n - pods\n - nodes\n verbs:\n - get\n - list\n - watch\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/defaults/main.yml", "content": "---\nnode_feature_discovery_enabled: false\nnode_feature_discovery_namespace: node-feature-discovery\nnode_feature_discovery_enable_nodefeature_api: true\nnode_feature_discovery_gc_replicas: 1\nnode_feature_discovery_gc_interval: 1h\nnode_feature_discovery_gc_sa_name: node-feature-discovery-gc\nnode_feature_discovery_gc_sa_create: true\nnode_feature_discovery_master_replicas: 1\nnode_feature_discovery_master_crd_controller: null\nnode_feature_discovery_master_instance: null\nnode_feature_discovery_master_config: null\nnode_feature_discovery_worker_sa_name: node-feature-discovery-worker\nnode_feature_discovery_worker_sa_create: true\nnode_feature_discovery_worker_config: null\nnode_feature_discovery_worker_tolerations: null\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/tasks/main.yml", "content": "---\n- name: Node Feature Discovery | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/node_feature_discovery\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Node Feature Discovery | Templates list\n set_fact:\n node_feature_discovery_templates:\n - { name: nfd-ns, file: nfd-ns.yaml, type: ns }\n - { name: nfd-api-crd, file: nfd-api-crds.yaml, type: crd }\n - { name: nfd-serviceaccount, file: nfd-serviceaccount.yaml, type: sa }\n - { name: nfd-role, file: nfd-role.yaml, type: role }\n - { name: nfd-clusterrole, file: nfd-clusterrole.yaml, type: clusterrole }\n - { name: nfd-rolebinding, file: nfd-rolebinding.yaml, type: rolebinding }\n - { name: nfd-clusterrolebinding, file: nfd-clusterrolebinding.yaml, type: clusterrolebinding }\n - { name: nfd-master-conf, file: nfd-master-conf.yaml, type: cm }\n - { name: nfd-worker-conf, file: nfd-worker-conf.yaml, type: cm }\n - { name: nfd-topologyupdater-conf, file: nfd-topologyupdater-conf.yaml, type: cm }\n - { name: nfd-gc, file: nfd-gc.yaml, type: deploy }\n - { name: nfd-master, file: nfd-master.yaml, type: deploy }\n - { name: nfd-worker, file: nfd-worker.yaml, type: ds }\n - { name: nfd-service, file: nfd-service.yaml, type: srv }\n\n- name: Node Feature Discovery | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ node_feature_discovery_templates }}\"\n register: node_feature_discovery_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Node Feature Discovery | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ node_feature_discovery_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.14.0\n name: nodefeatures.nfd.k8s-sigs.io\nspec:\n group: nfd.k8s-sigs.io\n names:\n kind: NodeFeature\n listKind: NodeFeatureList\n plural: nodefeatures\n singular: nodefeature\n scope: Namespaced\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: |-\n NodeFeature resource holds the features discovered for one node in the\n cluster.\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: Specification of the NodeFeature, containing features discovered\n for a node.\n properties:\n features:\n description: Features is the full \"raw\" features data that has been\n discovered.\n properties:\n attributes:\n additionalProperties:\n description: AttributeFeatureSet is a set of features having\n string value.\n properties:\n elements:\n additionalProperties:\n type: string\n description: Individual features of the feature set.\n type: object\n required:\n - elements\n type: object\n description: Attributes contains all the attribute-type features\n of the node.\n type: object\n flags:\n additionalProperties:\n description: FlagFeatureSet is a set of simple features only\n containing names without values.\n properties:\n elements:\n additionalProperties:\n description: Nil is a dummy empty struct for protobuf\n compatibility\n type: object\n description: Individual features of the feature set.\n type: object\n required:\n - elements\n type: object\n description: Flags contains all the flag-type features of the\n node.\n type: object\n instances:\n additionalProperties:\n description: InstanceFeatureSet is a set of features each of\n which is an instance having multiple attributes.\n properties:\n elements:\n description: Individual features of the feature set.\n items:\n description: InstanceFeature represents one instance of\n a complex features, e.g. a device.\n properties:\n attributes:\n additionalProperties:\n type: string\n description: Attributes of the instance feature.\n type: object\n required:\n - attributes\n type: object\n type: array\n required:\n - elements\n type: object\n description: Instances contains all the instance-type features\n of the node.\n type: object\n type: object\n labels:\n additionalProperties:\n type: string\n description: Labels is the set of node labels that are requested to\n be created.\n type: object\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.14.0\n name: nodefeaturegroups.nfd.k8s-sigs.io\nspec:\n group: nfd.k8s-sigs.io\n names:\n kind: NodeFeatureGroup\n listKind: NodeFeatureGroupList\n plural: nodefeaturegroups\n shortNames:\n - nfg\n singular: nodefeaturegroup\n scope: Namespaced\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: NodeFeatureGroup resource holds Node pools by featureGroup\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: Spec defines the rules to be evaluated.\n properties:\n featureGroupRules:\n description: List of rules to evaluate to determine nodes that belong\n in this group.\n items:\n description: GroupRule defines a rule for nodegroup filtering.\n properties:\n matchAny:\n description: MatchAny specifies a list of matchers one of which\n must match.\n items:\n description: MatchAnyElem specifies one sub-matcher of MatchAny.\n properties:\n matchFeatures:\n description: MatchFeatures specifies a set of matcher\n terms all of which must match.\n items:\n description: |-\n FeatureMatcherTerm defines requirements against one feature set. All\n requirements (specified as MatchExpressions) are evaluated against each\n element in the feature set.\n properties:\n feature:\n description: Feature is the name of the feature\n set to match against.\n type: string\n matchExpressions:\n additionalProperties:\n description: |-\n MatchExpression specifies an expression to evaluate against a set of input\n values. It contains an operator that is applied when matching the input and\n an array of values that the operator evaluates the input against.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n description: |-\n MatchExpressions is the set of per-element expressions evaluated. These\n match against the value of the specified elements.\n type: object\n matchName:\n description: |-\n MatchName in an expression that is matched against the name of each\n element in the feature set.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n required:\n - feature\n type: object\n type: array\n required:\n - matchFeatures\n type: object\n type: array\n matchFeatures:\n description: MatchFeatures specifies a set of matcher terms\n all of which must match.\n items:\n description: |-\n FeatureMatcherTerm defines requirements against one feature set. All\n requirements (specified as MatchExpressions) are evaluated against each\n element in the feature set.\n properties:\n feature:\n description: Feature is the name of the feature set to\n match against.\n type: string\n matchExpressions:\n additionalProperties:\n description: |-\n MatchExpression specifies an expression to evaluate against a set of input\n values. It contains an operator that is applied when matching the input and\n an array of values that the operator evaluates the input against.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n description: |-\n MatchExpressions is the set of per-element expressions evaluated. These\n match against the value of the specified elements.\n type: object\n matchName:\n description: |-\n MatchName in an expression that is matched against the name of each\n element in the feature set.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n required:\n - feature\n type: object\n type: array\n name:\n description: Name of the rule.\n type: string\n required:\n - name\n type: object\n type: array\n required:\n - featureGroupRules\n type: object\n status:\n description: |-\n Status of the NodeFeatureGroup after the most recent evaluation of the\n specification.\n properties:\n nodes:\n description: Nodes is a list of FeatureGroupNode in the cluster that\n match the featureGroupRules\n items:\n properties:\n name:\n description: Name of the node.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.14.0\n name: nodefeaturerules.nfd.k8s-sigs.io\nspec:\n group: nfd.k8s-sigs.io\n names:\n kind: NodeFeatureRule\n listKind: NodeFeatureRuleList\n plural: nodefeaturerules\n shortNames:\n - nfr\n singular: nodefeaturerule\n scope: Cluster\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: |-\n NodeFeatureRule resource specifies a configuration for feature-based\n customization of node objects, such as node labeling.\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: Spec defines the rules to be evaluated.\n properties:\n rules:\n description: Rules is a list of node customization rules.\n items:\n description: Rule defines a rule for node customization such as\n labeling.\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations to create if the rule matches.\n type: object\n extendedResources:\n additionalProperties:\n type: string\n description: ExtendedResources to create if the rule matches.\n type: object\n labels:\n additionalProperties:\n type: string\n description: Labels to create if the rule matches.\n type: object\n labelsTemplate:\n description: |-\n LabelsTemplate specifies a template to expand for dynamically generating\n multiple labels. Data (after template expansion) must be keys with an\n optional value ([=]) separated by newlines.\n type: string\n matchAny:\n description: MatchAny specifies a list of matchers one of which\n must match.\n items:\n description: MatchAnyElem specifies one sub-matcher of MatchAny.\n properties:\n matchFeatures:\n description: MatchFeatures specifies a set of matcher\n terms all of which must match.\n items:\n description: |-\n FeatureMatcherTerm defines requirements against one feature set. All\n requirements (specified as MatchExpressions) are evaluated against each\n element in the feature set.\n properties:\n feature:\n description: Feature is the name of the feature\n set to match against.\n type: string\n matchExpressions:\n additionalProperties:\n description: |-\n MatchExpression specifies an expression to evaluate against a set of input\n values. It contains an operator that is applied when matching the input and\n an array of values that the operator evaluates the input against.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n description: |-\n MatchExpressions is the set of per-element expressions evaluated. These\n match against the value of the specified elements.\n type: object\n matchName:\n description: |-\n MatchName in an expression that is matched against the name of each\n element in the feature set.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n required:\n - feature\n type: object\n type: array\n required:\n - matchFeatures\n type: object\n type: array\n matchFeatures:\n description: MatchFeatures specifies a set of matcher terms\n all of which must match.\n items:\n description: |-\n FeatureMatcherTerm defines requirements against one feature set. All\n requirements (specified as MatchExpressions) are evaluated against each\n element in the feature set.\n properties:\n feature:\n description: Feature is the name of the feature set to\n match against.\n type: string\n matchExpressions:\n additionalProperties:\n description: |-\n MatchExpression specifies an expression to evaluate against a set of input\n values. It contains an operator that is applied when matching the input and\n an array of values that the operator evaluates the input against.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n description: |-\n MatchExpressions is the set of per-element expressions evaluated. These\n match against the value of the specified elements.\n type: object\n matchName:\n description: |-\n MatchName in an expression that is matched against the name of each\n element in the feature set.\n properties:\n op:\n description: Op is the operator to be applied.\n enum:\n - In\n - NotIn\n - InRegexp\n - Exists\n - DoesNotExist\n - Gt\n - Lt\n - GtLt\n - IsTrue\n - IsFalse\n type: string\n value:\n description: |-\n Value is the list of values that the operand evaluates the input\n against. Value should be empty if the operator is Exists, DoesNotExist,\n IsTrue or IsFalse. Value should contain exactly one element if the\n operator is Gt or Lt and exactly two elements if the operator is GtLt.\n In other cases Value should contain at least one element.\n items:\n type: string\n type: array\n required:\n - op\n type: object\n required:\n - feature\n type: object\n type: array\n name:\n description: Name of the rule.\n type: string\n taints:\n description: Taints to create if the rule matches.\n items:\n description: |-\n The node this Taint is attached to has the \"effect\" on\n any pod that does not tolerate the Taint.\n properties:\n effect:\n description: |-\n Required. The effect of the taint on pods\n that do not tolerate the taint.\n Valid effects are NoSchedule, PreferNoSchedule and NoExecute.\n type: string\n key:\n description: Required. The taint key to be applied to\n a node.\n type: string\n timeAdded:\n description: |-\n TimeAdded represents the time at which the taint was added.\n It is only written for NoExecute taints.\n format: date-time\n type: string\n value:\n description: The taint value corresponding to the taint\n key.\n type: string\n required:\n - effect\n - key\n type: object\n type: array\n vars:\n additionalProperties:\n type: string\n description: |-\n Vars is the variables to store if the rule matches. Variables do not\n directly inflict any changes in the node object. However, they can be\n referenced from other rules enabling more complex rule hierarchies,\n without exposing intermediary output values as labels.\n type: object\n varsTemplate:\n description: |-\n VarsTemplate specifies a template to expand for dynamically generating\n multiple variables. Data (after template expansion) must be keys with an\n optional value ([=]) separated by newlines.\n type: string\n required:\n - name\n type: object\n type: array\n required:\n - rules\n type: object\n required:\n - spec\n type: object\n served: true\n storage: true\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: node-feature-discovery\nrules:\n- apiGroups:\n - \"\"\n resources:\n - nodes\n - nodes/status\n verbs:\n - get\n - patch\n - update\n - list\n- apiGroups:\n - nfd.k8s-sigs.io\n resources:\n - nodefeatures\n - nodefeaturerules\n - nodefeaturegroups\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - nfd.k8s-sigs.io\n resources:\n - nodefeaturegroup/status\n verbs:\n - patch\n - update\n- apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs:\n - create\n- apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n resourceNames:\n - \"nfd-master.nfd.kubernetes.io\"\n verbs:\n - get\n - update\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: node-feature-discovery-gc\nrules:\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - list\n - watch\n- apiGroups:\n - topology.node.k8s.io\n resources:\n - noderesourcetopologies\n verbs:\n - delete\n - list\n- apiGroups:\n - nfd.k8s-sigs.io\n resources:\n - nodefeatures\n verbs:\n - delete\n - list\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: node-feature-discovery\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: node-feature-discovery\nsubjects:\n- kind: ServiceAccount\n name: node-feature-discovery\n namespace: {{ node_feature_discovery_namespace }}\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: node-feature-discovery-gc\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: node-feature-discovery-gc\nsubjects:\n- kind: ServiceAccount\n name: {{ node_feature_discovery_gc_sa_name }}\n namespace: {{ node_feature_discovery_namespace }}\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: node-feature-discovery-gc\n namespace: {{ node_feature_discovery_namespace }}\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: gc\nspec:\n replicas: {{ node_feature_discovery_gc_replicas }}\n selector:\n matchLabels:\n app.kubernetes.io/name: node-feature-discovery\n role: gc\n template:\n metadata:\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: gc\n spec:\n serviceAccountName: {{ node_feature_discovery_gc_sa_name }}\n dnsPolicy: ClusterFirstWithHostNet\n containers:\n - name: gc\n image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }}\n imagePullPolicy: IfNotPresent\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n command:\n - \"nfd-gc\"\n args:\n - \"-gc-interval={{ node_feature_discovery_gc_interval }}\"\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n readOnlyRootFilesystem: true\n runAsNonRoot: true\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: node-feature-discovery-master-conf\n namespace: {{ node_feature_discovery_namespace }}\ndata:\n{% if node_feature_discovery_master_config %}\n nfd-master.conf: |-\n {{ node_feature_discovery_master_config | to_yaml(indent=2, width=1337) | indent(width=4) }}\n{% else %}\n nfd-master.conf: \"null\"\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2", "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: node-feature-discovery-master\n namespace: {{ node_feature_discovery_namespace }}\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: master\nspec:\n replicas: {{ node_feature_discovery_master_replicas }}\n selector:\n matchLabels:\n app.kubernetes.io/name: node-feature-discovery\n role: master\n template:\n metadata:\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: master\n spec:\n serviceAccountName: node-feature-discovery\n enableServiceLinks: false\n containers:\n - name: master\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n readOnlyRootFilesystem: true\n runAsNonRoot: true\n image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }}\n imagePullPolicy: IfNotPresent\n livenessProbe:\n grpc:\n port: 8082\n initialDelaySeconds: 10\n periodSeconds: 10\n readinessProbe:\n grpc:\n port: 8082\n initialDelaySeconds: 5\n periodSeconds: 10\n failureThreshold: 10\n ports:\n - containerPort: 8082\n name: grpc\n - containerPort: 8081\n name: metrics\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n command:\n - \"nfd-master\"\n args:\n - \"-port=8080\"\n{% if not node_feature_discovery_enable_nodefeature_api %}\n - \"-enable-nodefeature-api=false\"\n{% elif node_feature_discovery_master_replicas > 1 %}\n - \"-enable-leader-election\"\n{% endif %}\n{% if node_feature_discovery_master_crd_controller != none %}\n - \"-crd-controller={{ node_feature_discovery_master_crd_controller }}\"\n{% else %}\n{% if node_feature_discovery_master_instance %}\n ## By default, disable crd controller for other than the default instances\n - \"-crd-controller=false\"\n{% else %}\n ## By default, disable crd controller for other than the default instances\n - \"-crd-controller=true\"\n{% endif %}\n{% endif %}\n - \"-metrics=8081\"\n volumeMounts:\n - name: nfd-master-conf\n mountPath: \"/etc/kubernetes/node-feature-discovery\"\n readOnly: true\n volumes:\n - name: nfd-master-conf\n configMap:\n name: node-feature-discovery-master-conf\n items:\n - key: nfd-master.conf\n path: nfd-master.conf\n affinity:\n nodeAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - preference:\n matchExpressions:\n - key: node-role.kubernetes.io/master\n operator: In\n values:\n - \"\"\n weight: 1\n - preference:\n matchExpressions:\n - key: node-role.kubernetes.io/control-plane\n operator: In\n values:\n - \"\"\n weight: 1\n tolerations:\n - effect: NoSchedule\n key: node-role.kubernetes.io/master\n operator: Equal\n - effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n operator: Equal\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ node_feature_discovery_namespace }}\n labels:\n name: {{ node_feature_discovery_namespace }}\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: node-feature-discovery-worker\n namespace: {{ node_feature_discovery_namespace }}\nrules:\n- apiGroups:\n - nfd.k8s-sigs.io\n resources:\n - nodefeatures\n verbs:\n - create\n - get\n - update\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - get\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: node-feature-discovery-worker\n namespace: {{ node_feature_discovery_namespace }}\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: node-feature-discovery-worker\nsubjects:\n- kind: ServiceAccount\n name: {{ node_feature_discovery_worker_sa_name }}\n namespace: {{ node_feature_discovery_namespace }}\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-service.yaml.j2", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: node-feature-discovery-master\n namespace: {{ node_feature_discovery_namespace }}\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: master\nspec:\n type: ClusterIP\n ports:\n - port: 8080\n targetPort: grpc\n protocol: TCP\n name: grpc\n selector:\n app.kubernetes.io/name: node-feature-discovery\n role: master\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-serviceaccount.yaml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: node-feature-discovery\n namespace: {{ node_feature_discovery_namespace }}\n{% if node_feature_discovery_gc_sa_create %}\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: {{ node_feature_discovery_gc_sa_name }}\n namespace: {{ node_feature_discovery_namespace }}\n{% endif %}\n{% if node_feature_discovery_worker_sa_create %}\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: {{ node_feature_discovery_worker_sa_name }}\n namespace: {{ node_feature_discovery_namespace }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-topologyupdater-conf.yaml.j2", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: node-feature-discovery-topology-updater-conf\n namespace: {{ node_feature_discovery_namespace }}\ndata:\n nfd-topology-updater.conf: \"null\"\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker-conf.yaml.j2", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: node-feature-discovery-worker-conf\n namespace: {{ node_feature_discovery_namespace }}\ndata:\n{% if node_feature_discovery_worker_config %}\n nfd-worker.conf: |-\n {{ node_feature_discovery_worker_config | to_yaml(indent=2, width=1337) | indent(width=4) }}\n{% else %}\n nfd-worker.conf: \"null\"\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker.yaml.j2", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: node-feature-discovery-worker\n namespace: {{ node_feature_discovery_namespace }}\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: worker\nspec:\n selector:\n matchLabels:\n app.kubernetes.io/name: node-feature-discovery\n role: worker\n template:\n metadata:\n labels:\n app.kubernetes.io/name: node-feature-discovery\n role: worker\n spec:\n dnsPolicy: ClusterFirstWithHostNet\n serviceAccountName: {{ node_feature_discovery_worker_sa_name }}\n containers:\n - name: worker\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n readOnlyRootFilesystem: true\n runAsNonRoot: true\n image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }}\n imagePullPolicy: IfNotPresent\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n command:\n - \"nfd-worker\"\n args:\n - \"-server=node-feature-discovery-master:8080\"\n{% if not node_feature_discovery_enable_nodefeature_api %}\n - \"-enable-nodefeature-api=false\"\n{% endif %}\n - \"-metrics=8081\"\n ports:\n - name: metrics\n containerPort: 8081\n volumeMounts:\n - name: host-boot\n mountPath: \"/host-boot\"\n readOnly: true\n - name: host-os-release\n mountPath: \"/host-etc/os-release\"\n readOnly: true\n - name: host-sys\n mountPath: \"/host-sys\"\n readOnly: true\n - name: host-usr-lib\n mountPath: \"/host-usr/lib\"\n readOnly: true\n - name: host-lib\n mountPath: \"/host-lib\"\n readOnly: true\n - name: source-d\n mountPath: \"/etc/kubernetes/node-feature-discovery/source.d/\"\n readOnly: true\n - name: features-d\n mountPath: \"/etc/kubernetes/node-feature-discovery/features.d/\"\n readOnly: true\n - name: nfd-worker-conf\n mountPath: \"/etc/kubernetes/node-feature-discovery\"\n readOnly: true\n volumes:\n - name: host-boot\n hostPath:\n path: \"/boot\"\n - name: host-os-release\n hostPath:\n path: \"/etc/os-release\"\n - name: host-sys\n hostPath:\n path: \"/sys\"\n - name: host-usr-lib\n hostPath:\n path: \"/usr/lib\"\n - name: host-lib\n hostPath:\n path: \"/lib\"\n - name: source-d\n hostPath:\n path: \"/etc/kubernetes/node-feature-discovery/source.d/\"\n - name: features-d\n hostPath:\n path: \"/etc/kubernetes/node-feature-discovery/features.d/\"\n - name: nfd-worker-conf\n configMap:\n name: node-feature-discovery-worker-conf\n items:\n - key: nfd-worker.conf\n path: nfd-worker.conf\n{% if node_feature_discovery_worker_tolerations %}\n tolerations:\n {{ node_feature_discovery_worker_tolerations | to_yaml(indent=2, width=1337) | indent(width=8) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml", "content": "---\n# To restrict which AZ the volume should be provisioned in\n# set this value to true and set the list of relevant AZs\n# For it to work, the flag aws_ebs_csi_enable_volume_scheduling\n# in AWS EBS Driver must be true\nrestrict_az_provisioning: false\naws_ebs_availability_zones:\n - eu-west-3c\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml", "content": "---\n- name: Kubernetes Persistent Volumes | Copy AWS EBS CSI Storage Class template\n template:\n src: \"aws-ebs-csi-storage-class.yml.j2\"\n dest: \"{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml\"\n mode: \"0644\"\n register: manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Persistent Volumes | Add AWS EBS CSI Storage Class\n kube:\n name: aws-ebs-csi\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: StorageClass\n filename: \"{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - manifests.changed\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2", "content": "kind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n name: ebs-sc\nprovisioner: ebs.csi.aws.com\nvolumeBindingMode: WaitForFirstConsumer\nparameters:\n csi.storage.k8s.io/fstype: xfs\n type: gp2\n{% if restrict_az_provisioning %}\nallowedTopologies:\n- matchLabelExpressions:\n - key: topology.ebs.csi.aws.com/zone\n values:\n{% for value in aws_ebs_availability_zones %}\n - {{ value }}\n{% endfor %}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml", "content": "---\n## Available values: Standard_LRS, Premium_LRS, StandardSSD_LRS, UltraSSD_LRS\nstorage_account_type: StandardSSD_LRS\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml", "content": "---\n- name: Kubernetes Persistent Volumes | Copy Azure CSI Storage Class template\n template:\n src: \"azure-csi-storage-class.yml.j2\"\n dest: \"{{ kube_config_dir }}/azure-csi-storage-class.yml\"\n mode: \"0644\"\n register: manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Persistent Volumes | Add Azure CSI Storage Class\n kube:\n name: azure-csi\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: StorageClass\n filename: \"{{ kube_config_dir }}/azure-csi-storage-class.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - manifests.changed\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2", "content": "---\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: disk.csi.azure.com\nprovisioner: disk.csi.azure.com\nparameters:\n skuname: {{ storage_account_type }}\n{% if azure_csi_tags is defined %}\n tags: {{ azure_csi_tags }}\n{% endif %}\nreclaimPolicy: Delete\nvolumeBindingMode: Immediate\nallowVolumeExpansion: true\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml", "content": "---\nstorage_classes:\n - name: cinder-csi\n is_default: false\n parameters:\n availability: nova\n allowVolumeExpansion: false\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml", "content": "---\n- name: Kubernetes Persistent Volumes | Copy Cinder CSI Storage Class template\n template:\n src: \"cinder-csi-storage-class.yml.j2\"\n dest: \"{{ kube_config_dir }}/cinder-csi-storage-class.yml\"\n mode: \"0644\"\n register: manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Persistent Volumes | Add Cinder CSI Storage Class\n kube:\n name: cinder-csi\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: StorageClass\n filename: \"{{ kube_config_dir }}/cinder-csi-storage-class.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - manifests.changed\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2", "content": "{% for class in storage_classes %}\n---\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n name: \"{{ class.name }}\"\n annotations:\n storageclass.kubernetes.io/is-default-class: \"{{ class.is_default | default(false) | ternary(\"true\",\"false\") }}\"\nprovisioner: cinder.csi.openstack.org\nvolumeBindingMode: WaitForFirstConsumer\nparameters:\n{% for key, value in (class.parameters | default({})).items() %}\n \"{{ key }}\": \"{{ value }}\"\n{% endfor %}\n{% if cinder_topology is defined and cinder_topology is sameas true %}\nallowedTopologies:\n- matchLabelExpressions:\n - key: topology.cinder.csi.openstack.org/zone\n values:\n{% for zone in cinder_topology_zones %}\n - \"{{ zone }}\"\n{% endfor %}\n{% endif %}\nallowVolumeExpansion: {{ expand_persistent_volumes }}\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/defaults/main.yml", "content": "---\n# Choose between pd-standard and pd-ssd\ngcp_pd_csi_volume_type: pd-standard\ngcp_pd_regional_replication_enabled: false\ngcp_pd_restrict_zone_replication: false\ngcp_pd_restricted_zones:\n - europe-west1-b\n - europe-west1-c\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/tasks/main.yml", "content": "---\n- name: Kubernetes Persistent Volumes | Copy GCP PD CSI Storage Class template\n template:\n src: \"gcp-pd-csi-storage-class.yml.j2\"\n dest: \"{{ kube_config_dir }}/gcp-pd-csi-storage-class.yml\"\n mode: \"0644\"\n register: manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Persistent Volumes | Add GCP PD CSI Storage Class\n kube:\n name: gcp-pd-csi\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: StorageClass\n filename: \"{{ kube_config_dir }}/gcp-pd-csi-storage-class.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - manifests.changed\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/templates/gcp-pd-csi-storage-class.yml.j2", "content": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: csi-gce-pd\nprovisioner: pd.csi.storage.gke.io\nparameters:\n type: {{ gcp_pd_csi_volume_type }}\n{% if gcp_pd_regional_replication_enabled %}\n replication-type: regional-pd\n{% endif %}\nvolumeBindingMode: WaitForFirstConsumer\n{% if gcp_pd_restrict_zone_replication %}\nallowedTopologies:\n- matchLabelExpressions:\n - key: topology.gke.io/zone\n values:\n{% for value in gcp_pd_restricted_zones %}\n - {{ value }}\n{% endfor %}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/persistent_volumes/cinder-csi\n when:\n - cinder_csi_enabled\n tags:\n - persistent_volumes_cinder_csi\n - cinder-csi-driver\n\n - role: kubernetes-apps/persistent_volumes/aws-ebs-csi\n when:\n - aws_ebs_csi_enabled\n tags:\n - persistent_volumes_aws_ebs_csi\n - aws-ebs-csi-driver\n\n - role: kubernetes-apps/persistent_volumes/azuredisk-csi\n when:\n - azure_csi_enabled\n tags:\n - persistent_volumes_azure_csi\n - azure-csi-driver\n\n - role: kubernetes-apps/persistent_volumes/gcp-pd-csi\n when:\n - gcp_pd_csi_enabled\n tags:\n - persistent_volumes_gcp_pd_csi\n - gcp-pd-csi-driver\n\n - role: kubernetes-apps/persistent_volumes/upcloud-csi\n when:\n - upcloud_csi_enabled\n tags:\n - persistent_volumes_upcloud_csi\n - upcloud-csi-driver\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/upcloud-csi/defaults/main.yml", "content": "---\nstorage_classes:\n - name: standard\n is_default: true\n expand_persistent_volumes: true\n parameters:\n tier: maxiops\n - name: hdd\n is_default: false\n expand_persistent_volumes: true\n parameters:\n tier: hdd\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/upcloud-csi/tasks/main.yml", "content": "---\n- name: Kubernetes Persistent Volumes | Copy UpCloud CSI Storage Class template\n template:\n src: \"upcloud-csi-storage-class.yml.j2\"\n dest: \"{{ kube_config_dir }}/upcloud-csi-storage-class.yml\"\n mode: \"0644\"\n register: manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Persistent Volumes | Add UpCloud CSI Storage Class\n kube:\n name: upcloud-csi\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: StorageClass\n filename: \"{{ kube_config_dir }}/upcloud-csi-storage-class.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - manifests.changed\n" }, { "path": "roles/kubernetes-apps/persistent_volumes/upcloud-csi/templates/upcloud-csi-storage-class.yml.j2", "content": "{% for class in storage_classes %}\n---\nkind: StorageClass\napiVersion: storage.k8s.io/v1\nmetadata:\n name: \"{{ class.name }}\"\n annotations:\n storageclass.kubernetes.io/is-default-class: \"{{ class.is_default | default(false) | ternary(\"true\",\"false\") }}\"\nprovisioner: storage.csi.upcloud.com\nreclaimPolicy: Delete\nparameters:\n{% for key, value in (class.parameters | default({})).items() %}\n \"{{ key }}\": \"{{ value }}\"\n{% endfor %}\nallowVolumeExpansion: {{ class.expand_persistent_volumes | default(true) | ternary(\"true\",\"false\") }}\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/policy_controller/calico/defaults/main.yml", "content": "---\n# Limits for calico apps\ncalico_policy_controller_cpu_limit: 1000m\ncalico_policy_controller_memory_limit: 256M\ncalico_policy_controller_cpu_requests: 30m\ncalico_policy_controller_memory_requests: 64M\ncalico_policy_controller_deployment_nodeselector: \"kubernetes.io/os: linux\"\ncalico_policy_controller_log_level: info\n\n# SSL\ncalico_cert_dir: \"/etc/calico/certs\"\n" }, { "path": "roles/kubernetes-apps/policy_controller/calico/tasks/main.yml", "content": "---\n- name: Create calico-kube-controllers manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: calico-kube-controllers, file: calico-kube-controllers.yml, type: deployment}\n - {name: calico-kube-controllers, file: calico-kube-sa.yml, type: sa}\n - {name: calico-kube-controllers, file: calico-kube-cr.yml, type: clusterrole}\n - {name: calico-kube-controllers, file: calico-kube-crb.yml, type: clusterrolebinding}\n register: calico_kube_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - rbac_enabled or item.type not in rbac_resources\n\n- name: Start of Calico kube controllers\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"kube-system\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ calico_kube_manifests.results }}\"\n register: calico_kube_controller_start\n until: calico_kube_controller_start is succeeded\n retries: 4\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n" }, { "path": "roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n replicas: 1\n strategy:\n type: Recreate\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n template:\n metadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\n spec:\n nodeSelector:\n {{ calico_policy_controller_deployment_nodeselector }}\n{% if calico_datastore == \"etcd\" %}\n hostNetwork: true\n{% endif %}\n serviceAccountName: calico-kube-controllers\n tolerations:\n - key: CriticalAddonsOnly\n operator: Exists\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n{% if policy_controller_extra_tolerations is defined %}\n {{ policy_controller_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}\n{% endif %}\n priorityClassName: system-cluster-critical\n containers:\n - name: calico-kube-controllers\n image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n resources:\n limits:\n cpu: {{ calico_policy_controller_cpu_limit }}\n memory: {{ calico_policy_controller_memory_limit }}\n requests:\n cpu: {{ calico_policy_controller_cpu_requests }}\n memory: {{ calico_policy_controller_memory_requests }}\n livenessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -l\n periodSeconds: 10\n initialDelaySeconds: 10\n failureThreshold: 6\n readinessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -r\n periodSeconds: 10\n securityContext:\n runAsNonRoot: true\n env:\n - name: LOG_LEVEL\n value: {{ calico_policy_controller_log_level }}\n{% if calico_datastore == \"kdd\" %}\n - name: ENABLED_CONTROLLERS\n value: node\n - name: DATASTORE_TYPE\n value: kubernetes\n{% else %}\n - name: ENABLED_CONTROLLERS\n value: policy,namespace,serviceaccount,workloadendpoint,node\n - name: ETCD_ENDPOINTS\n value: \"{{ etcd_access_addresses }}\"\n - name: ETCD_CA_CERT_FILE\n value: \"{{ calico_cert_dir }}/ca_cert.crt\"\n - name: ETCD_CERT_FILE\n value: \"{{ calico_cert_dir }}/cert.crt\"\n - name: ETCD_KEY_FILE\n value: \"{{ calico_cert_dir }}/key.pem\"\n volumeMounts:\n - mountPath: {{ calico_cert_dir }}\n name: etcd-certs\n readOnly: true\n volumes:\n - hostPath:\n path: {{ calico_cert_dir }}\n name: etcd-certs\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2", "content": "---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\nrules:\n{% if calico_datastore == \"etcd\" %}\n # Pods are monitored for changing labels.\n # The node controller monitors Kubernetes nodes.\n # Namespace and serviceaccount labels are used for policy.\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n - serviceaccounts\n verbs:\n - watch\n - list\n - get\n # Watch for changes to Kubernetes NetworkPolicies.\n - apiGroups: [\"networking.k8s.io\"]\n resources:\n - networkpolicies\n verbs:\n - watch\n - list\n # Services are monitored for service LoadBalancer IP allocation\n - apiGroups: [\"\"]\n resources:\n - services\n - services/status\n verbs:\n - get\n - list\n - update\n - watch\n{% elif calico_datastore == \"kdd\" %}\n # Nodes are watched to monitor for deletions.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - watch\n - list\n - get\n # Pods are queried to check for existence.\n - apiGroups: [\"\"]\n resources:\n - pods\n verbs:\n - watch\n - list\n - get\n # IPAM resources are manipulated when nodes are deleted.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ipreservations\n verbs:\n - list\n # Pools are watched to maintain a mapping of blocks to IP pools.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n verbs:\n - list\n - watch\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n - tiers\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - watch\n # kube-controllers manages hostendpoints.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - hostendpoints\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - watch\n # Needs access to update clusterinformations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - clusterinformations\n verbs:\n - get\n - list\n - create\n - update\n - watch\n # KubeControllersConfiguration is where it gets its config\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - kubecontrollersconfigurations\n verbs:\n # read its own config\n - get\n - list\n # create a default if none exists\n - create\n # update status\n - update\n # watch for changes\n - watch\n # Services are monitored for service LoadBalancer IP allocation\n - apiGroups: [\"\"]\n resources:\n - services\n - services/status\n verbs:\n - get\n - list\n - update\n - watch\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-crb.yml.j2", "content": "---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-kube-controllers\nsubjects:\n- kind: ServiceAccount\n name: calico-kube-controllers\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-sa.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/policy_controller/meta/main.yml", "content": "---\ndependencies:\n - role: policy_controller/calico\n when:\n - kube_network_plugin in ['calico']\n - enable_network_policy\n tags:\n - policy-controller\n" }, { "path": "roles/kubernetes-apps/registry/defaults/main.yml", "content": "---\nregistry_namespace: \"kube-system\"\nregistry_storage_class: \"\"\nregistry_storage_access_mode: \"ReadWriteOnce\"\nregistry_disk_size: \"10Gi\"\nregistry_port: 5000\nregistry_replica_count: 1\n\n# type of service: ClusterIP, LoadBalancer or NodePort\nregistry_service_type: \"ClusterIP\"\n# you can specify your cluster IP address when registry_service_type is ClusterIP\nregistry_service_cluster_ip: \"\"\n# you can specify your cloud provider assigned loadBalancerIP when registry_service_type is LoadBalancer\nregistry_service_loadbalancer_ip: \"\"\n# annotations for managing Cloud Load Balancers\nregistry_service_annotations: {}\n# you can specify the node port when registry_service_type is NodePort\nregistry_service_nodeport: \"\"\n\n# name of kubernetes secret for registry TLS certs\nregistry_tls_secret: \"\"\n\nregistry_htpasswd: \"\"\n\n# registry configuration\n# see: https://docs.docker.com/registry/configuration/#list-of-configuration-options\nregistry_config:\n version: 0.1\n log:\n fields:\n service: registry\n storage:\n cache:\n blobdescriptor: inmemory\n http:\n addr: :{{ registry_port }}\n headers:\n X-Content-Type-Options: [nosniff]\n health:\n storagedriver:\n enabled: true\n interval: 10s\n threshold: 3\n\nregistry_ingress_annotations: {}\nregistry_ingress_host: \"\"\n# name of kubernetes secret for registry ingress TLS certs\nregistry_ingress_tls_secret: \"\"\n" }, { "path": "roles/kubernetes-apps/registry/tasks/main.yml", "content": "---\n- name: Registry | check registry_service_type value\n fail:\n msg: \"registry_service_type can only be 'ClusterIP', 'LoadBalancer' or 'NodePort'\"\n when: registry_service_type not in ['ClusterIP', 'LoadBalancer', 'NodePort']\n\n- name: Registry | Stop if registry_service_cluster_ip is defined when registry_service_type is not 'ClusterIP'\n fail:\n msg: \"registry_service_cluster_ip support only compatible with ClusterIP.\"\n when:\n - registry_service_cluster_ip is defined and registry_service_cluster_ip | length > 0\n - registry_service_type != \"ClusterIP\"\n\n- name: Registry | Stop if registry_service_loadbalancer_ip is defined when registry_service_type is not 'LoadBalancer'\n fail:\n msg: \"registry_service_loadbalancer_ip support only compatible with LoadBalancer.\"\n when:\n - registry_service_loadbalancer_ip is defined and registry_service_loadbalancer_ip | length > 0\n - registry_service_type != \"LoadBalancer\"\n\n- name: Registry | Stop if registry_service_nodeport is defined when registry_service_type is not 'NodePort'\n fail:\n msg: \"registry_service_nodeport support only compatible with NodePort.\"\n when:\n - registry_service_nodeport is defined and registry_service_nodeport | length > 0\n - registry_service_type != \"NodePort\"\n\n- name: Registry | Create addon dir\n file:\n path: \"{{ kube_config_dir }}/addons/registry\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n\n- name: Registry | Templates list\n set_fact:\n registry_templates:\n - { name: registry-ns, file: registry-ns.yml, type: ns }\n - { name: registry-sa, file: registry-sa.yml, type: sa }\n - { name: registry-svc, file: registry-svc.yml, type: svc }\n - { name: registry-secrets, file: registry-secrets.yml, type: secrets }\n - { name: registry-cm, file: registry-cm.yml, type: cm }\n - { name: registry-rs, file: registry-rs.yml, type: rs }\n\n- name: Registry | Append ingress templates to Registry Templates list when ALB ingress enabled\n set_fact:\n registry_templates: \"{{ registry_templates + [item] }}\"\n with_items:\n - [{ name: registry-ing, file: registry-ing.yml, type: ing }]\n when: ingress_alb_enabled\n\n- name: Registry | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/registry/{{ item.file }}\"\n mode: \"0644\"\n with_items: \"{{ registry_templates }}\"\n register: registry_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Registry | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"{{ registry_namespace }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/registry/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ registry_manifests.results }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Registry | Create PVC manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/addons/registry/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - { name: registry-pvc, file: registry-pvc.yml, type: pvc }\n register: registry_manifests\n when:\n - registry_storage_class != none and registry_storage_class\n - registry_disk_size != none and registry_disk_size\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Registry | Apply PVC manifests\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"{{ registry_namespace }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/addons/registry/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ registry_manifests.results }}\"\n when:\n - registry_storage_class != none and registry_storage_class\n - registry_disk_size != none and registry_disk_size\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-cm.yml.j2", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: registry-config\n namespace: {{ registry_namespace }}\n{% if registry_config %}\ndata:\n config.yml: |-\n {{ registry_config | to_yaml(indent=2, width=1337) | indent(width=4) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-ing.yml.j2", "content": "apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: registry\n namespace: {{ registry_namespace }}\n{% if registry_ingress_annotations %}\n annotations:\n {{ registry_ingress_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }}\n{% endif %}\nspec:\n{% if registry_ingress_tls_secret %}\n tls:\n - hosts:\n - {{ registry_ingress_host }}\n secretName: {{ registry_ingress_tls_secret }}\n{% endif %}\n rules:\n - host: {{ registry_ingress_host }}\n http:\n paths:\n - path: /\n pathType: Prefix\n backend:\n service:\n name: registry\n port:\n number: {{ registry_port }}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-ns.yml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ registry_namespace }}\n labels:\n name: {{ registry_namespace }}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2", "content": "---\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: registry-pvc\n namespace: {{ registry_namespace }}\n labels:\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n accessModes:\n - {{ registry_storage_access_mode }}\n storageClassName: {{ registry_storage_class }}\n resources:\n requests:\n storage: {{ registry_disk_size }}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-rs.yml.j2", "content": "---\napiVersion: apps/v1\nkind: ReplicaSet\nmetadata:\n name: registry\n namespace: {{ registry_namespace }}\n labels:\n k8s-app: registry\n version: v{{ registry_image_tag }}\n addonmanager.kubernetes.io/mode: Reconcile\nspec:\n{% if registry_storage_class != \"\" and registry_storage_access_mode == \"ReadWriteMany\" %}\n replicas: {{ registry_replica_count }}\n{% else %}\n replicas: 1\n{% endif %}\n selector:\n matchLabels:\n k8s-app: registry\n version: v{{ registry_image_tag }}\n template:\n metadata:\n labels:\n k8s-app: registry\n version: v{{ registry_image_tag }}\n spec:\n priorityClassName: {% if registry_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}\n serviceAccountName: registry\n securityContext:\n fsGroup: 1000\n runAsUser: 1000\n containers:\n - name: registry\n image: {{ registry_image_repo }}:{{ registry_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command:\n - /bin/registry\n - serve\n - /etc/docker/registry/config.yml\n env:\n - name: REGISTRY_HTTP_ADDR\n value: :{{ registry_port }}\n - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY\n value: /var/lib/registry\n{% if registry_htpasswd != \"\" %}\n - name: REGISTRY_AUTH\n value: \"htpasswd\"\n - name: REGISTRY_AUTH_HTPASSWD_REALM\n value: \"Registry Realm\"\n - name: REGISTRY_AUTH_HTPASSWD_PATH\n value: \"/auth/htpasswd\"\n{% endif %}\n{% if registry_tls_secret != \"\" %}\n - name: REGISTRY_HTTP_TLS_CERTIFICATE\n value: /etc/ssl/docker/tls.crt\n - name: REGISTRY_HTTP_TLS_KEY\n value: /etc/ssl/docker/tls.key\n{% endif %}\n volumeMounts:\n - name: registry-pvc\n mountPath: /var/lib/registry\n - name: registry-config\n mountPath: /etc/docker/registry\n{% if registry_htpasswd != \"\" %}\n - name: auth\n mountPath: /auth\n readOnly: true\n{% endif %}\n{% if registry_tls_secret != \"\" %}\n - name: tls-cert\n mountPath: /etc/ssl/docker\n readOnly: true\n{% endif %}\n ports:\n - containerPort: {{ registry_port }}\n name: registry\n protocol: TCP\n livenessProbe:\n httpGet:\n{% if registry_tls_secret != \"\" %}\n scheme: HTTPS\n{% endif %}\n path: /\n port: {{ registry_port }}\n readinessProbe:\n httpGet:\n{% if registry_tls_secret != \"\" %}\n scheme: HTTPS\n{% endif %}\n path: /\n port: {{ registry_port }}\n volumes:\n - name: registry-pvc\n{% if registry_storage_class != \"\" %}\n persistentVolumeClaim:\n claimName: registry-pvc\n{% else %}\n emptyDir: {}\n{% endif %}\n - name: registry-config\n configMap:\n name: registry-config\n{% if registry_htpasswd != \"\" %}\n - name: auth\n secret:\n secretName: registry-secret\n items:\n - key: htpasswd\n path: htpasswd\n{% endif %}\n{% if registry_tls_secret != \"\" %}\n - name: tls-cert\n secret:\n secretName: {{ registry_tls_secret }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-sa.yml.j2", "content": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: registry\n namespace: {{ registry_namespace }}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-secrets.yml.j2", "content": "apiVersion: v1\nkind: Secret\nmetadata:\n name: registry-secret\n namespace: {{ registry_namespace }}\ntype: Opaque\ndata:\n{% if registry_htpasswd != \"\" %}\n htpasswd: {{ registry_htpasswd | b64encode }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/registry/templates/registry-svc.yml.j2", "content": "---\napiVersion: v1\nkind: Service\nmetadata:\n name: registry\n namespace: {{ registry_namespace }}\n labels:\n k8s-app: registry\n addonmanager.kubernetes.io/mode: Reconcile\n kubernetes.io/name: \"KubeRegistry\"\n{% if registry_service_annotations %}\n annotations:\n {{ registry_service_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }}\n{% endif %}\nspec:\n selector:\n k8s-app: registry\n type: {{ registry_service_type }}\n{% if registry_service_type == \"ClusterIP\" and registry_service_cluster_ip != \"\" %}\n clusterIP: {{ registry_service_cluster_ip }}\n{% endif %}\n{% if registry_service_type == \"LoadBalancer\" and registry_service_loadbalancer_ip != \"\" %}\n loadBalancerIP: {{ registry_service_loadbalancer_ip }}\n{% endif %}\n ports:\n - name: registry\n port: {{ registry_port }}\n protocol: TCP\n targetPort: {{ registry_port }}\n{% if registry_service_type == \"NodePort\" and registry_service_nodeport != \"\" %}\n nodePort: {{ registry_service_nodeport }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/defaults/main.yml", "content": "---\nscheduler_plugins_enabled: false\n\nscheduler_plugins_namespace: scheduler-plugins\n\nscheduler_plugins_controller_replicas: 1\n\nscheduler_plugins_scheduler_replicas: 1\n\n# The default is determined by the number of control plane nodes.\nscheduler_plugins_scheduler_leader_elect: \"{{ ((groups['kube_control_plane'] | length) > 1) }}\"\n\n# Plugins to enable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info.\nscheduler_plugins_enabled_plugins:\n - Coscheduling\n - CapacityScheduling\n - NodeResourceTopologyMatch\n - NodeResourcesAllocatable\n\n# Plugins to disable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info.\nscheduler_plugins_disabled_plugins:\n - PrioritySort\n\n# Customize the enabled plugins' config.\n# Refer to the \"pluginConfig\" section of https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests//scheduler-config.yaml.\nscheduler_plugins_plugin_config:\n - name: Coscheduling\n args:\n permitWaitingTimeSeconds: 10 # default is 60\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/tasks/main.yml", "content": "---\n- name: Scheduler Plugins | Ensure dir exists\n file:\n path: \"{{ kube_config_dir }}/scheduler-plugins\"\n state: directory\n owner: root\n group: root\n mode: \"0755\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - scheduler_plugins\n\n- name: Scheduler Plugins | Create manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/scheduler-plugins/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - { name: appgroup, file: appgroup.diktyo.x-k8s.io_appgroups.yaml, type: crd }\n - { name: networktopology, file: networktopology.diktyo.x-k8s.io_networktopologies.yaml, type: crd }\n - { name: elasticquotas, file: scheduling.x-k8s.io_elasticquotas.yaml, type: crd }\n - { name: podgroups, file: scheduling.x-k8s.io_podgroups.yaml, type: crd }\n - { name: noderesourcetopologies, file: topology.node.k8s.io_noderesourcetopologies.yaml, type: crd }\n - { name: namespace, file: namespace.yaml, type: namespace }\n - { name: sa, file: sa-scheduler-plugins.yaml, type: serviceaccount }\n - { name: rbac, file: rbac-scheduler-plugins.yaml, type: rbac }\n - { name: cm, file: cm-scheduler-plugins.yaml, type: configmap }\n - { name: deploy, file: deploy-scheduler-plugins.yaml, type: deployment }\n register: scheduler_plugins_manifests\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - scheduler_plugins\n\n- name: Scheduler Plugins | Apply manifests\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/scheduler-plugins/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ scheduler_plugins_manifests.results }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - scheduler_plugins\n\n- name: Scheduler Plugins | Wait for controller pods to be ready\n command: \"{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l app=scheduler-plugins-controller -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'\" # noqa ignore-errors\n register: controller_pods_not_ready\n until: controller_pods_not_ready.stdout.find(\"scheduler-plugins-controller\")==-1\n retries: 30\n delay: 10\n ignore_errors: true\n changed_when: false\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - scheduler_plugins\n\n- name: Scheduler Plugins | Wait for scheduler pods to be ready\n command: \"{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l component=scheduler -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'\" # noqa ignore-errors\n register: scheduler_pods_not_ready\n until: scheduler_pods_not_ready.stdout.find(\"scheduler-plugins-scheduler\")==-1\n retries: 30\n delay: 10\n ignore_errors: true\n changed_when: false\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags:\n - scheduler_plugins\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: appgroups.appgroup.diktyo.x-k8s.io\nspec:\n group: appgroup.diktyo.x-k8s.io\n names:\n kind: AppGroup\n listKind: AppGroupList\n plural: appgroups\n shortNames:\n - ag\n singular: appgroup\n scope: Namespaced\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: AppGroup is a collection of Pods belonging to the same application.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: AppGroupSpec defines the number of Pods and which Pods belong\n to the group.\n properties:\n numMembers:\n description: NumMembers defines the number of Pods belonging to the\n App Group\n format: int32\n minimum: 1\n type: integer\n topologySortingAlgorithm:\n description: The preferred Topology Sorting Algorithm\n type: string\n workloads:\n description: Workloads defines the workloads belonging to the group\n items:\n description: AppGroupWorkload represents the Workloads belonging\n to the App Group.\n properties:\n dependencies:\n description: Dependencies of the Workload.\n items:\n description: DependenciesInfo contains information about one\n dependency.\n properties:\n maxNetworkCost:\n description: Max Network Cost between workloads\n format: int64\n maximum: 10000\n minimum: 0\n type: integer\n minBandwidth:\n anyOf:\n - type: integer\n - type: string\n description: MinBandwidth between workloads\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n workload:\n description: Workload reference Info.\n properties:\n apiVersion:\n description: ApiVersion defines the versioned schema\n of an object.\n type: string\n kind:\n description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"'\n type: string\n name:\n description: 'Name represents the workload, info:\n http://kubernetes.io/docs/user-guide/identifiers#names'\n type: string\n namespace:\n description: Namespace of the workload\n type: string\n selector:\n description: Selector defines how to find Pods related\n to the Workload (key = workload). (e.g., workload=w1)\n type: string\n required:\n - kind\n - name\n - selector\n type: object\n required:\n - workload\n type: object\n type: array\n workload:\n description: Workload reference Info.\n properties:\n apiVersion:\n description: ApiVersion defines the versioned schema of\n an object.\n type: string\n kind:\n description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"'\n type: string\n name:\n description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names'\n type: string\n namespace:\n description: Namespace of the workload\n type: string\n selector:\n description: Selector defines how to find Pods related to\n the Workload (key = workload). (e.g., workload=w1)\n type: string\n required:\n - kind\n - name\n - selector\n type: object\n required:\n - workload\n type: object\n type: array\n required:\n - numMembers\n - topologySortingAlgorithm\n - workloads\n type: object\n status:\n description: AppGroupStatus defines the observed use.\n properties:\n runningWorkloads:\n description: The number of actively running workloads (e.g., number\n of pods).\n format: int32\n minimum: 0\n type: integer\n scheduleStartTime:\n description: ScheduleStartTime of the group\n format: date-time\n type: string\n topologyCalculationTime:\n description: TopologyCalculationTime of the group\n format: date-time\n type: string\n topologyOrder:\n description: Topology order for TopSort plugin (QueueSort)\n items:\n description: AppGroupTopologyInfo represents the calculated order\n for a given Workload.\n properties:\n index:\n description: Topology index.\n format: int32\n type: integer\n workload:\n description: Workload reference Info.\n properties:\n apiVersion:\n description: ApiVersion defines the versioned schema of\n an object.\n type: string\n kind:\n description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"'\n type: string\n name:\n description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names'\n type: string\n namespace:\n description: Namespace of the workload\n type: string\n selector:\n description: Selector defines how to find Pods related to\n the Workload (key = workload). (e.g., workload=w1)\n type: string\n required:\n - kind\n - name\n - selector\n type: object\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: scheduler-config\n namespace: {{ scheduler_plugins_namespace }}\ndata:\n scheduler-config.yaml: |\n apiVersion: kubescheduler.config.k8s.io/v1\n kind: KubeSchedulerConfiguration\n leaderElection:\n leaderElect: {{ scheduler_plugins_scheduler_leader_elect | bool | lower }}\n profiles:\n # Compose all plugins in one profile\n - schedulerName: scheduler-plugins-scheduler\n plugins:\n multiPoint:\n enabled:\n{% for enabeld_plugin in scheduler_plugins_enabled_plugins %}\n - name: {{ enabeld_plugin }}\n{% endfor %}\n disabled:\n{% for disabled_plugin in scheduler_plugins_disabled_plugins %}\n - name: {{ disabled_plugin }}\n{% endfor %}\n{% if scheduler_plugins_plugin_config is defined and scheduler_plugins_plugin_config | length != 0 %}\n pluginConfig:\n{{ scheduler_plugins_plugin_config | to_nice_yaml(indent=2, width=256) | indent(6, true) }}\n{% endif %}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2", "content": "kind: Deployment\napiVersion: apps/v1\nmetadata:\n name: scheduler-plugins-controller\n namespace: {{ scheduler_plugins_namespace }}\n labels:\n app: scheduler-plugins-controller\nspec:\n replicas: {{ scheduler_plugins_controller_replicas }}\n selector:\n matchLabels:\n app: scheduler-plugins-controller\n template:\n metadata:\n labels:\n app: scheduler-plugins-controller\n spec:\n serviceAccountName: scheduler-plugins-controller\n containers:\n - name: scheduler-plugins-controller\n image: {{ scheduler_plugins_controller_image_repo }}:{{ scheduler_plugins_controller_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n labels:\n component: scheduler\n name: scheduler-plugins-scheduler\n namespace: {{ scheduler_plugins_namespace }}\nspec:\n selector:\n matchLabels:\n component: scheduler\n replicas: {{ scheduler_plugins_scheduler_replicas }}\n template:\n metadata:\n labels:\n component: scheduler\n spec:\n serviceAccountName: scheduler-plugins-scheduler\n containers:\n - command:\n - /bin/kube-scheduler\n - --config=/etc/kubernetes/scheduler-config.yaml\n image: {{ scheduler_plugins_scheduler_image_repo }}:{{ scheduler_plugins_scheduler_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n livenessProbe:\n httpGet:\n path: /healthz\n port: 10259\n scheme: HTTPS\n initialDelaySeconds: 15\n name: scheduler-plugins-scheduler\n readinessProbe:\n httpGet:\n path: /healthz\n port: 10259\n scheme: HTTPS\n resources:\n requests:\n cpu: '0.1'\n securityContext:\n privileged: false\n volumeMounts:\n - name: scheduler-config\n mountPath: /etc/kubernetes\n readOnly: true\n hostNetwork: false\n hostPID: false\n volumes:\n - name: scheduler-config\n configMap:\n name: scheduler-config\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ scheduler_plugins_namespace }}\n labels:\n name: {{ scheduler_plugins_namespace }}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: networktopologies.networktopology.diktyo.x-k8s.io\nspec:\n group: networktopology.diktyo.x-k8s.io\n names:\n kind: NetworkTopology\n listKind: NetworkTopologyList\n plural: networktopologies\n shortNames:\n - nt\n singular: networktopology\n scope: Namespaced\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: NetworkTopology defines network costs in the cluster between\n regions and zones\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: NetworkTopologySpec defines the zones and regions of the\n cluster.\n properties:\n configmapName:\n description: ConfigmapName to be used for cost calculation\n type: string\n weights:\n description: The manual defined weights of the cluster\n items:\n description: WeightInfo contains information about all network costs\n for a given algorithm.\n properties:\n name:\n description: Algorithm Name for network cost calculation (e.g.,\n userDefined)\n type: string\n topologyList:\n description: TopologyList owns Costs between origins\n items:\n description: TopologyInfo contains information about network\n costs for a particular Topology Key.\n properties:\n originList:\n description: OriginList for a particular origin.\n items:\n description: OriginInfo contains information about network\n costs for a particular Origin.\n properties:\n costList:\n description: Costs for the particular origin.\n items:\n description: CostInfo contains information about\n networkCosts.\n properties:\n bandwidthAllocated:\n anyOf:\n - type: integer\n - type: string\n description: Bandwidth allocated between origin\n and destination.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n bandwidthCapacity:\n anyOf:\n - type: integer\n - type: string\n description: Bandwidth capacity between origin\n and destination.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n destination:\n description: Name of the destination (e.g.,\n Region Name, Zone Name).\n type: string\n networkCost:\n description: Network Cost between origin and\n destination (e.g., Dijkstra shortest path,\n etc)\n format: int64\n minimum: 0\n type: integer\n required:\n - destination\n - networkCost\n type: object\n type: array\n origin:\n description: Name of the origin (e.g., Region Name,\n Zone Name).\n type: string\n required:\n - origin\n type: object\n type: array\n topologyKey:\n description: Topology key (e.g., \"topology.kubernetes.io/region\",\n \"topology.kubernetes.io/zone\").\n type: string\n required:\n - originList\n - topologyKey\n type: object\n type: array\n required:\n - name\n - topologyList\n type: object\n type: array\n required:\n - configmapName\n - weights\n type: object\n status:\n description: NetworkTopologyStatus defines the observed use.\n properties:\n nodeCount:\n description: The total number of nodes in the cluster\n format: int64\n minimum: 0\n type: integer\n weightCalculationTime:\n description: The calculation time for the weights in the network topology\n CRD\n format: date-time\n type: string\n type: object\n type: object\n served: true\n storage: true\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2", "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: scheduler-plugins-scheduler\nrules:\n- apiGroups: [\"\"]\n resources: [\"namespaces\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\", \"events.k8s.io\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\", \"update\"]\n- apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"create\"]\n- apiGroups: [\"coordination.k8s.io\"]\n resourceNames: [\"kube-scheduler\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"endpoints\"]\n verbs: [\"create\"]\n- apiGroups: [\"\"]\n resourceNames: [\"kube-scheduler\"]\n resources: [\"endpoints\"]\n verbs: [\"get\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n- apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"delete\", \"get\", \"list\", \"watch\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"bindings\", \"pods/binding\"]\n verbs: [\"create\"]\n- apiGroups: [\"\"]\n resources: [\"pods/status\"]\n verbs: [\"patch\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"replicationcontrollers\", \"services\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"apps\", \"extensions\"]\n resources: [\"replicasets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"apps\"]\n resources: [\"statefulsets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"policy\"]\n resources: [\"poddisruptionbudgets\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\", \"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\", \"update\"]\n- apiGroups: [\"authentication.k8s.io\"]\n resources: [\"tokenreviews\"]\n verbs: [\"create\"]\n- apiGroups: [\"authorization.k8s.io\"]\n resources: [\"subjectaccessreviews\"]\n verbs: [\"create\"]\n- apiGroups: [\"storage.k8s.io\"]\n resources: [\"csinodes\", \"storageclasses\" , \"csidrivers\" , \"csistoragecapacities\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"topology.node.k8s.io\"]\n resources: [\"noderesourcetopologies\"]\n verbs: [\"get\", \"list\", \"watch\"]\n# resources need to be updated with the scheduler plugins used\n- apiGroups: [\"scheduling.x-k8s.io\"]\n resources: [\"podgroups\", \"elasticquotas\", \"podgroups/status\", \"elasticquotas/status\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\", \"update\", \"patch\"]\n# for network-aware plugins add the following lines (scheduler-plugins v0.27.8)\n#- apiGroups: [ \"appgroup.diktyo.x-k8s.io\" ]\n# resources: [ \"appgroups\" ]\n# verbs: [ \"get\", \"list\", \"watch\", \"create\", \"delete\", \"update\", \"patch\" ]\n#- apiGroups: [ \"networktopology.diktyo.x-k8s.io\" ]\n# resources: [ \"networktopologies\" ]\n# verbs: [ \"get\", \"list\", \"watch\", \"create\", \"delete\", \"update\", \"patch\" ]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: scheduler-plugins-scheduler\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: scheduler-plugins-scheduler\nsubjects:\n- kind: ServiceAccount\n name: scheduler-plugins-scheduler\n namespace: {{ scheduler_plugins_namespace }}\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: scheduler-plugins-controller\nrules:\n- apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n- apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"create\", \"patch\", \"update\"]\n- apiGroups: [\"\"]\n resources: [\"nodes\"]\n verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n- apiGroups: [\"topology.node.k8s.io\"]\n resources: [\"noderesourcetopologies\"]\n verbs: [\"get\", \"list\", \"watch\"]\n# resources need to be updated with the scheduler plugins used\n- apiGroups: [\"scheduling.x-k8s.io\"]\n resources: [\"podgroups\", \"elasticquotas\", \"podgroups/status\", \"elasticquotas/status\"]\n verbs: [\"get\", \"list\", \"watch\", \"create\", \"delete\", \"update\", \"patch\"]\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: scheduler-plugins-controller\nsubjects:\n- kind: ServiceAccount\n name: scheduler-plugins-controller\n namespace: {{ scheduler_plugins_namespace }}\nroleRef:\n kind: ClusterRole\n name: scheduler-plugins-controller\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: sched-plugins::extension-apiserver-authentication-reader\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: extension-apiserver-authentication-reader\nsubjects:\n- kind: ServiceAccount\n name: scheduler-plugins-scheduler\n namespace: {{ scheduler_plugins_namespace }}\n- kind: ServiceAccount\n name: scheduler-plugins-controller\n namespace: {{ scheduler_plugins_namespace }}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2", "content": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: scheduler-plugins-scheduler\n namespace: {{ scheduler_plugins_namespace }}\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: scheduler-plugins-controller\n namespace: {{ scheduler_plugins_namespace }}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/52\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: elasticquotas.scheduling.x-k8s.io\nspec:\n group: scheduling.x-k8s.io\n names:\n kind: ElasticQuota\n listKind: ElasticQuotaList\n plural: elasticquotas\n shortNames:\n - eq\n - eqs\n singular: elasticquota\n scope: Namespaced\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: ElasticQuota sets elastic quota restrictions per namespace\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: ElasticQuotaSpec defines the Min and Max for Quota.\n properties:\n max:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: Max is the set of desired max limits for each named resource.\n The usage of max is based on the resource configurations of successfully\n scheduled pods.\n type: object\n min:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: Min is the set of desired guaranteed limits for each\n named resource.\n type: object\n type: object\n status:\n description: ElasticQuotaStatus defines the observed use.\n properties:\n used:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: Used is the current observed total usage of the resource\n in the namespace.\n type: object\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/50\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: podgroups.scheduling.x-k8s.io\nspec:\n group: scheduling.x-k8s.io\n names:\n kind: PodGroup\n listKind: PodGroupList\n plural: podgroups\n shortNames:\n - pg\n - pgs\n singular: podgroup\n scope: Namespaced\n versions:\n - name: v1alpha1\n schema:\n openAPIV3Schema:\n description: PodGroup is a collection of Pod; used for batch workload.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: Specification of the desired behavior of the pod group.\n properties:\n minMember:\n description: MinMember defines the minimal number of members/tasks\n to run the pod group; if there's not enough resources to start all\n tasks, the scheduler will not start anyone.\n format: int32\n type: integer\n minResources:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: MinResources defines the minimal resource of members/tasks\n to run the pod group; if there's not enough resources to start all\n tasks, the scheduler will not start anyone.\n type: object\n scheduleTimeoutSeconds:\n description: ScheduleTimeoutSeconds defines the maximal time of members/tasks\n to wait before run the pod group;\n format: int32\n type: integer\n type: object\n status:\n description: Status represents the current information about a pod group.\n This data may not be up to date.\n properties:\n failed:\n description: The number of pods which reached phase Failed.\n format: int32\n type: integer\n occupiedBy:\n description: OccupiedBy marks the workload (e.g., deployment, statefulset)\n UID that occupy the podgroup. It is empty if not initialized.\n type: string\n phase:\n description: Current phase of PodGroup.\n type: string\n running:\n description: The number of actively running pods.\n format: int32\n type: integer\n scheduleStartTime:\n description: ScheduleStartTime of the group\n format: date-time\n type: string\n succeeded:\n description: The number of pods which reached phase Succeeded.\n format: int32\n type: integer\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n" }, { "path": "roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870\n controller-gen.kubebuilder.io/version: v0.11.1\n creationTimestamp: null\n name: noderesourcetopologies.topology.node.k8s.io\nspec:\n group: topology.node.k8s.io\n names:\n kind: NodeResourceTopology\n listKind: NodeResourceTopologyList\n plural: noderesourcetopologies\n shortNames:\n - node-res-topo\n singular: noderesourcetopology\n scope: Cluster\n versions:\n - name: v1alpha2\n schema:\n openAPIV3Schema:\n description: NodeResourceTopology describes node resources and their topology.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n attributes:\n description: AttributeList contains an array of AttributeInfo objects.\n items:\n description: AttributeInfo contains one attribute of a Zone.\n properties:\n name:\n type: string\n value:\n type: string\n required:\n - name\n - value\n type: object\n type: array\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n topologyPolicies:\n description: 'DEPRECATED (to be removed in v1beta1): use top level attributes\n if needed'\n items:\n type: string\n type: array\n zones:\n description: ZoneList contains an array of Zone objects.\n items:\n description: Zone represents a resource topology zone, e.g. socket,\n node, die or core.\n properties:\n attributes:\n description: AttributeList contains an array of AttributeInfo objects.\n items:\n description: AttributeInfo contains one attribute of a Zone.\n properties:\n name:\n type: string\n value:\n type: string\n required:\n - name\n - value\n type: object\n type: array\n costs:\n description: CostList contains an array of CostInfo objects.\n items:\n description: CostInfo describes the cost (or distance) between\n two Zones.\n properties:\n name:\n type: string\n value:\n format: int64\n type: integer\n required:\n - name\n - value\n type: object\n type: array\n name:\n type: string\n parent:\n type: string\n resources:\n description: ResourceInfoList contains an array of ResourceInfo\n objects.\n items:\n description: ResourceInfo contains information about one resource\n type.\n properties:\n allocatable:\n anyOf:\n - type: integer\n - type: string\n description: Allocatable quantity of the resource, corresponding\n to allocatable in node status, i.e. total amount of this\n resource available to be used by pods.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n available:\n anyOf:\n - type: integer\n - type: string\n description: Available is the amount of this resource currently\n available for new (to be scheduled) pods, i.e. Allocatable\n minus the resources reserved by currently running pods.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n capacity:\n anyOf:\n - type: integer\n - type: string\n description: Capacity of the resource, corresponding to capacity\n in node status, i.e. total amount of this resource that\n the node has.\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n name:\n description: Name of the resource.\n type: string\n required:\n - allocatable\n - available\n - capacity\n - name\n type: object\n type: array\n type:\n type: string\n required:\n - name\n - type\n type: object\n type: array\n required:\n - zones\n type: object\n served: true\n storage: true\n" }, { "path": "roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml", "content": "---\nsnapshot_classes:\n - name: cinder-csi-snapshot\n is_default: false\n force_create: true\n deletionPolicy: Delete\n" }, { "path": "roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml", "content": "---\n- name: Kubernetes Snapshots | Copy Cinder CSI Snapshot Class template\n template:\n src: \"cinder-csi-snapshot-class.yml.j2\"\n dest: \"{{ kube_config_dir }}/cinder-csi-snapshot-class.yml\"\n mode: \"0644\"\n register: manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kubernetes Snapshots | Add Cinder CSI Snapshot Class\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/cinder-csi-snapshot-class.yml\"\n state: \"latest\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - manifests.changed\n" }, { "path": "roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2", "content": "{% for class in snapshot_classes %}\n---\nkind: VolumeSnapshotClass\napiVersion: snapshot.storage.k8s.io/v1\nmetadata:\n name: \"{{ class.name }}\"\n annotations:\n storageclass.kubernetes.io/is-default-class: \"{{ class.is_default | default(false) | ternary(\"true\",\"false\") }}\"\ndriver: cinder.csi.openstack.org\ndeletionPolicy: \"{{ class.deletionPolicy | default(\"Delete\") }}\"\nparameters:\n force-create: \"{{ class.force_create }}\"\n{% endfor %}\n" }, { "path": "roles/kubernetes-apps/snapshots/meta/main.yml", "content": "---\ndependencies:\n - role: kubernetes-apps/snapshots/snapshot-controller\n when:\n - cinder_csi_enabled or csi_snapshot_controller_enabled\n tags:\n - snapshot-controller\n\n - role: kubernetes-apps/snapshots/cinder-csi\n when:\n - cinder_csi_enabled\n tags:\n - snapshot\n - cinder-csi-driver\n" }, { "path": "roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml", "content": "---\nsnapshot_controller_replicas: 1\nsnapshot_controller_namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml", "content": "---\n- name: Check if snapshot namespace exists\n register: snapshot_namespace_exists\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n name: \"{{ snapshot_controller_namespace }}\"\n resource: \"namespace\"\n state: \"exists\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n tags: snapshot-controller\n\n- name: Snapshot Controller | Generate Manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: snapshot-ns, file: snapshot-ns.yml, apply: not snapshot_namespace_exists}\n - {name: rbac-snapshot-controller, file: rbac-snapshot-controller.yml}\n - {name: snapshot-controller, file: snapshot-controller.yml}\n register: snapshot_controller_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - item.apply | default(True) | bool\n tags: snapshot-controller\n\n- name: Snapshot Controller | Apply Manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ snapshot_controller_manifests.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n tags: snapshot-controller\n" }, { "path": "roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2", "content": "# RBAC file for the snapshot controller.\n#\n# The snapshot controller implements the control loop for CSI snapshot functionality.\n# It should be installed as part of the base Kubernetes distribution in an appropriate\n# namespace for components implementing base system functionality. For installing with\n# Vanilla Kubernetes, kube-system makes sense for the namespace.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: snapshot-controller\n namespace: {{ snapshot_controller_namespace }}\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: snapshot-controller-runner\nrules:\n - apiGroups: [\"\"]\n resources: [\"persistentvolumes\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"\"]\n resources: [\"persistentvolumeclaims\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\"]\n - apiGroups: [\"\"]\n resources: [\"events\"]\n verbs: [\"list\", \"watch\", \"create\", \"update\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\", \"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshotcontents/status\"]\n verbs: [\"patch\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\", \"delete\"]\n - apiGroups: [\"snapshot.storage.k8s.io\"]\n resources: [\"volumesnapshots/status\"]\n verbs: [\"update\", \"patch\"]\n\n - apiGroups: [\"groupsnapshot.storage.k8s.io\"]\n resources: [\"volumegroupsnapshotclasses\"]\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"groupsnapshot.storage.k8s.io\"]\n resources: [\"volumegroupsnapshotcontents\"]\n verbs: [\"create\", \"get\", \"list\", \"watch\", \"update\", \"delete\", \"patch\"]\n - apiGroups: [\"groupsnapshot.storage.k8s.io\"]\n resources: [\"volumegroupsnapshotcontents/status\"]\n verbs: [\"patch\"]\n - apiGroups: [\"groupsnapshot.storage.k8s.io\"]\n resources: [\"volumegroupsnapshots\"]\n verbs: [\"get\", \"list\", \"watch\", \"update\", \"patch\"]\n - apiGroups: [\"groupsnapshot.storage.k8s.io\"]\n resources: [\"volumegroupsnapshots/status\"]\n verbs: [\"update\", \"patch\"]\n\n # Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true\n # - apiGroups: [\"\"]\n # resources: [\"nodes\"]\n # verbs: [\"get\", \"list\", \"watch\"]\n\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: snapshot-controller-role\nsubjects:\n - kind: ServiceAccount\n name: snapshot-controller\n namespace: {{ snapshot_controller_namespace }}\nroleRef:\n kind: ClusterRole\n name: snapshot-controller-runner\n apiGroup: rbac.authorization.k8s.io\n\n---\nkind: Role\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: snapshot-controller-leaderelection\n namespace: {{ snapshot_controller_namespace }}\nrules:\n - apiGroups: [\"coordination.k8s.io\"]\n resources: [\"leases\"]\n verbs: [\"get\", \"watch\", \"list\", \"delete\", \"update\", \"create\"]\n\n---\nkind: RoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: snapshot-controller-leaderelection\n namespace: {{ snapshot_controller_namespace }}\nsubjects:\n - kind: ServiceAccount\n name: snapshot-controller\n namespace: {{ snapshot_controller_namespace }}\nroleRef:\n kind: Role\n name: snapshot-controller-leaderelection\n apiGroup: rbac.authorization.k8s.io\n" }, { "path": "roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2", "content": "# This YAML file shows how to deploy the snapshot controller\n\n# The snapshot controller implements the control loop for CSI snapshot functionality.\n# It should be installed as part of the base Kubernetes distribution in an appropriate\n# namespace for components implementing base system functionality. For installing with\n# Vanilla Kubernetes, kube-system makes sense for the namespace.\n\n---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: snapshot-controller\n namespace: {{ snapshot_controller_namespace }}\nspec:\n replicas: {{ snapshot_controller_replicas }}\n selector:\n matchLabels:\n app.kubernetes.io/name: snapshot-controller\n # The snapshot controller won't be marked as ready if the v1 CRDs are unavailable.\n # The flag --retry-crd-interval-max is used to determine how long the controller\n # will wait for the CRDs to become available before exiting. The default is 30 seconds\n # so minReadySeconds should be set slightly higher than the flag value.\n minReadySeconds: 35\n strategy:\n rollingUpdate:\n maxSurge: 0\n maxUnavailable: 1\n type: RollingUpdate\n template:\n metadata:\n labels:\n app.kubernetes.io/name: snapshot-controller\n spec:\n serviceAccountName: snapshot-controller\n containers:\n - name: snapshot-controller\n image: {{ snapshot_controller_image_repo }}:{{ snapshot_controller_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - \"--v=5\"\n - \"--leader-election={{ 'true' if snapshot_controller_replicas > 1 else 'false' }}\"\n" }, { "path": "roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-ns.yml.j2", "content": "---\napiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ snapshot_controller_namespace }}\n labels:\n name: {{ snapshot_controller_namespace }}\n" }, { "path": "roles/kubernetes-apps/utils/defaults/main.yml", "content": "---\nk8s_namespace: kube-system\n" }, { "path": "roles/kubernetes-apps/utils/vars/main.yml", "content": "---\n_kubectl_apply_stdin:\n- \"{{ kubectl }}\"\n- apply\n- -f\n- \"-\"\n- -n\n- \"{{ k8s_namespace }}\"\n- --server-side=\"{{ server_side_apply | lower }}\"\n# TODO: switch to default SSA\nserver_side_apply: false\nkubectl_apply_stdin: \"{{ _kubectl_apply_stdin | join(' ') }}\"\n" }, { "path": "roles/kubespray-defaults/tasks/main.yml", "content": "---\n- name: Warn for usage of deprecated role\n fail:\n msg: kubespray-defaults is deprecated, switch to kubespray_defaults\n ignore_errors: true # noqa ignore-errors\n run_once: true\n\n- name: Compat for direct role import\n import_role:\n name: kubespray_defaults\n" }, { "path": "roles/kubespray_defaults/defaults/main/download.yml", "content": "---\nlocal_release_dir: /tmp/releases\ndownload_cache_dir: /tmp/kubespray_cache\n\n# If this is true, debug information will be displayed but\n# may contain some private data, so it is recommended to set it to false\n# in the production environment.\n# false by default, unless we're running in CI. (CI_PROJECT_URL should be globally unique even if kubespray happens to run\n# in gitlab-ci in other contexts\nunsafe_show_logs: \"{{ lookup('env', 'CI_PROJECT_URL') == 'https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray' }}\"\n\n# do not delete remote cache files after using them\n# NOTE: Setting this parameter to TRUE is only really useful when developing kubespray\ndownload_keep_remote_cache: false\n\n# Only useful when download_run_once is false: Localy cached files and images are\n# uploaded to kubernetes nodes. Also, images downloaded on those nodes are copied\n# back to the ansible runner's cache, if they are not yet preset.\ndownload_force_cache: false\n\n# Used to only evaluate vars from download role\nskip_downloads: false\n\n# Optionally skip kubeadm images download\nskip_kubeadm_images: false\nkubeadm_images: {}\n\n# if this is set to true will only download files once. Doesn't work\n# on Flatcar Container Linux by Kinvolk unless the download_localhost is true and localhost\n# is running another OS type. Default compress level is 1 (fastest).\ndownload_run_once: false\ndownload_compress: 1\n\n# if this is set to true will download container\ndownload_container: true\n\n# if this is set to true, uses the localhost for download_run_once mode\n# (requires docker and sudo to access docker). You may want this option for\n# local caching of docker images or for Flatcar Container Linux by Kinvolk cluster nodes.\n# Otherwise, uses the first node in the kube_control_plane group to store images\n# in the download_run_once mode.\ndownload_localhost: false\n\n# Always pull images if set to True. Otherwise check by the repo's tag/digest.\ndownload_always_pull: false\n\n# Some problems may occur when downloading files over https proxy due to ansible bug\n# https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable\n# SSL validation of get_url module. Note that kubespray will still be performing checksum validation.\ndownload_validate_certs: true\n\n# Use the first kube_control_plane if download_localhost is not set\ndownload_delegate: \"{% if download_localhost %}localhost{% else %}{{ groups['kube_control_plane'][0] }}{% endif %}\"\n\n# Allow control the times of download retries for files and containers\ndownload_retries: 4\n\n# The docker_image_info_command might seems weird but we are using raw/endraw and `{{ `{{` }}` to manage the double jinja2 processing\ndocker_image_pull_command: \"{{ docker_bin_dir }}/docker pull\"\ndocker_image_info_command: \"{{ docker_bin_dir }}/docker images -q | xargs -i {{ '{{' }} docker_bin_dir }}/docker inspect -f {% raw %}'{{ '{{' }} if .RepoTags }}{{ '{{' }} join .RepoTags \\\",\\\" }}{{ '{{' }} end }}{{ '{{' }} if .RepoDigests }},{{ '{{' }} join .RepoDigests \\\",\\\" }}{{ '{{' }} end }}' {% endraw %} {} | tr '\\n' ','\"\nnerdctl_image_info_command: \"{{ bin_dir }}/nerdctl -n k8s.io images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' 2>/dev/null | grep -v ^:$ | tr '\\n' ','\"\nnerdctl_image_pull_command: \"{{ bin_dir }}/nerdctl -n k8s.io pull --quiet\"\ncrictl_image_info_command: \"{{ bin_dir }}/crictl images --verbose | awk -F ': ' '/RepoTags|RepoDigests/ {print $2}' | tr '\\n' ','\"\ncrictl_image_pull_command: \"{{ bin_dir }}/crictl pull\"\n\nimage_command_tool: \"{%- if container_manager == 'containerd' -%}nerdctl{%- elif container_manager == 'crio' -%}crictl{%- else -%}{{ container_manager }}{%- endif -%}\"\nimage_command_tool_on_localhost: \"{{ image_command_tool }}\"\n\nimage_pull_command: \"{{ lookup('vars', image_command_tool + '_image_pull_command') }}\"\nimage_info_command: \"{{ lookup('vars', image_command_tool + '_image_info_command') }}\"\nimage_pull_command_on_localhost: \"{{ lookup('vars', image_command_tool_on_localhost + '_image_pull_command') }}\"\nimage_info_command_on_localhost: \"{{ lookup('vars', image_command_tool_on_localhost + '_image_info_command') }}\"\n\n# Arch of Docker images and needed packages\nimage_arch: \"{{ host_architecture | default('amd64') }}\"\n\n# Versions\ncrun_version: \"{{ (crun_checksums['amd64'] | dict2items)[0].key }}\"\nrunc_version: \"{{ (runc_checksums['amd64'] | dict2items)[0].key }}\"\nkata_containers_version: \"{{ (kata_containers_binary_checksums['amd64'] | dict2items)[0].key }}\"\nyouki_version: \"{{ (youki_checksums['amd64'] | dict2items)[0].key }}\"\ngvisor_version: \"{{ (gvisor_runsc_binary_checksums['amd64'] | dict2items)[0].key }}\"\ncontainerd_version: \"{{ (containerd_archive_checksums['amd64'] | dict2items)[0].key }}\"\ncri_dockerd_version: \"{{ (cri_dockerd_archive_checksums['amd64'] | dict2items)[0].key }}\"\nargocd_version: \"{{ (argocd_install_checksums.no_arch | dict2items)[0].key }}\"\n\n# this is relevant when container_manager == 'docker'\ndocker_containerd_version: 1.6.32\n\n# gcr and kubernetes image repo define\ngcr_image_repo: \"gcr.io\"\nkube_image_repo: \"registry.k8s.io\"\nkubeadm_image_repo: \"{{ kube_image_repo }}\"\n\n# docker image repo define\ndocker_image_repo: \"docker.io\"\n\n# quay image repo define\nquay_image_repo: \"quay.io\"\n\n# github image repo define (ex multus only use that)\ngithub_image_repo: \"ghcr.io\"\n\n# TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults\n# after migration to container download\ncalico_version: \"{{ (calicoctl_binary_checksums['amd64'] | dict2items)[0].key }}\"\ncalico_ctl_version: \"{{ calico_version }}\"\ncalico_cni_version: \"{{ calico_version }}\"\ncalico_policy_version: \"{{ calico_version }}\"\ncalico_typha_version: \"{{ calico_version }}\"\ncalico_apiserver_version: \"{{ calico_version }}\"\ntypha_enabled: false\ncalico_apiserver_enabled: false\n\nflannel_version: 0.27.3\nflannel_cni_version: 1.7.1-flannel1\ncni_version: \"{{ (cni_binary_checksums['amd64'] | dict2items)[0].key }}\"\n\ncilium_version: \"1.19.1\"\ncilium_cli_version: \"{{ (ciliumcli_binary_checksums['amd64'] | dict2items)[0].key }}\"\ncilium_enable_hubble: false\n\nkube_ovn_version: \"1.12.21\"\nkube_ovn_dpdk_version: \"19.11-v{{ kube_ovn_version }}\"\nkube_router_version: \"2.1.1\"\nmultus_version: \"4.2.2\"\nhelm_version: \"{{ (helm_archive_checksums['amd64'] | dict2items)[0].key }}\"\nnerdctl_version: \"{{ (nerdctl_archive_checksums['amd64'] | dict2items)[0].key }}\"\nskopeo_version: \"{{ (skopeo_binary_checksums['amd64'] | dict2items)[0].key }}\"\n\npod_infra_version: \"{{ pod_infra_supported_versions[kube_major_version] }}\"\netcd_version: \"{{ etcd_supported_versions[kube_major_version] }}\"\ncrictl_version: \"{{ (crictl_checksums['amd64'].keys() | select('version', kube_major_next_version, '<'))[0] }}\"\ncrio_version: \"{{ (crio_archive_checksums['amd64'].keys() | select('version', kube_major_next_version, '<'))[0] }}\"\n\n# Scheduler plugins doesn't build for K8s 1.29 yet\nscheduler_plugins_supported_versions:\n '1.31': 0\n '1.30': 0\n '1.29': 0\nscheduler_plugins_version: \"{{ scheduler_plugins_supported_versions[kube_major_version] }}\"\n\nyq_version: \"{{ (yq_checksums['amd64'] | dict2items)[0].key }}\"\n\ngateway_api_version: \"{{ (gateway_api_standard_crds_checksums.no_arch | dict2items)[0].key }}\"\ngateway_api_channel: \"standard\"\n\nprometheus_operator_crds_version: \"{{ (prometheus_operator_crds_checksums.no_arch | dict2items)[0].key }}\"\n\ngithub_url: https://github.com\ndl_k8s_io_url: https://dl.k8s.io\nstorage_googleapis_url: https://storage.googleapis.com\nget_helm_url: https://get.helm.sh\n\n# Download URLs\nkubelet_download_url: \"{{ dl_k8s_io_url }}/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet\"\nkubectl_download_url: \"{{ dl_k8s_io_url }}/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl\"\nkubeadm_download_url: \"{{ dl_k8s_io_url }}/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm\"\netcd_download_url: \"{{ github_url }}/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz\"\ncni_download_url: \"{{ github_url }}/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz\"\ncalicoctl_download_url: \"{{ github_url }}/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}\"\ncalico_crds_download_url: \"{{ github_url }}/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml\"\nciliumcli_download_url: \"{{ github_url }}/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz\"\ncrictl_download_url: \"{{ github_url }}/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz\"\ncrio_download_url: \"{{ storage_googleapis_url }}/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz\"\nhelm_download_url: \"{{ get_helm_url }}/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz\"\nrunc_download_url: \"{{ github_url }}/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}\"\ncrun_download_url: \"{{ github_url }}/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}\"\nyouki_download_url: \"{{ github_url }}/youki-dev/youki/releases/download/v{{ youki_version }}/youki-{{ youki_version }}-{{ ansible_architecture }}-gnu.tar.gz\"\nkata_containers_download_url: \"{{ github_url }}/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz\"\n# gVisor only supports amd64 and uses x86_64 to in the download link\ngvisor_runsc_download_url: \"{{ storage_googleapis_url }}/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc\"\ngvisor_containerd_shim_runsc_download_url: \"{{ storage_googleapis_url }}/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1\"\nnerdctl_download_url: \"{{ github_url }}/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz\"\ncontainerd_download_url: \"{{ github_url }}/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ 'static-' if containerd_static_binary }}{{ containerd_version }}-linux-{{ image_arch }}.tar.gz\"\ncri_dockerd_download_url: \"{{ github_url }}/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz\"\nskopeo_download_url: \"{{ github_url }}/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}\"\nyq_download_url: \"{{ github_url }}/mikefarah/yq/releases/download/v{{ yq_version }}/yq_linux_{{ image_arch }}\"\nargocd_install_url: \"https://raw.githubusercontent.com/argoproj/argo-cd/v{{ argocd_version }}/manifests/install.yaml\"\ngateway_api_crds_download_url: \"{{ github_url }}/kubernetes-sigs/gateway-api/releases/download/v{{ gateway_api_version }}/{{ gateway_api_channel }}-install.yaml\"\nprometheus_operator_crds_download_url: \"{{ github_url }}/prometheus-operator/prometheus-operator/releases/download/v{{ prometheus_operator_crds_version }}/stripped-down-crds.yaml\"\n\netcd_binary_checksum: \"{{ etcd_binary_checksums[image_arch][etcd_version] }}\"\ncni_binary_checksum: \"{{ cni_binary_checksums[image_arch][cni_version] }}\"\nkubelet_binary_checksum: \"{{ kubelet_checksums[image_arch][kube_version] }}\"\nkubectl_binary_checksum: \"{{ kubectl_checksums[image_arch][kube_version] }}\"\nkubeadm_binary_checksum: \"{{ kubeadm_checksums[image_arch][kube_version] }}\"\nyq_binary_checksum: \"{{ yq_checksums[image_arch][yq_version] }}\"\nargocd_install_checksum: \"{{ argocd_install_checksums.no_arch[argocd_version] }}\"\ncalicoctl_binary_checksum: \"{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}\"\nciliumcli_binary_checksum: \"{{ ciliumcli_binary_checksums[image_arch][cilium_cli_version] }}\"\ncrictl_binary_checksum: \"{{ crictl_checksums[image_arch][crictl_version] }}\"\ncrio_archive_checksum: \"{{ crio_archive_checksums[image_arch][crio_version] }}\"\ncri_dockerd_archive_checksum: \"{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}\"\nhelm_archive_checksum: \"{{ helm_archive_checksums[image_arch][helm_version] }}\"\nrunc_binary_checksum: \"{{ runc_checksums[image_arch][runc_version] }}\"\ncrun_binary_checksum: \"{{ crun_checksums[image_arch][crun_version] }}\"\nyouki_archive_checksum: \"{{ youki_checksums[image_arch][youki_version] }}\"\nkata_containers_binary_checksum: \"{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}\"\ngvisor_runsc_binary_checksum: \"{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}\"\ngvisor_containerd_shim_binary_checksum: \"{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}\"\nnerdctl_archive_checksum: \"{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}\"\ncontainerd_archive_checksum: \"{{ containerd_archive_checksums[image_arch][containerd_version] }}\"\ncontainerd_static_archive_checksum: \"{{ containerd_static_archive_checksums[image_arch][containerd_version] }}\"\ncontainerd_checksum: \"{{ containerd_static_archive_checksum if containerd_static_binary else containerd_archive_checksum }}\"\nskopeo_binary_checksum: \"{{ skopeo_binary_checksums[image_arch][skopeo_version] }}\"\n\n# Containers\n# In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,\n# it helps a lot for local private development or bare metal environment.\n# So you need define --registry-mirror or --insecure-registry, and modify the following url address.\n# example:\n# You need to deploy kubernetes cluster on local private development.\n# Also provide the address of your own private registry.\n# And use --insecure-registry options for docker\nkube_proxy_image_repo: \"{{ kube_image_repo }}/kube-proxy\"\netcd_image_repo: \"{{ quay_image_repo }}/coreos/etcd\"\netcd_image_tag: \"v{{ etcd_version }}\"\nflannel_image_repo: \"{{ docker_image_repo }}/flannel/flannel\"\nflannel_image_tag: \"v{{ flannel_version }}\"\nflannel_init_image_repo: \"{{ docker_image_repo }}/flannel/flannel-cni-plugin\"\nflannel_init_image_tag: \"v{{ flannel_cni_version }}\"\ncalico_node_image_repo: \"{{ quay_image_repo }}/calico/node\"\ncalico_node_image_tag: \"v{{ calico_version }}\"\ncalico_cni_image_repo: \"{{ quay_image_repo }}/calico/cni\"\ncalico_cni_image_tag: \"v{{ calico_cni_version }}\"\ncalico_policy_image_repo: \"{{ quay_image_repo }}/calico/kube-controllers\"\ncalico_policy_image_tag: \"v{{ calico_policy_version }}\"\ncalico_typha_image_repo: \"{{ quay_image_repo }}/calico/typha\"\ncalico_typha_image_tag: \"v{{ calico_typha_version }}\"\ncalico_apiserver_image_repo: \"{{ quay_image_repo }}/calico/apiserver\"\ncalico_apiserver_image_tag: \"v{{ calico_apiserver_version }}\"\npod_infra_image_repo: \"{{ kube_image_repo }}/pause\"\npod_infra_image_tag: \"{{ pod_infra_version }}\"\ncilium_image_repo: \"{{ quay_image_repo }}/cilium/cilium\"\ncilium_image_tag: \"v{{ cilium_version }}\"\ncilium_operator_image_repo: \"{{ quay_image_repo }}/cilium/operator\"\ncilium_operator_image_tag: \"v{{ cilium_version }}\"\ncilium_hubble_relay_image_repo: \"{{ quay_image_repo }}/cilium/hubble-relay\"\ncilium_hubble_relay_image_tag: \"v{{ cilium_version }}\"\ncilium_hubble_certgen_image_repo: \"{{ quay_image_repo }}/cilium/certgen\"\ncilium_hubble_certgen_image_tag: \"v0.2.4\"\ncilium_hubble_ui_image_repo: \"{{ quay_image_repo }}/cilium/hubble-ui\"\ncilium_hubble_ui_image_tag: \"v0.13.3\"\ncilium_hubble_ui_backend_image_repo: \"{{ quay_image_repo }}/cilium/hubble-ui-backend\"\ncilium_hubble_ui_backend_image_tag: \"v0.13.3\"\ncilium_hubble_envoy_image_repo: \"{{ quay_image_repo }}/cilium/cilium-envoy\"\ncilium_hubble_envoy_image_tag: \"v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5\"\nkube_ovn_container_image_repo: \"{{ docker_image_repo }}/kubeovn/kube-ovn\"\nkube_ovn_container_image_tag: \"v{{ kube_ovn_version }}\"\nkube_ovn_vpc_container_image_repo: \"{{ docker_image_repo }}/kubeovn/vpc-nat-gateway\"\nkube_ovn_vpc_container_image_tag: \"v{{ kube_ovn_version }}\"\nkube_ovn_dpdk_container_image_repo: \"{{ docker_image_repo }}/kubeovn/kube-ovn-dpdk\"\nkube_ovn_dpdk_container_image_tag: \"{{ kube_ovn_dpdk_version }}\"\nkube_router_image_repo: \"{{ docker_image_repo }}/cloudnativelabs/kube-router\"\nkube_router_image_tag: \"v{{ kube_router_version }}\"\nmultus_image_repo: \"{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni\"\nmultus_image_tag: \"v{{ multus_version }}\"\nexternal_openstack_cloud_controller_image_repo: \"{{ kube_image_repo }}/provider-os/openstack-cloud-controller-manager\"\nexternal_openstack_cloud_controller_image_tag: \"v1.35.0\"\n\nkube_vip_version: 1.0.3\nkube_vip_image_repo: \"{{ github_image_repo }}/kube-vip/kube-vip{{ '-iptables' if kube_vip_lb_fwdmethod == 'masquerade' else '' }}\"\nkube_vip_image_tag: \"v{{ kube_vip_version }}\"\nnginx_image_repo: \"{{ docker_image_repo }}/library/nginx\"\nnginx_image_tag: 1.28.2-alpine\nhaproxy_image_repo: \"{{ docker_image_repo }}/library/haproxy\"\nhaproxy_image_tag: 3.2.13-alpine\n\n# Coredns version should be supported by corefile-migration (or at least work with)\n# bundle with kubeadm; if not 'basic' upgrade can sometimes fail\n\ncoredns_supported_versions:\n '1.35': 1.12.4\n '1.34': 1.12.1\n '1.33': 1.12.0\ncoredns_version: \"{{ coredns_supported_versions[kube_major_version] }}\"\ncoredns_image_repo: \"{{ kube_image_repo }}{{ '/coredns' if coredns_version is version('1.7.1', '>=') else '' }}/coredns\"\ncoredns_image_tag: \"{{ 'v' if coredns_version is version('1.7.1', '>=') else '' }}{{ coredns_version }}\"\n\nnodelocaldns_version: \"1.25.0\"\nnodelocaldns_image_repo: \"{{ kube_image_repo }}/dns/k8s-dns-node-cache\"\nnodelocaldns_image_tag: \"{{ nodelocaldns_version }}\"\n\ndnsautoscaler_version: 1.8.8\ndnsautoscaler_image_repo: \"{{ kube_image_repo }}/cpa/cluster-proportional-autoscaler\"\ndnsautoscaler_image_tag: \"v{{ dnsautoscaler_version }}\"\n\nscheduler_plugins_controller_image_repo: \"{{ kube_image_repo }}/scheduler-plugins/controller\"\nscheduler_plugins_controller_image_tag: \"v{{ scheduler_plugins_version }}\"\nscheduler_plugins_scheduler_image_repo: \"{{ kube_image_repo }}/scheduler-plugins/kube-scheduler\"\nscheduler_plugins_scheduler_image_tag: \"v{{ scheduler_plugins_version }}\"\n\nregistry_version: \"2.8.1\"\nregistry_image_repo: \"{{ docker_image_repo }}/library/registry\"\nregistry_image_tag: \"{{ registry_version }}\"\nmetrics_server_version: \"0.8.0\"\nmetrics_server_image_repo: \"{{ kube_image_repo }}/metrics-server/metrics-server\"\nmetrics_server_image_tag: \"v{{ metrics_server_version }}\"\nlocal_volume_provisioner_version: \"2.5.0\"\nlocal_volume_provisioner_image_repo: \"{{ kube_image_repo }}/sig-storage/local-volume-provisioner\"\nlocal_volume_provisioner_image_tag: \"v{{ local_volume_provisioner_version }}\"\nlocal_path_provisioner_version: \"0.0.32\"\nlocal_path_provisioner_image_repo: \"{{ docker_image_repo }}/rancher/local-path-provisioner\"\nlocal_path_provisioner_image_tag: \"v{{ local_path_provisioner_version }}\"\nalb_ingress_image_repo: \"{{ docker_image_repo }}/amazon/aws-alb-ingress-controller\"\nalb_ingress_image_tag: \"v1.1.9\"\ncert_manager_version: \"1.15.3\"\ncert_manager_controller_image_repo: \"{{ quay_image_repo }}/jetstack/cert-manager-controller\"\ncert_manager_controller_image_tag: \"v{{ cert_manager_version }}\"\ncert_manager_cainjector_image_repo: \"{{ quay_image_repo }}/jetstack/cert-manager-cainjector\"\ncert_manager_cainjector_image_tag: \"v{{ cert_manager_version }}\"\ncert_manager_webhook_image_repo: \"{{ quay_image_repo }}/jetstack/cert-manager-webhook\"\ncert_manager_webhook_image_tag: \"v{{ cert_manager_version }}\"\n\ncsi_attacher_image_repo: \"{{ kube_image_repo }}/sig-storage/csi-attacher\"\ncsi_attacher_image_tag: \"v4.4.2\"\ncsi_provisioner_image_repo: \"{{ kube_image_repo }}/sig-storage/csi-provisioner\"\ncsi_provisioner_image_tag: \"v3.6.2\"\ncsi_snapshotter_image_repo: \"{{ kube_image_repo }}/sig-storage/csi-snapshotter\"\ncsi_snapshotter_image_tag: \"v6.3.2\"\ncsi_resizer_image_repo: \"{{ kube_image_repo }}/sig-storage/csi-resizer\"\ncsi_resizer_image_tag: \"v1.9.2\"\ncsi_node_driver_registrar_image_repo: \"{{ kube_image_repo }}/sig-storage/csi-node-driver-registrar\"\ncsi_node_driver_registrar_image_tag: \"v2.4.0\"\ncsi_livenessprobe_image_repo: \"{{ kube_image_repo }}/sig-storage/livenessprobe\"\ncsi_livenessprobe_image_tag: \"v2.11.0\"\n\nsnapshot_controller_supported_versions:\n '1.35': \"v7.0.2\"\n '1.34': \"v7.0.2\"\n '1.33': \"v7.0.2\"\nsnapshot_controller_image_repo: \"{{ kube_image_repo }}/sig-storage/snapshot-controller\"\nsnapshot_controller_image_tag: \"{{ snapshot_controller_supported_versions[kube_major_version] }}\"\n\ncinder_csi_plugin_version: \"1.30.0\"\ncinder_csi_plugin_image_repo: \"{{ kube_image_repo }}/provider-os/cinder-csi-plugin\"\ncinder_csi_plugin_image_tag: \"v{{ cinder_csi_plugin_version }}\"\n\naws_ebs_csi_plugin_version: \"0.5.0\"\naws_ebs_csi_plugin_image_repo: \"{{ docker_image_repo }}/amazon/aws-ebs-csi-driver\"\naws_ebs_csi_plugin_image_tag: \"v{{ aws_ebs_csi_plugin_version }}\"\n\ngcp_pd_csi_plugin_version: \"1.9.2\"\ngcp_pd_csi_plugin_image_repo: \"{{ kube_image_repo }}/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver\"\ngcp_pd_csi_plugin_image_tag: \"v{{ gcp_pd_csi_plugin_version }}\"\n\nazure_csi_image_repo: \"mcr.microsoft.com/oss/kubernetes-csi\"\nazure_csi_provisioner_image_tag: \"v2.2.2\"\nazure_csi_attacher_image_tag: \"v3.3.0\"\nazure_csi_resizer_image_tag: \"v1.3.0\"\nazure_csi_livenessprobe_image_tag: \"v2.5.0\"\nazure_csi_node_registrar_image_tag: \"v2.4.0\"\nazure_csi_snapshotter_image_tag: \"v3.0.3\"\nazure_csi_plugin_version: \"1.10.0\"\nazure_csi_plugin_image_repo: \"mcr.microsoft.com/k8s/csi\"\nazure_csi_plugin_image_tag: \"v{{ azure_csi_plugin_version }}\"\n\ngcp_pd_csi_image_repo: \"gke.gcr.io\"\ngcp_pd_csi_driver_image_tag: \"v0.7.0-gke.0\"\ngcp_pd_csi_provisioner_image_tag: \"v1.5.0-gke.0\"\ngcp_pd_csi_attacher_image_tag: \"v2.1.1-gke.0\"\ngcp_pd_csi_resizer_image_tag: \"v0.4.0-gke.0\"\ngcp_pd_csi_registrar_image_tag: \"v1.2.0-gke.0\"\n\nmetallb_speaker_image_repo: \"{{ quay_image_repo }}/metallb/speaker\"\nmetallb_controller_image_repo: \"{{ quay_image_repo }}/metallb/controller\"\nmetallb_version: 0.13.9\nmetallb_image_tag: \"v{{ metallb_version }}\"\n\nnode_feature_discovery_version: 0.16.4\nnode_feature_discovery_image_repo: \"{{ kube_image_repo }}/nfd/node-feature-discovery\"\nnode_feature_discovery_image_tag: \"v{{ node_feature_discovery_version }}\"\n\ndownloads:\n etcd:\n container: \"{{ etcd_deployment_type != 'host' }}\"\n file: \"{{ etcd_deployment_type == 'host' }}\"\n enabled: true\n dest: \"{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz\"\n repo: \"{{ etcd_image_repo }}\"\n tag: \"{{ etcd_image_tag }}\"\n checksum: >-\n {{ etcd_binary_checksum if (etcd_deployment_type == 'host')\n else etcd_digest_checksum | d(None) }}\n url: \"{{ etcd_download_url }}\"\n unarchive: \"{{ etcd_deployment_type == 'host' }}\"\n owner: \"root\"\n mode: \"0755\"\n groups:\n - etcd\n\n cni:\n enabled: true\n file: true\n dest: \"{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz\"\n checksum: \"{{ cni_binary_checksum }}\"\n url: \"{{ cni_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n kubeadm:\n enabled: true\n file: true\n dest: \"{{ local_release_dir }}/kubeadm-{{ kube_version }}-{{ image_arch }}\"\n checksum: \"{{ kubeadm_binary_checksum }}\"\n url: \"{{ kubeadm_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n kubelet:\n enabled: true\n file: true\n dest: \"{{ local_release_dir }}/kubelet-{{ kube_version }}-{{ image_arch }}\"\n checksum: \"{{ kubelet_binary_checksum }}\"\n url: \"{{ kubelet_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n kubectl:\n enabled: true\n file: true\n dest: \"{{ local_release_dir }}/kubectl-{{ kube_version }}-{{ image_arch }}\"\n checksum: \"{{ kubectl_binary_checksum }}\"\n url: \"{{ kubectl_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n crictl:\n file: true\n enabled: true\n dest: \"{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz\"\n checksum: \"{{ crictl_binary_checksum }}\"\n url: \"{{ crictl_download_url }}\"\n unarchive: true\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n crio:\n file: true\n enabled: \"{{ container_manager == 'crio' }}\"\n dest: \"{{ local_release_dir }}/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz\"\n checksum: \"{{ crio_archive_checksum }}\"\n url: \"{{ crio_download_url }}\"\n unarchive: true\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n cri_dockerd:\n file: true\n enabled: \"{{ container_manager == 'docker' }}\"\n dest: \"{{ local_release_dir }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tar.gz\"\n checksum: \"{{ cri_dockerd_archive_checksum }}\"\n url: \"{{ cri_dockerd_download_url }}\"\n unarchive: true\n unarchive_extra_opts:\n - --strip=1\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n crun:\n file: true\n enabled: \"{{ crun_enabled }}\"\n dest: \"{{ local_release_dir }}/crun-{{ crun_version }}-{{ image_arch }}\"\n checksum: \"{{ crun_binary_checksum }}\"\n url: \"{{ crun_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n youki:\n file: true\n enabled: \"{{ youki_enabled }}\"\n dest: \"{{ local_release_dir }}/youki-{{ youki_version }}-{{ ansible_architecture }}.tar.gz\"\n checksum: \"{{ youki_archive_checksum }}\"\n url: \"{{ youki_download_url }}\"\n unarchive: true\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n runc:\n file: true\n enabled: \"{{ container_manager == 'containerd' }}\"\n dest: \"{{ local_release_dir }}/runc-{{ runc_version }}.{{ image_arch }}\"\n checksum: \"{{ runc_binary_checksum }}\"\n url: \"{{ runc_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n kata_containers:\n enabled: \"{{ kata_containers_enabled }}\"\n file: true\n dest: \"{{ local_release_dir }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz\"\n checksum: \"{{ kata_containers_binary_checksum }}\"\n url: \"{{ kata_containers_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n containerd:\n enabled: \"{{ container_manager == 'containerd' }}\"\n file: true\n dest: \"{{ local_release_dir }}/containerd-{{ 'static-' if containerd_static_binary }}{{ containerd_version }}-linux-{{ image_arch }}.tar.gz\"\n checksum: \"{{ containerd_checksum }}\"\n url: \"{{ containerd_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n gvisor_runsc:\n enabled: \"{{ gvisor_enabled }}\"\n file: true\n dest: \"{{ local_release_dir }}/gvisor-runsc-{{ gvisor_version }}-{{ ansible_architecture }}\"\n checksum: \"{{ gvisor_runsc_binary_checksum }}\"\n url: \"{{ gvisor_runsc_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: 755\n groups:\n - k8s_cluster\n\n gvisor_containerd_shim:\n enabled: \"{{ gvisor_enabled }}\"\n file: true\n dest: \"{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1-{{ gvisor_version }}-{{ ansible_architecture }}\"\n checksum: \"{{ gvisor_containerd_shim_binary_checksum }}\"\n url: \"{{ gvisor_containerd_shim_runsc_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: 755\n groups:\n - k8s_cluster\n\n nerdctl:\n file: true\n enabled: \"{{ container_manager == 'containerd' }}\"\n dest: \"{{ local_release_dir }}/nerdctl-{{ nerdctl_version }}-linux-{{ image_arch }}.tar.gz\"\n checksum: \"{{ nerdctl_archive_checksum }}\"\n url: \"{{ nerdctl_download_url }}\"\n unarchive: true\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n skopeo:\n file: true\n enabled: \"{{ container_manager == 'crio' }}\"\n dest: \"{{ local_release_dir }}/skopeo-{{ skopeo_version }}-{{ image_arch }}\"\n checksum: \"{{ skopeo_binary_checksum }}\"\n url: \"{{ skopeo_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n cilium:\n enabled: \"{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally }}\"\n container: true\n repo: \"{{ cilium_image_repo }}\"\n tag: \"{{ cilium_image_tag }}\"\n checksum: \"{{ cilium_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n cilium_operator:\n enabled: \"{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally }}\"\n container: true\n repo: \"{{ cilium_operator_image_repo }}\"\n tag: \"{{ cilium_operator_image_tag }}\"\n checksum: \"{{ cilium_operator_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n cilium_hubble_relay:\n enabled: \"{{ cilium_enable_hubble }}\"\n container: true\n repo: \"{{ cilium_hubble_relay_image_repo }}\"\n tag: \"{{ cilium_hubble_relay_image_tag }}\"\n checksum: \"{{ cilium_hubble_relay_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n cilium_hubble_certgen:\n enabled: \"{{ cilium_enable_hubble }}\"\n container: true\n repo: \"{{ cilium_hubble_certgen_image_repo }}\"\n tag: \"{{ cilium_hubble_certgen_image_tag }}\"\n checksum: \"{{ cilium_hubble_certgen_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n cilium_hubble_ui:\n enabled: \"{{ cilium_enable_hubble }}\"\n container: true\n repo: \"{{ cilium_hubble_ui_image_repo }}\"\n tag: \"{{ cilium_hubble_ui_image_tag }}\"\n checksum: \"{{ cilium_hubble_ui_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n cilium_hubble_ui_backend:\n enabled: \"{{ cilium_enable_hubble }}\"\n container: true\n repo: \"{{ cilium_hubble_ui_backend_image_repo }}\"\n tag: \"{{ cilium_hubble_ui_backend_image_tag }}\"\n checksum: \"{{ cilium_hubble_ui_backend_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n cilium_hubble_envoy:\n enabled: \"{{ cilium_enable_hubble }}\"\n container: true\n repo: \"{{ cilium_hubble_envoy_image_repo }}\"\n tag: \"{{ cilium_hubble_envoy_image_tag }}\"\n checksum: \"{{ cilium_hubble_envoy_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n ciliumcli:\n enabled: \"{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally }}\"\n file: true\n dest: \"{{ local_release_dir }}/cilium-{{ cilium_cli_version }}-{{ image_arch }}.tar.gz\"\n checksum: \"{{ ciliumcli_binary_checksum }}\"\n url: \"{{ ciliumcli_download_url }}\"\n unarchive: true\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n multus:\n enabled: \"{{ kube_network_plugin_multus }}\"\n container: true\n repo: \"{{ multus_image_repo }}\"\n tag: \"{{ multus_image_tag }}\"\n checksum: \"{{ multus_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n flannel:\n enabled: \"{{ kube_network_plugin == 'flannel' }}\"\n container: true\n repo: \"{{ flannel_image_repo }}\"\n tag: \"{{ flannel_image_tag }}\"\n checksum: \"{{ flannel_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n flannel_init:\n enabled: \"{{ kube_network_plugin == 'flannel' }}\"\n container: true\n repo: \"{{ flannel_init_image_repo }}\"\n tag: \"{{ flannel_init_image_tag }}\"\n checksum: \"{{ flannel_init_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n calicoctl:\n enabled: \"{{ kube_network_plugin == 'calico' }}\"\n file: true\n dest: \"{{ local_release_dir }}/calicoctl-{{ calico_ctl_version }}-{{ image_arch }}\"\n checksum: \"{{ calicoctl_binary_checksum }}\"\n url: \"{{ calicoctl_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - k8s_cluster\n\n calico_node:\n enabled: \"{{ kube_network_plugin == 'calico' }}\"\n container: true\n repo: \"{{ calico_node_image_repo }}\"\n tag: \"{{ calico_node_image_tag }}\"\n checksum: \"{{ calico_node_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n calico_cni:\n enabled: \"{{ kube_network_plugin == 'calico' }}\"\n container: true\n repo: \"{{ calico_cni_image_repo }}\"\n tag: \"{{ calico_cni_image_tag }}\"\n checksum: \"{{ calico_cni_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n calico_policy:\n enabled: \"{{ enable_network_policy and kube_network_plugin in ['calico'] }}\"\n container: true\n repo: \"{{ calico_policy_image_repo }}\"\n tag: \"{{ calico_policy_image_tag }}\"\n checksum: \"{{ calico_policy_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n calico_typha:\n enabled: \"{{ typha_enabled }}\"\n container: true\n repo: \"{{ calico_typha_image_repo }}\"\n tag: \"{{ calico_typha_image_tag }}\"\n checksum: \"{{ calico_typha_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n calico_apiserver:\n enabled: \"{{ calico_apiserver_enabled }}\"\n container: true\n repo: \"{{ calico_apiserver_image_repo }}\"\n tag: \"{{ calico_apiserver_image_tag }}\"\n checksum: \"{{ calico_apiserver_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n calico_crds:\n file: true\n enabled: \"{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}\"\n dest: \"{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/crds.yaml\"\n checksum: \"{{ calico_crds_checksums.no_arch[calico_version] }}\"\n url: \"{{ calico_crds_download_url }}\"\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n kube_ovn:\n enabled: \"{{ kube_network_plugin == 'kube-ovn' }}\"\n container: true\n repo: \"{{ kube_ovn_container_image_repo }}\"\n tag: \"{{ kube_ovn_container_image_tag }}\"\n checksum: \"{{ kube_ovn_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n kube_router:\n enabled: \"{{ kube_network_plugin == 'kube-router' }}\"\n container: true\n repo: \"{{ kube_router_image_repo }}\"\n tag: \"{{ kube_router_image_tag }}\"\n checksum: \"{{ kube_router_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n pod_infra:\n enabled: true\n container: true\n repo: \"{{ pod_infra_image_repo }}\"\n tag: \"{{ pod_infra_image_tag }}\"\n checksum: \"{{ pod_infra_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n kube-vip:\n enabled: \"{{ kube_vip_enabled }}\"\n container: true\n repo: \"{{ kube_vip_image_repo }}\"\n tag: \"{{ kube_vip_image_tag }}\"\n checksum: \"{{ kube_vip_digest_checksum | default(None) }}\"\n groups:\n - kube_control_plane\n\n nginx:\n enabled: \"{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'nginx' }}\"\n container: true\n repo: \"{{ nginx_image_repo }}\"\n tag: \"{{ nginx_image_tag }}\"\n checksum: \"{{ nginx_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n haproxy:\n enabled: \"{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'haproxy' }}\"\n container: true\n repo: \"{{ haproxy_image_repo }}\"\n tag: \"{{ haproxy_image_tag }}\"\n checksum: \"{{ haproxy_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n coredns:\n enabled: \"{{ dns_mode in ['coredns', 'coredns_dual'] }}\"\n container: true\n repo: \"{{ coredns_image_repo }}\"\n tag: \"{{ coredns_image_tag }}\"\n checksum: \"{{ coredns_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n nodelocaldns:\n enabled: \"{{ enable_nodelocaldns }}\"\n container: true\n repo: \"{{ nodelocaldns_image_repo }}\"\n tag: \"{{ nodelocaldns_image_tag }}\"\n checksum: \"{{ nodelocaldns_digest_checksum | default(None) }}\"\n groups:\n - k8s_cluster\n\n dnsautoscaler:\n enabled: \"{{ dns_mode in ['coredns', 'coredns_dual'] and enable_dns_autoscaler }}\"\n container: true\n repo: \"{{ dnsautoscaler_image_repo }}\"\n tag: \"{{ dnsautoscaler_image_tag }}\"\n checksum: \"{{ dnsautoscaler_digest_checksum | default(None) }}\"\n groups:\n - kube_control_plane\n\n helm:\n enabled: \"{{ helm_enabled }}\"\n file: true\n dest: \"{{ local_release_dir }}/helm-{{ helm_version }}/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz\"\n checksum: \"{{ helm_archive_checksum }}\"\n url: \"{{ helm_download_url }}\"\n unarchive: true\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n registry:\n enabled: \"{{ registry_enabled }}\"\n container: true\n repo: \"{{ registry_image_repo }}\"\n tag: \"{{ registry_image_tag }}\"\n checksum: \"{{ registry_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n metrics_server:\n enabled: \"{{ metrics_server_enabled }}\"\n container: true\n repo: \"{{ metrics_server_image_repo }}\"\n tag: \"{{ metrics_server_image_tag }}\"\n checksum: \"{{ metrics_server_digest_checksum | default(None) }}\"\n groups:\n - kube_control_plane\n\n local_volume_provisioner:\n enabled: \"{{ local_volume_provisioner_enabled }}\"\n container: true\n repo: \"{{ local_volume_provisioner_image_repo }}\"\n tag: \"{{ local_volume_provisioner_image_tag }}\"\n checksum: \"{{ local_volume_provisioner_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n local_path_provisioner:\n enabled: \"{{ local_path_provisioner_enabled }}\"\n container: true\n repo: \"{{ local_path_provisioner_image_repo }}\"\n tag: \"{{ local_path_provisioner_image_tag }}\"\n checksum: \"{{ local_path_provisioner_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n ingress_alb_controller:\n enabled: \"{{ ingress_alb_enabled }}\"\n container: true\n repo: \"{{ alb_ingress_image_repo }}\"\n tag: \"{{ alb_ingress_image_tag }}\"\n checksum: \"{{ ingress_alb_controller_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n cert_manager_controller:\n enabled: \"{{ cert_manager_enabled }}\"\n container: true\n repo: \"{{ cert_manager_controller_image_repo }}\"\n tag: \"{{ cert_manager_controller_image_tag }}\"\n checksum: \"{{ cert_manager_controller_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n cert_manager_cainjector:\n enabled: \"{{ cert_manager_enabled }}\"\n container: true\n repo: \"{{ cert_manager_cainjector_image_repo }}\"\n tag: \"{{ cert_manager_cainjector_image_tag }}\"\n checksum: \"{{ cert_manager_cainjector_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n cert_manager_webhook:\n enabled: \"{{ cert_manager_enabled }}\"\n container: true\n repo: \"{{ cert_manager_webhook_image_repo }}\"\n tag: \"{{ cert_manager_webhook_image_tag }}\"\n checksum: \"{{ cert_manager_webhook_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n gateway_api_crds:\n enabled: \"{{ gateway_api_enabled }}\"\n file: true\n version: \"{{ gateway_api_version }}\"\n dest: \"{{ local_release_dir }}/gateway-api-{{ gateway_api_channel }}-install.yaml\"\n checksum: \"{{ lookup('vars', 'gateway_api_' + gateway_api_channel + '_crds_checksums').no_arch[gateway_api_version] }}\"\n url: \"{{ gateway_api_crds_download_url }}\"\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n prometheus_operator_crds:\n enabled: \"{{ prometheus_operator_crds_enabled }}\"\n file: true\n version: \"{{ prometheus_operator_crds_version }}\"\n dest: \"{{ local_release_dir }}/prometheus-operator-crds.yaml\"\n checksum: \"{{ prometheus_operator_crds_checksums.no_arch[prometheus_operator_crds_version] }}\"\n url: \"{{ prometheus_operator_crds_download_url }}\"\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n csi_attacher:\n enabled: \"{{ cinder_csi_enabled or aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ csi_attacher_image_repo }}\"\n tag: \"{{ csi_attacher_image_tag }}\"\n checksum: \"{{ csi_attacher_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n csi_provisioner:\n enabled: \"{{ cinder_csi_enabled or aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ csi_provisioner_image_repo }}\"\n tag: \"{{ csi_provisioner_image_tag }}\"\n checksum: \"{{ csi_provisioner_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n csi_snapshotter:\n enabled: \"{{ cinder_csi_enabled or aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ csi_snapshotter_image_repo }}\"\n tag: \"{{ csi_snapshotter_image_tag }}\"\n checksum: \"{{ csi_snapshotter_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n snapshot_controller:\n enabled: \"{{ csi_snapshot_controller_enabled }}\"\n container: true\n repo: \"{{ snapshot_controller_image_repo }}\"\n tag: \"{{ snapshot_controller_image_tag }}\"\n checksum: \"{{ snapshot_controller_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n csi_resizer:\n enabled: \"{{ cinder_csi_enabled or aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ csi_resizer_image_repo }}\"\n tag: \"{{ csi_resizer_image_tag }}\"\n checksum: \"{{ csi_resizer_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n csi_livenessprobe:\n enabled: \"{{ cinder_csi_enabled or aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ csi_livenessprobe_image_repo }}\"\n tag: \"{{ csi_livenessprobe_image_tag }}\"\n checksum: \"{{ csi_livenessprobe_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n csi_node_driver_registrar:\n enabled: \"{{ cinder_csi_enabled or aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ csi_node_driver_registrar_image_repo }}\"\n tag: \"{{ csi_node_driver_registrar_image_tag }}\"\n checksum: \"{{ csi_node_driver_registrar_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n cinder_csi_plugin:\n enabled: \"{{ cinder_csi_enabled }}\"\n container: true\n repo: \"{{ cinder_csi_plugin_image_repo }}\"\n tag: \"{{ cinder_csi_plugin_image_tag }}\"\n checksum: \"{{ cinder_csi_plugin_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n aws_ebs_csi_plugin:\n enabled: \"{{ aws_ebs_csi_enabled }}\"\n container: true\n repo: \"{{ aws_ebs_csi_plugin_image_repo }}\"\n tag: \"{{ aws_ebs_csi_plugin_image_tag }}\"\n checksum: \"{{ aws_ebs_csi_plugin_digest_checksum | default(None) }}\"\n groups:\n - kube_node\n\n metallb_speaker:\n enabled: \"{{ metallb_speaker_enabled }}\"\n container: true\n repo: \"{{ metallb_speaker_image_repo }}\"\n tag: \"{{ metallb_image_tag }}\"\n checksum: \"{{ metallb_speaker_digest_checksum | default(None) }}\"\n groups:\n - kube_control_plane\n\n metallb_controller:\n enabled: \"{{ metallb_enabled }}\"\n container: true\n repo: \"{{ metallb_controller_image_repo }}\"\n tag: \"{{ metallb_image_tag }}\"\n checksum: \"{{ metallb_controller_digest_checksum | default(None) }}\"\n groups:\n - kube_control_plane\n\n yq:\n enabled: \"{{ argocd_enabled }}\"\n file: true\n version: \"{{ yq_version }}\"\n dest: \"{{ local_release_dir }}/yq-{{ yq_version }}-{{ image_arch }}\"\n checksum: \"{{ yq_binary_checksum }}\"\n url: \"{{ yq_download_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0755\"\n groups:\n - kube_control_plane\n\n argocd_install:\n enabled: \"{{ argocd_enabled }}\"\n file: true\n version: \"{{ argocd_version }}\"\n dest: \"{{ local_release_dir }}/argocd-install.yml\"\n checksum: \"{{ argocd_install_checksum }}\"\n url: \"{{ argocd_install_url }}\"\n unarchive: false\n owner: \"root\"\n mode: \"0644\"\n groups:\n - kube_control_plane\n\ndownload_defaults:\n container: false\n file: false\n repo: None\n tag: None\n enabled: false\n dest: None\n url: None\n unarchive: false\n owner: \"{{ kube_owner }}\"\n mode: None\n" }, { "path": "roles/kubespray_defaults/defaults/main/main.yml", "content": "---\n# Use proxycommand if bastion host is in group all\n# This change obseletes editing ansible.cfg file depending on bastion existence\nansible_ssh_common_args: \"{% if 'bastion' in groups['all'] %} -o ProxyCommand='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p -p {{ hostvars['bastion']['ansible_port'] | default(22) }} {{ hostvars['bastion']['ansible_user'] }}@{{ hostvars['bastion']['ansible_host'] }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} ' {% endif %}\"\n\n# selinux state\npreinstall_selinux_state: permissive\n\n# Check if access_ip responds to ping. Set false if your firewall blocks ICMP.\nping_access_ip: true\n\n# Setting this value to false will fail\n# For details, read this comment https://github.com/kubernetes-sigs/kubespray/pull/11016#issuecomment-2004985001\n# if kube_api_anonymous_auth: \"{{ undef() }}\", remove --anonymous-auth argument\nkube_api_anonymous_auth: true\n\n# Default value, but will be set to true automatically if detected\nis_fedora_coreos: false\n\n# Kubernetes 1.35+: fail on cgroup v1 by default\nkubelet_fail_cgroup_v1: true\n\n# Swap settings\nkubelet_fail_swap_on: true\nkubelet_swap_behavior: LimitedSwap\n\n## Change this to use another Kubernetes version, e.g. a current beta release\nkube_version: \"{{ (kubelet_checksums['amd64'] | dict2items)[0].key }}\"\n\n## The minimum version working\nkube_version_min_required: \"{{ (kubelet_checksums['amd64'] | dict2items)[-1].key }}\"\n\n## Kube Proxy mode One of ['ipvs', 'iptables', 'nftables']\nkube_proxy_mode: ipvs\n\n# Debugging option for the kubeadm config validate command\n# Set to false only for development and testing scenarios where validation is expected to fail (pre-release Kubernetes versions, etc.)\nkubeadm_config_validate_enabled: true\n\n## The timeout for init first control-plane\nkubeadm_init_timeout: 300s\n\n# TODO: remove this\nkube_reserved_cgroups_for_service_slice: kube.slice\n\n## List of kubeadm init phases that should be skipped during control plane setup\n## By default 'addon/coredns' is skipped\n## 'addon/kube-proxy' gets skipped for some network plugins\nkubeadm_init_phases_skip_default: [ \"addon/coredns\" ]\nkubeadm_init_phases_skip: >-\n {%- if kube_network_plugin == 'kube-router' and (kube_router_run_service_proxy is defined and kube_router_run_service_proxy) -%}\n {{ kubeadm_init_phases_skip_default + [\"addon/kube-proxy\"] }}\n {%- elif kube_network_plugin == 'cilium' and (cilium_kube_proxy_replacement is defined and (cilium_kube_proxy_replacement == 'strict' or (cilium_kube_proxy_replacement | bool) or (cilium_kube_proxy_replacement | string | lower == 'true') )) -%}\n {{ kubeadm_init_phases_skip_default + [\"addon/kube-proxy\"] }}\n {%- elif kube_network_plugin == 'calico' and (calico_bpf_enabled is defined and calico_bpf_enabled) -%}\n {{ kubeadm_init_phases_skip_default + [\"addon/kube-proxy\"] }}\n {%- elif kube_proxy_remove is defined and kube_proxy_remove -%}\n {{ kubeadm_init_phases_skip_default + [\"addon/kube-proxy\"] }}\n {%- else -%}\n {{ kubeadm_init_phases_skip_default }}\n {%- endif -%}\n\n# List of kubeadm phases that should be skipped when joining a new node\n# You may need to set this to ['preflight'] for air-gaped deployments to avoid failing connectivity tests.\nkubeadm_join_phases_skip_default: []\nkubeadm_join_phases_skip: >-\n {{ kubeadm_join_phases_skip_default }}\n\n# List of kubeadm upgrade node phases that should be skipped when upgrading a secondary control plane node (supports different phases than kubeadm init and kubeadm upgrade apply)\nkubeadm_upgrade_node_phases_skip_default: []\nkubeadm_upgrade_node_phases_skip: >-\n {%- if kube_version is version('1.32.0', '>=') -%}\n {{ kubeadm_upgrade_node_phases_skip_default + kubeadm_init_phases_skip }}\n {%- else -%}\n {{ kubeadm_upgrade_node_phases_skip_default }}\n {%- endif -%}\n\n# Set to true to remove the role binding to anonymous users created by kubeadm\nremove_anonymous_access: false\n\n# A string slice of values which specify the addresses to use for NodePorts.\n# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).\n# The default empty string slice ([]) means to use all local addresses.\n# kube_proxy_nodeport_addresses_cidr is retained for legacy config\nkube_proxy_nodeport_addresses: >-\n {%- if kube_proxy_nodeport_addresses_cidr is defined -%}\n [{{ kube_proxy_nodeport_addresses_cidr }}]\n {%- else -%}\n []\n {%- endif -%}\n\n# Set to true to allow pre-checks to fail and continue deployment\nignore_assert_errors: false\n\n# kube-vip\nkube_vip_enabled: false\nkube_vip_lb_fwdmethod: local\nkube_vip_address:\n\n# nginx-proxy configure\nnginx_config_dir: \"/etc/nginx\"\n\n# haproxy configure\nhaproxy_config_dir: \"/etc/haproxy\"\n\n# Directory where the binaries will be installed\nbin_dir: /usr/local/bin\ndocker_bin_dir: /usr/bin\ncontainerd_bin_dir: \"{{ bin_dir }}\"\netcd_data_dir: /var/lib/etcd\n# Where the binaries will be downloaded.\n# Note: ensure that you've enough disk space (about 1G)\nlocal_release_dir: \"/tmp/releases\"\n# Random shifts for retrying failed ops like pushing/downloading\nretry_stagger: 5\n\n# DNS configuration.\n# Kubernetes cluster name, also will be used as DNS domain\ncluster_name: cluster.local\n# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods\nndots: 2\n# Default resolv.conf options\ndocker_dns_options:\n- ndots:{{ ndots }}\n- timeout:2\n- attempts:2\n# Can be coredns, coredns_dual, manual, or none\ndns_mode: coredns\n\n# Enable dns autoscaler\nenable_dns_autoscaler: true\n\n# DNS servers added after the cluster DNS\n# These will also be used as upstream by Coredns for out-cluster queries\nupstream_dns_servers: []\n\n# Enable nodelocal dns cache\nenable_nodelocaldns: true\nenable_nodelocaldns_secondary: false\nnodelocaldns_ip: 169.254.25.10\nnodelocaldns_health_port: 9254\nnodelocaldns_second_health_port: 9256\nnodelocaldns_bind_metrics_host_ip: false\nnodelocaldns_secondary_skew_seconds: 5\n\n# Should be set to a cluster IP if using a custom cluster DNS\nmanual_dns_server: \"\"\n\n# Can be host_resolvconf, docker_dns or none\nresolvconf_mode: host_resolvconf\n# Ip address of the kubernetes DNS service (called skydns for historical reasons)\nskydns_server: \"{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}\"\nskydns_server_secondary: \"{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}\"\ndns_domain: \"{{ cluster_name }}\"\ndocker_dns_search_domains:\n- 'default.svc.{{ dns_domain }}'\n- 'svc.{{ dns_domain }}'\nsearchdomains: []\n\nkube_dns_servers:\n coredns: [\"{{ skydns_server }}\"]\n coredns_dual: \"{{ [skydns_server] + [skydns_server_secondary] }}\"\n manual: [\"{{ manual_dns_server }}\"]\n\ndns_servers: \"{{ kube_dns_servers[dns_mode] }}\"\n\nenable_coredns_k8s_external: false\ncoredns_k8s_external_zone: k8s_external.local\n\nenable_coredns_k8s_endpoint_pod_names: false\n\n# Kubernetes configuration dirs and system namespace.\n# Those are where all the additional config stuff goes\n# the kubernetes normally puts in /srv/kubernetes.\n# This puts them in a sane location and namespace.\n# Editing those values will almost surely break something.\nkube_config_dir: /etc/kubernetes\nkube_script_dir: \"{{ bin_dir }}/kubernetes-scripts\"\nkube_manifest_dir: \"{{ kube_config_dir }}/manifests\"\n\n# Kubectl command\n# This is for consistency when using kubectl command in roles, and ensure\nkubectl: \"{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf\"\n\n# This is where all the cert scripts and certs will be located\nkube_cert_dir: \"{{ kube_config_dir }}/ssl\"\n\n# compatibility directory for kubeadm\nkube_cert_compat_dir: \"/etc/kubernetes/pki\"\n\n# This is where all of the bearer tokens will be stored\nkube_token_dir: \"{{ kube_config_dir }}/tokens\"\n\n# This is the user that owns the cluster installation.\nkube_owner: kube\n\n# This is the group that the cert creation scripts chgrp the\n# cert files to. Not really changeable...\nkube_cert_group: kube-cert\n\n# Set to true when the CAs are managed externally.\n# When true, disables all tasks manipulating certificates. Ensure before the kubespray run that:\n# - Certificates and CAs are present in kube_cert_dir\n# - Kubeconfig files are present in kube_config_dir\nkube_external_ca_mode: false\n\n# Cluster Loglevel configuration\nkube_log_level: 2\n\n# Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin)\n# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing\nkube_network_plugin: calico\nkube_network_plugin_multus: false\n\n## Network plugin options with dependencies across the whole playbook\n\n# Deploy cilium even if kube_network_plugin is not cilium.\n# This enables to deploy cilium alongside another CNI to replace kube-proxy.\ncilium_deploy_additionally: false\n\n# Identity allocation mode selects how identities are shared between cilium\n# nodes by setting how they are stored. The options are \"crd\" or \"kvstore\".\n# - \"crd\" stores identities in kubernetes as CRDs (custom resource definition).\n# These can be queried with:\n# `kubectl get ciliumid`\n# - \"kvstore\" stores identities in an etcd kvstore.\n# - In order to support External Workloads, \"crd\" is required\n# - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta\n# - KVStore operations are only required when cilium-operator is running with any of the below options:\n# - --synchronize-k8s-services\n# - --synchronize-k8s-nodes\n# - --identity-allocation-mode=kvstore\n# - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations\ncilium_identity_allocation_mode: crd\n\n# Determines if calico_rr group exists\npeer_with_calico_rr: \"{{ 'calico_rr' in groups and groups['calico_rr'] | length > 0 }}\"\n\n# Choose data store type for calico: \"etcd\" or \"kdd\" (kubernetes datastore)\ncalico_datastore: \"kdd\"\n\n# Kubernetes internal network for services, unused block of space.\nkube_service_addresses: 10.233.0.0/18\n\n# internal network. When used, it will assign IP\n# addresses from this range to individual pods.\n# This network must be unused in your network infrastructure!\nkube_pods_subnet: 10.233.64.0/18\n\n# internal network node size allocation (optional). This is the size allocated\n# to each node for pod IP address allocation. Note that the number of pods per node is\n# also limited by the kubelet_max_pods variable which defaults to 110.\n#\n# Example:\n# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node:\n# - kube_pods_subnet: 10.233.64.0/18\n# - kube_network_node_prefix: 24\n# - kubelet_max_pods: 110\n#\n# Example:\n# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node:\n# - kube_pods_subnet: 10.233.64.0/18\n# - kube_network_node_prefix: 25\n# - kubelet_max_pods: 110\nkube_network_node_prefix: 24\n\n# Configure Dual Stack networking (i.e. both IPv4 and IPv6)\n# enable_dual_stack_networks: false # deprecated\n\n# Configure IPv4 Stack networking\nipv4_stack: true\n# Configure IPv6 Stack networking\nipv6_stack: \"{{ enable_dual_stack_networks | default(false) }}\"\n\n# Kubernetes internal network for IPv6 services, unused block of space.\n# This is only used if ipv6_stack is set to true\n# This provides 4096 IPv6 IPs\nkube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116\n\n# Internal network. When used, it will assign IPv6 addresses from this range to individual pods.\n# This network must not already be in your network infrastructure!\n# This is only used if ipv6_stack is set to true.\n# This provides room for 256 nodes with 254 pods per node.\nkube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n\n# IPv6 subnet size allocated to each for pods.\n# This is only used if ipv6_stack is set to true\n# This provides room for 254 pods per node.\nkube_network_node_prefix_ipv6: 120\n\n\n# The virtual cluster IP, real host IPs and ports the API Server will be\n# listening on.\n# NOTE: loadbalancer_apiserver_localhost somewhat alters the final API enpdoint\n# access IP value (automatically evaluated below)\nkube_apiserver_ip: \"{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}\"\n\n# NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost\n# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.\nkube_apiserver_bind_address: \"::\"\n\n# https\nkube_apiserver_port: 6443\n\n# If non-empty, will use this string as identification instead of the actual hostname\nkube_override_hostname: \"{{ inventory_hostname }}\"\n\n# define kubelet config dir for dynamic kubelet\n# kubelet_config_dir:\ndefault_kubelet_config_dir: \"{{ kube_config_dir }}/dynamic_kubelet_dir\"\n\n# Aggregator\nkube_api_aggregator_routing: false\n\n# Profiling\nkube_profiling: false\n\n# Graceful Node Shutdown\nkubelet_shutdown_grace_period: 60s\n# kubelet_shutdown_grace_period_critical_pods should be less than kubelet_shutdown_grace_period\n# to give normal pods time to be gracefully evacuated\nkubelet_shutdown_grace_period_critical_pods: 20s\n\n# Cloud Provider\n# This variable can only be set to \"external\" or empty string, otherwise the check will fail.\ncloud_provider: \"\"\n# External Cloud Controller Manager (Formerly known as cloud provider)\n# cloud_provider must be \"external\", otherwise this setting is invalid.\n# Supported external cloud controllers are: 'openstack', 'vsphere', 'oci', 'huaweicloud', 'hcloud' and 'manual'\n# 'manual' does not install the cloud controller manager used by Kubespray.\n# If you fill in a value other than the above, the check will fail.\nexternal_cloud_provider: \"\"\n\n# Whether to deploy the container engine\ndeploy_container_engine: \"{{ 'k8s_cluster' in group_names or etcd_deployment_type == 'docker' }}\"\n\n# Container for runtime\ncontainer_manager: containerd\n\n# Enable Node Resource Interface plugin for containerd\nnri_enabled: \"{{ container_manager == 'containerd' }}\"\n\n# Enable Kata Containers as additional container runtime\n# When enabled, it requires `container_manager` different than Docker\nkata_containers_enabled: false\n\n# Enable gVisor as an additional container runtime\n# gVisor is only supported with container_manager Docker or containerd\ngvisor_enabled: false\n\n# Enable runc as additional container runtime\n# When enabled, it requires container_manager=crio\nrunc_enabled: false\n\n# Enable crun as additional container runtime\n# When enabled, it requires container_manager=crio\ncrun_enabled: false\n\n# Enable youki as additional container runtime\n# When enabled, it requires container_manager=crio\nyouki_enabled: false\n\n# Container on localhost (download images when download_localhost is true)\ncontainer_manager_on_localhost: \"{{ container_manager }}\"\n\n# CRI socket path\ncri_socket: >-\n {%- if container_manager == 'crio' -%}\n unix:///var/run/crio/crio.sock\n {%- elif container_manager == 'containerd' -%}\n unix:///var/run/containerd/containerd.sock\n {%- elif container_manager == 'docker' -%}\n unix:///var/run/cri-dockerd.sock\n {%- endif -%}\n\ncrio_insecure_registries: []\n\n## Uncomment this if you want to force overlay/overlay2 as docker storage driver\n## Please note that overlay2 is only supported on newer kernels\n# docker_storage_options: -s overlay2\n\n## Only set this if you have more than 3 nameservers:\n## If true Kubespray will only use the first 3, otherwise it will fail\ndocker_dns_servers_strict: false\n\n# Path used to store Docker data\ndocker_daemon_graph: \"/var/lib/docker\"\n\n## Used to set docker daemon iptables options to true\ndocker_iptables_enabled: \"false\"\n\n# Docker log options\n# Rotate container stderr/stdout logs at 50m and keep last 5\ndocker_log_opts: \"--log-opt max-size=50m --log-opt max-file=5\"\n\n## A list of insecure docker registries (IP address or domain name), for example\n## to allow insecure-registry access to self-hosted registries. Empty by default.\n# docker_insecure_registries:\n# - mirror.registry.io\n# - 172.19.16.11\ndocker_insecure_registries: []\n\n## A list of additional registry mirrors, for example China registry mirror. Empty by default.\n# docker_registry_mirrors:\n# - https://registry.docker-cn.com\n# - https://mirror.aliyuncs.com\ndocker_registry_mirrors: []\n\n## If non-empty will override default system MounFlags value.\n## This option takes a mount propagation flag: shared, slave\n## or private, which control whether mounts in the file system\n## namespace set up for docker will receive or propagate mounts\n## and unmounts. Leave empty for system default\n# docker_mount_flags:\n\n## A string of extra options to pass to the docker daemon.\n# docker_options: \"\"\n\n## A list of plugins to install using 'docker plugin install --grant-all-permissions'\n## Empty by default so no plugins will be installed.\ndocker_plugins: []\n\n# Containerd options - thse are relevant when container_manager == 'containerd'\ncontainerd_use_systemd_cgroup: true\n\n# Use static containerd binary for compatibility with older distributions (e.g., Debian 11).\ncontainerd_static_binary: false\n\n# Containerd conf default dir\ncontainerd_storage_dir: \"/var/lib/containerd\"\ncontainerd_state_dir: \"/run/containerd\"\ncontainerd_systemd_dir: \"/etc/systemd/system/containerd.service.d\"\ncontainerd_cfg_dir: \"/etc/containerd\"\n\n# Settings for containerized control plane (etcd/kubelet/secrets)\n# deployment type for legacy etcd mode\netcd_deployment_type: host\ncert_management: script\n\n# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts\nkubeconfig_localhost: false\n# Download kubectl onto the host that runs Ansible in {{ bin_dir }}\nkubectl_localhost: false\n\n# Define credentials_dir here so it can be overridden\ncredentials_dir: \"{{ inventory_dir }}/credentials\"\n\n# K8s image pull policy (imagePullPolicy)\nk8s_image_pull_policy: IfNotPresent\n\n# Addons which can be enabled\nhelm_enabled: false\nregistry_enabled: false\nmetrics_server_enabled: false\nenable_network_policy: true\nlocal_path_provisioner_enabled: false\nlocal_volume_provisioner_enabled: false\nlocal_volume_provisioner_directory_mode: \"0700\"\ncinder_csi_enabled: false\naws_ebs_csi_enabled: false\nazure_csi_enabled: false\ngcp_pd_csi_enabled: false\nvsphere_csi_enabled: false\nupcloud_csi_enabled: false\ncsi_snapshot_controller_enabled: false\npersistent_volumes_enabled: false\ningress_alb_enabled: false\ncert_manager_enabled: false\nexpand_persistent_volumes: false\nmetallb_enabled: false\nmetallb_speaker_enabled: \"{{ metallb_enabled }}\"\nargocd_enabled: false\ngateway_api_enabled: false\nprometheus_operator_crds_enabled: false\n\n## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)\n# openstack_blockstorage_version: \"v1/v2/auto (default)\"\nopenstack_blockstorage_ignore_volume_az: \"{{ volume_cross_zone_attachment | default('false') }}\"\n# set max volumes per node (cinder-csi), default not set\n# node_volume_attach_limit: 25\n# Cinder CSI topology, when false volumes can be cross-mounted between availability zones\n# cinder_topology: false\n# Set Cinder topology zones (can be multiple zones, default not set)\n# cinder_topology_zones:\n# - nova\ncinder_csi_ignore_volume_az: \"{{ volume_cross_zone_attachment | default('false') }}\"\n\n## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.\nopenstack_lbaas_enabled: false\n# openstack_lbaas_subnet_id: \"Neutron subnet ID (not network ID) to create LBaaS VIP\"\n## To enable automatic floating ip provisioning, specify a subnet.\n# openstack_lbaas_floating_network_id: \"Neutron network ID (not subnet ID) to get floating IP from, disabled by default\"\n## Override default LBaaS behavior\n# openstack_lbaas_use_octavia: False\n# openstack_lbaas_method: \"ROUND_ROBIN\"\n# openstack_lbaas_provider: \"haproxy\"\nopenstack_lbaas_create_monitor: \"yes\"\nopenstack_lbaas_monitor_delay: \"1m\"\nopenstack_lbaas_monitor_timeout: \"30s\"\nopenstack_lbaas_monitor_max_retries: \"3\"\nopenstack_cacert: \"{{ lookup('env', 'OS_CACERT') }}\"\n\n# Default values for the external OpenStack Cloud Controller\nexternal_openstack_lbaas_enabled: true\nexternal_openstack_network_ipv6_disabled: false\nexternal_openstack_network_internal_networks: []\nexternal_openstack_network_public_networks: []\n\n# Default values for the external Hcloud Cloud Controller\nexternal_hcloud_cloud:\n hcloud_api_token: \"\"\n token_secret_name: hcloud\n\n service_account_name: cloud-controller-manager\n\n controller_image_tag: \"latest\"\n ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset\n ## Format:\n ## external_hcloud_cloud.controller_extra_args:\n ## arg1: \"value1\"\n ## arg2: \"value2\"\n controller_extra_args: {}\n\n## List of authorization modes that must be configured for\n## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and\n## 'RBAC' modes are tested. Order is important.\nauthorization_modes: ['Node', 'RBAC']\n\n## Structured authorization config\n## Structured AuthorizationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authorization modes with a structured configuration file.\n## AuthorizationConfiguration files offer features not available with the `--authorization-mode` flag, although Kubespray supports both methods and authorization-mode remains the default for now.\n## Note: Because the `--authorization-config` and `--authorization-mode` flags are mutually exclusive, the `authorization_modes` ansible variable is ignored when `kube_apiserver_use_authorization_config_file` is set to true. The two features cannot be used at the same time.\n## Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/#configuring-the-api-server-using-an-authorization-config-file\n## Examples: https://kubernetes.io/blog/2024/04/26/multi-webhook-and-modular-authorization-made-much-easier/\n## KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3221-structured-authorization-configuration\nkube_apiserver_use_authorization_config_file: false\nkube_apiserver_authorization_config_api_version: \"{{ 'v1alpha1' if kube_version is version('1.30.0', '<') else 'v1beta1' if kube_version is version('1.32.0', '<') else 'v1' }}\"\nkube_apiserver_authorization_config_authorizers:\n- type: Node\n name: node\n- type: RBAC\n name: rbac\n## Example for use with kube_webhook_authorization: true\n# - type: Webhook\n# name: webhook\n# webhook:\n# connectionInfo:\n# type: KubeConfigFile\n# kubeConfigFile: \"{{ kube_config_dir }}/webhook-authorization-config.yaml\"\n# subjectAccessReviewVersion: v1beta1\n# matchConditionSubjectAccessReviewVersion: v1\n# timeout: 3s\n# failurePolicy: NoOpinion\n# matchConditions:\n# # Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/\n# # only send resource requests to the webhook\n# - expression: has(request.resourceAttributes)\n# # Don't intercept requests from kube-system service accounts\n# - expression: \"!('system:serviceaccounts:kube-system' in request.groups)\"\n# ## Below expressions avoid issues with kubeadm init and other system components that should be authorized by Node and RBAC\n# # Don't process node and bootstrap token requests with the webhook\n# - expression: \"!('system:nodes' in request.groups)\"\n# - expression: \"!('system:bootstrappers' in request.groups)\"\n# - expression: \"!('system:bootstrappers:kubeadm:default-node-token' in request.groups)\"\n# # Don't process kubeadm requests with the webhook\n# - expression: \"!('kubeadm:cluster-admins' in request.groups)\"\n# - expression: \"!('system:masters' in request.groups)\"\n\n## Two workarounds are required to use AuthorizationConfiguration with kubeadm v1.29.x:\n## 1. Enable the StructuredAuthorizationConfiguration feature gate:\n# kube_apiserver_feature_gates:\n# - StructuredAuthorizationConfiguration=true\n## 2. Use the following kubeadm_patches to remove defaulted authorization-mode flags (Workaround for a kubeadm defaulting bug on v1.29.x. fixed in 1.30+ via: https://github.com/kubernetes/kubernetes/pull/123654)\n# kubeadm_patches:\n# - target: kube-apiserver\n# type: strategic\n# patch:\n# spec:\n# containers:\n# - name: kube-apiserver\n# $deleteFromPrimitiveList/command:\n# - --authorization-mode=Node,RBAC\n\nrbac_enabled: \"{{ ('RBAC' in authorization_modes and not kube_apiserver_use_authorization_config_file) or (kube_apiserver_use_authorization_config_file and kube_apiserver_authorization_config_authorizers | selectattr('type', 'equalto', 'RBAC') | list | length > 0) }}\"\n\n# When enabled, API bearer tokens (including service account tokens) can be used to authenticate to the kubelet's HTTPS endpoint\nkubelet_authentication_token_webhook: true\n\n# When enabled, access to the kubelet API requires authorization by delegation to the API server\nkubelet_authorization_mode_webhook: true\n\n# kubelet uses certificates for authenticating to the Kubernetes API\n# Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration\nkubelet_rotate_certificates: true\n# kubelet can also request a new server certificate from the Kubernetes API\nkubelet_rotate_server_certificates: false\n\n# If set to true, kubelet errors if any of kernel tunables is different than kubelet defaults\nkubelet_protect_kernel_defaults: true\n\n# Set additional sysctl variables to modify Linux kernel variables, for example:\n# additional_sysctl:\n# - { name: kernel.pid_max, value: 131072 }\n#\nadditional_sysctl: []\n\n## List of key=value pairs that describe feature gates for\n## the k8s cluster.\nkube_feature_gates: []\nkube_apiserver_feature_gates: []\nkube_controller_feature_gates: []\nkube_scheduler_feature_gates: []\nkube_proxy_feature_gates: []\nkubelet_feature_gates: []\nkubeadm_feature_gates: []\n\n# Local volume provisioner storage classes\n# Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted\n# see https://github.com/ansible/ansible/issues/17324\nlocal_volume_provisioner_storage_classes: |\n {\n \"{{ local_volume_provisioner_storage_class | default('local-storage') }}\": {\n \"host_dir\": \"{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}\",\n \"mount_dir\": \"{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}\",\n \"volume_mode\": \"Filesystem\",\n \"fs_type\": \"ext4\"\n\n }\n }\n\nssl_ca_dirs: |-\n [\n {% if ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] -%}\n '/usr/share/ca-certificates',\n {% elif ansible_os_family == 'RedHat' -%}\n '/etc/pki/tls',\n '/etc/pki/ca-trust',\n {% elif ansible_os_family == 'Debian' -%}\n '/usr/share/ca-certificates',\n {% endif -%}\n ]\n\n# used for delegating tasks on a working control plane node\nfirst_kube_control_plane: \"{{ groups['kube_control_plane'] | first }}\"\n# Vars for pointing to kubernetes api endpoints\nkube_apiserver_count: \"{{ groups['kube_control_plane'] | length }}\"\nkube_apiserver_address: \"{{ hostvars[inventory_hostname]['main_ip'] }}\"\nkube_apiserver_access_address: \"{{ hostvars[inventory_hostname]['main_access_ip'] }}\"\nfirst_kube_control_plane_address: \"{{ hostvars[groups['kube_control_plane'][0]]['main_access_ip'] }}\"\nloadbalancer_apiserver_localhost: \"{{ loadbalancer_apiserver is not defined }}\"\nloadbalancer_apiserver_type: \"nginx\"\n# applied if only external loadbalancer_apiserver is defined, otherwise ignored\napiserver_loadbalancer_domain_name: \"{{ 'localhost' if loadbalancer_apiserver_localhost else (loadbalancer_apiserver.address | d(undef())) }}\"\nkube_apiserver_global_endpoint: |-\n {% if loadbalancer_apiserver is defined -%}\n https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}\n {%- elif loadbalancer_apiserver_localhost -%}\n https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }}\n {%- else -%}\n https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}\n {%- endif %}\nkube_apiserver_endpoint: |-\n {% if loadbalancer_apiserver is defined -%}\n https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}\n {%- elif ('kube_control_plane' not in group_names) and loadbalancer_apiserver_localhost -%}\n https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }}\n {%- elif 'kube_control_plane' in group_names -%}\n https://{{ kube_apiserver_bind_address | regex_replace('::', '127.0.0.1') | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}\n {%- else -%}\n https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}\n {%- endif %}\nkube_apiserver_client_cert: \"{{ kube_cert_dir }}/ca.crt\"\nkube_apiserver_client_key: \"{{ kube_cert_dir }}/ca.key\"\n\n# Set to true to deploy etcd-events cluster\netcd_events_cluster_enabled: false\n\n# etcd group can be empty when kubeadm manages etcd\netcd_hosts: \"{{ groups['etcd'] | default(groups['kube_control_plane']) }}\"\n\n# Vars for pointing to etcd endpoints\netcd_address: \"{{ hostvars[inventory_hostname]['main_ip'] }}\"\netcd_access_address: \"{{ hostvars[inventory_hostname]['main_access_ip'] }}\"\netcd_events_access_address: \"{{ hostvars[inventory_hostname]['main_access_ip'] }}\"\netcd_peer_url: \"https://{{ etcd_access_address | ansible.utils.ipwrap }}:2380\"\netcd_client_url: \"https://{{ etcd_access_address | ansible.utils.ipwrap }}:2379\"\netcd_events_peer_url: \"https://{{ etcd_events_access_address | ansible.utils.ipwrap }}:2382\"\netcd_events_client_url: \"https://{{ etcd_events_access_address | ansible.utils.ipwrap }}:2383\"\netcd_access_addresses: |-\n {% for item in etcd_hosts -%}\n https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:2379{% if not loop.last %},{% endif %}\n {%- endfor %}\netcd_events_access_addresses_list: |-\n [\n {% for item in etcd_hosts -%}\n 'https://{{ hostvars[item].main_access_ip | ansible.utils.ipwrap }}:2383'{% if not loop.last %},{% endif %}\n {%- endfor %}\n ]\netcd_metrics_addresses: |-\n {% for item in etcd_hosts -%}\n https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:{{ etcd_metrics_port | default(2381) }}{% if not loop.last %},{% endif %}\n {%- endfor %}\netcd_events_access_addresses: \"{{ etcd_events_access_addresses_list | join(',') }}\"\netcd_events_access_addresses_semicolon: \"{{ etcd_events_access_addresses_list | join(';') }}\"\n# user should set etcd_member_name in inventory/mycluster/hosts.ini\netcd_member_name: |-\n {% for host in groups['etcd'] %}\n {% if inventory_hostname == host %}{{ hostvars[host].etcd_member_name | default(\"etcd\" + loop.index | string) }}{% endif %}\n {% endfor %}\netcd_peer_addresses: |-\n {% for item in groups['etcd'] -%}\n {{ hostvars[item].etcd_member_name | default(\"etcd\" + loop.index | string) }}=https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:2380{% if not loop.last %},{% endif %}\n {%- endfor %}\netcd_events_peer_addresses: |-\n {% for item in groups['etcd'] -%}\n {{ hostvars[item].etcd_member_name | default(\"etcd\" + loop.index | string) }}-events=https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:2382{% if not loop.last %},{% endif %}\n {%- endfor %}\n\netcd_heartbeat_interval: \"250\"\netcd_election_timeout: \"5000\"\netcd_snapshot_count: \"100000\"\n\ncertificates_key_size: 2048\ncertificates_duration: 36500\n\netcd_config_dir: /etc/ssl/etcd\netcd_events_data_dir: \"/var/lib/etcd-events\"\netcd_cert_dir: \"{{ etcd_config_dir }}/ssl\"\n\ntypha_enabled: false\n\ncalico_apiserver_enabled: false\n\n_host_architecture_groups:\n x86_64: amd64\n aarch64: arm64\n armv7l: arm\nhost_architecture: >-\n {%- if ansible_architecture in _host_architecture_groups -%}\n {{ _host_architecture_groups[ansible_architecture] }}\n {%- else -%}\n {{ ansible_architecture }}\n {%- endif -%}\n\n_host_os_groups:\n Linux: linux\n Darwin: darwin\n Win32NT: windows\nhost_os: >-\n {%- if ansible_system in _host_os_groups -%}\n {{ _host_os_groups[ansible_system] }}\n {%- else -%}\n {{ ansible_system }}\n {%- endif -%}\n\n# Sets the eventRecordQPS parameter in kubelet-config.yaml.\n# Setting it to 0 allows unlimited requests per second.\nkubelet_event_record_qps: 50\n\nproxy_env_defaults:\n http_proxy: \"{{ http_proxy | default('') }}\"\n HTTP_PROXY: \"{{ http_proxy | default('') }}\"\n https_proxy: \"{{ https_proxy | default('') }}\"\n HTTPS_PROXY: \"{{ https_proxy | default('') }}\"\n no_proxy: \"{{ no_proxy | default('') }}\"\n NO_PROXY: \"{{ no_proxy | default('') }}\"\n\n# If we use SSL_CERT_FILE: {{ omit }} it cause in value __omit_place_holder__ and break environments\n# Combine dict is avoiding the problem with omit placeholder. Maybe it can be better solution?\nproxy_env: \"{{ proxy_env_defaults | combine({'SSL_CERT_FILE': https_proxy_cert_file}) if https_proxy_cert_file is defined else proxy_env_defaults }}\"\n\nproxy_disable_env:\n ALL_PROXY: ''\n FTP_PROXY: ''\n HTTPS_PROXY: ''\n HTTP_PROXY: ''\n NO_PROXY: ''\n all_proxy: ''\n ftp_proxy: ''\n http_proxy: ''\n https_proxy: ''\n no_proxy: ''\n\n# sysctl_file_path to add sysctl conf to\nsysctl_file_path: \"/etc/sysctl.d/99-sysctl.conf\"\n\n# ignore sysctl errors about unknown keys\nsysctl_ignore_unknown_keys: false\n\nsystem_upgrade: false\nsystem_upgrade_reboot: on-upgrade # never, always\n\n# Enables or disables the scheduler plugins.\nscheduler_plugins_enabled: false\n\n## NTP Settings\n# Start the ntpd or chrony service and enable it at system boot.\nntp_enabled: false\n\n# TODO: Refactor NTP package selection to integrate with the general package installation system\n# instead of using a separate variable approach\n\n# The package to install which provides NTP functionality.\n# The default is ntp for most platforms, or chrony on RHEL/CentOS 7 and later.\n# The ntp_package can be one of ['ntp', 'ntpsec', 'chrony']\nntp_package: >-\n {% if ansible_os_family == \"RedHat\" -%}\n chrony\n {%- else -%}\n ntp\n {%- endif -%}\n" }, { "path": "roles/kubespray_defaults/vars/main/checksums.yml", "content": "---\ncrictl_checksums:\n arm64:\n 1.35.0: sha256:519071de89b64c43e2a1661bb5489c6c3fd5e9e5fcef75e50e542b0c891f1118\n 1.34.0: sha256:c31d252e203df5f4cf37f314bd3092eb79087e791631c1e607087c74b6d0423f\n 1.33.0: sha256:e1f34918d77d5b4be85d48f5d713ca617698a371b049ea1486000a5e86ab1ff3\n amd64:\n 1.35.0: sha256:2e141e5b22cb189c40365a11807d69b76b9b3caced89fac2f4ec879408ce2177\n 1.34.0: sha256:a8ff2a3edb37a98daf3aba7c3b284fe0aa5bff24166d896ab9ef64c8913c9f51\n 1.33.0: sha256:8307399e714626e69d1213a4cd18c8dec3d0201ecdac009b1802115df8973f0f\n ppc64le:\n 1.35.0: sha256:786522b14d684604c8b435312a310972bc1b460cddb1bb216a298098cd86b22e\n 1.34.0: sha256:1da50181f2f6f6f6332b9dbc7d7cc020457ccd542620167953c0e288535acc93\n 1.33.0: sha256:4224acfef4d1deba2ba456b7d93fa98feb0a96063ef66024375294f1de2b064f\ncrio_archive_checksums:\n arm64:\n 1.35.0: sha256:e57175a4d00387b78adfbe248d087d8127bed625afb529e34b2c90d08cfdaf87\n 1.34.5: sha256:999a5dc2dc9854222aeff8a20897e0b34f0ba02c9b260b611d66c62e00e279e0\n 1.34.4: sha256:d176f6256d606a3fc279f9f2994ef4a4c4cbaaa0601f4d1bba1a19bec5674ce9\n 1.34.3: sha256:314595247054b53767a736e24bc3030a5f7c17552944c62b2e190c9e95fe4ca6\n 1.34.2: sha256:ac7530f7fc9d531a87bfdfcae9cf8bf81a8bbdb75e63a046ed96911aa7b68ebd\n 1.34.1: sha256:41a71cab6a61ae429ec447d572fd1cdea0a7e33d62aaa58c3b07467665b50b9f\n 1.34.0: sha256:3006658270477c5fb1e88e9124e40982d2ba7b34495fcc12f0fecd33bbab9a5a\n 1.33.9: sha256:bfcd534db3d1a9380dd7007d623e1eb3250ba64f7c4657e79e9e99b1d874f8f1\n 1.33.8: sha256:59c91726535dcadd0372df0c6aa8595e4d59590994b598b2d97ea2510b216359\n 1.33.7: sha256:af3ea22d3d6944c9a907c6c13d77e9fc4dbcf3972ffbde18dd6f37f1c2ffbd0d\n 1.33.6: sha256:6ee49e746d1a5be1a664a6f801c68b169cb181a9aaf12218eed121e2b151bfdb\n 1.33.5: sha256:ef1b5e2162b0f55722e0966db0cfe387f3ba7cb91d6a803f627121733132792d\n 1.33.4: sha256:6a04cb1ab2020508927d7237ff1174bb330211a1076683417b30642a9c8e4996\n 1.33.3: sha256:39cfbb196326952e554e0fb5f95ebcb6cc1735cf6d56a88b8ecd17d89fbc6c26\n 1.33.2: sha256:0a161cb1437a50fbdb04bf5ca11dbec8bfc567871d0597a5676737278a945a36\n 1.33.1: sha256:6bf135db438937f0ab7a533af64564a0fb1d2079a43723ce9255ecbf9556ae05\n 1.33.0: sha256:8a0dbee2879495d5b33e6fdeac32e5d86c356897bdcf3a94cd602851620ce8b5\n amd64:\n 1.35.0: sha256:55b6d3e9fc9a5864ab5cdf0b24d54b1dcbaf6d4919274b3b9eb37bfc4b0b8cb5\n 1.34.5: sha256:d6606fb6d686b8f814dfec801f0f3cf2ded974c194fa90facefda36075b6fab2\n 1.34.4: sha256:f6348a781c34b433fe1c5150da3408e51e828b610eacbe734405e9c31136d810\n 1.34.3: sha256:e269914f3bc4f36ac87cd593d74daaa43c390571994062180019248be32cc6f7\n 1.34.2: sha256:3a0012938ed389e9270a208bb73b250062d5f1be5798472b1728403d55ddc1da\n 1.34.1: sha256:22c1e4d68d9339aa58a1b0f1b40a8944102934a7505105abe461dc8a7e3de540\n 1.34.0: sha256:5a8bc5c3b8072cb9bde1cf025d5597f75bf21018712c5b72d5cb0657948595c8\n 1.33.9: sha256:81c20a12866d9a7c08c6e381ed326141c917454b696a05b46ae27665fe3c5cfa\n 1.33.8: sha256:537adda39074377893f1f650a71b576ba487b3c4d2ee55e9b22f4e95fc188594\n 1.33.7: sha256:e2999436a272c77370241a4f962c80737698dd8c2400fe75e5c7cf2142c96001\n 1.33.6: sha256:4d0d446f73d9db6d5bf2c03ecdc39d9d702836886f4715886c15dc2f461cc810\n 1.33.5: sha256:b8883e51837ee7fd45c88c762f37ca4b96d80ec6a7b46ec989381089e762aa7f\n 1.33.4: sha256:8f6d14828659b85da7c83bad798d50c2f7e0311742615fb7ed305f77bab54e50\n 1.33.3: sha256:2ee843fd1bbdf32607015771a2e1320b46829f22516e559a49dc7c4e29bb756e\n 1.33.2: sha256:6e82739bbbeae12d571a277a88d85e8a0e23dbc87529414a91ee5f2e23792dcf\n 1.33.1: sha256:036063194028d24c75b9ce080e475ad97bacc955de796b7c895845294db8edbf\n 1.33.0: sha256:dad0cec9e09368b37b35ce824b0ef517a1b33365c4bb164fe82310c73c886f7e\n ppc64le:\n 1.35.0: sha256:081ab73a6970ac3c68893dea9a03b0732ca22ab44a2aa8794fddac0bd4dfa749\n 1.34.5: sha256:3a10d4c1406df01bd9ab88750eabc1273964e9c5f24c7d4a0b719ae77e6cfec2\n 1.34.4: sha256:dca59a28fe9b0b9163418eca1545c9ed01cf514179f108d14e462c6074fd103c\n 1.34.3: sha256:4dd782484eeb460b9a95e6e2e07474216fc02ad45a27ba871799d18f2b6ee0ae\n 1.34.2: sha256:d4c3c9ba24b1b0eabf3c11ddec98801dda7a87b0529706e9ede18b8cc9e4182a\n 1.34.1: sha256:cba0ac74e7202fe28cf8aa895b83f7a30d78b148666add78e19215259f629bb0\n 1.34.0: sha256:e9e41d14439db0ca88cf2cd8533038203f379c25cd612f37635c17908e050ebf\n 1.33.9: sha256:c0a9e60800f66f85c70615128fec5a8358ffde0f715a4058163707dbcca8eb94\n 1.33.8: sha256:1d69c01512e8ebdd51fc70fc64473a31d492e8db095c0ee5d3ee58722048150c\n 1.33.7: sha256:076e7519bfff72a43fb1121ce836eee3cc1fec5bb5a59a11747c514e9d162d26\n 1.33.6: sha256:3643eefe295604288f5b652fb9c672a60f96dc803e63edaf9ee64ed4047a50dd\n 1.33.5: sha256:cf85062f39d755418da0ee4f869c7a4817bf95daee6e35df53010ad29be37c88\n 1.33.4: sha256:2b1594dad9af944e29ee74e788a8d28e1304e3f435f2efb61e5c38f20c2106f7\n 1.33.3: sha256:4293bc74f348db58adb0b0dd6affb918abee999cbaf0e42ea8a33427b8d278a5\n 1.33.2: sha256:8ed65404a57262a9f8eb75b61afa37fcec134472eb1a6d81f1889a74ff32c651\n 1.33.1: sha256:12646aca33f65fe335c27d3af582c599584d3f51185f01044e7ddd0668bb2b4c\n 1.33.0: sha256:b4fa46b25538d8145197f8bf2e935486392c0ca2a9fa609aedd02b9f106d37a6\nkubelet_checksums:\n arm64:\n 1.35.1: sha256:73475c6db8fd8a9780b1b378fa2f917875e6146166c24603c1abc6eafd4493a8\n 1.35.0: sha256:aa658d077348b43d238f50966a583f4244b2a7d45590c77b3b165b7d44983ab8\n 1.34.4: sha256:c78845473c434ee85a2444eeab87f8b20f524e3ab6854a078f79468f44aad8f5\n 1.34.3: sha256:765b740e3ad9c590852652a2623424ec60e2dddce2c6280d7f042f56c8c98619\n 1.34.2: sha256:3e31b1bee9ab32264a67af8a19679777cd372b1c3a04b5d7621289cf137b357c\n 1.34.1: sha256:6a66bc08d6c637fcea50c19063cf49e708fde1630a7f1d4ceca069a45a87e6f1\n 1.34.0: sha256:e45a7795391cd62ee226666039153832d3096c0f892266cd968936e18b2b40b0\n 1.33.8: sha256:e835f15be6d8b7b27b963a46c4a054f7663c26741f17e003bfcb8271350cf882\n 1.33.7: sha256:3035c44e0d429946d6b4b66c593d371cf5bbbfc85df39d7e2a03c422e4fe404a\n 1.33.6: sha256:7d8b7c63309cfe2da2331a1ae13cce070b9ba01e487099e7881a4281667c131d\n 1.33.5: sha256:c6ad0510c089d49244eede2638b4a4ff125258fd29a0649e7eef05c7f79c737f\n 1.33.4: sha256:623329b1a5f4858e3a5406d3947807b75144f4e71dde11ef1a71362c3a8619cc\n 1.33.3: sha256:3f69bb32debfaf25fce91aa5e7181e1e32f3550f3257b93c17dfb37bed621a9c\n 1.33.2: sha256:0fa15aca9b90fe7aef1ed3aad31edd1d9944a8c7aae34162963a6aaaf726e065\n 1.33.1: sha256:10540261c311ae005b9af514d83c02694e12614406a8524fd2d0bad75296f70d\n 1.33.0: sha256:ae5a4fc6d733fc28ff198e2d80334e21fcb5c34e76b411c50fff9cb25accf05a\n amd64:\n 1.35.1: sha256:e7343310e03ff0d424df4397bdfa4468947d6d1f0f93dac586c1e8d6e4086d5d\n 1.35.0: sha256:2f4ed7778681649b81244426c29c5d98df60ccabf83d561d69e61c1cbb943ddf\n 1.34.4: sha256:03b8fea715a7ef82eeaf518dee34c72670c57cc7bc40dc1320c04fbf4f15172f\n 1.34.3: sha256:0e759f40bbc717c05227ae3994b77786f58f59ffa0137a34958c6b26fa5bcbbd\n 1.34.2: sha256:9c5e717b774ee9b9285ce47e7d2150c29e84837eb19a7eaa24b60b1543c9d58f\n 1.34.1: sha256:5a72c596c253ea0b0e5bcc6f29903fd41d1d542a7cadf3700c165a2a041a8d82\n 1.34.0: sha256:5c0d28cea2a3a5c91861dda088a29d56c1b027e184dae1d792686f0710750076\n 1.33.8: sha256:1caa69c5328cfa774218f75f0621a6f10a1b97e095af85015f468aeb8fdf956a\n 1.33.7: sha256:2cea40c8c6929330e799f8fc73233a4b61e63f208739669865e2a23a39c3a007\n 1.33.6: sha256:10cd08fe1f9169fd7520123bcdfff87e37b8a4e21c39481faa382f00355b6973\n 1.33.5: sha256:8f6106b970259486c5af5cbee404d4f23406d96d99dfb92a6965b299c2a4db0e\n 1.33.4: sha256:109bd2607b054a477ede31c55ae814eae8e75543126dc4cea40b04424d843489\n 1.33.3: sha256:37f9093ed2b4669cccf5474718e43ec412833e1267c84b01e662df2c4e5d7aaa\n 1.33.2: sha256:77fa5d29995653fe7e2855759a909caf6869c88092e2f147f0b84cbdba98c8f3\n 1.33.1: sha256:f7224648451dd4f9f2c4f79416f9874223c286ce41727788965fd0341ddb59c4\n 1.33.0: sha256:dd416d94850c342226d3dcdce838518b040ccea16548bfeaf2595934af88ef60\n ppc64le:\n 1.35.1: sha256:ec8b7f870043f711b5d73e528342af1705d6ad7f8308d7f31d74d967986b54f6\n 1.35.0: sha256:f24eb1244878a3876fe180e6052822cc9998033850478b2f4776e5c3b09baecd\n 1.34.4: sha256:fab75e3eb1e0edf15aef7e8ba219256b44f047544ac421737d1778784fa46676\n 1.34.3: sha256:67dcceb6d91710e4da7af720eda7b20fd4e8c24237fc345602bb54439ad8ccca\n 1.34.2: sha256:a195f278b9bac26803f1e26b0f608e0dce66aad033e8c043e8555775612530c9\n 1.34.1: sha256:c4782dbf1987680e9b2baa3ecf5db9e66395772e82b251eb73a150fbfbe0b906\n 1.34.0: sha256:ed663fa4ff3e305276dd889885303e07989dfab073e95ef2da931b975f6686e8\n 1.33.8: sha256:392ed39b6c037bc5c510412c9b5cfd29238d31dd67d1a3cbae7ef4a274304c63\n 1.33.7: sha256:f96dd4272ca8eccf1f93fb5162323840b9286c5a42a5305fcc1b4d47889534d3\n 1.33.6: sha256:00ae91297503518efd237d40900af4de0067597ae4f2ab8250ddb629ffb6df05\n 1.33.5: sha256:1d785ead3f6709f66a105c629a020b9dfe6dff775fae42f7d147edec2d178351\n 1.33.4: sha256:5133077024e5a59ece48d2e6d0fdaeed5c4f90c5e781f25c89c984ee4da396a6\n 1.33.3: sha256:bb4123e09734348d4b553c031bfe7710adcffffe79ed9973a526e36d87aa19fe\n 1.33.2: sha256:be8412cb9bf30125e3a88ecb9bfca4df1ff5d4e650947c46222683071f1a17d7\n 1.33.1: sha256:c1bc01115a513eaec76d56dc52a52aeb05f866a6d07c55335c1fff56c868543d\n 1.33.0: sha256:6fa5abbc14d65b943b00fcfc8a6ac7eb39fd7e924271738c6f17e0b7e74c665b\nkubectl_checksums:\n arm:\n 1.35.1: sha256:dbe14e5b12184d72978b17b167aedc3f42f4a1faf249180025d6359eebcd983e\n 1.35.0: sha256:dca28f6af03b31ca6043baa1da7332472c7a3df743606a758534b9ac3ed7ecce\n 1.34.4: sha256:3a6e631bdbb79e633d23055dadc97b45f45d325105ddf40e696de2a324a254c0\n 1.34.3: sha256:e0cf1eddede6abfd539e30ccbb4e50f65b2d6ff44b3bb9d9107ea8775a90a7e4\n 1.34.2: sha256:18e03c1c6ab1dbff6d2a648bf944213f627369d1daeea5b43a7890181ab33abf\n 1.34.1: sha256:ca6218ae8bf366bd8ccdcb440b756c67422a4e04936163845f74d8c056e786ee\n 1.34.0: sha256:69d2ce88274caf9d9117b359cc27656fb6f9dd6517c266cfd93c6513043968b8\n 1.33.8: sha256:734dea07663751c8b45926c843e2c250f13473d65f396555a1ecfe0c9c502fa8\n 1.33.7: sha256:f6b9ac99f4efb406c5184d0a51d9ed896690c80155387007291309cbb8cdd847\n 1.33.6: sha256:89bcef827ac8662781740d092cff410744c0653d828b68cc14051294fcd717e6\n 1.33.5: sha256:5a3a416a85cfc9f7a348c0c0e6334b7449e00a57288ab5a57286ccf68a4d06af\n 1.33.4: sha256:eefd3864ce5440e0ba648b12d53ccffaad97f1c049781b1aa21af6a5278f035f\n 1.33.3: sha256:0124dba9e9091b872591cabcbaea7df07069cb132d38d95f3c7bc8d5b8b621a9\n 1.33.2: sha256:f3992382aa0ea21f71a976b6fd6a213781c9b58be60c42013950110cf2184f2a\n 1.33.1: sha256:6b1cd6e2bf05c6adaa76b952f9c4ea775f5255913974ccdb12145175d4809e93\n 1.33.0: sha256:bbb4b4906d483f62b0fc3a0aea3ddac942820984679ad11635b81ee881d69ab3\n arm64:\n 1.35.1: sha256:706256e21a4e9192ee62d1a007ac0bfcff2b0b26e92cc7baad487a6a5d08ff82\n 1.35.0: sha256:58f82f9fe796c375c5c4b8439850b0f3f4d401a52434052f2df46035a8789e25\n 1.34.4: sha256:5b982c0644ab1e27780246b9085a5886651b4a7ed86243acbb2bacc1bea01dda\n 1.34.3: sha256:46913a7aa0327f6cc2e1cc2775d53c4a2af5e52f7fd8dacbfbfd098e757f19e9\n 1.34.2: sha256:95df604e914941f3172a93fa8feeb1a1a50f4011dfbe0c01e01b660afc8f9b85\n 1.34.1: sha256:420e6110e3ba7ee5a3927b5af868d18df17aae36b720529ffa4e9e945aa95450\n 1.34.0: sha256:00b182d103a8a73da7a4d11e7526d0543dcf352f06cc63a1fde25ce9243f49a0\n 1.33.8: sha256:76e284669f1f6343bd9fe2a011757809c8c01cf51da9f85ee6ef4eb93c8393a8\n 1.33.7: sha256:fa7ee98fdb6fba92ae05b5e0cde0abd5972b2d9a4a084f7052a1fd0dce6bc1de\n 1.33.6: sha256:3ab32d945a67a6000ba332bf16382fc3646271da6b7d751608b320819e5b8f38\n 1.33.5: sha256:6db7c5d846c3b3ddfd39f3137a93fe96af3938860eefdbf2429805ee1656e381\n 1.33.4: sha256:76cd7a2aa59571519b68c3943521404cbce55dafb7d8866f8d0ea2995b396eef\n 1.33.3: sha256:3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd\n 1.33.2: sha256:54dc02c8365596eaa2b576fae4e3ac521db9130e26912385e1e431d156f8344d\n 1.33.1: sha256:d595d1a26b7444e0beb122e25750ee4524e74414bbde070b672b423139295ce6\n 1.33.0: sha256:48541d119455ac5bcc5043275ccda792371e0b112483aa0b29378439cf6322b9\n amd64:\n 1.35.1: sha256:36e2f4ac66259232341dd7866952d64a958846470f6a9a6a813b9117bd965207\n 1.35.0: sha256:a2e984a18a0c063279d692533031c1eff93a262afcc0afdc517375432d060989\n 1.34.4: sha256:d50c359d95e0841eaad08ddc27c7be37cba8fdccfba5c8e2ded65e121ff112db\n 1.34.3: sha256:ab60ca5f0fd60c1eb81b52909e67060e3ba0bd27e55a8ac147cbc2172ff14212\n 1.34.2: sha256:9591f3d75e1581f3f7392e6ad119aab2f28ae7d6c6e083dc5d22469667f27253\n 1.34.1: sha256:7721f265e18709862655affba5343e85e1980639395d5754473dafaadcaa69e3\n 1.34.0: sha256:cfda68cba5848bc3b6c6135ae2f20ba2c78de20059f68789c090166d6abc3e2c\n 1.33.8: sha256:7f9c3faab7c9f9cc3f318d49eb88efc60eb3b3a7ce9eee5feb39b1280e108a29\n 1.33.7: sha256:471d94e208a89be62eb776700fc8206cbef11116a8de2dc06fc0086b0015375b\n 1.33.6: sha256:d25d9b63335c038333bed785e9c6c4b0e41d791a09cac5f3e8df9862c684afbe\n 1.33.5: sha256:6a12d6c39e4a611a3687ee24d8c733961bb4bae1ae975f5204400c0a6930c6fc\n 1.33.4: sha256:c2ba72c115d524b72aaee9aab8df8b876e1596889d2f3f27d68405262ce86ca1\n 1.33.3: sha256:2fcf65c64f352742dc253a25a7c95617c2aba79843d1b74e585c69fe4884afb0\n 1.33.2: sha256:33d0cdec6967817468f0a4a90f537dfef394dcf815d91966ca651cc118393eea\n 1.33.1: sha256:5de4e9f2266738fd112b721265a0c1cd7f4e5208b670f811861f699474a100a3\n 1.33.0: sha256:9efe8d3facb23e1618cba36fb1c4e15ac9dc3ed5a2c2e18109e4a66b2bac12dc\n ppc64le:\n 1.35.1: sha256:bced44e491ce52cce11e2b4bd4bd9181f4f963ffe868438778d028d56485c5d9\n 1.35.0: sha256:8989809d0ac771244dabe50ed742249ac60eeb6d385cd234ee151eb40b7c32c4\n 1.34.4: sha256:b083c39879483816f34d1f7e2e31e70ec48984fcc1753c79f4b846cfedbf41ac\n 1.34.3: sha256:ae239b7f6f071e47014e1b5b20aa60626e06b32922a6b5054562ae2c5fa82c18\n 1.34.2: sha256:49a985986a9add6c229c628bf2a83addebbdeeef40469fce2a54e51b6f1bb05b\n 1.34.1: sha256:45499f0728b4a3428400db289edb444609d41787061f09b66f18028c0a73652f\n 1.34.0: sha256:1773805a0c128f4d267b2e11f4c74cac287e9a07fffaecc3f7af6df9c8aaf82c\n 1.33.8: sha256:aa079f403c80ba6017449c230733fed4e5d7b0a8700bd6590ee202161b8b12af\n 1.33.7: sha256:0807c38a1342ab8dea6435f33d5897a01527d348a968a5c4ca2929769f3d54f2\n 1.33.6: sha256:4b056b1749c619fab6a855247c3bd04123f2b61cf136ca6bddf69ff97a727e32\n 1.33.5: sha256:37e2204d371bbbb90fd693049a7a45b81991ca8bcc9b8baf041a7c9f23e9035c\n 1.33.4: sha256:fa61404b9c3d76f342f2ad05616753475739ab488e0beffd22942e0cb266cfa9\n 1.33.3: sha256:e0261727822c685f63902f7d78ddfcd112bfde4619692b6c1aae68d162245f67\n 1.33.2: sha256:d1cdf13cb786c1ee6d5bf6d85034f496aa2fee97b287028043eb14c5dc74993f\n 1.33.1: sha256:f922dd8f558dc616ebaa34908ceb7964ebb8caadd7c48699d0b791ffff2be1aa\n 1.33.0: sha256:580d076c891711ec37afaf5994f72a8aad9d45c25413e6e94648e988a5a9933a\nkubeadm_checksums:\n arm64:\n 1.35.1: sha256:80097a3c4ef824f4edfe131d2bd429772c4be3a460c42a44f2320164a917de32\n 1.35.0: sha256:1dac7dc2c6a56548bbc6bf8a7ecf4734f2e733fb336d7293d84541ebe52d0e50\n 1.34.4: sha256:d8028b7e8c8d6c9b3fc3da6bc88d4d0cfb33df1b4b026a7d6e8c35d1471c9f6e\n 1.34.3: sha256:697cf3aa54f1a5740b883a3b18a5d051b4032fd68ba89af626781a43ec9bccc3\n 1.34.2: sha256:065f7de266c59831676cc48b50f404fd18d1f6464502d53980957158e4cab3a7\n 1.34.1: sha256:b0dc5cf091373caf87d069dc3678e661464837e4f10156f1436bd35a9a7db06b\n 1.34.0: sha256:6b7108016bb2b74132f7494e200501d6522682c01759db91892051a052079c77\n 1.33.8: sha256:b5248b51e66e4716261f2c926fe2f08a293795e6863099e7792b4d57dbb9109e\n 1.33.7: sha256:b24eeeff288f9565e11a2527e5aed42c21386596110537adb805a5a2a7b3e9ce\n 1.33.6: sha256:ef80c198ca15a0850660323655ebf5c32cc4ab00da7a5a59efe95e4bcf8503ab\n 1.33.5: sha256:b1c00657649e35771569d095e531d826bd19baf57bcb53cccf3f91d7d60b7808\n 1.33.4: sha256:ef471b454d68ee211e279ddeaebde6ee7a8e14b66ae58e0d0184e967c3595892\n 1.33.3: sha256:bf8ed3bc3952e04f29863c6910ae84b359fe7ac1e642ed4d742ceb396e62c6f2\n 1.33.2: sha256:21efc1ba54a1cf25ac68208b7dde2e67f6d0331259f432947d83e70b975ad4cc\n 1.33.1: sha256:5b3e3a1e18d43522fdee0e15be13a42cee316e07ddcf47ef718104836edebb3e\n 1.33.0: sha256:746c0ee45f4d32ec5046fb10d4354f145ba1ff0c997f9712d46036650ad26340\n amd64:\n 1.35.1: sha256:8a7ff344eef1bfba88f9a74b3fdc9ea4448c94f1b3cefb8c0aeeaf1f96e05053\n 1.35.0: sha256:729e7fb34e4f1bfcf2bdaf2a14891ed64bd18c47aaab42f8cc5030875276cfed\n 1.34.4: sha256:b967f1fa0e36621c402d38bb560eb4a943954d5cf5a00e5150842f6f5da73455\n 1.34.3: sha256:f9ce265434d306e59d800b26f3049b8430ba71f815947f4bacdcdc33359417fb\n 1.34.2: sha256:6a2346006132f6e1ed0b5248e518098cf5abbce25bf11b8926fb1073091b83f4\n 1.34.1: sha256:20654fd7c5155057af5c30b86c52c9ba169db6229eee6ac7abab4309df4172e7\n 1.34.0: sha256:aecc23726768d1753fd417f6e7395cb1a350373295e8e9d9f80e95ed3618e38e\n 1.33.8: sha256:8259af514dc3655e8abec1a69b637f31cce2ecb940a80ae4a268e5287890f009\n 1.33.7: sha256:c10813d54f58ef33bbe6675f3d39c8bd401867743ebc729afdd043265040c31d\n 1.33.6: sha256:c1b84cb3482dd79e26629012f432541ccb505c17f5073aa1fdbca26b1e4909fd\n 1.33.5: sha256:6761219749c6c67a56a5668dfe65d669e0c1f34d4b280b72de6d74d47c601f1e\n 1.33.4: sha256:a109ebcb68e52d3dd605d92f92460c884dcc8b68aebe442404af19b6d9d778ec\n 1.33.3: sha256:baaa1f7621c9c239cd4ac3be5b7e427df329d7e1e15430db5f6ea5bb7a15a02b\n 1.33.2: sha256:5c623ec9a9b8584beba510da5c2b775c41cf51c0accdfb43af093bc084563845\n 1.33.1: sha256:9a481b0a5f1cee1e071bc9a0867ca0aad5524408c2580596c00767ba1a7df0bd\n 1.33.0: sha256:5a65cfec0648cabec124c41be8c61040baf2ba27a99f047db9ca08cac9344987\n ppc64le:\n 1.35.1: sha256:eec12948cfabc18115636c44aca894bf9abef3b2ea73cba180314ee3c218dcca\n 1.35.0: sha256:77a466e1b6a8e28362a729541269de0a7c4a6b9e7770cccefcd745502e656b90\n 1.34.4: sha256:69f1065e718ef2aa5f0287444ef97bd4a5fb8841fc0662f54ca8992a39865391\n 1.34.3: sha256:2b8b48b3b0eb657e04122a158cb7fcad964fba5bd2d8e07f8eeec6f856a63ecf\n 1.34.2: sha256:bea4ed6d971523da794a802de15910b08c09e23bc4c850ee3b953c4bdb0b7976\n 1.34.1: sha256:ddb6bd80bee0719924ae901672b99205226badab74fb13a9e1bb6d3de49fbb21\n 1.34.0: sha256:7201ba36f44187f408a036c4a545e2a3cd12943b1297092687bb66c9a1a9fed6\n 1.33.8: sha256:d618fa97b5782b57512e0a8ab9ed17af190236907af7bd3c9c0776d81c78273f\n 1.33.7: sha256:db2e20d0c20928ae7d68d7603020f8ffd89dcdac4fdc160ef83f1da663868bed\n 1.33.6: sha256:58aaec7b5066b6e3705e0493a2f51c7f101b17165ce714c4d52a2b53861c078b\n 1.33.5: sha256:b1e261109a4e22e0a417d10724bed7f71ba12c2acc167a55d89211e49c2e5eee\n 1.33.4: sha256:eb4f3b7a875ffe06aadd5b5ff7b3dccec125933b7ba6fcb5baed39c9c01220c4\n 1.33.3: sha256:d9f30f0eb538be98cd07603b945611b056be5e5871369b16e23090545ef8cdfa\n 1.33.2: sha256:1b818900ac7af72a14f50300d6c6ad600eecdc578c37b75fa488cc654ca08c25\n 1.33.1: sha256:a772834ba22478c9119f03ecca2a27a70234623d74ff1d7671ee85675a4e830b\n 1.33.0: sha256:26cb7ac57d522a59c84c4784b176097d23c7b4e61874fab84ae719d0e43ac0bc\netcd_binary_checksums:\n arm64:\n 3.6.8: sha256:438f56a700d17ce761510a3e63e6fa5c1d587b2dd4d7a22c179c09a649366760\n 3.6.7: sha256:ef5fc443cf7cc5b82738f3c28363704896551900af90a6d622cae740b5644270\n 3.6.6: sha256:8a15f5427c111ff4692753682374970fb68878401d946c2c28bdad6857db652f\n 3.6.5: sha256:7010161787077b07de29b15b76825ceacbbcedcb77fe2e6832f509be102cab6b\n 3.6.4: sha256:323421fa279f4f3d7da4c7f2dfa17d9e49529cb2b4cdf40899a7416bccdde42d\n 3.6.3: sha256:4b39989093699da7502d1cdd649c412055a2bddd26b3d80ed87d0db31957075c\n 3.6.2: sha256:79d0a2488967aa07ecfde79158b1dab458158522f834810c2827eecac4695a31\n 3.6.1: sha256:5f8ed6e314df44128c218decbf0d146cf882583d05c6f6d9023ce905d232aaec\n 3.6.0: sha256:81477b120ef66ff338fe7de63d894e5feec17e6e1f1d98507676832e089d9b58\n 3.5.27: sha256:1277309f540c5a0329c428f95455c9f76d24f768c8d28fd2753e891c379053fa\n 3.5.26: sha256:93ac1667df0e178ea6d152476ce4088df4075604fe4bc7f85f4719e863cd030b\n 3.5.25: sha256:419dce0b679df31cc45201ef2449b7a6a48e9d241af01741957c9ac86a35badc\n 3.5.24: sha256:efc01f6b3fbef0f000cb53bcad4845c116d7fdd8769ca39d9c40d2fe4d2e509f\n 3.5.23: sha256:d95118595a9556a29775f99c1ef2ee16f6be113df76b7178643a2bfd6a6f37c3\n 3.5.22: sha256:22aca5a253c4a9f2850300b0a1dd209587586ff0e985f5cb1c34e9e5edc07848\n 3.5.21: sha256:95bf6918623a097c0385b96f139d90248614485e781ec9bee4768dbb6c79c53f\n 3.5.20: sha256:f034232e6fb64b0d89c45fd78b8b4c3e9fb8d69605dddddcdebf5d7cd96a1531\n 3.5.19: sha256:a786fd2c92c3c0404586ffedf1b318e4944a17aefed1fa6566f5712ddb8359ad\n 3.5.18: sha256:c2bcaf465537d6d47c8bb82a69e31f786f32257050e3bca445bc4e63479ec714\n 3.5.17: sha256:7d717a62520bf39fa1115dfbb1df79479ff74b5eda0914f4132bfa60a48b9549\n 3.5.16: sha256:8e68c55e6d72b791a9e98591c755af36f6f55aa9eca63767822cd8a3817fdb23\n 3.5.15: sha256:5aa28b435edb1f22bf6455f754e16a13e3e4eb1ad8fc7c22ad47aa8e722febf2\n 3.5.14: sha256:90510c79c4aae3c3313691f5770fc53b3ac883338fc0254bf8d22460acd3c19d\n 3.5.13: sha256:2854993bf622764ccdeb7a146fae2965cb0fcba93c5c8f391e0d5f153c8a7a02\n 3.5.12: sha256:31f30c01918771ece28d6e553e0f33be9483ced989896ecf6bbe1edb07786141\n 3.5.11: sha256:6edf0cddc8fa2d7674129abe2e44d5a37cc3a6e3b500c13c6cbc2ed2ecf08bf4\n 3.5.10: sha256:ff74a6018d9b2a1320bff30e5a11b4f2f5c2a3d147df8a8bad53c01b9f800ee1\n 3.5.9: sha256:bb201c106a61bbab59e2d9f37f4bdff99d50201f513c66b4578741eab581fb28\n 3.5.8: sha256:3f4441b293a2d0d4d2f8b2cd9504376e15818f7b865ef4b436e8e6f865f895ff\n 3.5.7: sha256:1a35314900da7db006b198dd917e923459b462128101736c63a3cda57ecdbf51\n 3.5.6: sha256:888e25c9c94702ac1254c7655709b44bb3711ebaabd3cb05439f3dd1f2b51a87\n amd64:\n 3.6.8: sha256:cf9cfe91a4856cb90eed9c99e6aee4b708db2c7888b88a6f116281f04b0ea693\n 3.6.7: sha256:cf8af880c5a01ee5363cefa14a3e0cb7e5308dcf4ed17a6973099c9a7aee5a9a\n 3.6.6: sha256:887afaa4a99f22d802ccdfbe65730a5e79aa5c9ce2c8799c67e9d804c50ecedb\n 3.6.5: sha256:66bad39ed920f6fc15fd74adcb8bfd38ba9a6412f8c7852d09eb11670e88cac3\n 3.6.4: sha256:4d5f3101daa534e45ccaf3eec8d21c19b7222db377bcfd5e5a9144155238c105\n 3.6.3: sha256:3f3b4aa9785d86322c50b296eebdc7a0a57b27065190154b5858bf6a7512ac10\n 3.6.2: sha256:4b5d55d61e2218fab7c1cc1c00b341c469159ecde8cedd575fa858683f67e9f4\n 3.6.1: sha256:1324664bfe56d178d1362a57462ca5a7b26a6d2cbe9e1c94b6820e32cb82d673\n 3.6.0: sha256:42305b0dcbba7b6fdff0382d0c7b99c42026c88c44847a619ab58cde216725d9\n 3.5.27: sha256:0aad9a9e4e0817a021e933f9806a2b2960a62f949ad5a3d6436d8886945cb1bc\n 3.5.26: sha256:0a682a91201dc8351d507210bc30b021a11e254eab806f03224b51e8fad29abb\n 3.5.25: sha256:168af82b59772e1811a9af7b358d42f5c6df44e0d9767afb006ecf12c4bbd607\n 3.5.24: sha256:042497e2ddcee06f22e5d486d81f58affa26b53ee423e2a6aaca3d3ea98c8191\n 3.5.23: sha256:8b50d62d38cb2de005b42227dd14b33f6e01758970f248f9257789ecfaf634c9\n 3.5.22: sha256:20174ab70a6f3df94da13ecac4610f42c47d25af82426f21c112c2c841ec499a\n 3.5.21: sha256:adddda4b06718e68671ffabff2f8cee48488ba61ad82900e639d108f2148501c\n 3.5.20: sha256:9ac85616fb8c0e45f485074dde0258ca2b7b42f1dd5320821af5a8b66daf7072\n 3.5.19: sha256:16ae742def5f330800590e8d505d72830a3b0b7012e559e6bd76f0bc9864bf42\n 3.5.18: sha256:6ddde039a7a506badf34e7edfb38e1ea90e36f05c8cfceba602045df623d86fa\n 3.5.17: sha256:eff6ac621d41711085d0f38fab17d8fa3705f6326c3ff11301a1f5a71fc94edd\n 3.5.16: sha256:b414b27a5ad05f7cb01395c447c85d3227e3fb1c176e51757a283b817f645ccc\n 3.5.15: sha256:3f6b48d8c2844699f2b19c1880508ecf63e1489769ed37ebb97495d5cd848a89\n 3.5.14: sha256:b0b34298f53f6830f08e7ddc57fc74dc45563216a66e94d9e6b0b9e0b0281b34\n 3.5.13: sha256:31e6fcbee0e8c3df27cf1ba69b522e338377f5ed6447f5d05700aee367f3b7e7\n 3.5.12: sha256:f2ff0cb43ce119f55a85012255609b61c64263baea83aa7c8e6846c0938adca5\n 3.5.11: sha256:e256885e753dc99001335e099d3c2eb8cf21a865a087ee4d7e3665752ae5929a\n 3.5.10: sha256:26e90d024fa2310bc52bb40e7f2132e81640b55f8fc446c00ae07e30af2a44fd\n 3.5.9: sha256:d59017044eb776597eca480432081c5bb26f318ad292967029af1f62b588b042\n 3.5.8: sha256:d4c1b8d90ad53658f12ffc293afc5694b7bc6cb093af609188649a799e1cc8dc\n 3.5.7: sha256:a43119af79c592a874e8f59c4f23832297849d0c479338f9df36e196b86bc396\n 3.5.6: sha256:4db32e3bc06dd0999e2171f76a87c1cffed8369475ec7aa7abee9023635670fb\n ppc64le:\n 3.6.8: sha256:3b9bb486b0eb8d79b30410749ec26e174db075956c9ecb533b313b9263e7ba78\n 3.6.7: sha256:de3b1ed50fc8868cdd56b12b0cd81d6740bf53edbca570400a78e530e4829b7b\n 3.6.6: sha256:e4f528b63a731e9b96f5d10f55ce096223fb4e1bc1778aa2535a3d47e9a129e5\n 3.6.5: sha256:3cf99879c7c5b8678a0ec2edf9102b268ea934584db2850f049d89ed8e36b61c\n 3.6.4: sha256:2910fc73e42e1eeb9cc7da8080b821c7649558465e0e6122e49afce832e4b9da\n 3.6.3: sha256:de8ee412ee2669483fd9c730e915c5bd4fe113ba33be4a70305d13ff35e1f919\n 3.6.2: sha256:bf79b9d4c7e9f86e611e73de9fe54a195bc0ad54aeb17200b1c8bda3c4119705\n 3.6.1: sha256:bb87fcd0ea4b9fabf502703512c416ca1d9f4082679cb7f6dbc34bed3dfc13f6\n 3.6.0: sha256:1180d06e3a3787ab65078d9a488f778a4712c59cc82d614abde80c5d06efe38f\n 3.5.27: sha256:b41d488dcd579e780f49f5bd747e9386e17e1376ffb77bfff061f7944818a678\n 3.5.26: sha256:9678ddaced9fcd4878b76b0b76c9c2a3638a70bdc362c9f4cb25ecc48de2c6d3\n 3.5.25: sha256:0dee64e99a43a06dd9541a40a18b52c7309eb1682a2a32740d4bdf358296c007\n 3.5.24: sha256:4b252266a59a00c0f608f481c836fb469d2cd0f60ecbc119c4f1fe0611910ab1\n 3.5.23: sha256:9b9caa29715f387633c6f3639efd3b85284665f7e8632551c904c9859edbb7a8\n 3.5.22: sha256:4dbe98bf5fd82a0f9295bd7ab47429381ab0a80f1e83861c50b01452fa515353\n 3.5.21: sha256:6fb6ecb3d1b331eb177dc610a8efad3aceb1f836d6aeb439ba0bfac5d5c2a38c\n 3.5.20: sha256:563bdac64fc92442cf366c02294dff1cbbd3885a86dbcf7f2e87d9388c3b3223\n 3.5.19: sha256:cc8651929f4d5794892eeeabf612a243ea6233125bc5f8b0f711118736e2710f\n 3.5.18: sha256:ad90260978a9a94572c8aedfa3c4ab225a451e84ab01a1df35e4124863672999\n 3.5.17: sha256:5c737b586a1ebcc12bf0d68a2b56583764f4aba82ab4934629626da93d4a9ecc\n 3.5.16: sha256:33322806f4a2aa3d4947a4d42ec6a120296535e7f00e2f9d74e515e9386333e0\n 3.5.15: sha256:6512f7308a0d0af7c9bb46bca1c4f4e816304304d838d1045ae21d9772c5490e\n 3.5.14: sha256:01681d4d33bba5130c9cffca42c35b0f68e0d991b0b4ee65dab6fd36568d4fee\n 3.5.13: sha256:f372b524e2c118dbb0dbe1097474a072fc93bc30da65efa92999137303bcd9a7\n 3.5.12: sha256:ebd8060508d572678d8d1e4f90f87863e3a6cfcba856ceca32379b03251c0597\n 3.5.11: sha256:a2e70b291811fa8ccc34cc7d297bf7d31e3af790bc31e54cad034a49e9db2eb7\n 3.5.10: sha256:10cd8e4ecf6718b9712bf2edfac2e4924d7f21dbe58d368e6e10578c85bd8c01\n 3.5.9: sha256:551539ebb344ebdc77f170ea51512a6cda35877ffdcbd8b3316b2495a8b2bd87\n 3.5.8: sha256:20e28302c1424b1a3daf7d817f2662e4c64e395a82765d1696cb53cb6bc37a4e\n 3.5.7: sha256:e861aa6acd4d326ec01bfa06fffb80d33f3f8c26e0eb8b73e4424578d149bd04\n 3.5.6: sha256:e235cb885996b8aac133975e0077eaf0a2f8dc7062ad052fa7395668a365906b\ncni_binary_checksums:\n arm:\n 1.8.0: sha256:7ed51af2ee5c7784c8b978b293ad1cf6c3022d7792a2e79197fb9cdc98f3f752\n 1.7.1: sha256:1df4fa20a0fe279bda5a671d172911de2c1a81813bfe8fb0398b46fa9e49d0fb\n 1.6.2: sha256:ee31b0117206dc6242a0349443020025a71752d20d683c08caa70b32de179c4b\n 1.6.0: sha256:10cd1b6b0f7c1e6faf18b2e46ee338beb1e1cce253efc7086f8bc1f4e1061d1a\n arm64:\n 1.8.0: sha256:57ce466fc3b79db1f19b8f4c63e07a1112306efa53c94fe810a2150dd9e07ddb\n 1.7.1: sha256:119fcb508d1ac2149e49a550752f9cd64d023a1d70e189b59c476e4d2bf7c497\n 1.6.2: sha256:01e0e22acc7f7004e4588c1fe1871cc86d7ab562cd858e1761c4641d89ebfaa4\n 1.6.0: sha256:db09ab057ecf60b05ba05cbec38d55b95cc139c7f1078e2e4857cc13af158cee\n amd64:\n 1.8.0: sha256:ab3bda535f9d90766cccc90d3dddb5482003dd744d7f22bcf98186bf8eea8be6\n 1.7.1: sha256:1a28a0506bfe5bcdc981caf1a49eeab7e72da8321f1119b7be85f22621013098\n 1.6.2: sha256:b8e811578fb66023f90d2e238d80cec3bdfca4b44049af74c374d4fae0f9c090\n 1.6.0: sha256:682b49ff8933a997a52107161f1745f8312364b4c7f605ccdf7a77499130d89d\n ppc64le:\n 1.8.0: sha256:63f762df723eb7dbee83e3751167ff1e18cf7b86ef5b48eb620c91af2def434a\n 1.7.1: sha256:15a4070b20c4d6c8bc9b3db52d8303d0520ff7a89cd2e87a2fdf1e9b8dd69373\n 1.6.2: sha256:66dcb90886a039e919904f2fb761b88e03dedd25fb718196855432771eaaa325\n 1.6.0: sha256:d8d4bd74247407c8c73de057bc00adac28bb1ed2d2ee60a9dda278e3b398bcc2\ncalicoctl_binary_checksums:\n arm64:\n 3.30.6: sha256:47ecc00bdd797f82e4bac0ff3904c3a5143ba2d61e8ae1cbbce286ca76d3790a\n 3.30.5: sha256:7611343e7a56e770b95e2bb882dda787efbbd4331b1dd6316ff8ea189238dfaa\n 3.30.4: sha256:b21fbbc55b6f5d50c1c0faae714242cae3e013185cb8e26ce56981bd10da260d\n 3.30.3: sha256:2ae0474b88a6042e5489d7410d2669a9d443c9d5c51e2bdc8ebe4d6dd98f2475\n 3.30.2: sha256:b7aaa3ef1d3f280f835f3e339cbc32d4c6cd6493c7adf46b1ecae29fca848e7f\n 3.30.1: sha256:e3ae853fecfbf09bb2cb8d5ecded553b2d8d1a7229cfbf911fb3ff13c3ab5412\n 3.30.0: sha256:3b97d2e15a0ffc42755571ca0749a1722c7d87d045d2345606eec8e18e95ccc6\n 3.29.7: sha256:484479aa441401adf0db2effb911562d25a4c9a3f022aa6a3b082d027108edec\n 3.29.6: sha256:db3da30af65b3610ee87c8d1bbefb54ca6c1982ed2a0a950215f5eb812bf217f\n 3.29.5: sha256:b121a5fb297271c72ba252f6650d4752c67772f16dd57022d69439866e947957\n 3.29.4: sha256:3139c83cdd3e648d9605c2cebb4657871e41310aad0c7b4bf198e3a8393c5cec\n 3.29.3: sha256:d6cba570af9162dff56714ac5e22dfdd170742bc58a51211f587875a3de79fc4\n 3.29.2: sha256:3a9b80335338b7f4af762d4a7cf68e67b40839e50711fbe6e67f9a62b69bafdd\n 3.29.1: sha256:6f662d316a267854dc5487242ca7ec8ca70c35b52bed258aafb76c2d113643c2\n 3.29.0: sha256:ab23afb283fcdffcf0e1156cdced68d05b6c2b70fd4ea2cbc3189d0ecd43bdfd\n 3.28.5: sha256:691baba17e6a50d0ceee7f95569b864729e436f474fce1e9842041b33cc14316\n 3.28.4: sha256:48887a6dd715f7340511788c3f311810326e61dcce5a6c1554e365cd372ffab1\n 3.28.3: sha256:b61b5206bc7795793edf792040acf5c52d48ff5de701001d0dbbd850edd0c077\n 3.28.2: sha256:8ebe965424ac94084499182b2853de62e5d18cdc346a3b8974e991d8b7a9592d\n 3.28.1: sha256:c062d13534498a427c793a4a9190be4df3cf796a3feb29e4a501e1d6f48daa7c\n 3.28.0: sha256:c4ca8563d2a920729116a3a30171c481580c8c447938ce974ce14d7ce25a31bf\n amd64:\n 3.30.6: sha256:2017e19727dca689d8bb73a9d8dff3c6a8ba7d8c75049f99ee207272161b5749\n 3.30.5: sha256:6cdfb17b0276f648f4fdb051a5d75617a50b3c328d4cccfc40d087b96c361d80\n 3.30.4: sha256:7e2e5e75b25c55683b68eabeb9b00390b1d359e72bf57f7ec2b76bb006fd175f\n 3.30.3: sha256:a7d017d1abf6ef5d6e03267187c0dd68c32f5e937b64decd29d003be44fa6b94\n 3.30.2: sha256:a89e03de84194ab445a998940768959aaabaa5d92be31fe00814b21f86f48c76\n 3.30.1: sha256:85208c36a4fb4e251322a4e544a7b8fc9bd6b9ba2922685a2a2b0aa7dc75d743\n 3.30.0: sha256:a8d385b40526e117a622eb2134f7ed0ce77b9048dbd560dfa2be49fbb4ee6f25\n 3.29.7: sha256:c4e281a404d14da561c65838195b2a4bcf18aebca1bede5ef6b99e0bf78cb8a0\n 3.29.6: sha256:42ee55f279a9df0ef205d001bdde1a4cd4ad3f4dce4d0a3cb7d04ef10f1fed6b\n 3.29.5: sha256:0b118c7efa08e84751a6c7f6647d61ecac58793343d949ad75b71a1d55480a9c\n 3.29.4: sha256:f2a6da6e97052da3b8b787aaea61fa83298586e822af8b9ec5f3858859de759c\n 3.29.3: sha256:8101eef6d31ca80db0c64c7ab8930f657dafc1f8696f145ef5d5f162026eedda\n 3.29.2: sha256:6076d6745c4d60c0c4322961cbb256a0ffa8476cf7f8dbe5de4ae82c55bca020\n 3.29.1: sha256:2ac849181cb1fb40c61c06d075711025cdb909d80562d078cc548d50a0edcd3d\n 3.29.0: sha256:df5048549d72a1f7ea4f61c655699d3b16d8a45873f28c3855c39597b73e8a3d\n 3.28.5: sha256:fe4702ce171045728b6c37b2c01e6f903780997aea7e695b35735754eeeeaf64\n 3.28.4: sha256:ff07f5ac4dbf9a849adb12db20e7b35857869fb98b23e802404dbb4a8a98e013\n 3.28.3: sha256:b7dc6d01407ea04c110b8d50312591d7a7c3aa5239c875354ced83ac6b924137\n 3.28.2: sha256:d7f30447f0f59262051b95bdc656407442c4f71066dc37ddd3b676108fab569d\n 3.28.1: sha256:22ec5727c38dbe19001792b4ca64ac760a6e2985d5c1a231d919dbebe5bca171\n 3.28.0: sha256:4ea270699e67ca29e5533ddb0a68d370cb0005475796c7e841f83047da6297b6\n ppc64le:\n 3.30.6: sha256:9a9c368499b1e3d08418dfbb566379483e15c50d08dd1bcaf6148c115d82ed36\n 3.30.5: sha256:5b6de49da1af2633549bff5e8f4d8a573a175b65c47c29d327ef6a0760d39a93\n 3.30.4: sha256:8fc8ef492d463e184e714bc6d31b05f9066c8af3445928efef233850f036bb92\n 3.30.3: sha256:ccd13ced62baf633fb4347fbe6c9fdc0d3b1b7deb1794c83c015507a0cb8238e\n 3.30.2: sha256:6364bbadfcaf5935427a5f6ef431435b39ccacc22c8a1b5f9071fda00bdfd8ec\n 3.30.1: sha256:d35943a9eccf7def1f3f92b782b221331a2e9163625233b98a72189e2f1b8a6b\n 3.30.0: sha256:5694651bbc6a50dc5537f1f0acf4d4c3897eec5f3b114bdc7cb6bce21df7ba83\n 3.29.7: sha256:b76f1bd1feb3a83a6591196d403b35e1ce97dfdc88babcf1f52f905c0c3cc94e\n 3.29.6: sha256:f1369d48577ff441f893ed08197987e17851cd19b99911d9999ecda667567a69\n 3.29.5: sha256:ec2ab56be3ffdc7ee41f14f3a5fe17564c13a8103939113437c124e2a9900a0e\n 3.29.4: sha256:c7798ef7817bb67f84d54aef661066e64f957977fc80f88b49be2b13a3492d06\n 3.29.3: sha256:edb98d2a0d3f8afbf98eb000f0d535d4678af39dd6e10a09ea5615a4824f692f\n 3.29.2: sha256:6f3fd72be26fcf52605d9ece716363a73bb194ca59ee34a257156d30fa5c1542\n 3.29.1: sha256:ef6064f2ec1a09b5eb8c43ab0c64bd42785c24f5b22b950583fb5074f472c2b7\n 3.29.0: sha256:c9c2a29a349c6f681aa79b5f5d6aee738305d95aa7f158b6217f487808758e53\n 3.28.5: sha256:1aa3b36f198aecdad312664c7b2dd2b15daced54fd4e2db56563d57431fb10d3\n 3.28.4: sha256:9646b8b66981ed68017d30291f44e3e4ff1f6ce318c88c1e837097c061e2bb79\n 3.28.3: sha256:08bfe47df894ae22f2a1256f28b46345cc1718cd9c936ca8248ae5b761c33dab\n 3.28.2: sha256:9889a2f9c26ae82a501b33440b3a0772f552a4ece128cd57a21e395452b4238f\n 3.28.1: sha256:985caad36fed7b883a2cd4cf91e556974bcca95fe4e6b7ff4cb64d8d8fbe9223\n 3.28.0: sha256:0789cb0d1478ec3f0a44db265b19042be9dfc18bc1776343c7ea8d246561d12b\nciliumcli_binary_checksums:\n arm64:\n 0.18.9: sha256:eaa2b3570d3737592ec912505a247173e25fc7bca92d16b32d72b3aca94a743f\n 0.18.8: sha256:44e6dd188336b9168986945c99f8e0882ec4e54a4b6422d44d8e36ec449ba929\n 0.18.7: sha256:dbaa2ab4b1969f4402adf430d6a1bd914c5ab52475ec68f50b3af6fa7fe2fecc\n 0.18.6: sha256:7639c69b410c26d0276fe1297d53e9573f094b56822bd01e85153acb3ca7dd43\n 0.18.5: sha256:54a517c05e1f60c4d3230191edcdc80647dc666173896a8e724817694aae4194\n 0.18.4: sha256:ee438745fedf3ffeb289a12e5d1a8b1a4e8f931a66835ad705bd491461051087\n 0.18.3: sha256:e0588268fc9ab6e0b7a363c4e15ecf69ed2a4cade956ab272745262e456f0e54\n 0.18.2: sha256:db3fae09ba005d6d345858655777bb5c972c9c841f98dc3fad3455d3084dba61\n 0.18.1: sha256:e6556fc7ccd071d7612446945d361c869dfeb423e0738147e0b46b2550bc2bf9\n 0.18.0: sha256:fd20a79875c8089694fb9b5dc3a0bf89d51711f9239637931ff0ace76ce78816\n 0.17.0: sha256:dee29ad27f3958882b450019e2021698282e8fcf8b136c27397798102cc1ad13\n 0.16.24: sha256:cf7f1276bbcf4aa5e6347d5619efe990cf1340d5898f8405931e277a1f76c670\n 0.16.23: sha256:7973302bead01c3f2e1d0f03e2766a0d6e76d3c52c666c750b9871a28b9afb32\n 0.16.22: sha256:b70c15e40b36ac34d59597f2448c5b4e0033964c517f926dbb9654aa07fb1e5b\n 0.16.21: sha256:7b710ddc0d3250f8c5d49a4cefb459b98c3c6651da9389d65cc727451fe3c4e6\n 0.16.20: sha256:4e3f50a6021fa8985232cebb0d9d33ba63e51f43f8fe5d657fc3e91179f8b96a\n 0.16.19: sha256:4a7a401d671054efb98dc9a8028aea82d66bed0bfc86ff008d823d1200d20030\n 0.16.18: sha256:8cd07955edf821ab72f7b765d9b23c196c0e69189e38864ec2470987c2b234fc\n 0.16.17: sha256:3a2ce18f06b4e88944b232fe1ef30678ab6157846628303da56c15e423d609fe\n 0.16.16: sha256:934232e6524f0236edc497664c491a84c7599eb909b7806c789e96b72bb92238\n 0.16.15: sha256:af297814428dbcf2da4c6c5c03a359f0e5592a01f882ddf39aef0dcd0542223f\n 0.16.14: sha256:f335a6af685d6cc47fe4b53bb3fe1eef4bb567d02eff11dc44f4c33fb058e131\n 0.16.13: sha256:2465d104b4e463cdd01021f04e064cef075dd93918eaf1ef4c6b02273de772a6\n 0.16.12: sha256:a093cd04332be79510a78ee6d7a35f61cb8dc068b7d77441fccef7e1b2dfe36f\n 0.16.11: sha256:b118df2e75b950335385c9ead4c59dff977c1e36c845320fe5308d4a2c8ce39d\n 0.16.10: sha256:625e880a9a5a9f38931a3e129fe2c811bd4883105b6f4dbe562734d48a082f7a\n 0.16.9: sha256:3417584bd63f36617b0556e0f327d45ad35aa462b4f3dd2907603f5c9040fb80\n 0.16.8: sha256:a88d984cf962a07a60bbe3da75494661cf9acd4166575513d7875c960349174a\n 0.16.7: sha256:9b6ef0f1097e3937127a05e44c9506fb476ef2cad88ecacd1b2fd9c70de680cd\n 0.16.6: sha256:3c36f52012fa3ebfed5c3959cc7e429ed6c3da640a5ffdd28bc0d9818db383dc\n 0.16.5: sha256:5d5692970af1abcdc22fcb38aec5fec770483ed4b5c1e17d26ffd4923f07a851\n 0.16.4: sha256:d83461431b5c853888c4213f9276991a95806960a87af64f2131b7e445ee9ce0\n 0.16.3: sha256:c6d5f4dec27a5937814afb2750194d15c6caaed130aabb539793fce42698f860\n 0.16.2: sha256:b5c48449c7d7d80bc537d9d231e1242e92fa0db9c7fef3c5e4198e21b3965e8b\n 0.16.1: sha256:07f3f2ba4d772140e46004ee7fa239134acc27afc4f79fc301ee7037617babc1\n 0.16.0: sha256:fe16bcd447fc6fe764ca75712f5832d7504845e9f782684ff09c9f52548237fe\n amd64:\n 0.18.9: sha256:15978aaf82373b0682aa87ab217848b3fb6e3cd80adad365d34696fe92543923\n 0.18.8: sha256:422940f0b7eb6eabc1e126945d1772e3f824c3f4f9fbb0df0dbbf00a271311ba\n 0.18.7: sha256:7f5ff96f1793ee389ef77435a72debf122f5ec253d41418fca99d2c21472016a\n 0.18.6: sha256:075bcc605308ff40a488d75a9a8555713dc0139d36536e032d3ebc2a1e7a9f4d\n 0.18.5: sha256:e63893745b67f58032d9b4f142ae7d6e97286df66af27ff24cd72dc81efc9ff9\n 0.18.4: sha256:6d7d2831380e8741cd46ef92e8f5132c66864311432fae5ab572608ca2d69353\n 0.18.3: sha256:5fe565f3b98b5846b867319aa76bc057fca37894d80db56edc20e4e809d10b25\n 0.18.2: sha256:1b4bd5fd5c96ab1195cd4eb56841c983a21149c62ee39922b7955f1cd0eda23a\n 0.18.1: sha256:c472639d460173e8d807a3f57048f9d1bcdb325e9edba320550d7ec62b72f956\n 0.18.0: sha256:3ac8bd270763e40a7853c73f8c7ec9e49707e1723801884a083dc25469b6b4ba\n 0.17.0: sha256:4ba0687ff7d47e182a7328409fb0eae123e64fa6099cd6f8b9bf240c0012ecf4\n 0.16.24: sha256:019c9c765222b3db5786f7b3a0bff2cd62944a8ce32681acfb47808330f405a7\n 0.16.23: sha256:e7cd3b982eca9b6214226536a147490ebb6ea3caad40d5a724daeea0bec5e3be\n 0.16.22: sha256:8bd9faae272aef2e75c686a55de782018013098b66439a1ee0c8ff1e05c5d32c\n 0.16.21: sha256:f5c4fed6f78b340a6678793c299cc5a8907e55bde9c9e0cc1759c26f6f37db05\n 0.16.20: sha256:8d0e75bf3a867880cb82bbec57027864879775997e3824737e36cf16467d9d04\n 0.16.19: sha256:211c994c9cabe1cb194f308510b78d8498681ee88771fd7c503e259e74eeb541\n 0.16.18: sha256:aa77d11150fac1e59431e5a50d5977219aae6c4bd9ccb051899c2ab99256f30e\n 0.16.17: sha256:77391d4c157e26685e030ec97c075e8cf82dda7fb11875ea9ee4df3bcb0e0070\n 0.16.16: sha256:2d493a70e08d0df947a19d6065f40bd3886554a7fad30b02a7839aeee7400c02\n 0.16.15: sha256:20a5296a9eca3aa50ad9bfa061805b5f332df9d25829f2fd34ecd42d0bf26caf\n 0.16.14: sha256:65c6fc67cf7cff7d958266642032386b4a9a715c502f016aff1761ec2c736b85\n 0.16.13: sha256:9e7597f97192f3d2cbe61a25d1a4aec33a43ef9c307bff27c3c1e81af0011891\n 0.16.12: sha256:535d3a72771f4b1c732701289dbde74ec9efae2e6b56aafc2b448773bd6d815b\n 0.16.11: sha256:f3afc4f32aebeedbdf082c0aa69e243809bfcfba8f60cb1bbf19abc56b68a750\n 0.16.10: sha256:aeb9d7c56108283a9fb9b370ec36b33f28f3126f6c4e6b4176a15cc6b2d3fc70\n 0.16.9: sha256:c6c381044025167a2dec943dfae26887cfa4ade0462f463466fcf82da5df8446\n 0.16.8: sha256:1903abd3055d2ddfccd5a4f6a2a45a00943d6fe9a14c889bf22714d46601f318\n 0.16.7: sha256:7acd60b565428203c3015a3273fae431bad0e833bf465a7460a5d4b432a2badd\n 0.16.6: sha256:3172f3ca66df44233f39bcc2c43c4c149ed90da29acd1da44e8efd498828cf93\n 0.16.5: sha256:699ac1b5f1793145477005cb6b7bd0205458057120e660bed4bc617d3457e261\n 0.16.4: sha256:cbc30655a886ee05c8ffb5eff3ad6f92d37f129c49f4515ce258658f9078fb10\n 0.16.3: sha256:0ff241d8c31d9f4a10d570dc2e9c8d35648bd9e5e21242d07e40a0c55c9485f1\n 0.16.2: sha256:9cfb7ee5b1d6bd125174675e8aab6f9a09283cde699b5ac201475784ecb864f8\n 0.16.1: sha256:9c3bb746e554efc788631629932ab00d13979c75799d957f6d3c17219deb6d45\n 0.16.0: sha256:da98675f961833d4ffd68b1046d907b228a7d394ded2abd70a50b20eaca171c4\ncalico_crds_checksums:\n no_arch:\n 3.30.6: sha256:b0eb83f6d70afac27e8830f22642cd12b0692e4d1a1b5060caa9231a951e736a\n 3.30.5: sha256:68bbe7f44693374f1379aa3fa55f254e9a689d070c26d0de26b2c9fb8d1166ab\n 3.30.4: sha256:be1d346a966a0be79cad7c9856ee5fc0eef8d88b70eba8d4d0bc4be057138bd3\n 3.30.3: sha256:f813232c182229da17658f59db511acb3997e51973eeb293b57ac0dbb5ccf791\n 3.30.2: sha256:57ccedd965f3dcbfd2a38a53b6b9e84e07232205fb83d23ed2afcee94590eed2\n 3.30.1: sha256:af066bd48e68c391dec3645b94d11a1ca513398ee6c56b5a67f0eb13f13fe21e\n 3.30.0: sha256:ee795478b6ab659ee172de74e1cf974871b37f10290dfc75f4d5eae887ba4ce0\n 3.29.7: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5\n 3.29.6: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5\n 3.29.5: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5\n 3.29.4: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5\n 3.29.3: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5\n 3.29.2: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5\n 3.29.1: sha256:aaa336bf0ef87495eccecae7eb65acaf59508a7f0a44dbeec933e05d73bbe0a0\n 3.29.0: sha256:ed35a2bd383674f4d61b013f2588be1ee08b5e7a26eb3208ba6a5565ebf0175c\n 3.28.5: sha256:541635bf3e0cd409ff2f5b9b78363ac8901da4565fffaeb4c1507e19461bf4c7\n 3.28.4: sha256:541635bf3e0cd409ff2f5b9b78363ac8901da4565fffaeb4c1507e19461bf4c7\n 3.28.3: sha256:541635bf3e0cd409ff2f5b9b78363ac8901da4565fffaeb4c1507e19461bf4c7\n 3.28.2: sha256:f09dbaf5b25419659af654f3b50edb3a2b1ebcfeab80b0e56f7fbc79721e8ec3\n 3.28.1: sha256:f09dbaf5b25419659af654f3b50edb3a2b1ebcfeab80b0e56f7fbc79721e8ec3\n 3.28.0: sha256:f09dbaf5b25419659af654f3b50edb3a2b1ebcfeab80b0e56f7fbc79721e8ec3\nhelm_archive_checksums:\n arm:\n 3.18.4: sha256:34ea88aef15fd822e839da262176a36e865bb9cfdb89b1f723811c0cc527f981\n 3.18.3: sha256:5ec62879f57d6acc0436440c88459d2a5c8de233273e73ff6498d79fd2d92653\n 3.18.2: sha256:a848c9db5e51f7cc4975bcfbba415c30cdfb67e141b6efc0e8b3a66cd89e8607\n 3.18.1: sha256:6a58bf4091f798ac1a6fa7e21ea142fe138f5b688f2e3af0c49147f99210fff6\n 3.18.0: sha256:88f6264801fd9c5bb3d2d24c7b3da4e239d137b39bacd18d25b22823e6bd31f7\n 3.17.3: sha256:60d76d1e12d3e058a9e9a8209eff748a6fab5948028a1f0860f48e141243d33d\n 3.17.2: sha256:0b13ec8580dd5498b5a2d7cb34146e098049f59500a266db1bb98f59649eb90a\n 3.17.1: sha256:1dc5ed54350f4f7ae87441e878be4f4fd9b727a86b11b1d20b1001358c83bed3\n 3.16.4: sha256:432e774d1087d3773737888d384c62477b399227662b42cbf0c32e95e6e72556\n 3.16.3: sha256:02ba2f3b1524113f49be6df25a0b4b3190010d6e218c8e2b2fde4578a8439a9c\n 3.16.2: sha256:f0f606d0806a518b749bd82e8dbfe6a803aa33340215590ef3977c60e366ba82\n 3.16.1: sha256:a15a8ddfc373628b13cd2a987206756004091a1f6a91c3b9ee8de6f0b1e2ce90\n 3.16.0: sha256:73efcd63d1b7f1d9db6afc2e039e03a638fe43d3633735363184692ebc8eef02\n arm64:\n 3.18.4: sha256:c0a45e67eef0c7416a8a8c9e9d5d2d30d70e4f4d3f7bea5de28241fffa8f3b89\n 3.18.3: sha256:3382ebdc6d6e027371551a63fc6e0a3073a1aec1061e346692932da61cfd8d24\n 3.18.2: sha256:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d\n 3.18.1: sha256:5ddc8fbd4b17857754a95be799543ceafa5aa9532b05f738ee590a76bb049988\n 3.18.0: sha256:489c9d2d3ea4e095331249d74b4407fb5ac1d338c28429d70cdedccfe6e2b029\n 3.17.3: sha256:7944e3defd386c76fd92d9e6fec5c2d65a323f6fadc19bfb5e704e3eee10348e\n 3.17.2: sha256:d78d76ec7625a94991e887ac049d93f44bd70e4876200b945f813c9e1ed1df7c\n 3.17.1: sha256:c86c9b23602d4abbfae39d9634e25ab1d0ea6c4c16c5b154113efe316a402547\n 3.16.4: sha256:d3f8f15b3d9ec8c8678fbf3280c3e5902efabe5912e2f9fcf29107efbc8ead69\n 3.16.3: sha256:5bd34ed774df6914b323ff84a0a156ea6ff2ba1eaf0113962fa773f3f9def798\n 3.16.2: sha256:1888301aeb7d08a03b6d9f4d2b73dcd09b89c41577e80e3455c113629fc657a4\n 3.16.1: sha256:780b5b86f0db5546769b3e9f0204713bbdd2f6696dfdaac122fbe7f2f31541d2\n 3.16.0: sha256:fc121590b532d7f2037ae5cdd39d2b56dec96069d8bc613a08965f29a156e84f\n amd64:\n 3.18.4: sha256:f8180838c23d7c7d797b208861fecb591d9ce1690d8704ed1e4cb8e2add966c1\n 3.18.3: sha256:6ec85f306dd8fe9eb05c61ba4593182b2afcfefb52f21add3fe043ebbdc48e39\n 3.18.2: sha256:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce\n 3.18.1: sha256:b1c7e8e261fd30f34c617282813ecafc63628fcd59a255a9fc51b1fe43394c05\n 3.18.0: sha256:961e587fc2c03807f8a99ac25ef063fa9e6915f1894729399cbb95d2a79af931\n 3.17.3: sha256:ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd\n 3.17.2: sha256:90c28792a1eb5fb0b50028e39ebf826531ebfcf73f599050dbd79bab2f277241\n 3.17.1: sha256:3b66f3cd28409f29832b1b35b43d9922959a32d795003149707fea84cbcd4469\n 3.16.4: sha256:fc307327959aa38ed8f9f7e66d45492bb022a66c3e5da6063958254b9767d179\n 3.16.3: sha256:f5355c79190951eed23c5432a3b920e071f4c00a64f75e077de0dd4cb7b294ea\n 3.16.2: sha256:9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb\n 3.16.1: sha256:e57e826410269d72be3113333dbfaac0d8dfdd1b0cc4e9cb08bdf97722731ca9\n 3.16.0: sha256:327cfbc7ddc5a3eb644039ceb0cff66394628654c4f5a76bf715ed15b893983b\n ppc64le:\n 3.18.4: sha256:dbd74c59e7710f26e058596723abbf73662b553e01f40dfb08508ffffaeb7b81\n 3.18.3: sha256:ca5ab0bb205488276095881f04b72bfed5c0ddb92f715940dde6a7ccae72818c\n 3.18.2: sha256:1d21b2acdd79a13d20585b61fe90bababdd3f7047530d66aad650869c772b5c3\n 3.18.1: sha256:4d03617f742e4774ddf2170840ec385c67abf0db93e5df1aa9a036bb1275988e\n 3.18.0: sha256:559036fe183593488275a19796ca0b13f56e9d586b697a0d968e8b1e24472d7c\n 3.17.3: sha256:b821885a502b2fa159e3ef3afe9cde6e6c9876d4a623f18868829c3ee4a3c64c\n 3.17.2: sha256:6bb1c83078bdd5e9acad5793dfc9ab3b5b56d410723a660ff1da61dbdff3207b\n 3.17.1: sha256:4223394f3fca82a7f8e8d083caf6faf0ee0639d8f235071334579237078a2c2e\n 3.16.4: sha256:0ba4375a6dcf6117a8e7729fbed36d9220f8ad98dbc7aabc16186f22917caead\n 3.16.3: sha256:266f7698c56a724fddd3a2f2b862ad496c4338dce79f0282fdbc6e23e1738608\n 3.16.2: sha256:32a1b6073064a4a86d2a684180b6662ea202d1294b09ca52a6ba9d4cf071fec7\n 3.16.1: sha256:9f0178957c94516eff9a3897778edb93d78fab1f76751bd282883f584ea81c23\n 3.16.0: sha256:d13a4b87b31a5b50c8d93dd9988dfb312a61e56504102f466a4004e5a3ab8e9e\ncri_dockerd_archive_checksums:\n arm64:\n 0.3.24: sha256:c783a03735887c4a8fc894bd4cf7a1c0defef3ecf50a4d79ff31eed45c26b17e\n 0.3.23: sha256:a78037d2d2e9c52c48372a5cbba7b94b1c57be5759449beef29cfe03cbe6f14b\n 0.3.22: sha256:3260b214c9b12dbf0cbf4d60410c45aacfc31ba52aa7b74164135968e8950cb6\n 0.3.21: sha256:35de6b1e8eba11d8ba6d71fa7499cb3d610a1e7b866c9d43b7f87029e3a769cd\n 0.3.20: sha256:e6b4661c51c832ee1cbbb75d1c8b086fa803acc153d400454c3b8cf324547d89\n 0.3.18: sha256:d16204a4f01685ba67319adb3acc6a6f3e62d8bcfd87bc67f5e08f7332515a9d\n 0.3.17: sha256:761ee6db946ff2c8da04e57833dbbca8e5c38bc3f5b8e84bda689b1b7260c36f\n 0.3.16: sha256:0ab930b6f7ab87697cd94c954d01e19fdd5b5d65810662effb957c5db49a55d4\n 0.3.15: sha256:39e430378c9c08a1d5056426e3c772ae50416ffeadbe720fa63a853c475fd5a4\n 0.3.14: sha256:12c5538887ccea5ec363337dbb50ff52f3b020c93d65649ee1a906c4ec994bf7\n 0.3.13: sha256:a4a3d18f26902631b8323c6b6d87925f8ceff626b21dbc1d160aca84454f9b69\n 0.3.12: sha256:23520fc42dcdcb9f44b0edb555a5d760eedde52a230750e21227fbb258f514b5\n 0.3.11: sha256:877f635a7005b393f7aab24ca4b1cd7bdfb3b967d055e858408240c86e3cab9a\n 0.3.10: sha256:24d2d9cdbb4ed4bda4b0838edb52104ac7a4e2212a0ee05b177de0ae5b6a4a9a\n 0.3.9: sha256:f5051002b4f95b0e8fe7fbd5f8de4493350e010834d2a8b647f2b26c45c6c203\n 0.3.8: sha256:64286af171785f0facb72cf364867600b4db19f43a01db49b8b364f5d04aadae\n 0.3.7: sha256:8da54563ee7ddee36b1adf1f96b3b7b97ec2bc0ec23559b89d9af8eae5e62d9e\n 0.3.6: sha256:793b8f57cecf734c47bface10387a8e90994c570b516cb755900f21ebd0a663b\n 0.3.5: sha256:c20014dc5a71e6991a3bd7e1667c744e3807b5675b1724b26bb7c70093582cfe\n amd64:\n 0.3.24: sha256:dd4b7f514c248a3aaca398f467430a4c58aae9a77ea8b96a2f5b5d6fba0948d1\n 0.3.23: sha256:c7fe5db7f9396186193b58ded0e62a31eca7b3c58ad8691d57017986f96482ee\n 0.3.22: sha256:6621a96a885c82844d12318de00f510eae3459871cf1ad47317f38dd242f9a03\n 0.3.21: sha256:6c35838bc4b1aef74f9113670e114ca729a5f295f9457b226791e18e86e91698\n 0.3.20: sha256:2ce46d6bbd7f6a7e06e211836c201fdc2311111913eccc63a03f6ef4fe1958fc\n 0.3.18: sha256:937578ddcdb28c71afded3fda25d555e0c9e6d396668977ff98228d55886dc79\n 0.3.17: sha256:5568d571c2cfee7a31ce0b35b7fcc65b96c85b573ee6645151c4d022ed92a626\n 0.3.16: sha256:cc7f181ce850130dc375515c54cd8a27e1e862252abc5b7eade7b4a03ddabd8e\n 0.3.15: sha256:4779b7c3663f002871e79ecf6aa8eb48d0bb74df035baecf56b816deb21d12c4\n 0.3.14: sha256:89096dff11b60c8d4f061ee88751c895650a3ef6db5c19d63ab22a1aeb42a5b3\n 0.3.13: sha256:b902eef9bd02529e73a03078a1234c038cd9a12cd86af19970e4f8596f95df26\n 0.3.12: sha256:bf0890018cb339a313b6d52e994fe2628f5b6d5f005cec39adb171f055691c96\n 0.3.11: sha256:b2475988f3b86d85c7835269121171e35c92454ad5f4cd6252183b0fccd74d63\n 0.3.10: sha256:3e19ef525e02d2d1dfd42e8d661ee45b4bc8a49a6dcafd8baa578bdb3a23aeb6\n 0.3.9: sha256:a6d9b4b796e9eff830311a2349d259507302cb3955dd07b78296b91e40e8b433\n 0.3.8: sha256:e12ea6df8228b7d0794c930d32117c4e5a3dcf25a56c3facdf7006289ec6383c\n 0.3.7: sha256:518c5d5345085f36d311f274208705d7fdb79337a80c256871ce941d5a7d47a1\n 0.3.6: sha256:cf271d65abee88c0c0a6d9dacb151913bf37d25d45913a7e04b09efe408eae18\n 0.3.5: sha256:30d47bd89998526d51a8518f9e8ef10baed408ab273879ee0e30350702092938\nrunc_checksums:\n arm64:\n 1.3.4: sha256:d6dcab36d1b6af1b72c7f0662e5fcf446a291271ba6006532b95c4144e19d428\n 1.3.3: sha256:3c9a8e9e6dafd00db61f4611692447ebab4a56388bae4f82192aed67b66df712\n 1.3.2: sha256:06fbccb4528ecd490f3f333d6dcf22c876bd72a024813a0c0a46312121f4c5fd\n 1.3.1: sha256:e531c276312ec701be7edf2b998ac4220c858d02d274bef5546945964792d074\n 1.3.0: sha256:85c5e4e4f72e442c8c17bac07527cd4f961ee48e4f2b71797f7533c94f4a52b9\n 1.2.9: sha256:c155ae4716480be73a1c3bbedfa23df179800ba39ca83eb59f5b1b9b72a7ecbd\n 1.2.8: sha256:7c463d49d83f2da9f5ba6db395600c8da6c5445fd4597c2de8e8b0f82eb1bebb\n 1.2.7: sha256:b540e52e2e2d285b7ac9336d788195eb832292e4aa311dbd936ff7e8514e6cb7\n 1.2.6: sha256:12c612e2ebe6ca198de676ce75ed557e79fe6109032209bb8e25166c967fe170\n 1.2.5: sha256:bfc6575f4c601740539553b639ad6f635c23f76695ed484171bd864df6a23f76\n 1.2.4: sha256:285f6c4c3de1d78d9f536a0299ae931219527b2ebd9ad89df5a1072896b7e82a\n 1.2.3: sha256:4ef19ab21ce1ae5a01e1d3fa5b005e45cdf59f5d3ab32541c9e262cb2b2d3451\n 1.2.2: sha256:bfd3e6c58bd6060eaa725520c31cbc8f6386ac7606e65bfa7fe9084100aa1789\n 1.2.1: sha256:8c0d81c80ffdaab986629a9c787d8468ab41851e7aab8f9617a4c3674e192aaa\n 1.2.0: sha256:3d4f66dc1d91f1b2a46713d185a506a604f1fe9f2f2b89c281eb1c5c13677ff0\n 1.1.15: sha256:c680f8c470ffb228944ca80e1a4dbb6768b3ad97057350852e128847f9dd10bc\n 1.1.14: sha256:050ee97c266bf7d31e1474568ffcbb2a3ff2208087aaa238c8bbe7e398414126\n 1.1.13: sha256:4b93701752f5338ed51592b38e039aef8c1a59856d1225df21eba84c2830743c\n 1.1.12: sha256:879f910a05c95c10c64ad8eb7d5e3aa8e4b30e65587b3d68e009a3565aed5bb8\n 1.1.11: sha256:9f1ee53f06b78cc4a115ca6ae4eec10567999539ce828a22c5351edba043ed12\n 1.1.10: sha256:4830afd426bdeacbdf9cb8729524aa2ed51790b8c4b28786995925593708f1c8\n 1.1.9: sha256:b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0\n 1.1.8: sha256:7c22cb618116d1d5216d79e076349f93a672253d564b19928a099c20e4acd658\n amd64:\n 1.3.4: sha256:5966ca40b6187b30e33bfc299c5f1fe72e8c1aa01cf3fefdadf391668f47f103\n 1.3.3: sha256:8781ab9f71c12f314d21c8e85f13ca1a82d90cf475aa5131a7b543fcc5487543\n 1.3.2: sha256:e7a8e30bd6d248f494aae9163521ff4eb112a30602ac56ada0871e3531269c2d\n 1.3.1: sha256:53bfce31ca047e537e0767b21c9d529d4b5b3e1cb9c590ca81654f9a5615d80d\n 1.3.0: sha256:028986516ab5646370edce981df2d8e8a8d12188deaf837142a02097000ae2f2\n 1.2.9: sha256:b21fda1d5a1510b19c3f6fb43ecb3a92887654a9d0c2b64e84b8b08f24ff0e30\n 1.2.8: sha256:09d2b25f31d3a819c97854c223ff9b36be7fdb0ce57fec709af499e1e19e8e89\n 1.2.7: sha256:7c6564ab7b562a275778e59dd6bd43404768e5462e31cbac6b15e36e14693ca8\n 1.2.6: sha256:0774f49d1b1eebb5849e644db5e4dc6f2b06cee05f13b3d17d5d6ba62d6f2ebc\n 1.2.5: sha256:fbd851fce6a8e0d67a9d184ea544c2abf67c9fd29b80fcc1adf67dfe9eb036a1\n 1.2.4: sha256:e83565aa78ec8f52a4d2b4eb6c4ca262b74c5f6770c1f43670c3029c20175502\n 1.2.3: sha256:e6e8c8049b1910fce58fa68c057aaa5f42cee2a73834df5e59e5da7612d2739d\n 1.2.2: sha256:a34f5ab4fc1df1f456293c3d797a76f2d41cf3cd970bb49fc53ba94bbc8a5cf6\n 1.2.1: sha256:b106d49c60e688022f5909432a77bd3260f29687199d47213ed87269588af781\n 1.2.0: sha256:3bbb68e49bc89dd2607f11d2ff0fa699963ebada39c32ad8a6aab0d40435c1ed\n 1.1.15: sha256:d218e1f8be4dcb1f288dea754faee342375a36f695eac5ab37fc8b7270a78763\n 1.1.14: sha256:a83c0804ebc16826829e7925626c4793da89a9b225bbcc468f2b338ea9f8e8a8\n 1.1.13: sha256:bcfc299c1ab255e9d045ffaf2e324c0abaf58f599831a7c2c4a80b33f795de94\n 1.1.12: sha256:aadeef400b8f05645768c1476d1023f7875b78f52c7ff1967a6dbce236b8cbd8\n 1.1.11: sha256:77ae134de014613c44d25e6310a57a219a7a91155cd47d069a0f22a2cad5caea\n 1.1.10: sha256:81f73a59be3d122ab484d7dfe9ddc81030f595cc59968f61c113a9a38a2c113a\n 1.1.9: sha256:b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43\n 1.1.8: sha256:1d05ed79854efc707841dfc7afbf3b86546fc1d0b3a204435ca921c14af8385b\n ppc64le:\n 1.3.4: sha256:268d9be1188f3efa82cad0d8e6b938d8da0d741427660d874ca9386c68d72937\n 1.3.3: sha256:c42394e7cf7cd508a91b090b72d57ff4df262effde742d5e29ea607e65f38b43\n 1.3.2: sha256:9373062bc547b5afe44fb0122a12aaa980763969d4b69dd17134a6a292838ce5\n 1.3.1: sha256:a1365d76be84e7ec06340a7e9fd31c0dc89f4e704b7198f2fb0c35788dbacc6f\n 1.3.0: sha256:156601012e6c473f2a5c7dbabbd08d8c56f151256433e6010bf4e5f6e569b5b6\n 1.2.9: sha256:cb1b63c25a102d7ccd7c590ebb57b530551134f2cd40fc9cfce39e1741b68c9f\n 1.2.8: sha256:36fa5415fc001bbf0b702c249819dd20d33d739199e0f5c65502fd213fbf5670\n 1.2.7: sha256:09fc9f9cfdca8820227a8fa76536b2f08488a51d34e987686b7f5d9d2d9c2d3c\n 1.2.6: sha256:0d7fffba4f89920edd3246afd4f07b18a975d0d97193ffae418e8418c236c168\n 1.2.5: sha256:3764385971ac719535425629e1ac4d451934392993779ee9e8e8ed7566715f5f\n 1.2.4: sha256:141fa41c1f382483ccf374827f99c7843414fceb95e8ceb710aba8bac984d016\n 1.2.3: sha256:6d1b771096000a14faae660465faf9626a76afe994cbe60581ec4eac1718f12d\n 1.2.2: sha256:9af46fe0bdc654c72593a937806ca034ffbbf4f62f25c1de7a40b5b0f4374de7\n 1.2.1: sha256:652920e145b461151b7e87b28b339594e62129cfc87370b03651a37c39bbc0df\n 1.2.0: sha256:0bd876309958ec00a0e86df3f549f025ad7ae32d981536c1a2932465b479be70\n 1.1.15: sha256:b4c8dbbd4973b1cc69fcade012db690e26e0b354d5fcdf04a12ffe972d5ab098\n 1.1.14: sha256:31630474455c4208594623f1337a06556c7fe42d3983000ca800ad3bef6d8164\n 1.1.13: sha256:4675d51dc0b08ad8e17d3065f2e4ce47760728945f33d3092385e792357e6519\n 1.1.12: sha256:4069d1d57724126e116ad6dbd84409082d1b0afee1ee960b17558f146a742bb6\n 1.1.11: sha256:e3d1da41f97db1bb7e9a8d96c9092747c14ee53bc9f160048828e63f3a2d0896\n 1.1.10: sha256:94a091c06c363e4af7be398dc31fa6e02576d5ecda6de1cbf3a08fe8662bf678\n 1.1.9: sha256:065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6\n 1.1.8: sha256:a816cd654e804249c4f757cc6bf2aa2c128e4b8e6a993067d44c63c891c081ab\ncrun_checksums:\n arm64:\n '1.17': sha256:3049017b99208f5ecd15c1366f47a77dace87f42dccf317ad40a07f1a867518c\n 1.16.1: sha256:973817340e6da12c90c751b011c797396940cca965cefa74557bd1c0939f4042\n '1.16': sha256:4595ff16487b16d2158fa8c3452bc0e1ecdc177ab2ace40fc02cd6e49838ff67\n amd64:\n '1.17': sha256:e9512a3e034e781b2396d068fd24eafcd5788e410403da886df9dc8871d504a5\n 1.16.1: sha256:7b6f1791fb9b2c49ec959b9384b3c4e2ec8c69945fd5292a179d23eb62422eb3\n '1.16': sha256:7f53bffd6b0e216f8f6d6472bb73dc4c6c4ea2c2e7342c52d4bee2972798ce68\n ppc64le:\n '1.17': sha256:ca8ee0fabcac57b61b80f6c234ae20b3b9821433fdf1a6306be5defeac11930e\n 1.16.1: sha256:9590ce79697c5509731f8e58d1733b7051c36f92104925221ca8bda800afee41\n '1.16': sha256:fc7199a2faac1ca0e3e58dee4dd369b9065aa0d95f3257d8803e521213f1bd9b\nyouki_checksums:\n amd64:\n 0.5.7: sha256:10077b1a4f013990a416acf15e6b397cf64b5d62008516a9711abc22730d8203\n 0.5.6: sha256:c372ad70d4107174064225848e686e84ddb597cb44ac7a540e0c47a86e6e6081\n 0.5.5: sha256:7a6844c2e529daa54c6d7558572269508faa26e3b14b5440f9502c413fef558c\n 0.5.4: sha256:69709a8739c868071e8e7e4bdcbcab41b28524c2adb2f0af0c1a4d6ba4691ae0\n 0.5.3: sha256:173b8998cd0abf22e38e36611b34cc19a16431b353dd893e3d988cfc77b4e6ac\n 0.5.2: sha256:361c7187939eab02039fa4289d33158fe4bf4d21ab8cc7139ace8f52081524ba\n 0.5.1: sha256:554dea487dcb54b34fff003fc047458cecf073337d20b77e69c22918c2986aaf\n 0.5.0: sha256:d7ec0cd5f69ecd96b41df13cc38a970ab217aa9c4b2262ca4843a68595975d91\n 0.4.1: sha256:6f7ea3651b284a808f344ad40f9f8315b3b02b76a3ee6c6af7bfff65753284c9\n 0.4.0: sha256:7cf3ae3d1be19a731378b289000cb36ba9cedcbff8a0cabe38edb782e8c55f72\n arm64:\n 0.5.7: sha256:d51990751e796391254c36d40f05b0624017cb544284f9721a645d05747b3d84\n 0.5.6: sha256:df078232fd5e4af822be2d35e0614715dfa781a3e5790b66b71d1d89a2247692\n 0.5.5: sha256:83069dd0ac11dca04f5a2376f27538f86ae5304db0ba148b20d342afe62d6069\n 0.5.4: sha256:2a8d55a7127c751d088ab9ef989496b700afde414296139d5461606527869c42\n 0.5.3: sha256:a15dfe9a1eec2d595b9a972a8a0fa1a919ee3d3523e77ca8c22099bfadf7e88d\n 0.5.2: sha256:8df7e4898088e6fcf942f2e145b0a50c1ad81f5109044e48897c0af9bb3f83e6\n 0.5.1: sha256:ed8e2f5ac03553d0a8ac641339a9f94f42baee652e2afdc1afae08529807a796\n 0.5.0: sha256:1e504e01caf396e8ab87de24bcdde0adaf6251ff2a3dcd0d8622cba0132caadc\n 0.4.1: sha256:c954397353340a013d77d63cda9b074e5f1de57e06fe6f8d44f508b5dc03c998\n 0.4.0: sha256:0d897debaee98a937188317304a9caafb0da5bf798717aefedf113c108f73263\nkata_containers_binary_checksums:\n amd64:\n 3.7.0: sha256:bebf218cafdc082476c7dabbcc5439aee6a41d6dda24dd3cfffbe0a6ae94e23d\n 3.6.0: sha256:d418859e123d04edd71e79c8f532213e5281ec7170b97cdfe3cf85487187d60f\n 3.5.0: sha256:cd0e6a70828db3f9e27390f08c863edda36824c201c35eaa0a4c7498b7e4c9e3\n arm64:\n 3.7.0: sha256:23e0d5c4e38c6944729a11c631f947abc5aab19427a44d8691bfb9ad1c38b831\n 3.6.0: sha256:38277c8d1b9f3c4ea1089338f9251238551b88a5259e8e8ed45fe392ec1ed4bd\n 3.5.0: sha256:fa4cf67d010244c4f8d0e6d450d04e28d1bbce5ad1a3cbc0154adff628d56c0c\ngvisor_runsc_binary_checksums:\n arm64:\n '20260209.1': sha512:e95170b4f70688d014c795ffa9b3d583753f865edfd8afb4e2969490869bdb46b60672f641741f788e2ffee8f29751a017e9a68b98c1e44f5194da9a64b0ff28\n '20260202.0': sha512:5fbb9c68efdf3a404217fb57be55051b4b5f8b83ca631101204615b87ff5b6ea8680cd6599e434f1d87fecb9071367b65e90cd8ad5df3f0b9f0101796ecc8c43\n '20260126.0': sha512:c1b42f5789c09a68eb006964048448c058776440477fac83c7fd9cef879cec40878fb2f5f2450315ca0e7f568889f0b52c842b84929784a57023961f6eb77d04\n '20260112.0': sha512:3b7925d26d71fdcb8cb552950c88bcfed658c06ad6b1211906bfe86d13bc56d8005ac90a4d9ab4c8b6a48eb62ec51ebcdfd45a64067ac5190274e710961e51ea\n '20260105.0': sha512:cc98ad73e8d181f4738c97883180bc76cf8b2eb773c11f3a44f1636d0b0e00f2ee9228e4eecd414f94d6410f4877e6c93260b8070130fba767583026115d1038\n '20251215.0': sha512:5e7d6206bce4164c9109d37dfb0b169d1c59cc256910de42799a868c3f9ba5560ef5c05c0de3fad4f0856f906463588ff25c9bce3b25e0d3f20874521dffe767\n '20251208.0': sha512:db07dc2def9b1e0b13e17bec5f98e9cd794159955ac999432fad16d1ec747924a05cd5e854b4d45f11147c090208c0ce7d915a0734cf2960047bd4daaba0465b\n '20251201.0': sha512:fb527cea4d165478f297a918734f10acabf5230a4a0d29b19709cb6a69a389d32c2a0da328146f72ef0d8776aca35d97647db82ff46be60e85ad02305f631896\n '20251118.0': sha512:80a2970cb966d69d59313ec64583174189293db24605f8309a9e8b230e3be6f0e7e387bc11dd5db1896a8c308dd81da8778c0f0418d7ebfdda6b26f03c8d499f\n '20251110.0': sha512:297d42a46463d5b68c4786bcd448fe0914d7af91cd62f3c51b494b94e1c91d7eda68c6f21cfb1a66ac4d45aadcf20f7410291c0f3f17b090799f9cfa676cb563\n '20251103.0': sha512:a124f892b6f937ff88f9833cba78bc22d2cf869a205e20694374cbad575c9c9dd501cd4b8897aa4338622b6171d7cf4ce8d5a9b3b259c60450a6ade5b3fbc4f5\n '20251020.0': sha512:683b465e68310c9e633213b8663095a8967529ffc1b1b44f7be31eabc53805b46f590b5da1782390f039fa71cd9566eeeebf0c58def1eb59482ca2ff0c7601df\n '20251013.0': sha512:5b27f834190173fcae4ac7e1f0658bed093b1aa18884338474989977f252974ab856242abe5e4fff3e40e9e7e10f15391745805f17dd9414f6ec8f52bc1f2d7c\n '20251006.0': sha512:c4d3eab7b556d5f8108083b398f8a967da26ea28a7f61dd56d690aca4a9123f629278a1a63bf8e93d586830fbd496760a412d7103c406226c9dea9443dbb5a0c\n '20250922.0': sha512:a31a71a24d20e5800c76bf44a9cf33af0dec063c8657fac65805994eef6b136bc66bf4780a4c8acfe0c9d87733c30432b1e4c31cf9994a23d448c8af0865dcd7\n '20250915.0': sha512:a31a71a24d20e5800c76bf44a9cf33af0dec063c8657fac65805994eef6b136bc66bf4780a4c8acfe0c9d87733c30432b1e4c31cf9994a23d448c8af0865dcd7\n '20250820.0': sha512:02767f2682b088558d55b2874732e2668803eee06608654a27429b2f17428b469ea3d166f0d281fd0e081e2b57df86e50dea5bd9fb2286d08edc120b26516ead\n '20250811.0': sha512:f37d5629be1877937ab906a1a5047430a9c16bf15688e3068b589e3659c4ad176faced985f16abdb120ecdceb1c149439f4c2ff557c40a7699502f11edeff9d0\n '20250804.0': sha512:18c66aff5fa090f7929c57a30157b577ec5322718693caa4d9fb404e13ab531e0e77fc55ad3470217b3808ee68876670f38ec950147d9ac13857ded758f09528\n '20250721.0': sha512:e419753ffe2fea7ba74a033156d445236f531a52cf4976b4b40d50a4071fc1b29ac5faac9604fd069e1a6d9553b66f75f649b3437c6cc5afe9e7d071c1415dbe\n '20250715.0': sha512:e49ef9b557c9800f16156c9b79b44b2c3ab2779c0bf4ed5ed2a9a666c4e63684bd954485e4db94dde8cb69f46fb59794d9716c453921b8c69ddd40d8b01c1e40\n '20250707.0': sha512:c2f7539edb022c01912eee358f57f0fdee2c7333323e46f53a3892a3d367746cbb5cb472070a3ebabbfde4491009613a663f9ab5f2bdea7059e21ea2764614a0\n '20250625.0': sha512:3016f92fdcf4e0badc801ffa5502751c0746efce50dfbc2e58613ee0aaeb44e4ea2dffc01a52703d24348f815c1c16af1eb77ec463b4ad0f790ac54fffd041bf\n '20250616.0': sha512:b1e9af177411faaab3349e1e04924ad4b7b19103c974962fe65235943a5856e87b8f9f9261bc6b1f014237dfe443c833c1ae12139228ddb49c4745e6c848f414\n '20250611.0': sha512:6eba75bd816329bdb909c1d7f80dfd01aea88aa8f880740a1eefa348d2211b1460975da559d851609e3b68008ebf89643a23a8854413b032e5855e32fbb8246a\n '20250512.0': sha512:00e9edeb4a9ae702c9617a583f2978a042f20a05807acfa992bc76de4fae2e6e1e994d34ad6f21c826d2cfdea89f6a163c69c0750cf4d90135146438587a3a8c\n '20250505.0': sha512:1611599c6788d3c3f7495b5054aaf9ec81e7a714061582f913359886452fa14f8e65b2bd2d139bc24b5955749167f0db03aefaa6b3ae175296b56814f53d7898\n '20250429.0': sha512:bd58d212088263ad998fa62dbc7f2a8f74ea3914e8a7a319813c3e461f297dcdbc3e85069aacbcaa8c2e573b0e7b17d730d21ab96f8c3ca9516bd43acc070330\n '20250421.0': sha512:647127e139c77d5d360db915d64a21f461fc11ea47d3660feb48952a70639155cd8c19e2bbe16d190a1666c6f689c45bda2aa5d3440596ef174983fe41d8539d\n '20250414.0': sha512:d1ba68b20057622e58e886f472e021a473222590c936a86951005d7b97366b446ef0342b91457ffc0d7e543d54c9c06a363f2883bdd6c594799c4ca1091dabd5\n '20250407.0': sha512:cb590f72b0fbda45e89a2300e9247f12ff295a8c52653c8cf815c662d3fbbc774f9b915cdd4fad59e30694d8cc8737fe2a1a8186ab5136f7701bd6e6877a1662\n amd64:\n '20260209.1': sha512:1e0e42f7d3f4b3eded4e96be5af4dcdbecc9bca7ce40f5b9fa191210690397d71771c7c0e0835c32221261b004250fe513a9265447e62d9bf92fb6a5f7276a68\n '20260202.0': sha512:f7bb9cc5e3f5e36a6788f959361415f6d7f7cd0225b8b4d99728da4b1ac7e5c7ce9c72b4c61e424ba93db77c983109d56b54907a3b2e2b982b34058410611023\n '20260126.0': sha512:cce974fa832c50d26c6ccc08ce50b4972921cd0818ebe8007587211d360cbc828ceea4ec8296703200afa208b679437d24f27a6dca31887b3c0fc6ee8be5eb05\n '20260112.0': sha512:b36de90cdad4cfe0b9b66318407da79c035dd6dcf4c1374250011f34e511c0a29e335fe04eabb0d3fe7140131925f619f724a4702b37c49557bdeb25924b4dc8\n '20260105.0': sha512:15c8adabc9f1006d469177b0ec3962d4993e01c85be17d381a4979029eacc7db37ef354e3eafd279573135a1adf81baffc5c19f2bbfac932c79386f6ac74e52f\n '20251215.0': sha512:ea82bb66ce61a80adb6edaa61e2f2b1cd6339c504a55dd6663555010ed7f96c6234ac787bd9ecdb29ed4058e806e829fa45f14093466913dafc44d56055a5acb\n '20251208.0': sha512:4b9a29a6f887aedbc10de5f5f0900eb64026c3472b5522ee21a6d2b3d30ac3ebc084a78b97e371d3bf830dcba4f61a5809922ea768650d52ef120221b4a9b19f\n '20251201.0': sha512:8534bc833d9b1e286b8876abb17dd6fe202c40a75a36dd62b0ce892bf9dceb1773e71447848e7acab120ce99283c22d2f4e4a6171008c9c5f3d5fe6ad6f1cc75\n '20251118.0': sha512:cc95eec3e22a574ac533278ee8c72672542edf0ab467a89c13f02abea6404ffe20ea4a538a3482b072a8e45222a13cefbf9e7f44bada35b436769e04b12ab970\n '20251110.0': sha512:40d9ec839850cb1994321f0716026b6149adb712bab576b157be2c31b832e68e11475647b2776499fca4c52e96fc7489877fff2fb1985d5f1e128d14f776bd6b\n '20251103.0': sha512:01a465bb5bb37d3c6343a33420b6badfe6e5d8a5ff522f1fb2c183a6e24559cc660373137adc8f5a8c8c362c573a2d01ef6936e126c74c810b16bf9bd19bfd04\n '20251020.0': sha512:8ca31116bdaf6ed4ed6f5475438ac55392b822e98227338a967e4d47bad84c4a96a194f9f33659b7d52c18417a4d55b727b2573a1c8eb8db569cc65e85c38984\n '20251013.0': sha512:923362961cfca5594468669a3ba59fa3f0e210ff9d24b175ad4ff767334d1440fda08423a4a598c64ba2b3e29c09a82eb947a2134d8eb8156e96e699a3b692ae\n '20251006.0': sha512:2086280196df63be2a983e15b594bd9dfbb90a177160e3dfcbfc73bf38dcd31705e62a4ac6d2a3c17e9d2afbd6699dc9780e4cc2bb0ae877ea375d10925882cd\n '20250922.0': sha512:aa008e497d50cc97eb3103645a80ede5f40f13b0d55c675ef3774ecc60cac299e3941253daf9fcbbd097294d7826a2dcf2343eec39a6b52646ec9deaceb3c9cc\n '20250915.0': sha512:aa008e497d50cc97eb3103645a80ede5f40f13b0d55c675ef3774ecc60cac299e3941253daf9fcbbd097294d7826a2dcf2343eec39a6b52646ec9deaceb3c9cc\n '20250820.0': sha512:d6c12a4cb4f714bfcba6fd6611ad4ca73fd88dce790a083d2ceda807cd7e074c0131d5dd2a3490399e8be91feed9afe450793e9708dacddc4afc99ee6e5c3d2e\n '20250811.0': sha512:95cc8973a8ba6fdea608c36288afe83e17a890398d387de89dfd1457e902ab1d73fd3bd52a4fc2b923accd36ad5d1e76b5ea373e9c68d9821efb1785f830892d\n '20250804.0': sha512:c9572008a35e812277b158933e7f549b734ac0e52349398067f3de1bd42572dee6a0911e85d2737b33e8adbf94515b630d57936058cdbb199b02a290a906ae5b\n '20250721.0': sha512:cafd0256341a5e6b32b81cdb8664943d82a80f55ce10fb4d5061e1ac0c8a767946d8665055bb1529226813b215dc545067a3a428b9673aaec85a32455a82a11a\n '20250715.0': sha512:06b3c36e6230a4f106914b7a82c5218724ca574b6c4590bb9f50796e4aa257bfae8b621b346561ed9a6e31c1328d447cf7594c6d0fc26532c3ba15171f18e7f0\n '20250707.0': sha512:348687c9a10c23a51da5dcffcd9e1866250b2a964aa1f599ccd706c41bf0b85823875d6edd5b4dbc1f2e9229eed6d9cb13193fc13b988d3ea614bed9b4ebe955\n '20250625.0': sha512:2193da9a4a2a072bfb1fb314964528ac0da4bd56482552458da6ecb557aaa2f2e939fede3545933608ce2daf43897fff173f1c272375a331647a2bd27588f3be\n '20250616.0': sha512:516e39e16ed05c69d4173a408b0a4ab9a24ac3d1129a35f01173c76e174d3aa9652371c85445e5d45aed6de1a0f37507b4f735184f35ee8e6f3d9af968a6659e\n '20250611.0': sha512:669c9eac780242ef966ac09804a9448faace00f91e2b1f2f5b79d88f214333e84f8bab4304ee48db59c715dc89685787adbc5fc34f8ad3bb379852a88b1d9bdb\n '20250512.0': sha512:981a554ad63f7ed082a43be646b8e910481e4bfc837c5ee5dd5a1353a47b0ae337f9b02700649a542db864ae35af6981e6bdef86c6a48a5e47dacfb97be9b7b0\n '20250505.0': sha512:25705616c3cfc82bb5772e815b2b6b030664dccba7a0db9babcfad5de46d16ce8bff8cd9cc11d366da4acd0f01fb04a0d95bbae070aa923f1492d2f142f271c3\n '20250429.0': sha512:b91d0351907290fe159cf041dbd332f8d2d4151d6a7aaafe161cd842452551b98fa1122e195e2cf42801eb8ff38716270de4f33331dde784cbfc452ec1e368a0\n '20250421.0': sha512:419f80c01cef46aaab0a0eaf9be4bc20fd3aba94e8d0dd8ceacd3b166139d5bc8e701964feb11bf6de7a4274924692a7d0b5bcf5de34f5dfaeec57f7f1ecd88f\n '20250414.0': sha512:cc629d16ed0483bfa42ec270f409349f02f3bbd27db1fa384c3146cb54e85cabbfeaf920d1fd189fcae2ab11a54f9ce481d3c4fb306ac18e035943aa02144fbe\n '20250407.0': sha512:097259d6d93548bf669e21cfec5ba6a47081e43f61d22c5d8a8a4c0c209c81ac9c4454162b826f98cec49e047bbdc29c270113ab6db5519ef3e6a90f302fa47b\ngvisor_containerd_shim_binary_checksums:\n arm64:\n '20260209.1': sha512:714ad3a53a28aa4acd891553d848278f5a873d0a1733836382eaf2bf701d62ece9cef324390602d2676af5e2e3a3d329486d2b18803c9cef5685220764757eb4\n '20260202.0': sha512:714ad3a53a28aa4acd891553d848278f5a873d0a1733836382eaf2bf701d62ece9cef324390602d2676af5e2e3a3d329486d2b18803c9cef5685220764757eb4\n '20260126.0': sha512:84abf41b68ba450ed2cbbdf544e7d347d30f6fd577572e2e58f2fa8e038689f557953148287e26c8f4ee5040c1e928670f113bebca6d81ed7ce014ec4e0ad256\n '20260112.0': sha512:3215952718bd1636173649c4742e3d8e1978c410abd71bb8252c8ad6d28130cb6d66684aa089f61a0eda0b8786553620a08a9f1b5ab824bb27b1b0cf47bfb25b\n '20260105.0': sha512:cfe8a07c304dca21171e5a76614ac3605f5b1ec8f9ed2eeac014a44bc00821864f219db0e25fcc1c56cedbe335bbf34a7fa6bc57335888dcd04278bc0263f5cc\n '20251215.0': sha512:2b3a00ec2d646a1c26c1944781b5caf039ce7035dd72281ccff8e244af55606e01667de311febee1a0a03ebd2633af6ebb0ad72d27b8a966743ffe31563b3a5a\n '20251208.0': sha512:f3a6d9ff32dae45c62ae831580e5dfbd28fed38f1ca9daf09e6a9960a5373da7e29fbf61e0846676102f053ed38f23a0ad41349f5326fb3a2991b296d33c853a\n '20251201.0': sha512:9546236a7ddad9a2ccd51c41f2f309b7f4016fdf489581f77b1b803ed73ca72501af2de3e3d0b58daa633384baa0d46ecd515760165ed51bfb6c0900649c6306\n '20251118.0': sha512:0179f0b049c882703758d5cba387e1e4fd0300aef20197e35e2886f480f0668fecb8deb3aa84341d0b874127d88b337fbcf609f563c3310b47520e8144e9d55a\n '20251110.0': sha512:8f2b16ad59e9ffdafd1218851cba9d007d4ffb15c5ec2003e0c691eb048935a82f9e8b578c051b05738e3b4e1f141ed893c73415313ec639f348fe989659b893\n '20251103.0': sha512:8f2b16ad59e9ffdafd1218851cba9d007d4ffb15c5ec2003e0c691eb048935a82f9e8b578c051b05738e3b4e1f141ed893c73415313ec639f348fe989659b893\n '20251020.0': sha512:8f2b16ad59e9ffdafd1218851cba9d007d4ffb15c5ec2003e0c691eb048935a82f9e8b578c051b05738e3b4e1f141ed893c73415313ec639f348fe989659b893\n '20251013.0': sha512:038c3a7180520538d488d6dead7c04555d82060d301e3d785b4b6315a33e51b20b7895caa8dedfcf8cfeb0f355ea7aea579c2ea34ea46c5aad408cc962dd282f\n '20251006.0': sha512:eb7b3aa338ad44a72f1944eb06edeb73c1a1633ec4f7961e4e6262420a288319e2bb30101802b497e1aaf5b34a7438dd1733571249db74f00846e7a20cacd681\n '20250922.0': sha512:038c3a7180520538d488d6dead7c04555d82060d301e3d785b4b6315a33e51b20b7895caa8dedfcf8cfeb0f355ea7aea579c2ea34ea46c5aad408cc962dd282f\n '20250915.0': sha512:038c3a7180520538d488d6dead7c04555d82060d301e3d785b4b6315a33e51b20b7895caa8dedfcf8cfeb0f355ea7aea579c2ea34ea46c5aad408cc962dd282f\n '20250820.0': sha512:0015d061af2369a8e5e21dad6f69f0e3cb03c6e396090d0e127c6638d646dac8f4e8bd60ab901ea87f6ebbd093e66db86efbccb1f225dc1dfccc76b8861976a7\n '20250811.0': sha512:70a3c4212bc4a5ced31313640d5696af5d3bd11fc06221af2ac5441b2527b82d0c1671ad76ba5c5b120108912744477a2a922cc8444661fc50594b72f146c2f7\n '20250804.0': sha512:a6f7c8e76969588de24f75965ddfb62eec592d03f1882bfb746d7cf7a8a86949f8f441a4a063d7a5c766e6cf576c8d2b9ca7a1918de4902d1053938b6cc382c0\n '20250721.0': sha512:ff38aa2218ae24f7747a85c8d5f5e8a466a68a73c2b7f8a55c1cc8cf7eb054e70924685b491f860bab613169bd67c7bc9113e45884028625d6598c9fd8dcf802\n '20250715.0': sha512:8a92f6334b93fabcbe251c3d0877d7e7dfc22cd26b317a758e20f4cf884002edbfbef3137fb899bf9515f72d2d8aeb4b115902e2decff0e136979f4045d0e7e9\n '20250707.0': sha512:2f891792ae6610b471111ad917ddf69b2504ba741490e8e120b268b5e1340c445af89512c52a5c3376ca31dd857722b69f5dd0e79e3cf81d9ecde021054a0f73\n '20250625.0': sha512:fad846df0dcd29ff84280fb2e01e3edf51a8648cb24baf53d9aad46e35891fef215863a4a3a42c4a96bc3c359495def4d3ee25985ae4e99df2db7bdbb318498b\n '20250616.0': sha512:5de1584ae09bf56d40af000f656f1cef9357d69d223ae40938a77b05fcc8326a3ef5058f7a9860471205df072923c2e1a776b85ddafdddad0d20f33810456307\n '20250611.0': sha512:b84bd331fb9abb13c5f2260de04d536b134b3e71e9b6575700eed64441023c34cc1164125bc0485014782683e4d177b3c85c19e5c1ed218820a505c57e286922\n '20250512.0': sha512:43daf4b8b0e094ebf2cede8bbbf89ee0695ff31924e140bdfcff529296e8f004b457485b9f991ae9ec93cf6150535e297db00a92be8a054589b3316841fbc056\n '20250505.0': sha512:42cd72f9b2011a8ad166d9dc246fdb46ef602aae43127373750a7ff65be84f8b300c50e4977495ce59670af5fc5f92c3c5ba96c5d751cb4e6e2fcce373210e06\n '20250429.0': sha512:9a9a2c351789e6a14896ec5e56ebe7ca1dc7424087d13c175e38a4522a7e6f1533ac8ef5aeea0bcbd554cd5e4d6b6d7ac3df2dfebcbea3e7164bf00fa823c310\n '20250421.0': sha512:c86577ddb8b7b46b5b050000e242dc09bebeffa7cb9d21acb84c4ef896cfa340f024e2b9f463fd4f7945683854c524f4a45de3ff3917f4ba65552cede4229974\n '20250414.0': sha512:33b9c67bc7b73ca49154aff48da52029414a707b6a3a25eb4f71e861a94dec8fce220e63a162841670ddd4876f45b0e39abdf9f8c3235019c89f209684d3007d\n '20250407.0': sha512:1c3838e10c905af0cb52697712bf6bd76b94c9e9d3d07a7643cd43dc2f8dab03b4ed4693c117e555e07a158e04ee583b6b1f1cf2fb9705244ffa5fdc4af67248\n amd64:\n '20260209.1': sha512:bd21b80502be25484d8b43168c88d66b6f3e853c78c0ae5b5206c5625e2a365e98c8b3ba259453d18c01d1aa08fb7c8c1e7f122fdcd7ef806bfc2f44f5837b5e\n '20260202.0': sha512:bd21b80502be25484d8b43168c88d66b6f3e853c78c0ae5b5206c5625e2a365e98c8b3ba259453d18c01d1aa08fb7c8c1e7f122fdcd7ef806bfc2f44f5837b5e\n '20260126.0': sha512:51c3b4bc21cb5c3d4e3baf9f43e5fecd86c327abf0c84d492510f480cdfb38c90d43f3b0dbf1887ada8846d3806da79a73729acaedc570894ba6ed7cf9e083ed\n '20260112.0': sha512:89f55750488559796fe51d2c10c289a8b0617fb9f6498714c026825268eeed449941d23e8cd5b285b69c1b032005ddeec278345198301c50d89ff6d3f66871a5\n '20260105.0': sha512:7f3f5a864fda5f4e2de9db20dd5edad60b6aa467cc7c22d13f40cdce811783d66018f2c28fb74b907c6d6ac0e39f6d0e1047f1f33447b8a8682f1fbaa25edeb4\n '20251215.0': sha512:538a04d88a39de1679afd9868806bd5fdc63737a4871955fc8a8c8e183942c6cc3dbd6b34b2f5589f5f474b4826427f149d5c6abec4ca8d09db363ff5f149b4f\n '20251208.0': sha512:8f1e41374785bdfdf69c5798cfbdec53a833ff6724d36dd644a387b2f6e151c513389b2e5b3c0d5347c5c2ff214910db3c8c164b4d5bb2fe963c5a5eb70ca1a4\n '20251201.0': sha512:216a937437cb1747d5e84edd9ae7274c5a2c4f712f4601e7e0ca06e0a688bebfac267707028b78845276302023d305ec9a93f5b200c9c3c3cdf86a2f41817703\n '20251118.0': sha512:b0f0fa1ee431c63cfbb9007a62c49a374bcbfbbbf5997e63c827d1673f6933d65044ca4f06608bb494f870ced97295dc065810f5e905dfd4a632fe4d61faff7f\n '20251110.0': sha512:56a27dab74191db97f888c936b53861248851a2579d838073f528db7cb9353da5a919a27a38a48447b0a81bd42ab92873c480be769a9818a464ba9cf27872581\n '20251103.0': sha512:56a27dab74191db97f888c936b53861248851a2579d838073f528db7cb9353da5a919a27a38a48447b0a81bd42ab92873c480be769a9818a464ba9cf27872581\n '20251020.0': sha512:56a27dab74191db97f888c936b53861248851a2579d838073f528db7cb9353da5a919a27a38a48447b0a81bd42ab92873c480be769a9818a464ba9cf27872581\n '20251013.0': sha512:e5478ab26ed5e20a503661ccf23b39a45a2ef593e5ad2d8db36c4d44917747efdc22f6af8e76e0c0eeec91e28cc8efedd9c0ed640cde7c9d1843293d47b19196\n '20251006.0': sha512:5849b045a3aa9d26e5850b18b96db8566781398a26db43f02087089e388b6fad05072dc16f1e6024ff725a1d60dd14c3428a3611c781060f973b196db594c304\n '20250922.0': sha512:e5478ab26ed5e20a503661ccf23b39a45a2ef593e5ad2d8db36c4d44917747efdc22f6af8e76e0c0eeec91e28cc8efedd9c0ed640cde7c9d1843293d47b19196\n '20250915.0': sha512:e5478ab26ed5e20a503661ccf23b39a45a2ef593e5ad2d8db36c4d44917747efdc22f6af8e76e0c0eeec91e28cc8efedd9c0ed640cde7c9d1843293d47b19196\n '20250820.0': sha512:06ae1d7647d3ae2155fe374339ad7ec2fcc0d84b3d8d4cef72e1c788d6fe17d6aa227e6dfe1214335617d8d42a06aeb495c65fa53cfb4444e01dc7ab2f22a86b\n '20250811.0': sha512:e8fbe414831e50d20c9f7e3046674a2b609b01a8a7814334ba33e1d6d4ab370525cb494069a782f180de82db5fb6a72c35d9e8cdfd9eba2b43937c474f8c59e0\n '20250804.0': sha512:474113f2c6454738b276023e96548f4fa1bb84017160bf06bb0aa2c6c5b27c12fd1d5fa0d5f9c39e0e91ccbb69525b303018a13b24eec18c3139db94128c0a74\n '20250721.0': sha512:c6305e9e5b0f20ef11e681f0901c6fdd245a79880fe75861f8489a816b4c26f6080d54e210cb827150a4347167c25f2a698d586bed8cc265095bbbcd678d28fe\n '20250715.0': sha512:c6ffccc15f47a9ff69c6f9a5843c93025e8bd2684846a973a8b483a2e8adabd153029864124404de66606a5707706ec79eaa5d0a9ed5df77a3bf2730f488c7c2\n '20250707.0': sha512:c171aeba73e56ad81e55dbd31b518401ca88ae3bcfa9d8209d3b28122b06e5c79e0fdb8bc25a748bc0426241bf5225f466f5ad138f61b488046e258bcf70a30d\n '20250625.0': sha512:61f19bc35df8b15669d822df1fc2ef1e0e9fbac1451be4aa16979df42bd100889e960df689dba9d40d35c700408e1071da41c9dddb426332daef6f2939c6067f\n '20250616.0': sha512:1622ac317d7dff4ca7a5e56bff396a50fbc90f061b12289944d3946acdafb9814a2ea0bd2900e90f0d924f9c6f9450a29dd9bd83a7d976b30ce3cb3a85a2c168\n '20250611.0': sha512:04c9ff5f2e344fe45d743c093ec80948817e7b9af8de5899bd1d3de893cb29bfbcad42f244afea030ac64ddf86a33ee45e22c956162d9991fc6b388cc96c81af\n '20250512.0': sha512:eb7acb5bbd24dd208643b0e91b2195fabd1ca3887612ad33bc34d62a86e4944f3ad80e7592ee5a49cbd6a12aeaa466127a7a220722c2ea64f37df96bebba4ac2\n '20250505.0': sha512:11a1b003a73b2ae8924b03adc557966d815b79d756c9e40adc505c11ffe6f8e30153e5d133566bced39797fbd41651680fc17c0d7686d2ab3cf63b466e68dcc1\n '20250429.0': sha512:42b16d541d589d96075c29e4bf7005bc429c28f411c928412fcd18f093b98f3a7969d799d567730e08f379ce9c2ba7c02bb1e8d10b7fa72179349ac2f40c8d7f\n '20250421.0': sha512:eda25a84342130d3fe7f23ec3abad56de0fb08ac36c430b423c2d51cc21a75e902a4671ffac9481bf04f8985ded12110e65aa8a2032bda2699083d1b9b07a672\n '20250414.0': sha512:93fba15f7bd00dec79fbf132d586497f7981cb4bc3ef6cd77e150367cb45f5f2b8aedb00952ff652ec7438d742b773a19ee2b6eae1bf98facd192a5c233d38a3\n '20250407.0': sha512:09acdc895cea6706ba528939da2e6ddab148dfee56addb0d52d7af74378454f4e05cfd47cbb29ad0569139c49cf298be9d4b94a3c2d28b75c05f713e425746e8\nnerdctl_archive_checksums:\n arm:\n 2.2.1: sha256:8d49681ac806dd3acb2477675daba3574b7d019aea26513ac1960549473738df\n 2.2.0: sha256:91f286e4babbd6e000e743f55e2ec6fd6b93f5b227386175f7932d247ab5a431\n 2.1.6: sha256:9523cce6ed87d379fe06d9b043936398f1b047917037d0faae151de83acb3b4d\n 2.1.5: sha256:a946db17dca42c0835d4ef891057afda03998501f600db3825f8f421f9c0180e\n 2.1.4: sha256:dcb2786c45913903e84ee2cbc064f9812b3f7e82bb208b14d622cd4707882063\n 2.1.3: sha256:5176c4fc55d2d769f619da6cc577c5e4b1fa64d5ff9911df269980084fd9708b\n 2.1.2: sha256:8dab9e2cfe58b94e3e0de837c2ba4dec6864c014848f59bc764a2b8db44322b5\n 2.1.1: sha256:06ec5d079166a69ea0326af7f30dd7306c03a25dcf27b186bc125230f5d827c3\n 2.0.5: sha256:128bb220999c69cad054cf6394cd70d0665891257db4a7dcbedc5ced409eccb6\n 2.0.4: sha256:89e540cb1ac0ed37ec50afc578970a0c9b6a7f1c1b684368da9a726259a3d359\n 2.0.3: sha256:d95f238738623ae1f4fb01b6a7f287436ba85493700a9de263b3efbff57424d4\n 2.0.2: sha256:910619da11b90d71758e6843543ab2106c20b5149f353289bd6d553151b540ca\n 2.0.1: sha256:8f42611dc1554b29dfe990f058ed12920be9cdd78798dcfc6b3845e613eb1252\n 2.0.0: sha256:c68d19a66b6163e67290397ff24660b63c8e6ba6ae3ce4deff33b5a9f7df51d3\n 1.7.7: sha256:26582565426152dc426fd47d8090547128e415314f36710bc58ab475160ab0a4\n 1.7.6: sha256:4c48463659b09636aa23b50825f85cdc38901b6c42e321f69a589d89f6e1d0d5\n 1.7.5: sha256:2d258a7d67e9fa808424ad42f9299a0feb318cafd2758f0287748acedeee4c0d\n 1.7.4: sha256:91d3a8bcc2247dd80f8f5769419e6f344dea412937de4c318f65d8e9bf01355b\n 1.7.3: sha256:44369f34a98e5955eb02e41779b1a470332194e4c2bef136fe471943eaf8057a\n 1.7.2: sha256:d952c1cbe3d25478bbed5f4ee7af4bb52fa4ed47e43802dc5eb2888a4c8da704\n 1.7.1: sha256:799d35de7a182da35d850308c7f1787cd7321404348ff2d5ba64ad43b06b395a\n 1.7.0: sha256:8b9e7cccbcc0a472685d1bc285f591f41005f8699e7265ea5438a3e06aefdcfd\n arm64:\n 2.2.1: sha256:abc83c9ac3d843c3442eedfb61c6456b8b59b1e4cd69f69598ca1582acc7c094\n 2.2.0: sha256:37b353122e0785578d1680fb1d7be546f4c64d0a4aed7875d3a216b2c44be76d\n 2.1.6: sha256:5c30be3ec118eb222bf635f8049c2f96b4c46d9343ec445058e9bc2ee9531c28\n 2.1.5: sha256:d8d7caea291e0ad828dfb885fed2b9ee6703432be54ff3b62ed81aedd8431f63\n 2.1.4: sha256:aaf5acbbb044d82038518780cebebcc2a901afde2db465d3581b8987c2d6f6fc\n 2.1.3: sha256:62e34ce6156c942368968c0d88c731cb2f3b341785d584e56a2c175916d5f88d\n 2.1.2: sha256:af6b8c9028fb1adc84b64c9a63da0f369bfa5cfc9176c8b7c58a573d30bd8a74\n 2.1.1: sha256:648a053c9b5f4e0da1b474fdec40ea7f5b2eb2c1ce88c7881a7c6c2cd11c6478\n 2.0.5: sha256:ecf57d202fe6fb1fadb68f25f3d1702a424490f8ca0817a902689f1e1fae9e4d\n 2.0.4: sha256:1f394e3aabc2b202ad17a5ece0495ca554ae3ca346a5dd0aa8344a891734763d\n 2.0.3: sha256:f2c3f12c99e112cd82ba19ca9b875045c44b2f5a19cecc295ed8d61d415e8851\n 2.0.2: sha256:c50ba98be0ef05684948f7873078558504a7cc46ff92cffc764c1625b1cd0d40\n 2.0.1: sha256:cbca59744f6e9dea962e1d3a754294b5e64b53b82f4f7f7d603a591f38545fd5\n 2.0.0: sha256:568b0383996cb51e1a5f97abb3973ebac2af1a26268fc7d9ae0d34165ba033e1\n 1.7.7: sha256:230ad8f4f88100774d123213a427d3d43071e0dcf9f70efbea50ab9efff1cd4c\n 1.7.6: sha256:4fa0a6e936c7a9cb9bb81e784fddaa593cb00afb48b08842e3f0503748c21348\n 1.7.5: sha256:a53d87fc7d1f4ffeec55e5e08d2397b02ada0d334874c3cece306ad36f828a6c\n 1.7.4: sha256:d8df47708ca57b9cd7f498055126ba7dcfc811d9ba43aae1830c93a09e70e22d\n 1.7.3: sha256:e4f16b78d884768f6997558130146ba9bd7846828b19fa2ca8e8eda988953fd7\n 1.7.2: sha256:de68d5380d65604cd26c164988547cf46b698f7819a5d51d98e3a0f031f5594d\n 1.7.1: sha256:46affa0564bb74f595a817e7d5060140099d9cfd9e00e1272b4dbe8b0b85c655\n 1.7.0: sha256:1255eea5bc2dbac9339d0a9acfb0651dda117504d52cd52b38cf3c2251db4f39\n amd64:\n 2.2.1: sha256:34144de7f12756aa4b9dc42a907fd95b0c5eb82a63566a650ca10c8abe7a26a0\n 2.2.0: sha256:1b3390a832eaeaa1459cf42357da983205da2dd72300a015ad018b3499fc455e\n 2.1.6: sha256:22857b373edea479a4534ba62cae1c77f2af38f0aac4c91c1c68cf09e29d6f9e\n 2.1.5: sha256:9ff862624084fa1b2c6272c4498754801bff3f4ce09421ac6eba58a760878544\n 2.1.4: sha256:d6e91d3e275bfb3404959b1f95ed25ff6cd83e3181d17b93afe2d39cd025501d\n 2.1.3: sha256:227390fb16c20d6ab3a7588e5ce72202df1bec7960a2a091599ce3c1bf3fd1d8\n 2.1.2: sha256:1a08c35d16a0db0b4ac298adb8e4dab4293803d492cbba7aaf862a48a04c463d\n 2.1.1: sha256:4fe308bbadf7dd079c058a34cd0bacef3b35c46da88c37f5f125044d90941595\n 2.0.5: sha256:a029af80fd4b3de096d1a18779e7ff8369fbd1285944ebc50bdd22ad41438b5d\n 2.0.4: sha256:2f9d22179868db4f0a1daf2fd65f58a24f8a78efd2d9b17659f56bcdce85efd0\n 2.0.3: sha256:95ff850688a73eace7453f19e74bf4cc8a1f3e458eeb97ef7a6b74de9825df16\n 2.0.2: sha256:1ba015dba039cf6ec2434e88d97707f0b715790e6b7f2e7b6ff7be9200f47bc1\n 2.0.1: sha256:96e5e3ed79f189a986cd33a40b0c817d7b6c7d9238f51a0737213f409e5d82af\n 2.0.0: sha256:b3920c4a04dc7972d9e2977147c1ef5c415551802b3c4b0b613ba9d0c46cfbbf\n 1.7.7: sha256:298bb95aee485b24d566115ef7e4e90951dd232447b05de5646a652a23db70a9\n 1.7.6: sha256:0326d6a42dbec5c104ed9d7aa8cbc62727433dbe000cf21cc29d06b22507e0f0\n 1.7.5: sha256:775f8bddd5e93acc4162c568dd6b84d5925549180991b83cfd4b7e33d4844921\n 1.7.4: sha256:71aee9d987b7fad0ff2ade50b038ad7e2356324edc02c54045960a3521b3e6a7\n 1.7.3: sha256:ee93ffe6f90e50bde153a9a0dd779594e0bc13a26949053965958b91b6dffdd0\n 1.7.2: sha256:aed7d33d645bfb97c8df978d952a1e1f7e02b0b3ed2c0089ee4285af7f8f971b\n 1.7.1: sha256:5fc0a6e8c3a71cbba95fbdb6833fb8a7cd8e78f53de10988362d4029c14b905a\n 1.7.0: sha256:844c47b175a3d6bc8eaad0c51f23624a5ef10c09e55607803ec2bc846fb04df9\n ppc64le:\n 2.2.1: sha256:05c3573e0468fbe6ccecce497b8129beec0fa1d8afadeba244e3d5ac63047fce\n 2.2.0: sha256:cc9f55ffec892498bb27db1f6b0eef16b591ee4ce873b61f2fd9a9a30930c620\n 2.1.6: sha256:807678bc5042cccf81dbe13b00bbe8e18dc24412481c3cc68eafa316ae43842d\n 2.1.5: sha256:18c72aa80d974394452058472e0dfcfe6200307969a17a33ffe1a85606a2663c\n 2.1.4: sha256:82b3c3ccbf314e27591977268c81f92660bf6b78fc568e4e7dc1b583f61b622d\n 2.1.3: sha256:a3ca5c287bc3dad0d23ee8c10a9acfd86910bc120737763bd4cefb872b33b958\n 2.1.2: sha256:2008c94b18900f3a58439ae9dcb7aa6659da82e2d0c9d67ffbecdaa152b9b0bd\n 2.1.1: sha256:f277823a3814c6309bf4d31e94e2b8fec24c99326e517ec81c325e081dd7c20a\n 2.0.5: sha256:01bec466e0184945d8138ef0bc7e6689e61c4c8f56bf0968825c7f5a956c076d\n 2.0.4: sha256:93c5bd5f32a3b821fd462a96c6943b06663ff03e8fb16327957fe99576855116\n 2.0.3: sha256:8d6283b1fe871e319a2f5cf96fe97aba649eeaac0a2a22c81b9b4d3c613c210a\n 2.0.2: sha256:1baed7f4312404da966155856aa1e4b4f48bae73d64fd2cf6c41ef9326a07b10\n 2.0.1: sha256:78a3846cacc570e8ee4a1d60928a55954fb4fd1b3b731c0c975a808134166fab\n 2.0.0: sha256:879efec2963c9854aa1491d80d341570c44dc12b9728084ce64a986ac4fa9bfd\n 1.7.7: sha256:8236524c4cc6c91007c794160065716516a4f7b63270dc00b796df11776b4f19\n 1.7.6: sha256:89906f9bcdf8d5bd866646c43e14c0ae15a83ba3ebed44b06c3629a11517e242\n 1.7.5: sha256:8e0891f608144d8d751070edea5cd98d2a76a053ad7fa6b9d4aae94a700aaea2\n 1.7.4: sha256:97c99ab6030ffac1fb780fe012de06a36512b17b13de5c99445468b5a5fe5a62\n 1.7.3: sha256:e63ae0a8f5ccd12877ff944b609d0a4c55c97ba79808ab16c7dc7e99fd8f3dd6\n 1.7.2: sha256:e5c01702d3cec0763d28bd3cf6ea9c3efc58662a93cb4e15669a839782af10d7\n 1.7.1: sha256:09fd0cbef25c98e08c5cc2d1e39da279cbf66c430fdf6c8738e56ce8f949dad9\n 1.7.0: sha256:e421ae655ff68461bad04b4a1a0ffe40c6f0fcfb0847d5730d66cd95a7fd10cd\ncontainerd_archive_checksums:\n arm64:\n 2.2.1: sha256:dac15a0d412a24be8bfe6a40cec8f51829062725169f1e72ac7d120a891ef5b6\n 2.2.0: sha256:8805c2123d3b7c7ee2030e9f8fc07a1167d8a3f871d6a7d7ec5d1deb0b51a4a7\n 2.1.6: sha256:88d6e32348c36628c8500a630c6dd4b3cb8c680b1d18dc8d1d19041f67757c6e\n 2.1.5: sha256:fe81122c0cc8222470fa3be51f42fa918ac29ffd956ccd2fc408c1997babd2ca\n 2.1.4: sha256:846d13bc2bf1c01ae2f20d13beb9b3a1e50b52c86e955b4ac7d658f5847f2b0e\n 2.1.3: sha256:7e423abc7bf52ff6cb724f44995cca335b40331efa727415a5efc99ca34ac8d5\n 2.1.2: sha256:57fa4005ed3bb648f4a2ff3ef2f9ce12b27ee1397225626e3165b9ef4af45530\n 2.1.1: sha256:4e3c8c0c2e61438bb393a9ea6bb94f8f56b559ec3243d7b1a2943117bca4dcb4\n 2.1.0: sha256:f6c3972347848177805eed8a9c282fca6aaec0e6fd28701579e63cb20bdbce07\n 2.0.7: sha256:e590e39956a451ff692454ea991961b08627c514dcc253e8845d2a57f9a78e2f\n 2.0.6: sha256:0f308f386b1ee24712875f02bded92ce7099a707ed43f57b3fb9c934dcb6bed1\n 2.0.5: sha256:36eaf77dc65df4b60d6e06204631a4105b4e942dd2704d618758a2aa0eecc264\n 2.0.4: sha256:0fde98b24bb55363a54150732e0ac99a43bccf2a9711371bd5470f32790316f2\n 2.0.3: sha256:3701008e72e983259afaa594cca5d8126e78e38cf0a586a1f6971cb3f61c4b6b\n 2.0.2: sha256:14a2a9f7f75f73e5bcfb8b183d0b84830c54b98ef8c5f6ed70e51f1a230c673e\n 2.0.1: sha256:b07120ae227b52edfdb54131d44b13b987b39e8c1f740b0c969b7701e0fad4fa\n 2.0.0: sha256:2a00b1553f38aa9e716d61316b661961c2fbfbb7aad7bd73b377be5725ecc0f1\n 1.7.30: sha256:a09c3b01b3b6935e839c8a9588b5528c57ebfca4747d816654a7d1e7575c0a63\n 1.7.29: sha256:176d523a6d6dc5520e0c35b8eb0de6e54bd4d00486d5fbbeb4163f30e2962f17\n 1.7.28: sha256:97457594ff8549cb82d664306593cafd3d2c781c706f9fffed885a46d8919bec\n 1.7.27: sha256:3f03ea60c7dacddf890be3ab18f7ef859d9d104b19627f52038d7984361912bc\n 1.7.26: sha256:adea067914e678ac37d5091ead66f1e36e5cced4d395bbd2be60772495e09eff\n 1.7.25: sha256:e9201d478e4c931496344b779eb6cb40ce5084ec08c8fff159a02cabb0c6b9bf\n 1.7.24: sha256:420406d2b34ebb422ab3755fbeede59bf3bfcfccf5cfa584b558c93769d99064\n 1.7.23: sha256:6a66b5e63a5e88ff7eeb478ccaca9083d44e51e1d7261ae183fe5951a6226ccd\n 1.7.22: sha256:48d0a8461ae829b12b07c3663b14b70287d0607a0792719c51b4e4dd700b02ce\n 1.7.21: sha256:7b6b67d998eb86856d23df5d57269c054539072bbb27677975cf78269b2c5c10\n 1.7.20: sha256:cf80cd305f7d1c23aaf0c57bc1c1e37089cad9130d533db6fe968cdebd16c759\n 1.7.19: sha256:1839e6f7cd7c62d9df3ef3deac3f404cdd5cd47bbdf8acfeb0b0f3776eb20002\n 1.7.18: sha256:e80ce87b469af03b3bdcf68b95f0f4a303787ae247581bcd42f04acf1ad4c24d\n 1.7.17: sha256:8d9749985796a208e860afe331ec77cb485566104e5cc7c0b5e9e82ec7681969\n 1.7.16: sha256:2d4373de40a6f58cd0f29377c0257b35697a987248e6268520586996771d7a75\n 1.7.15: sha256:5cc8bd8f3d9803ef0ef701596e89d62ad6850a2544e722842f4533642df36d87\n 1.7.14: sha256:44df66d0a0332465e7d15e90b974cd4f08d059dfa26652218ed9485390f47f9e\n 1.7.13: sha256:118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d\n 1.7.12: sha256:8a1b35a521d071a8828f63fe007a51e5b7ac863a1195f5dee32543b1a9d5f2b6\n 1.7.11: sha256:5eae27cce38a14be5390d4035127aa11416bc5ae592a9ff25b11870872ce1159\n 1.7.10: sha256:0667b12a04a896a61cf508a4a77190c280f4a1fa35f38c8a4ba63f605b5ec375\n 1.7.9: sha256:09ca326dee14e00c439137071747c15cc280480e2c26c1e82698c992dd1889c6\n 1.7.8: sha256:3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1\n 1.7.7: sha256:0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3\n 1.7.6: sha256:d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce\n 1.7.5: sha256:98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9\n 1.7.4: sha256:ea5a04379bd4252fc1e0b7b37f69cd516350c5269054483535d6eab7a0c79d2e\n 1.7.3: sha256:85d2eaedabff57ac1d7cd3884bf232155c4c46491f6b071982e4f7b684b74445\n 1.7.2: sha256:d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6\n 1.7.1: sha256:1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b\n 1.7.0: sha256:e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2\n amd64:\n 2.2.1: sha256:f5d8e90ecb6c1c7e33ecddf8cc268a93b9e5b54e0e850320d765511d76624f41\n 2.2.0: sha256:b9626a94ab93b00bcbcbf13d98deef972c6fb064690e57940632df54ad39ee71\n 2.1.6: sha256:4793dc5c1f34ebf8402990d0050f3c294aa3c794cd5a4baa403c1cf10602326d\n 2.1.5: sha256:403af72d9f956ed8a5ad5b0ac0f1e8e371a1488f2b9edf9b4ba13db0653936ea\n 2.1.4: sha256:316d510a0428276d931023f72c09fdff1a6ba81d6cc36f31805fea6a3c88f515\n 2.1.3: sha256:436cc160c33b37ec25b89fb5c72fc879ab2b3416df5d7af240c3e9c2f4065d3c\n 2.1.2: sha256:87c18b2686f38ee6f738492d04fc849f80567b7849d0710ee9d19fac3454adc4\n 2.1.1: sha256:918e88fd393c28c89424e6535df0546ca36c1dfa7d8a5d685dee70b449380a9b\n 2.1.0: sha256:0e5359e957b66b679be807563a543c7416e305e3aafcf56bad90ef87a917014d\n 2.0.7: sha256:6dc663cd245d19c3e68260ca86247469563a31decac925087b62a2a819dfcbee\n 2.0.6: sha256:a545471a67b8508a3c58ad01a4e6bb2921ace1e59e00a0a2d2e784c1f4fa8caa\n 2.0.5: sha256:88ab31f3e78e4d2fa12dcb933032122d11d441c83b79a89c6c8076f871e50df8\n 2.0.4: sha256:e1c64c5fd60ecd555e750744eaef150b6f78d7f750da5c08c52825aa6b791737\n 2.0.3: sha256:ac70856f1d8bd3aa9ca5d62db5516b86dfa0f934c1fd1d1c5fa4422dd12ba45e\n 2.0.2: sha256:9bd5b6a1bdf505d520d9a329c520258ed0a17faa9fe3db12712ee858ad59aae3\n 2.0.1: sha256:85061a5ce1b306292d5a64f85d5cd3aff93d0982737a1069d370dd6cb7bbfd09\n 2.0.0: sha256:6f8da716941f7e89315cefaa6e5a8f1ff10b323ff46611313c455df7ab1ebee1\n 1.7.30: sha256:ca0f27e34411504acd1cd24fcbaa71b9c47d31ce9408c47c54a2dc1810ceb1df\n 1.7.29: sha256:71d9f6e4ea4a9e108e2172b0e7f6fa137e086808db3e6874dbdb91c01102d3d4\n 1.7.28: sha256:7a8c262deb63becc877e82d23749e4f99f4a17e8e660f9b8c257ca87a5c056b6\n 1.7.27: sha256:5b038fb22ab5dbb1ce57dd3d8f102460cd8619ff2afc78870837b06e8c4e840a\n 1.7.26: sha256:fdf1fb17086b62fc861103da4e3fda3d79bc543b42d2acef5d07e76b13d35d19\n 1.7.25: sha256:02990fa281c0a2c4b073c6d2415d264b682bd693aa7d86c5d8eb4b86d684a18c\n 1.7.24: sha256:1a94f15139f37633f39e24f08a4071f4533b285df3cbee6478972d26147bcaef\n 1.7.23: sha256:8a0de43d9313aef2ebdccc0ffa49461a4a28139a2c0ef104c3c847f6f37c8119\n 1.7.22: sha256:f8b2d935d1f86003f4e0c1af3b9f0d2820bacabe6dc9f562785b74af24c5e468\n 1.7.21: sha256:3d1fcdfd0b141f4dc4916b7aee7f9a7773dc344baffc8954e1ca66b1adc5c120\n 1.7.20: sha256:e09410787b6f392748959177a84e024424f75d7aff33ea1c5b783f2260edce67\n 1.7.19: sha256:97f75e60f0ad19d335b1d23385835df721cad4492740d50576997f2717dc3f94\n 1.7.18: sha256:a24b05b341c155a0ec367d3d0fd1d437c09a0261dffdecc0e44e9abbf2d02aca\n 1.7.17: sha256:04cf937349f82d29fe98553ff45a7e9ea2ed6b81fe6514e3679cf263b50409ff\n 1.7.16: sha256:4f4f2c3c7d14fd59a404961a3a3341303c2fdeeba0e78808c209f606e828f99c\n 1.7.15: sha256:ea27e6454954bd9cb62a70b0a40eb085ae9c96cb8c075a74910102b33586e07d\n 1.7.14: sha256:48e0d9747cd51cb90e0b278d100397653d9f2e765effca194427e4796395b240\n 1.7.13: sha256:c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23\n 1.7.12: sha256:6a24d8b996533fa1b0d7348fe9813a78cd01fa16cff865a961ad0d556f5cd665\n 1.7.11: sha256:d66161d54546fad502fd50a13fcb79efff033fcd895adc9c44762680dcde4e69\n 1.7.10: sha256:eacb0296bff2ae5225a18492dcb32fb28ad4a1fe0a39ea9073367c7e43dc5838\n 1.7.9: sha256:ccd5b434393666f6ebbc90eea959ffd3e61958a1e3e1cc830a678f040142d4b0\n 1.7.8: sha256:5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9\n 1.7.7: sha256:371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba\n 1.7.6: sha256:58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe\n 1.7.5: sha256:33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25\n 1.7.4: sha256:fc070fabfe3539d46ae5db160d18381270928b3f912e2e800947e9fbd43f510c\n 1.7.3: sha256:de7f61aacba88ee647a7dcde1ca77672ec44ab9fb3e58ae90c0efc9b2d8f3068\n 1.7.2: sha256:2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7\n 1.7.1: sha256:9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85\n 1.7.0: sha256:b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0\n ppc64le:\n 2.2.1: sha256:3de0f215ea649952a9e99040cb3888d8059bd3d35b04edbe6afb916c763f9ea7\n 2.2.0: sha256:e4ecd0b03200864e117371b25cce5335e39ce0b0a168a01d2ba6562a05020f0b\n 2.1.6: sha256:aef2b639a14ae79f2bbe43356b25e84ecfb2c7f269c87f41e41585e724073e54\n 2.1.5: sha256:dc95edc01958d18f8475ab4d415e8c92cb3bad580167db8b0054374fd9004f78\n 2.1.4: sha256:d519e40e266f39cdd68f2c31e2e4e9b70eda09b96f3c3de343a7a3e11d49ad4c\n 2.1.3: sha256:e517a6d936ffb6d2292e9c6560aa363382b1457eba34cad8289f6f3f76201588\n 2.1.2: sha256:9d8a02413050ae234eeef4152fb703b3d093b5809411f0e905f098554fc066d6\n 2.1.1: sha256:36c90c9993e9f2142aa014cbd352bd7d3db6b8daa2990b4a9f59e706db78deaf\n 2.1.0: sha256:3e380629da9d21366c916ec3873022a2a5700584d409297a1bc183b11fcb0809\n 2.0.7: sha256:15186b11d555f61e495e94d5a99b2eb2c7f21a5e569eca02170c4bc12fa8c7f6\n 2.0.6: sha256:20df16cac3a912d6df34040dc81c7e5e2c95a06b2b21fc2ffb4372b6d41274e4\n 2.0.5: sha256:09773a42829c0ac9b8dd449753c755b3ba65cb7e8d06485950f99d32fd6c1e0d\n 2.0.4: sha256:ca970d9a53ae504bc36197d6daa931338c387c83b6948b9f9bfdd1a75e25dcf6\n 2.0.3: sha256:2f0faa0086ae81d00680367ee9d75aafd3c4ca4535362db83fea62dd19c47079\n 2.0.2: sha256:1b19d31bb8a7f9d26d9b50675e78f397d0b01fa635c33cca456f91c412fa6df1\n 2.0.1: sha256:09a25357343c7336fe519e5fd1a9dd0f22da869e9deda50c2bc61b6e8c9384be\n 2.0.0: sha256:2e7f4b15ac85c22c1ced102bbb424124078248f0af3183425ff335a998079809\n 1.7.30: sha256:2edba94f31ed0c32cde0cca2e79e4426224eece4ef4fa1f194b2df4c811417fe\n 1.7.29: sha256:8c08edb22b53a44cf7b5a5e31ca8fb86d1d6df378724048ae0d9890a3b66f081\n 1.7.28: sha256:e8f64abf81503aeee0db0d5682197e9ce377ffeb858313c5bc9fc3d7faa4b85f\n 1.7.27: sha256:ccdfa16e4bba3a993d74fac794d22ddadc1013d351cd099ea933827050ef05a0\n 1.7.26: sha256:34a86b1bd598b34e8c05956c5976fb0c0b347937d3cd0837edbcebc7f9e7e53f\n 1.7.25: sha256:0934176e32eace1c23dcb9edff0e78f872bf8f7152b5e6f622e9ccf1ddce8722\n 1.7.24: sha256:2ca4d527dac68132a2a6b3971d82ddfd18edc7fa838b7cfcfe6eb11efd017871\n 1.7.23: sha256:00dd8a1145d7392ffe1e2b74da147b896e4387afb5e73ed6e5cd3744add32826\n 1.7.22: sha256:6747b7291ffbfde2c0bf0031978985df92ac74414f09bf190afda0fc9e797146\n 1.7.21: sha256:5ce0c1125e8d9ca04e2b524a2bac8b1eb97876c073023d5e083f7da64fcd8207\n 1.7.20: sha256:dc611df0baa90509dda35e0be993da52f42b067514329fcf538d000b110364e8\n 1.7.19: sha256:f41c2f28ee933a9ca24ff02cca159099fbcf798850e56cf0b7a6047ebe21fa86\n 1.7.18: sha256:d6cfb3bc8fbdead7d435d5f3f6b1913b5896f7f97102c1bbad206f9123c2a5d3\n 1.7.17: sha256:873b76a507d362eec73887f61fa1400f3a892c7dbed1759f5dad2b654095b534\n 1.7.16: sha256:d0add7a55a5d4411cafb276469d2b78bc3ada11cb4b444b9e35f9ef60c00960d\n 1.7.15: sha256:b38641d9bd18139495cf9839999039b19941f53d36a6d72efe4577c489dfda0c\n 1.7.14: sha256:b84b523909b9dd0c0b2bc40bd2b9af543ec9f1186df69c220ae3749e34623dbb\n 1.7.13: sha256:89605ed2365d5eb779477d11947101236eb44e5244f1e58bb162a9e68d242798\n 1.7.12: sha256:80f16891b387d86712026234de7d4d0365a38106dbe5e51b65b1200b24822721\n 1.7.11: sha256:6f91c5dabdccd1fc75aae8687381bb185b9eb4200beb29d0993dea8175f5fa61\n 1.7.10: sha256:15a5191bf7c555956a8565d8786399d51b13f2718d59b1a5b2bd380fc420bf8a\n 1.7.9: sha256:174b8af2d878ad8410205b9ba44fa8d2a9683a521abf13f168f67b7f7375d5b3\n 1.7.8: sha256:2b563df9e1bddc96a99a023963c99b5faf3066d3fcbc23ff44ba24229e939444\n 1.7.7: sha256:0335e7447ed84757489337686a709e95ffa379a8780f238725abb10facaeaa7f\n 1.7.6: sha256:956fadb01b35c3214f2b6f82abc0dda3e1b754cb223cd24e818334b08cb09fb2\n 1.7.5: sha256:2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc\n 1.7.4: sha256:c3397f67fb5756e6336ff76eeb34dfad7ad66235877a4186d82044c7a4caf482\n 1.7.3: sha256:d1977922e74147782dd5bb488f260ee14d758d29a7651cd97bc2e6c7cc1a3cce\n 1.7.2: sha256:cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e\n 1.7.1: sha256:17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342\n 1.7.0: sha256:051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889\ncontainerd_static_archive_checksums:\n arm64:\n 2.2.1: sha256:6b3b011ee388638eace05ac3be0eb32dfb4a43086695c29d06e997febd336f2e\n 2.2.0: sha256:5f2a7f451231ff35d8306f874c51606fc9da1e2db56048834a23260f68a78eef\n 2.1.6: sha256:9da292010d36d80afa3bb48cbd15f65d3bf38177217060272a1c3fd65351cfa4\n 2.1.5: sha256:d1a1e64c4334e17d6f9f40093d5ff9f810b95ac34c7dcba55e7d2226d2a8ab79\n 2.1.4: sha256:c5f0957064e6ed5a67905ea3f8e451dadd16530334b86baaad678dd357205c30\n 2.1.3: sha256:74703e628223c6f19ab2df8497a061d08dc1b81c03c720cbc3d66fedaddf9ca5\n 2.1.2: sha256:3f1dcd2bc02cbf9e8fde308c144551153f19e15105a26cebf213c8e232a27f5e\n 2.1.1: sha256:f4525d8adc4445f8d623c6fd91e7dd750189b96539c29ede2636583ac5e4cf7b\n 2.1.0: sha256:1239f60717f4ec2e06e51d2ed86c43d8a0ee10880c73131e58e1e0689e9ad049\n 2.0.7: sha256:cb46d915f8dea6e94fccd204f29bdb3cbe502d7deb9220a6e662cb33f29784fb\n 2.0.6: sha256:4b6e773839b153a189935d76aace6e965811fbacce5cc938d74910582f2c0de3\n 2.0.5: sha256:302a57f36c80c547c57e439e5f4d5d4e6ce73a9c7266a6787c739a3b9d4dd813\n 2.0.4: sha256:3529558b49e87a220205b632fea1c4b2cdc60f1d93eab3d4c307f66b91df189d\n 2.0.3: sha256:f4946c0a73f966d47ea56dd806a4e466447f194d9d472fe3f59477c6b1510dee\n 2.0.2: sha256:da5631d38702257674da542969c285adfdbdccffc12c5ea39de4db6113de51bc\n 2.0.1: sha256:6022bd160b6d83f13fccd87b1c3854b0294a940335fdf014c10c80d71b279115\n 2.0.0: sha256:428cd0b08eab57003db8a98742d8404b4b69dcd335c5f0f66ceec5fe3b9b31b6\n 1.7.30: sha256:1458b3c4a20eb1305bb1d4680c660d0726fa7709acdd95ec7c0fd6bdd632c36a\n 1.7.29: sha256:d428ad3fb7036c07c141f6666de575d8c7fd862e658a28da5ae20b737fdf7d75\n 1.7.28: sha256:360307a347f1114d6f67fe261674033661c56e6deb3c2529705fc834aba98ff1\n 1.7.27: sha256:ce2b308e81dc1e633362a45b64fbefc58a7c521e7e060cf95a7709bc1704b402\n 1.7.26: sha256:f35d8eb8467b7875ab768b4b869c9905616d998549e1e0ed993a52eec319dc51\n 1.7.25: sha256:34bd6c26f07011af71a4b31d6637b28d693b0c3a0e847e9df10cac1ccfd17db1\n 1.7.24: sha256:16d13febaa1da15c3ca9fa662f6ae08ca7dc4274f1d6f15f478a61238470c2c0\n 1.7.23: sha256:e7b21b7a487f6102693e115d9150976a3d4812084ec5ec950ff0840fb97631c5\n 1.7.22: sha256:a4cc5c572b587799550d4fdb27b3f3f86bc0fc9d93a7b9b468e704df9d052c5c\n 1.7.21: sha256:5f36f5381fa5db7e35eec8e2bf342dd26ff9825f24caf7b41f72778a97e9c757\n 1.7.20: sha256:9082a99262f8b007bc19856ebef2c48b7b78d50c17b80f8835fffcbdd9f490ea\n 1.7.19: sha256:efd641f5443614b4c17b33c185b62b3e5f67dccc9d140c48d67783c685f62532\n 1.7.18: sha256:1eba98be0632d30911ae4bf6ceb27cda32007f5c0d7701a94623f59848dc7cdf\n 1.7.17: sha256:53fb903fa47b4816308297ae9a0a2ead8cbe3028fbba573d0d5054f722c70663\n 1.7.16: sha256:6d64cdda15b0c0a5de0a644e7f0b0704582eed2e388a75a88f8b6dce8f3f7ebc\n 1.7.15: sha256:5b8affc00938d2e74331ad54e15b89a209b4e0adf0c81256b07b4363ce0a657b\n 1.7.14: sha256:5f2add641528921a42451409db305f64c397421a4571466c800bef22f055a0f8\n 1.7.13: sha256:551bcbace1865c645285e2e8458cbb22ca5674de776e409ce75c81a2c2124f70\n 1.7.12: sha256:d1617a272be304d0b4ed423c0bfa905ea68d979fdac777ec4136eacafabdde5e\n 1.7.11: sha256:bcc0ce5b485ce4b82ed3e585ed7093f4dfda9c9828d1b4caf53efb414bc0ed18\n 1.7.10: sha256:4601af2e1cbb08125c1d72ae7c5dbd5dc1f6e47a2ff7b9d42409b494e8d2f75b\n 1.7.9: sha256:5b76cf84a4f7707ad9ea411c16d5d74b07b06faab38e089b145697ef615bdf3f\n 1.7.8: sha256:661fc140d9054c2a2717491ce268596619b16958a7344a144ba5401a8e7d536e\n 1.7.7: sha256:b9bb0aeaf86a5f70015033651e471d067266747a8a39d6fbfd96b8f9c654b891\n 1.7.6: sha256:6f0fa35e7b3ca528986ea6962fac883a250b06d2f86ac8a4e9f41384929d7648\n 1.7.5: sha256:851da2b4a43acfeaa6eb17296b20b5b7ddc4bf48e255209351ee8217cc7e99c7\n 1.7.4: sha256:4557030cf4d3fe7ce52c8b299fa365340302f027311e0968103b3df3d88dacb8\n 1.7.3: sha256:188ee47a27966608db91b147187a76c367357a3138add5c6ef35fa9bb1a4bb96\n 1.7.2: sha256:0dee8e1862f92af95ac4b29eb23738cc7eaec2e61cb05a93be27137014e4af97\n 1.7.1: sha256:f0435e7cda3c3abc40d3f27d403a8e24bd0b927a8a893a7e4dfaec5996fa9731\n 1.7.0: sha256:6e648cd832f026e23eb6998191e618da7c1ec0c0373263d503ff464e0ae3977a\n amd64:\n 2.2.1: sha256:af3e82bac6abed58d45956c653244aa2be583359a9753614278ef652012f2883\n 2.2.0: sha256:2d20037947cbb0def12b8ac0c572b212284c1832bf3c921df1e58975515d1d08\n 2.1.6: sha256:577900a5a8684c27e344aeeb1fc64e355745f58cba7f83c53649235ba25abbbf\n 2.1.5: sha256:9389a4bff0112258bd953c66382709c2a4a11e25e376c8c4c7075b39de156882\n 2.1.4: sha256:50e53500800f4f74d0d8b2e57e939297eab68b0fe11a0957b771d5faa61fae5b\n 2.1.3: sha256:1bb0c910e8fdf623fac2305ec66e72c4afbf612de282577dfdbebf08360937d5\n 2.1.2: sha256:5b2b8e82c5d3e6019428db55296b5169e748b94601c0c944c1d780904aa35543\n 2.1.1: sha256:9ab7df748ae3e9c2a513586668b46175c27290eda6028609223d9065eb777964\n 2.1.0: sha256:e8e5da8ca6586bf2023940ec5bc5d4e949924ac8dca2a9f0dc469bb9a297146f\n 2.0.7: sha256:23371338ea3d930f020f6ad476a608aa8196ecdd016f9f5cd511c87cf37f86b7\n 2.0.6: sha256:e6d69c5931af2a36c98cd066c96d60ee8b05a961d1f54d3d13d80c9afcd74a16\n 2.0.5: sha256:3506b47f8f0806c86ab315ee50e09580fb6f21e1ed3a30051105c98c5592c57b\n 2.0.4: sha256:b8785bc2ca5417fe62fcd5efc687e630cce11ca57e50edd6b9baca10d41b4c1c\n 2.0.3: sha256:866c05ef1c7f22512e1abed271a1b4b6a8ab9c02e8e802da536f7043767a0e8a\n 2.0.2: sha256:7cd4de9c8ad37f3248a45b9d6347b7628e4d77d1c8e35c1f80343450fa47dc00\n 2.0.1: sha256:22b2a7df86fe3e53e219af22a2b5e81d1b67e67d55ec3a18f89990b161fb2157\n 2.0.0: sha256:e72cc69db9984a8d46a34495c302d2b50188ee2dd5c7000a7b471d0350e14ad1\n 1.7.30: sha256:290eea517a7aa919aa319563f8270d284ab5197736362285da09b198b76a34b5\n 1.7.29: sha256:ba4a2e919d2882dba707d8c9c436ed3496e3bb99f5cfb13a31b29b497cf7ec2e\n 1.7.28: sha256:659eceb8a5831a704089825d980f63abfb686650646751e13f9f9fac780ea08e\n 1.7.27: sha256:e3ea27eb0e7b8dd92ba7a5ecdd363372ad3c30f9cdecbec75e60ae4a43ca93b9\n 1.7.26: sha256:e19d790ee965e39d0eaa03d471ac4d3a8dfffab0a7ebba9f5deab77b6eb66f0a\n 1.7.25: sha256:53f30807bceaefc3d294cc223893e3897062c99c4fc0ff6b0fd60a81e7ec0bee\n 1.7.24: sha256:c42230baff057c606cd4eb9b71007fb8e2e55e951ad901f0ae0d96bc55ccf85e\n 1.7.23: sha256:625c317ca556b39852bd1260196a2b9d18ab44c8980cea31f4fc58cf9bdc9b47\n 1.7.22: sha256:d42192685f9fc8b6761c648ed5f0a4ea10b21c4288444d575c5608b42b20eb17\n 1.7.21: sha256:24527d4b0696ef16717d19d983d70a031c77c979e235acb90653480c71b9ec6f\n 1.7.20: sha256:61929db32711d62d1c95400a44606daca2992c18a090135c2a42e41f75777b48\n 1.7.19: sha256:7a79ea6953d02dba76e08213fb5f9ecd6397440d832aadc621d255b25a62414e\n 1.7.18: sha256:ef5b06fbc09b2e83ee739e49e2ec71bd18bb9e334eb79c5c073cd23d800260e7\n 1.7.17: sha256:b4ccad6a3c14a30c9a14571527c2eab0ea2231e27d187750d9ac6d5f477938e5\n 1.7.16: sha256:ef763349371986fd9723eea2b7ce98610328283efc1fdd65754302cce4539db9\n 1.7.15: sha256:089901b4356d59f5e9ab8f06ee28e3f4a5896cd97b4aa76fdc35a891f4aff48e\n 1.7.14: sha256:091b374cdac4b0539b7362ce7167b79de0708a42d06baf558f67bd81a385af9a\n 1.7.13: sha256:b0a9bb64ee5c0347978f02b97dbb31665cee9254e674da54ffbc3bb4b5fda34a\n 1.7.12: sha256:1c1df44243bb6923ade7c22c8d9776470e538a683be6f68347bb4f0d82149152\n 1.7.11: sha256:f1d6a1b1bc1a6b7d4daadb3c474110cd2172071c9b34f90856ea305d1a2d9e18\n 1.7.10: sha256:2d4318cd0229b1dc757a3eaa37cd1f6df58bb876371ce59ecc3571e0f2fca446\n 1.7.9: sha256:1d550bd05e4f997e61ab6f3e9f4e19cdc660521e37f3373bd9c7d1804d37043a\n 1.7.8: sha256:3e13f8ee1657b874bf4741975d094d7a9df9051c400b3f822e310c91997c1441\n 1.7.7: sha256:e3c68cf9e01fb150d81764355718a5c31367d3129d76d108bb3b63b368ad2d1e\n 1.7.6: sha256:ee3d1e2a10e7b8b4deface422fc3b4c3700bcac8ae9497c0ade22d5b76649c71\n 1.7.5: sha256:c32df30af783dd67f7640458881b0dbb06f076b9ffe384d710d2d1a24cdad2b2\n 1.7.4: sha256:04f4b3034b2b39e40d621ef24abae8daadfe2d06f1bd480f633869944c379c9c\n 1.7.3: sha256:66eeb863c375dc832589e852798f9ec47bf5da8b53c7ab513f63a30adb13618f\n 1.7.2: sha256:56d2b16560592c42eee2eb758c51815a37f1de3aa3bf5c57073b4d6ded761358\n 1.7.1: sha256:8b4e8ed8a650ea435aa71e115fa1a70701ab98bc1836b3ed33341af35bf85a3a\n 1.7.0: sha256:64ad6428cc4aca486db3a6148682052955d1e3134b69f079edf686c21d123fcd\n ppc64le:\n 2.2.1: sha256:fc9235be9a3dd3005e7fe6a9d7bb80e42dbfbff4b119cdce6ea3ee66bc7ae9ca\n 2.2.0: sha256:d15a4edfe689ce71df8cc5a0c1837856f54aba8d7336170600e6592c2fbf3d8d\n 2.1.6: sha256:c64312b87181d900452b5c3360a90578acd39ec7664d0c2e060183b24a708766\n 2.1.5: sha256:4404b918b9e101274baa072188054766a1af16be8d22f02a51a5f6ee4e5d159f\n 2.1.4: sha256:9bc1ac45ba197873a4d47045313e0cc55910802937739bf57aded125abe55c8c\n 2.1.3: sha256:0084e26bcf5a2653278766662d5adc27cb00a17a21413cc3fbe1e99d9dacf174\n 2.1.2: sha256:4e61d5f9b5789e27c28b94fe0901552fb0070aa0c36aeaabc6e2786f3d5828ff\n 2.1.1: sha256:32c307e4b5cf014b594a9e7c22c610238143da8797f2bf4058dfb94225606ad4\n 2.1.0: sha256:df73c5ce713196be46f4b20d3ff081d67dabb5c730f61da7a3a65a9029b4f762\n 2.0.7: sha256:3f5993c7717216ae82083ce8c96af4d471da6e55affb03bfe4b86c59c6f54781\n 2.0.6: sha256:15a9af201730810fa2b2943c94a2651063a7d09b684f706520965c02000bbd0a\n 2.0.5: sha256:7e16bdcc5ceab50a880f6a9c7885b15f98ab60f0b963b3092928f89a7b26c8be\n 2.0.4: sha256:740018f44d2dccbd48d24c2362aae5098c459eda4602f5bc5f85738dccc2ead7\n 2.0.3: sha256:328c4647cee948a9062b8c17b5d1e75125cab1c025a2087dcc7bb1eef2d666de\n 2.0.2: sha256:4085c490ad5afb40b34782d427130396513a7eb35a49da8a6b7ade946dd309ce\n 2.0.1: sha256:6e8608e1993099f4ad44d77765be76fd67147108a6247f6886af33be08287a84\n 2.0.0: sha256:c389b68c9ca7774efdbc9e479d9a3be14b71d60a2a8c23f1d8764ec9598f3d55\n 1.7.30: sha256:95f12a656977f516b257f6447c197a30bd75b8b9d494a741a7f1f486a069de8c\n 1.7.29: sha256:5f4a351d4ecf0aedda743b2ea4b7304a72cf534f4f83c976de3a0d5adf006445\n 1.7.28: sha256:94dcce54f82bcd34a171af082cd035163cbb281923afc09454e0539eeef73974\n 1.7.27: sha256:6a140413dd38954078dddb1b1405e403a92173fe15d094fe0bcbac93a1ff7039\n 1.7.26: sha256:79fb1499e436265a71a31d1ecb69d8b5630df168555786fde8797c91320ad21f\n 1.7.25: sha256:6df5a58d88dba8e3b932fb6b5be1ff780848cdcdf32bddfaf5358ef3d45b62ab\n 1.7.24: sha256:c91562864a6f7e037b10689838fc1dd4c3051e82dc5c1ba0a409019d72816315\n 1.7.23: sha256:4757c76855adc70860a2bcfc583379ac97220c563f356c5eef06d663762b6931\n 1.7.22: sha256:b8fa787859cfb8e9c245036d21c9096ef835247349b0126eb15d6ae94a693924\n 1.7.21: sha256:544a2e9c265d643ad2c96aea2a8d33cb0709681f19186abc4a3bc0779dbe9360\n 1.7.20: sha256:b8dc431a83d46a29cdbda9ad1842e3409b9d8ec5fdf629d02a7e77f1f4e47dfb\n 1.7.19: sha256:e6d5ffcc15f80d506293eea7ca0e123fd63151f49f52b6b56c08d2b07eea97b7\n 1.7.18: sha256:978c938b0b02de4041be758207462a0337d798caa5bb04dd10ff7d2c4ce0c9e4\n 1.7.17: sha256:8154423f54a786bfd1cb6b5e41a47181e80ce7a52a9b28c205b6e81e4d6ec0c7\n 1.7.16: sha256:d02dce8cc44715bc437e546f669615f8c632788542845be224bc0604ad4de857\n 1.7.15: sha256:7b7a1c7d1d1483ff4b9849a6064aa709ee5bc4b8835dfe75714665550fdac89d\n 1.7.14: sha256:2dd693054275fe476a7d7efeae9b103557c51574aa702c38890b737f1a2be7f7\n 1.7.13: sha256:b97a754c5207562f70d30fa6a700cc90e65bb1bdfe876b5e42ba098e6f261b3b\n 1.7.12: sha256:e9ca2e6abb8993d04e41555fc5bdce6f44a4cebe3dc54ca9f113445716c1a71d\n 1.7.11: sha256:451578368acba17569ceb186511929afa7ad1c4a3af3d02983c0c1b5f5f7f21e\n 1.7.10: sha256:6c874d49da4a004158000611fc06d36432a8a888a7fefd19e524d4165bd86141\n 1.7.9: sha256:9b3afba45aa5facc3b0695db48d4da0acf94673a373b72e34ea45f8eeb946e5e\n 1.7.8: sha256:fe38a1d4c868b0756220ce6ea32862fcf9d980a98407740d8a4a1702807792ac\n 1.7.7: sha256:3a2e43b7324aedbf2ca49a533a956c83a448f99a2bb07b3320d407cf8e28ad55\n 1.7.6: sha256:5377b4fa4a8759c18d211afd7767d87b17b8f042bdc9901f9b2544261d28c5f3\n 1.7.5: sha256:258aa8a6dec6a80ec9fc152786e15cf4e2a165147c2eac9a78aeec869292f558\n 1.7.4: sha256:493cdd9e5ae57722ddb718b59a790ec07c5aa328254602f6df25b83f40e6a1cd\n 1.7.3: sha256:56ed334b3b8c1c191694c49efc123c4b8ee8244674ff7dbb2bd309da9eac4b57\n 1.7.2: sha256:d5c32307d7982a5314946e3724534739c433222ead3c36c7afeff2165653d2df\n 1.7.1: sha256:53c4fa2d544a74ee813e1881c06fa23cb7cf58d9fbd2dd7d5a20f38fc308ac44\n 1.7.0: sha256:316aca35c3e046d99adc2fb9771e6c3c42ebfc35332e42cfc467708d8bc0d60e\nskopeo_binary_checksums:\n arm64:\n 1.16.1: sha256:3272f15f469af843d325134ff8a77a069d647c5f247766715c098b8f0622b627\n 1.16.0: sha256:331b09b3b6e6550c178ea1c2fb2bdc5bdbd90c6f6e8d86a974f1117d6ab2fabe\n 1.15.2: sha256:f81487af3104e37537ff21f1b2527b294f5cc4e7988941a1655ded97c027ac1d\n 1.15.1: sha256:e20e34f96b5545bacd469b0d85ccce811ffbe2809db36248a3becb4638276959\n 1.15.0: sha256:bde8cc7e764d246281430d5da07ca906ee0838803199e3a6136a58802b2e0207\n 1.14.5: sha256:23e157de988c6020f1300b5d73d84d2fed2823ed61dbc6828de3552e9c77a6db\n 1.14.4: sha256:d825f93b28cf7502569fe75c46aa78187bb63b6bc06036621de7b63290b51058\n 1.14.3: sha256:e93a82b88e9bff46cbe4e68f96e265d934026a845b76ce51672c7cce26fba164\n 1.14.2: sha256:364c46085de31edf4b312f13587442f4eade1f181bc5a9ea2ab2ffab5b575916\n 1.14.1: sha256:fd4fc0adae14f27788fd52cf0d23be2cfd1963e184c4af689de30185455e29a6\n amd64:\n 1.16.1: sha256:8813fb7fcd7a723196ac287683dd929d280f6fe7f0782eace452fe1e3ff2b7eb\n 1.16.0: sha256:7bc31ed810d1366304d2e975c2910cea5e22cbd68f8316f14cacf44f6c0bd1d2\n 1.15.2: sha256:6b84d1158f29610f692f24c82459a865c2a21911647cc0cdf44027e7a59f73ba\n 1.15.1: sha256:d45a93dab851f072fe5d3f0419f5c8bb3ee48069b588c211cccebd023fd5ae3a\n 1.15.0: sha256:3cdbcde0163abb4c942f62d0302479d5aa4d31c5970d712841cf5d5f76edc594\n 1.14.5: sha256:180c2d7e8bc00685ba291572db6ddd90acadf03af7595521da17ae1f2c28f4b1\n 1.14.4: sha256:4c6f8f7c6e5f01675adff8c5bbb542d8d02b9bbdecf0d2abac1e99b8a34a9768\n 1.14.3: sha256:2db7e036e99ad3b808aaffbafc5267391bd3ba2f45ff03dd0090686eb3eb0f1e\n 1.14.2: sha256:51218f93a2b079e36a36f7fbe2d2d86778be0a6947653031b4f9e254e2469224\n 1.14.1: sha256:6b7776bcdf0c92af5d3f3c91a959d091011b42d839025b90f12b7201a083f308\n ppc64le:\n 1.16.1: sha256:248f8f601e4c40dd6d603b66ac26246f96d18451cc3642718c59afb6c2403cf7\n 1.16.0: sha256:24f1266d6146c27143b5002387c5b68086f1355de7db5c9bfe820928e3b8e298\n 1.15.2: sha256:5b123d38c34024e8b62b3bc94abfeea3007291743260bf7f62b2a1d935f1c3f9\n 1.15.1: sha256:39a4a6d77daca09a93a0b490285f48cd9040da1ba9c05b1f9709483e4f65c318\n 1.15.0: sha256:fb7f390f52f4b81f85d9bdce8715af5e27ee3969eff236b5f3c0f3a0b5a182e1\n 1.14.5: sha256:4ed476c46fabb3b320aac9b88ddc1b7a2665cb151a93482db7cb98e5768a768f\n 1.14.4: sha256:f1b37ad1b83bd43bada6e49518165cf41d727d0662351dc5fcc9a46f0c3b4482\n 1.14.3: sha256:9028b7c4aafe235f1ba4efd57435b97ace341e544d3a6807440ac3b0f32d7d73\nyq_checksums:\n arm:\n 4.42.1: sha256:4e3fe0c37793d28e96d465d9958fbf679d8c616e1857d0faf7980ad087f32aee\n 4.41.1: sha256:ccd50344652c02574ca7dd123c7d66a06b391838e8ca6088b688e6edf2e25d0c\n 4.40.7: sha256:fb922bb1e3974fbd15957feafb5e9bbabe43f4192999cf9b3e0e470815f2e0da\n 4.40.5: sha256:c587b2411e43d3fbcdd24c233fb558a362b5111a8446b23f9ce9a4a5665a7041\n 4.40.4: sha256:2ff3f17483f2172a20130b16328114bfe6abd7d3068d66d8194a5093079e8529\n 4.40.3: sha256:6a97856e8b4ef992ce08dcfdf97fec517cf612b1a89078406f401673f126c21c\n arm64:\n 4.42.1: sha256:16a57531a594b66c3e0981cd93f9e9cd4b684a347b86eaf5e3f409074ad67eb8\n 4.41.1: sha256:066aa930d74e39a25447b1900d8cbb3e1c7df72cd75bc203bc6ae5ee577a5b4a\n 4.40.7: sha256:a84f2c8f105b70cd348c3bf14048aeb1665c2e7314cbe9aaff15479f268b8412\n 4.40.5: sha256:9431f0fa39a0af03a152d7fe19a86e42e9ff28d503ed4a70598f9261ec944a97\n 4.40.4: sha256:79c61a1ebfedb165ec8c4678777775b52e2c581801f5d4cd80f97300852fe0f0\n 4.40.3: sha256:44a5cca10d33019b8a46212882197be4f961dfe7deddde0af497065aa980a6a4\n amd64:\n 4.42.1: sha256:1a95960dddd426321354d58d2beac457717f7c49a9ec0806749a5a9e400eb45e\n 4.41.1: sha256:ce0d5a61c256a463fd32f67f133e0c2948bc2cf77d44c42ff335a40e6bef34bf\n 4.40.7: sha256:4f13ee9303a49f7e8f61e7d9c87402e07cc920ae8dfaaa8c10d7ea1b8f9f48ed\n 4.40.5: sha256:0d6aaf1cf44a8d18fbc7ed0ef14f735a8df8d2e314c4cc0f0242d35c0a440c95\n 4.40.4: sha256:f9163412d9aa2aa55e888fdcaf2b4053ada20074be35f701424caa7163100704\n 4.40.3: sha256:6e9a5ed9591dbf1d13aaec4efaaf0ecdaf4945ea393b9ce01f4c3dea22311470\n ppc64le:\n 4.42.1: sha256:d0d1cdbd2c4a7e6995433baf879cadaa47f6f12290e1661ea11933ed90baccb6\n 4.41.1: sha256:eed2af79d0ad787878b2d5c7c592e43ac152208d9ed432b42a43663167e276e8\n 4.40.7: sha256:ac0e8d06a7ed9afc108b4e2e9d6900312b01757f61b75fcecb809f15c39b10e7\n 4.40.5: sha256:a1df9d2b872fbb30583526bf4f37f737dc1913b28606dfc1dafeaf56a8862b3d\n 4.40.4: sha256:c67379085a44558825a60a8af3b59b400852b168356070829bc0f45c70553f45\n 4.40.3: sha256:2fe818a0b141913a41548e0e727267479d0f755221c73f9e304788c8e9139a45\ngateway_api_standard_crds_checksums:\n no_arch:\n 1.4.1: sha256:73b91b77f6be023a8c92c969fc664e5bd3b1a28aea59eac9ebc904607354dad2\n 1.4.0: sha256:6a4029e661446d64add866a00ecdc40c14219b68777ab614c5cdaac0adb481f1\n 1.3.0: sha256:78796d5c51450fc55d8dc8092ba8137f8c807982d7508d7875d5c537a24082b9\n 1.2.1: sha256:97598bf6ab3b33b9b5c5432bdd24de091e4e9c3aa0575ebb0710a2a19cd64d64\n 1.2.0: sha256:38ed055bb25dc580c0366899c0bed9b9e92dfcd1c180a569133f3946026cf102\n 1.1.1: sha256:ffbfc11c5d1a11e8fd03de12a1b48f55ee782646d84b630068f48fdde86a60cf\n 1.1.0: sha256:c411805475d430a34242623a8e17153a7c40e946497bfd494e558b0d1a8858b3\n 1.0.0: sha256:23e4e1095c72a0587474f7fb3f85c319cdec77a083ab91237ffbdec1f1834d2a\ngateway_api_experimental_crds_checksums:\n no_arch:\n 1.4.1: sha256:553327e0ff32a1a2be446bf93823c8413cf9253ac6a6d5407eebd1e8d269f69e\n 1.4.0: sha256:0414b160767377e85fd362855501200c6b83b84758bcd532652e3fe1cc677e49\n 1.3.0: sha256:3e7a27e4456ff3d68606a6a8516306aaff354d6f0950b32bb31930669b7bf8b8\n 1.2.1: sha256:d3aa6723a3306770cffb601ee22af3d35da43acfa1ca547fc0d3bce08dad66e7\n 1.2.0: sha256:4369188e63b9ab5a35b5a83032c94d871159dece086b908b6ea18ea321ca06a9\n 1.1.1: sha256:529011bdf6c71ad6200bcd483ce4f248bc45309207d294bedf24e45a7563a9b0\n 1.1.0: sha256:10f322744a005d4e73e2b067e95fecd4cfec619dc7564930b488c296bfa3bec1\n 1.0.0: sha256:6c601dced7872a940d76fa667ae126ba718cb4c6db970d0bab49128ecc1192a3\nprometheus_operator_crds_checksums:\n no_arch:\n 0.88.1: sha256:b827b8ec478e6b31cc1b85c1736570a3575953fe9f470fc29d0ffdb2803d94c4\n 0.88.0: sha256:11ee66653657f3abc1bc8c41e17aa950eadb66035edb7f84cd3a1cbe4c67b2a4\n 0.87.1: sha256:62490f7c1863539d61295f53784e27d70deec96a3b465832ba3cf96120e298b5\n 0.87.0: sha256:a5282133ffa634405b0414d2fdc07e6fe393124d1d5072073af363689dac6a62\n 0.86.2: sha256:7c9d455333ac5ea7837d5f0e4edd966698e44edd79108bafdd8508f2da503b5b\n 0.86.1: sha256:9a30912ba9970a2968d7a8bf030a9f6579a5e8b312961018b5fe4c1153fc5fce\n 0.86.0: sha256:0d2a590b288c79a98515e9fc4315451cfbde964c7977eb527696f7c2ebf47f58\n 0.85.0: sha256:30e1b1b034ebc750d50a77dc19841176d698d524edf677276a760f9e228e1208\n 0.84.1: sha256:f4a186ac58f354793e27a0b4b6f8baf5a31a9d10045e5085c23b0570dbfd30dd\n 0.84.0: sha256:8990f6837ccff4461df9abe19d31d532fef11386d85d861b392249fff2502255\nargocd_install_checksums:\n no_arch:\n 2.14.21: sha256:ee39d40847bbb36154ebcdf2f5c93e0a9001ab60131afb60dbe42981a069699e\n 2.14.20: sha256:747db8bb6d1591b49bb6266412e6dcaccf3c7bd3bd81ef7d63b0ab5bfe6af951\n 2.14.19: sha256:c0f4c5331837a01a9764853ba451993a61f403e9e520986479e7777af5ef5a1f\n 2.14.18: sha256:ea09a2a920d234ee99468bcf8da2851f909906e9152178b5d06efb282af0f33e\n 2.14.17: sha256:d62bbea850dc8b515aa0d990fa52d9296d99542f8446b5d309c59e8a1fa29b21\n 2.14.16: sha256:f8788883040870835c008ce11942a8d5d8b250df4a314e3009dfb30cfb963725\n 2.14.15: sha256:0368b8a0adbb673408f2cc2367302ad1068d12cd9ab17cf6680bcb5fdba7c381\n 2.14.14: sha256:36315fcd4bc86e41b89f322daabf0bab20ffccc98976f882dba56f79c00e65ed\n 2.14.13: sha256:2b96310c5f52f23aad8643fc4536fc4ba45d2a41384bd71f191dc4ed2939b166\n 2.14.12: sha256:c1156a3bd9f76397757d3c42f024aaff67cac718518138c65e78b5997c37d6fc\n 2.14.11: sha256:9fd3317476ddad309954b7f99dbf53712b3611f0fc713390b5f7fdc68b536e2f\n 2.14.10: sha256:ddb73ff9b1ddfda4f882175a2069be163ba3ff7eb9c64622415778bb34e43955\n 2.14.9: sha256:b2c8093169039e3e516cb82cd98a64ca9ca0f116d7302a6ad074ffc7b28d85c2\n 2.14.8: sha256:f436b495681b2bdc1388c3ce0f708c6744b771ae637c9f93b2e50e166fac2dbd\n 2.14.7: sha256:4cffc79a9967cdd48dec48f0dd5f5b257057d0c193b3f4a9209b4d2ed614952d\n 2.14.6: sha256:549de9af3ff1a244e50e39ef523034788edc25e268c7be12afcf401e94cbc916\n 2.14.5: sha256:247ccda29c9faac4e0c8598680f5ebefff9911e957e3aeaf838eb4bbf455f2f4\n" }, { "path": "roles/kubespray_defaults/vars/main/main.yml", "content": "---\n# Internal version manipulation tooling\n\n# Get kubernetes major version (i.e. 1.17.4 => 1.17)\nkube_major_version: \"{{ (kube_version | split('.'))[:-1] | join('.') }}\"\nkube_next: \"{{ ((kube_version | split('.'))[1] | int) + 1 }}\"\nkube_major_next_version: \"1.{{ kube_next }}\"\n\npod_infra_supported_versions:\n '1.35': '3.10.1'\n '1.34': '3.10.1'\n '1.33': '3.10'\n\netcd_supported_versions:\n '1.35': \"{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.7', '<'))[0] }}\"\n '1.34': \"{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}\"\n '1.33': \"{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}\"\n# Kubespray constants\n\nkube_proxy_deployed: \"{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}\"\n\n# The lowest version allowed to upgrade from (same as calico_version in the previous branch)\ncalico_min_version_required: \"3.27.0\"\n\ncontainerd_min_version_required: \"1.3.7\"\n\n# mixed kube_service_addresses/kube_service_addresses_ipv6 for a variety of network stacks(dualstack, ipv6only, ipv4only)\nkube_service_subnets: >-\n {%- if ipv4_stack and ipv6_stack -%}\n {{ kube_service_addresses }},{{ kube_service_addresses_ipv6 }}\n {%- elif ipv4_stack -%}\n {{ kube_service_addresses }}\n {%- else -%}\n {{ kube_service_addresses_ipv6 }}\n {%- endif -%}\n\n# mixed kube_pods_subnet/kube_pods_subnet_ipv6 for a variety of network stacks(dualstack, ipv6only, ipv4only)\nkube_pods_subnets: >-\n {%- if ipv4_stack and ipv6_stack -%}\n {{ kube_pods_subnet }},{{ kube_pods_subnet_ipv6 }}\n {%- elif ipv4_stack -%}\n {{ kube_pods_subnet }}\n {%- else -%}\n {{ kube_pods_subnet_ipv6 }}\n {%- endif -%}\n" }, { "path": "roles/network_facts/defaults/main.yml", "content": "---\n# Additional string host to inject into NO_PROXY\nadditional_no_proxy: \"\"\nadditional_no_proxy_list: \"{{ additional_no_proxy | split(',') }}\"\nno_proxy_exclude_workers: false\n" }, { "path": "roles/network_facts/meta/main.yml", "content": "---\ndependencies:\n - role: kubespray_defaults\n" }, { "path": "roles/network_facts/tasks/main.yaml", "content": "---\n- name: Gather node IPs\n setup:\n gather_subset: '!all,!min,network'\n filter: \"ansible_default_ip*\"\n when: ansible_default_ipv4 is not defined or ansible_default_ipv6 is not defined\n ignore_unreachable: true\n\n- name: Set computed IPs variables\n vars:\n fallback_ip: \"{{ ansible_default_ipv4.address | d('127.0.0.1') }}\"\n fallback_ip6: \"{{ ansible_default_ipv6.address | d('::1') }}\"\n # Set 127.0.0.1 as fallback IP if we do not have host facts for host\n # ansible_default_ipv4 isn't what you think.\n _ipv4: \"{{ ip | default(fallback_ip) }}\"\n _access_ipv4: \"{{ access_ip | default(_ipv4) }}\"\n _ipv6: \"{{ ip6 | default(fallback_ip6) }}\"\n _access_ipv6: \"{{ access_ip6 | default(_ipv6) }}\"\n _access_ips:\n - \"{{ _access_ipv4 if ipv4_stack }}\"\n - \"{{ _access_ipv6 if ipv6_stack }}\"\n _ips:\n - \"{{ _ipv4 if ipv4_stack }}\"\n - \"{{ _ipv6 if ipv6_stack }}\"\n set_fact:\n cacheable: true\n main_access_ip: \"{{ _access_ipv4 if ipv4_stack else _access_ipv6 }}\"\n main_ip: \"{{ _ipv4 if ipv4_stack else _ipv6 }}\"\n # Mixed IPs - for dualstack\n main_access_ips: \"{{ _access_ips | select }}\"\n main_ips: \"{{ _ips | select }}\"\n\n- name: Set no_proxy to all assigned cluster IPs and hostnames\n when:\n - http_proxy is defined or https_proxy is defined\n - no_proxy is not defined\n vars:\n groups_with_no_proxy:\n - kube_control_plane\n - \"{{ '' if no_proxy_exclude_workers else 'kube_node' }}\" # TODO: exclude by a boolean in inventory rather than global variable\n - etcd\n - calico_rr\n hosts_with_no_proxy: \"{{ groups_with_no_proxy | select | map('extract', groups) | select('defined') | flatten }}\"\n _hostnames: \"{{ (hosts_with_no_proxy +\n (hosts_with_no_proxy | map('extract', hostvars, morekeys=['ansible_hostname'])\n | select('defined')))\n | unique }}\"\n no_proxy_prepare:\n - \"{{ apiserver_loadbalancer_domain_name | d('') }}\"\n - \"{{ loadbalancer_apiserver.address if loadbalancer_apiserver is defined else '' }}\"\n - \"{{ hosts_with_no_proxy | map('extract', hostvars, morekeys=['main_access_ip']) }}\"\n - \"{{ _hostnames }}\"\n - \"{{ _hostnames | map('regex_replace', '$', '.' + dns_domain ) }}\"\n - \"{{ additional_no_proxy_list }}\"\n - 127.0.0.1\n - localhost\n - \"{{ kube_service_subnets }}\"\n - \"{{ kube_pods_subnets }}\"\n - svc\n - \"svc.{{ dns_domain }}\"\n set_fact:\n no_proxy: \"{{ no_proxy_prepare | select | flatten | unique | join(',') }}\"\n run_once: true\n" }, { "path": "roles/network_plugin/calico/files/openssl.conf", "content": "req_extensions = v3_req\ndistinguished_name = req_distinguished_name\n\n[req_distinguished_name]\n\n[ v3_req ]\nbasicConstraints = CA:FALSE\nkeyUsage = digitalSignature, keyEncipherment\n\n[ ssl_client ]\nextendedKeyUsage = clientAuth, serverAuth\nbasicConstraints = CA:FALSE\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid,issuer\n\n[ v3_ca ]\nbasicConstraints = CA:TRUE\nkeyUsage = cRLSign, digitalSignature, keyCertSign\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always,issuer\n\n[ ssl_client_apiserver ]\nextendedKeyUsage = clientAuth, serverAuth\nbasicConstraints = CA:FALSE\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid,issuer\nsubjectAltName = DNS:calico-api.calico-apiserver.svc\n" }, { "path": "roles/network_plugin/calico/handlers/main.yml", "content": "---\n- name: Delete 10-calico.conflist\n file:\n path: /etc/cni/net.d/10-calico.conflist\n state: absent\n listen: Reset_calico_cni\n when: calico_cni_config is defined\n\n- name: Calico | delete calico-node docker containers\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f\"\n args:\n executable: /bin/bash\n register: docker_calico_node_remove\n until: docker_calico_node_remove is succeeded\n retries: 5\n when:\n - container_manager in [\"docker\"]\n - calico_cni_config is defined\n listen: Reset_calico_cni\n\n- name: Calico | delete calico-node crio/containerd containers\n shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name calico-node-* -q | xargs -I% --no-run-if-empty bash -c \"{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %\"'\n args:\n executable: /bin/bash\n register: crictl_calico_node_remove\n until: crictl_calico_node_remove is succeeded\n retries: 5\n when:\n - container_manager in [\"crio\", \"containerd\"]\n - calico_cni_config is defined\n listen: Reset_calico_cni\n" }, { "path": "roles/network_plugin/calico/meta/main.yml", "content": "---\ndependencies:\n - role: network_plugin/calico_defaults\n" }, { "path": "roles/network_plugin/calico/rr/defaults/main.yml", "content": "---\n# Global as_num (/calico/bgp/v1/global/as_num)\n# should be the same as in calico role\nglobal_as_num: \"64512\"\ncalico_baremetal_nodename: \"{{ kube_override_hostname | default(inventory_hostname) }}\"\n" }, { "path": "roles/network_plugin/calico/rr/tasks/main.yml", "content": "---\n- name: Calico-rr | Pre-upgrade tasks\n include_tasks: pre.yml\n\n- name: Calico-rr | Configuring node tasks\n include_tasks: update-node.yml\n\n- name: Calico-rr | Set label for route reflector\n command: >-\n {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }}\n 'i-am-a-route-reflector=true' --overwrite\n changed_when: false\n register: calico_rr_label\n until: calico_rr_label is succeeded\n delay: \"{{ retry_stagger | random + 3 }}\"\n retries: 10\n" }, { "path": "roles/network_plugin/calico/rr/tasks/pre.yml", "content": "---\n- name: Calico-rr | Disable calico-rr service if it exists\n service:\n name: calico-rr\n state: stopped\n enabled: false\n failed_when: false\n\n- name: Calico-rr | Delete obsolete files\n file:\n path: \"{{ item }}\"\n state: absent\n with_items:\n - /etc/calico/calico-rr.env\n - /etc/systemd/system/calico-rr.service\n" }, { "path": "roles/network_plugin/calico/rr/tasks/update-node.yml", "content": "---\n# Workaround to retry a block of tasks, ansible doesn't have a direct way to do it,\n# you can follow the block loop request in: https://github.com/ansible/ansible/issues/46203\n- name: Calico-rr | Configure route reflector\n block:\n - name: Set the retry count\n set_fact:\n retry_count: \"{{ 0 if retry_count is undefined else retry_count | int + 1 }}\"\n\n - name: Calico | Set label for route reflector # noqa command-instead-of-shell\n shell: \"{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-rr-id={{ calico_rr_id }} --overwrite\"\n changed_when: false\n register: calico_rr_id_label\n until: calico_rr_id_label is succeeded\n delay: \"{{ retry_stagger | random + 3 }}\"\n retries: 10\n when: calico_rr_id is defined\n\n - name: Calico-rr | Fetch current node object\n command: \"{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson\"\n changed_when: false\n register: calico_rr_node\n until: calico_rr_node is succeeded\n delay: \"{{ retry_stagger | random + 3 }}\"\n retries: 10\n\n - name: Calico-rr | Set route reflector cluster ID\n # noqa: jinja[spacing]\n set_fact:\n calico_rr_node_patched: >-\n {{ calico_rr_node.stdout | from_json | combine({ 'spec': { 'bgp':\n { 'routeReflectorClusterID': cluster_id }}}, recursive=True) }}\n\n - name: Calico-rr | Configure route reflector # noqa command-instead-of-shell\n shell: \"{{ bin_dir }}/calicoctl.sh replace -f-\"\n args:\n stdin: \"{{ calico_rr_node_patched | to_json }}\"\n\n rescue:\n - name: Fail if retry limit is reached\n fail:\n msg: Ended after 10 retries\n when: retry_count | int == 10\n\n - name: Retrying node configuration\n debug:\n msg: \"Failed to configure route reflector - Retrying...\"\n\n - name: Retry node configuration\n include_tasks: update-node.yml\n" }, { "path": "roles/network_plugin/calico/tasks/calico_apiserver_certs.yml", "content": "---\n- name: Calico | Check if calico apiserver exists\n command: \"{{ kubectl }} -n calico-apiserver get secret calico-apiserver-certs\"\n register: calico_apiserver_secret\n changed_when: false\n failed_when: false\n\n- name: Calico | Create ns manifests\n template:\n src: \"calico-apiserver-ns.yml.j2\"\n dest: \"{{ kube_config_dir }}/calico-apiserver-ns.yml\"\n mode: \"0644\"\n\n- name: Calico | Apply ns manifests\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/calico-apiserver-ns.yml\"\n state: \"latest\"\n\n- name: Calico | Ensure calico certs dir\n file:\n path: /etc/calico/certs\n state: directory\n mode: \"0755\"\n when: calico_apiserver_secret.rc != 0\n\n- name: Calico | Copy ssl script for apiserver certs\n template:\n src: make-ssl-calico.sh.j2\n dest: \"{{ bin_dir }}/make-ssl-apiserver.sh\"\n mode: \"0755\"\n when: calico_apiserver_secret.rc != 0\n\n- name: Calico | Copy ssl config for apiserver certs\n copy:\n src: openssl.conf\n dest: /etc/calico/certs/openssl.conf\n mode: \"0644\"\n when: calico_apiserver_secret.rc != 0\n\n- name: Calico | Generate apiserver certs\n command: >-\n {{ bin_dir }}/make-ssl-apiserver.sh\n -f /etc/calico/certs/openssl.conf\n -c {{ kube_cert_dir }}\n -d /etc/calico/certs\n -s apiserver\n when: calico_apiserver_secret.rc != 0\n\n- name: Calico | Create calico apiserver generic secrets\n command: >-\n {{ kubectl }} -n calico-apiserver\n create secret generic {{ item.name }}\n --from-file={{ item.cert }}\n --from-file={{ item.key }}\n with_items:\n - name: calico-apiserver-certs\n cert: /etc/calico/certs/apiserver.crt\n key: /etc/calico/certs/apiserver.key\n when: calico_apiserver_secret.rc != 0\n" }, { "path": "roles/network_plugin/calico/tasks/check.yml", "content": "---\n- name: Stop if legacy encapsulation variables are detected (ipip)\n assert:\n that:\n - ipip is not defined\n msg: \"'ipip' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Stop if legacy encapsulation variables are detected (ipip_mode)\n assert:\n that:\n - ipip_mode is not defined\n msg: \"'ipip_mode' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Stop if legacy encapsulation variables are detected (calcio_ipam_autoallocateblocks)\n assert:\n that:\n - calcio_ipam_autoallocateblocks is not defined\n msg: \"'calcio_ipam_autoallocateblocks' configuration variable is deprecated, it's a typo, please configure your inventory with 'calico_ipam_autoallocateblocks' set to 'true' or 'false' according to your specific needs\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n\n- name: Stop if supported Calico versions\n assert:\n that:\n - \"calico_version in calico_crds_checksums.no_arch.keys()\"\n msg: \"Calico version not supported {{ calico_version }} not in {{ calico_crds_checksums.no_arch.keys() }}\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Check if calicoctl.sh exists\n stat:\n path: \"{{ bin_dir }}/calicoctl.sh\"\n register: calicoctl_sh_exists\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Check if calico ready\n command: \"{{ bin_dir }}/calicoctl.sh get ClusterInformation default\"\n register: calico_ready\n run_once: true\n ignore_errors: true\n retries: 5\n delay: 10\n until: calico_ready.rc == 0\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when: calicoctl_sh_exists.stat.exists\n\n- name: Check that current calico version is enough for upgrade\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when: calicoctl_sh_exists.stat.exists and calico_ready.rc == 0\n block:\n - name: Get current calico version\n shell: \"set -o pipefail && {{ bin_dir }}/calicoctl.sh version | grep 'Client Version:' | awk '{ print $3}'\"\n args:\n executable: /bin/bash\n register: calico_version_on_server\n changed_when: false\n check_mode: false\n\n - name: Assert that current calico version is enough for upgrade\n assert:\n that:\n - calico_version_on_server.stdout.removeprefix('v') is version(calico_min_version_required, '>=')\n msg: >\n Your version of calico is not fresh enough for upgrade.\n Minimum version is {{ calico_min_version_required }} supported by the previous kubespray release.\n But current version is {{ calico_version_on_server.stdout }}.\n\n- name: \"Check that cluster_id is set and a valid IPv4 address if calico_rr enabled\"\n assert:\n that:\n - cluster_id is defined\n - cluster_id is ansible.utils.ipv4\n msg: \"A unique cluster_id is required if using calico_rr, and it must be a valid IPv4 address\"\n when:\n - peer_with_calico_rr\n - inventory_hostname == groups['kube_control_plane'][0]\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check that calico_rr nodes are in k8s_cluster group\"\n assert:\n that:\n - '\"k8s_cluster\" in group_names'\n msg: \"calico_rr must be a child group of k8s_cluster group\"\n when:\n - '\"calico_rr\" in group_names'\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check vars defined correctly\"\n assert:\n that:\n - \"calico_pool_name is defined\"\n - \"calico_pool_name is match('^[a-zA-Z0-9-_\\\\\\\\.]{2,63}$')\"\n msg: \"calico_pool_name contains invalid characters\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check calico network backend defined correctly\"\n assert:\n that:\n - \"calico_network_backend in ['bird', 'vxlan', 'none']\"\n msg: \"calico network backend is not 'bird', 'vxlan' or 'none'\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check ipip and vxlan mode defined correctly\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n assert:\n that:\n - \"calico_ipip_mode in ['Always', 'CrossSubnet', 'Never']\"\n - \"calico_vxlan_mode in ['Always', 'CrossSubnet', 'Never']\"\n msg: \"calico inter host encapsulation mode is not 'Always', 'CrossSubnet' or 'Never'\"\n\n- name: \"Check ipip and vxlan mode if simultaneously enabled\"\n assert:\n that:\n - \"calico_vxlan_mode in ['Never']\"\n msg: \"IP in IP and VXLAN mode is mutualy exclusive modes\"\n when:\n - \"calico_ipip_mode in ['Always', 'CrossSubnet']\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check ipip and vxlan mode if simultaneously enabled\"\n assert:\n that:\n - \"calico_ipip_mode in ['Never']\"\n msg: \"IP in IP and VXLAN mode is mutualy exclusive modes\"\n when:\n - \"calico_vxlan_mode in ['Always', 'CrossSubnet']\"\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Get Calico {{ calico_pool_name }} configuration\"\n command: \"{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }} -o json\"\n failed_when: false\n changed_when: false\n check_mode: false\n register: calico\n run_once: true\n when: ipv4_stack | bool\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Set calico_pool_conf\"\n set_fact:\n calico_pool_conf: '{{ calico.stdout | from_json }}'\n when:\n - ipv4_stack | bool\n - calico is defined\n - calico.rc == 0 and calico.stdout\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check if inventory match current cluster configuration\"\n assert:\n that:\n - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize | int\n - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet))\n - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode\n - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode\n msg: \"Your inventory doesn't match the current cluster configuration\"\n when:\n - ipv4_stack | bool\n - calico_pool_conf is defined\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Get Calico {{ calico_pool_name }}-ipv6 configuration\"\n command: \"{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }}-ipv6 -o json\"\n failed_when: false\n changed_when: false\n check_mode: false\n register: calico_ipv6\n run_once: true\n when: ipv6_stack | bool\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Set calico_pool_ipv6_conf\"\n set_fact:\n calico_pool_conf: '{{ calico_ipv6.stdout | from_json }}'\n when:\n - ipv6_stack | bool\n - alico_ipv6 is defined\n - calico_ipv6.rc == 0 and calico_ipv6.stdout\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check if ipv6 inventory match current cluster configuration\"\n assert:\n that:\n - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize_ipv6 | int\n - calico_pool_conf.spec.cidr == (calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6))\n - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode_ipv6\n - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode_ipv6\n msg: \"Your ipv6 inventory doesn't match the current cluster configuration\"\n when:\n - ipv6_stack | bool\n - calico_pool_ipv6_conf is defined\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check kdd calico_datastore if calico_apiserver_enabled\"\n assert:\n that: calico_datastore == \"kdd\"\n msg: \"When using calico apiserver you need to use the kubernetes datastore\"\n when:\n - calico_apiserver_enabled\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check kdd calico_datastore if typha_enabled\"\n assert:\n that: calico_datastore == \"kdd\"\n msg: \"When using typha you need to use the kubernetes datastore\"\n when:\n - typha_enabled\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: \"Check ipip mode is Never for calico ipv6\"\n assert:\n that:\n - \"calico_ipip_mode_ipv6 in ['Never']\"\n msg: \"Calico doesn't support ipip tunneling for the IPv6\"\n when: ipv6_stack | bool\n run_once: true\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n" }, { "path": "roles/network_plugin/calico/tasks/install.yml", "content": "---\n- name: Calico | Install Wireguard packages\n package:\n name: \"{{ item }}\"\n state: present\n with_items: \"{{ calico_wireguard_packages }}\"\n register: calico_package_install\n until: calico_package_install is succeeded\n retries: 4\n when: calico_wireguard_enabled\n\n- name: Calico | Copy calicoctl binary from download dir\n copy:\n src: \"{{ downloads.calicoctl.dest }}\"\n dest: \"{{ bin_dir }}/calicoctl\"\n mode: \"0755\"\n remote_src: true\n\n- name: Calico | Create calico certs directory\n file:\n dest: \"{{ calico_cert_dir }}\"\n state: directory\n mode: \"0750\"\n owner: root\n group: root\n when: calico_datastore == \"etcd\"\n\n- name: Calico | Link etcd certificates for calico-node\n file:\n src: \"{{ etcd_cert_dir }}/{{ item.s }}\"\n dest: \"{{ calico_cert_dir }}/{{ item.d }}\"\n state: hard\n mode: \"0640\"\n force: true\n with_items:\n - {s: \"{{ kube_etcd_cacert_file }}\", d: \"ca_cert.crt\"}\n - {s: \"{{ kube_etcd_cert_file }}\", d: \"cert.crt\"}\n - {s: \"{{ kube_etcd_key_file }}\", d: \"key.pem\"}\n when: calico_datastore == \"etcd\"\n\n- name: Calico | Generate typha certs\n include_tasks: typha_certs.yml\n when:\n - typha_secure\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Calico | Generate apiserver certs\n include_tasks: calico_apiserver_certs.yml\n when:\n - calico_apiserver_enabled\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Calico | Install calicoctl wrapper script\n template:\n src: \"calicoctl.{{ calico_datastore }}.sh.j2\"\n dest: \"{{ bin_dir }}/calicoctl.sh\"\n mode: \"0755\"\n owner: root\n group: root\n\n- name: Calico | wait for etcd\n uri:\n url: \"{{ etcd_access_addresses.split(',') | first }}/health\"\n validate_certs: false\n client_cert: \"{{ calico_cert_dir }}/cert.crt\"\n client_key: \"{{ calico_cert_dir }}/key.pem\"\n register: result\n until: result.status == 200 or result.status == 401\n retries: 10\n delay: 5\n run_once: true\n when: calico_datastore == \"etcd\"\n\n- name: Calico | Check if calico network pool has already been configured\n # noqa risky-shell-pipe - grep will exit 1 if no match found\n shell: >\n {{ bin_dir }}/calicoctl.sh get ippool | grep -w \"{{ calico_pool_cidr | default(kube_pods_subnet) }}\" | wc -l\n args:\n executable: /bin/bash\n register: calico_conf\n retries: 4\n until: calico_conf.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n changed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - ipv4_stack | bool\n\n- name: Calico | Ensure that calico_pool_cidr is within kube_pods_subnet when defined\n assert:\n that: \"[calico_pool_cidr] | ansible.utils.ipaddr(kube_pods_subnet) | length == 1\"\n msg: \"{{ calico_pool_cidr }} is not within or equal to {{ kube_pods_subnet }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - ipv4_stack | bool\n - calico_pool_cidr is defined\n - 'calico_conf.stdout == \"0\"'\n\n- name: Calico | Check if calico IPv6 network pool has already been configured\n # noqa risky-shell-pipe - grep will exit 1 if no match found\n shell: >\n {{ bin_dir }}/calicoctl.sh get ippool | grep -w \"{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}\" | wc -l\n args:\n executable: /bin/bash\n register: calico_conf_ipv6\n retries: 4\n until: calico_conf_ipv6.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n changed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - ipv6_stack\n\n- name: Calico | Ensure that calico_pool_cidr_ipv6 is within kube_pods_subnet_ipv6 when defined\n assert:\n that: \"[calico_pool_cidr_ipv6] | ansible.utils.ipaddr(kube_pods_subnet_ipv6) | length == 1\"\n msg: \"{{ calico_pool_cidr_ipv6 }} is not within or equal to {{ kube_pods_subnet_ipv6 }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - ipv6_stack | bool\n - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == \"0\"\n - calico_pool_cidr_ipv6 is defined\n\n- name: Calico | kdd specific configuration\n when:\n - ('kube_control_plane' in group_names)\n - calico_datastore == \"kdd\"\n block:\n - name: Calico | Create calico manifests for kdd\n copy:\n src: \"{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/crds.yaml\"\n dest: \"{{ kube_config_dir }}/kdd-crds.yml\"\n mode: \"0644\"\n remote_src: true\n\n - name: Calico | Create Calico Kubernetes datastore resources\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/kdd-crds.yml\"\n state: \"latest\"\n register: kubectl_result\n until: kubectl_result is succeeded\n retries: 5\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Calico | Configure Felix\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n block:\n - name: Calico | Get existing FelixConfiguration\n command: \"{{ bin_dir }}/calicoctl.sh get felixconfig default -o json\"\n register: _felix_cmd\n ignore_errors: true\n changed_when: false\n\n - name: Calico | Set kubespray FelixConfiguration\n set_fact:\n _felix_config: >\n {\n \"kind\": \"FelixConfiguration\",\n \"apiVersion\": \"projectcalico.org/v3\",\n \"metadata\": {\n \"name\": \"default\",\n },\n \"spec\": {\n \"ipipEnabled\": {{ calico_ipip_mode != 'Never' }},\n \"reportingInterval\": \"{{ calico_felix_reporting_interval }}\",\n \"bpfLogLevel\": \"{{ calico_bpf_log_level }}\",\n \"bpfEnabled\": {{ calico_bpf_enabled | bool }},\n \"bpfExternalServiceMode\": \"{{ calico_bpf_service_mode }}\",\n \"wireguardEnabled\": {{ calico_wireguard_enabled | bool }},\n \"logSeverityScreen\": \"{{ calico_felix_log_severity_screen }}\",\n \"vxlanEnabled\": {{ calico_vxlan_mode != 'Never' }},\n \"featureDetectOverride\": \"{{ calico_feature_detect_override }}\",\n \"floatingIPs\": \"{{ calico_felix_floating_ips }}\"\n }\n }\n\n - name: Calico | Process FelixConfiguration\n set_fact:\n _felix_config: \"{{ _felix_cmd.stdout | from_json | combine(_felix_config, recursive=True) }}\"\n when:\n - _felix_cmd is success\n\n - name: Calico | Configure calico FelixConfiguration\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ _felix_config is string | ternary(_felix_config, _felix_config | to_json) }}\"\n changed_when: false\n\n- name: Calico | Configure Calico IP Pool\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - ipv4_stack | bool\n block:\n - name: Calico | Get existing calico network pool\n command: \"{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }} -o json\"\n register: _calico_pool_cmd\n ignore_errors: true\n changed_when: false\n\n - name: Calico | Set kubespray calico network pool\n set_fact:\n _calico_pool: >\n {\n \"kind\": \"IPPool\",\n \"apiVersion\": \"projectcalico.org/v3\",\n \"metadata\": {\n \"name\": \"{{ calico_pool_name }}\",\n },\n \"spec\": {\n \"blockSize\": {{ calico_pool_blocksize }},\n \"cidr\": \"{{ calico_pool_cidr | default(kube_pods_subnet) }}\",\n \"ipipMode\": \"{{ calico_ipip_mode }}\",\n \"vxlanMode\": \"{{ calico_vxlan_mode }}\",\n \"natOutgoing\": {{ nat_outgoing | default(false) }}\n }\n }\n\n - name: Calico | Process calico network pool\n when:\n - _calico_pool_cmd is success\n block:\n - name: Calico | Get current calico network pool blocksize\n set_fact:\n _calico_blocksize: >\n {\n \"spec\": {\n \"blockSize\": {{ (_calico_pool_cmd.stdout | from_json).spec.blockSize }}\n }\n }\n - name: Calico | Merge calico network pool\n set_fact:\n _calico_pool: \"{{ _calico_pool_cmd.stdout | from_json | combine(_calico_pool, _calico_blocksize, recursive=True) }}\"\n\n - name: Calico | Configure calico network pool\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ _calico_pool is string | ternary(_calico_pool, _calico_pool | to_json) }}\"\n changed_when: false\n\n- name: Calico | Configure Calico IPv6 Pool\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - ipv6_stack | bool\n block:\n - name: Calico | Get existing calico ipv6 network pool\n command: \"{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }}-ipv6 -o json\"\n register: _calico_pool_ipv6_cmd\n ignore_errors: true\n changed_when: false\n\n - name: Calico | Set kubespray calico network pool\n set_fact:\n _calico_pool_ipv6: >\n {\n \"kind\": \"IPPool\",\n \"apiVersion\": \"projectcalico.org/v3\",\n \"metadata\": {\n \"name\": \"{{ calico_pool_name }}-ipv6\",\n },\n \"spec\": {\n \"blockSize\": {{ calico_pool_blocksize_ipv6 }},\n \"cidr\": \"{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}\",\n \"ipipMode\": \"{{ calico_ipip_mode_ipv6 }}\",\n \"vxlanMode\": \"{{ calico_vxlan_mode_ipv6 }}\",\n \"natOutgoing\": {{ nat_outgoing_ipv6 | default(false) }}\n }\n }\n\n - name: Calico | Process calico ipv6 network pool\n when:\n - _calico_pool_ipv6_cmd is success\n block:\n - name: Calico | Get current calico ipv6 network pool blocksize\n set_fact:\n _calico_blocksize_ipv6: >\n {\n \"spec\": {\n \"blockSize\": {{ (_calico_pool_ipv6_cmd.stdout | from_json).spec.blockSize }}\n }\n }\n - name: Calico | Merge calico ipv6 network pool\n set_fact:\n _calico_pool_ipv6: \"{{ _calico_pool_ipv6_cmd.stdout | from_json | combine(_calico_pool_ipv6, _calico_blocksize_ipv6, recursive=True) }}\"\n\n - name: Calico | Configure calico ipv6 network pool\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ _calico_pool_ipv6 is string | ternary(_calico_pool_ipv6, _calico_pool_ipv6 | to_json) }}\"\n changed_when: false\n\n- name: Populate Service External IPs\n set_fact:\n _service_external_ips: \"{{ _service_external_ips | default([]) + [{'cidr': item}] }}\"\n with_items: \"{{ calico_advertise_service_external_ips }}\"\n run_once: true\n\n- name: Populate Service LoadBalancer IPs\n set_fact:\n _service_loadbalancer_ips: \"{{ _service_loadbalancer_ips | default([]) + [{'cidr': item}] }}\"\n with_items: \"{{ calico_advertise_service_loadbalancer_ips }}\"\n run_once: true\n\n- name: \"Determine nodeToNodeMesh needed state\"\n set_fact:\n nodeToNodeMeshEnabled: \"false\"\n when:\n - peer_with_router | default(false) or peer_with_calico_rr | default(false)\n - ('k8s_cluster' in group_names)\n run_once: true\n\n- name: Calico | Configure Calico BGP\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n block:\n - name: Calico | Get existing BGP Configuration\n command: \"{{ bin_dir }}/calicoctl.sh get bgpconfig default -o json\"\n register: _bgp_config_cmd\n ignore_errors: true\n changed_when: false\n\n - name: Calico | Set kubespray BGP Configuration\n set_fact:\n # noqa: jinja[spacing]\n _bgp_config: >\n {\n \"kind\": \"BGPConfiguration\",\n \"apiVersion\": \"projectcalico.org/v3\",\n \"metadata\": {\n \"name\": \"default\",\n },\n \"spec\": {\n \"listenPort\": {{ calico_bgp_listen_port }},\n \"logSeverityScreen\": \"Info\",\n {% if not calico_no_global_as_num | default(false) %}\"asNumber\": {{ global_as_num }},{% endif %}\n \"nodeToNodeMeshEnabled\": {{ nodeToNodeMeshEnabled | default('true') }} ,\n {% if calico_advertise_cluster_ips | default(false) %}\n \"serviceClusterIPs\":\n {%- if ipv4_stack and ipv6_stack-%}\n [{\"cidr\": \"{{ kube_service_addresses }}\", \"cidr\": \"{{ kube_service_addresses_ipv6 }}\"}],\n {%- elif ipv6_stack-%}\n [{\"cidr\": \"{{ kube_service_addresses_ipv6 }}\"}],\n {%- else -%}\n [{\"cidr\": \"{{ kube_service_addresses }}\"}],\n {%- endif -%}\n {% endif %}\n {% if calico_advertise_service_loadbalancer_ips | length > 0 %}\"serviceLoadBalancerIPs\": {{ _service_loadbalancer_ips }},{% endif %}\n \"serviceExternalIPs\": {{ _service_external_ips | default([]) }}\n }\n }\n\n - name: Calico | Process BGP Configuration\n set_fact:\n _bgp_config: \"{{ _bgp_config_cmd.stdout | from_json | combine(_bgp_config, recursive=True) }}\"\n when:\n - _bgp_config_cmd is success\n\n - name: Calico | Set up BGP Configuration\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ _bgp_config is string | ternary(_bgp_config, _bgp_config | to_json) }}\"\n changed_when: false\n\n- name: Calico | Create calico manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: calico-config, file: calico-config.yml, type: cm}\n - {name: calico-node, file: calico-node.yml, type: ds}\n - {name: calico, file: calico-node-sa.yml, type: sa}\n - {name: calico, file: calico-cr.yml, type: clusterrole}\n - {name: calico, file: calico-crb.yml, type: clusterrolebinding}\n - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm }\n register: calico_node_manifests\n when:\n - ('kube_control_plane' in group_names)\n - rbac_enabled or item.type not in rbac_resources\n\n- name: Calico | Create calico manifests for typha\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: calico, file: calico-typha.yml, type: typha}\n register: calico_node_typha_manifest\n when:\n - ('kube_control_plane' in group_names)\n - typha_enabled\n\n- name: Calico | get calico apiserver caBundle\n command: \"{{ bin_dir }}/kubectl get secret -n calico-apiserver calico-apiserver-certs -o jsonpath='{.data.apiserver\\\\.crt}'\"\n changed_when: false\n register: calico_apiserver_cabundle\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - calico_apiserver_enabled\n\n- name: Calico | set calico apiserver caBundle fact\n set_fact:\n calico_apiserver_cabundle: \"{{ calico_apiserver_cabundle.stdout }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - calico_apiserver_enabled\n\n- name: Calico | Create calico manifests for apiserver\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: calico, file: calico-apiserver.yml, type: calico-apiserver}\n register: calico_apiserver_manifest\n when:\n - ('kube_control_plane' in group_names)\n - calico_apiserver_enabled\n\n- name: Start Calico resources\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"kube-system\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ calico_node_manifests.results }}\"\n - \"{{ calico_node_typha_manifest.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n\n- name: Start Calico apiserver resources\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"calico-apiserver\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items:\n - \"{{ calico_apiserver_manifest.results }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - not item is skipped\n loop_control:\n label: \"{{ item.item.file }}\"\n\n- name: Wait for calico kubeconfig to be created\n wait_for:\n path: /etc/cni/net.d/calico-kubeconfig\n timeout: \"{{ calico_kubeconfig_wait_timeout }}\"\n when:\n - inventory_hostname not in groups['kube_control_plane']\n - calico_datastore == \"kdd\"\n\n- name: Calico | Create Calico ipam manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: calico, file: calico-ipamconfig.yml, type: ipam}\n when:\n - ('kube_control_plane' in group_names)\n - calico_datastore == \"kdd\"\n\n- name: Calico | Create ipamconfig resources\n kube:\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/calico-ipamconfig.yml\"\n state: \"latest\"\n register: resource_result\n until: resource_result is succeeded\n retries: 4\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - calico_datastore == \"kdd\"\n\n- name: Calico | Peer with Calico Route Reflector\n include_tasks: peer_with_calico_rr.yml\n when:\n - peer_with_calico_rr | default(false)\n\n- name: Calico | Peer with the router\n include_tasks: peer_with_router.yml\n when:\n - peer_with_router | default(false)\n" }, { "path": "roles/network_plugin/calico/tasks/main.yml", "content": "---\n- name: Calico Pre tasks\n import_tasks: pre.yml\n\n- name: Calico repos\n import_tasks: repos.yml\n\n- name: Calico install\n include_tasks: install.yml\n" }, { "path": "roles/network_plugin/calico/tasks/peer_with_calico_rr.yml", "content": "---\n- name: Calico | Set label for groups nodes\n command: \"{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite\"\n changed_when: false\n register: calico_group_id_label\n until: calico_group_id_label is succeeded\n delay: \"{{ retry_stagger | random + 3 }}\"\n retries: 10\n when:\n - calico_group_id is defined\n\n- name: Calico | Configure peering with route reflectors at global scope\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n # revert when it's already a string\n stdin: \"{{ stdin is string | ternary(stdin, stdin | to_json) }}\"\n vars:\n stdin: >\n {\"apiVersion\": \"projectcalico.org/v3\",\n \"kind\": \"BGPPeer\",\n \"metadata\": {\n \"name\": \"{{ calico_rr_id }}-to-node\"\n },\n \"spec\": {\n \"peerSelector\": \"calico-rr-id == '{{ calico_rr_id }}'\",\n \"nodeSelector\": \"calico-group-id == '{{ calico_group_id }}'\"\n }}\n register: output\n retries: 4\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n when:\n - calico_rr_id is defined\n - calico_group_id is defined\n - ('calico_rr' in group_names)\n\n- name: Calico | Configure peering with route reflectors at global scope\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n # revert when it's already a string\n stdin: \"{{ stdin is string | ternary(stdin, stdin | to_json) }}\"\n vars:\n stdin: >\n {\"apiVersion\": \"projectcalico.org/v3\",\n \"kind\": \"BGPPeer\",\n \"metadata\": {\n \"name\": \"peer-to-rrs\"\n },\n \"spec\": {\n \"nodeSelector\": \"!has(i-am-a-route-reflector)\",\n \"peerSelector\": \"has(i-am-a-route-reflector)\"\n }}\n register: output\n retries: 4\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n with_items:\n - \"{{ groups['calico_rr'] | default([]) }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - calico_rr_id is not defined or calico_group_id is not defined\n\n- name: Calico | Configure route reflectors to peer with each other\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n # revert when it's already a string\n stdin: \"{{ stdin is string | ternary(stdin, stdin | to_json) }}\"\n vars:\n stdin: >\n {\"apiVersion\": \"projectcalico.org/v3\",\n \"kind\": \"BGPPeer\",\n \"metadata\": {\n \"name\": \"rr-mesh\"\n },\n \"spec\": {\n \"nodeSelector\": \"has(i-am-a-route-reflector)\",\n \"peerSelector\": \"has(i-am-a-route-reflector)\"\n }}\n register: output\n retries: 4\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n with_items:\n - \"{{ groups['calico_rr'] | default([]) }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n" }, { "path": "roles/network_plugin/calico/tasks/peer_with_router.yml", "content": "---\n- name: Calico | Configure peering with router(s) at global scope\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ stdin is string | ternary(stdin, stdin | to_json) }}\"\n vars:\n stdin: >\n {\"apiVersion\": \"projectcalico.org/v3\",\n \"kind\": \"BGPPeer\",\n \"metadata\": {\n \"name\": \"global-{{ item.name | default(item.router_id | replace(':', '-')) }}\"\n },\n \"spec\": {\n \"asNumber\": \"{{ item.as }}\",\n \"peerIP\": \"{{ item.router_id }}\"\n }}\n register: output\n retries: 4\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n with_items:\n - \"{{ peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'global') | list }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Calico | Get node for per node peering\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }}\"\n register: output_get_node\n when:\n - ('k8s_cluster' in group_names)\n - local_as is defined\n - groups['calico_rr'] | default([]) | length == 0\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Calico | Patch node asNumber for per node peering\n command:\n cmd: |-\n {{ bin_dir }}/calicoctl.sh patch node \"{{ inventory_hostname }}\" --patch '{{ patch is string | ternary(patch, patch | to_json) }}'\n vars:\n patch: >\n {\"spec\": {\n \"bgp\": {\n \"asNumber\": \"{{ local_as }}\"\n },\n \"orchRefs\": [{\"nodeName\": \"{{ inventory_hostname }}\", \"orchestrator\": \"k8s\"}]\n }}\n register: output\n retries: 0\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n when:\n - ('k8s_cluster' in group_names)\n - local_as is defined\n - groups['calico_rr'] | default([]) | length == 0\n - output_get_node.rc == 0\n\n- name: Calico | Configure node asNumber for per node peering\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ stdin is string | ternary(stdin, stdin | to_json) }}\"\n vars:\n stdin: >\n {\"apiVersion\": \"projectcalico.org/v3\",\n \"kind\": \"Node\",\n \"metadata\": {\n \"name\": \"{{ inventory_hostname }}\"\n },\n \"spec\": {\n \"bgp\": {\n \"asNumber\": \"{{ local_as }}\"\n },\n \"orchRefs\":[{\"nodeName\":\"{{ inventory_hostname }}\",\"orchestrator\":\"k8s\"}]\n }}\n register: output\n retries: 4\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n when:\n - ('k8s_cluster' in group_names)\n - local_as is defined\n - groups['calico_rr'] | default([]) | length == 0\n - output_get_node.rc != 0\n\n- name: Calico | Configure peering with router(s) at node scope\n command:\n cmd: \"{{ bin_dir }}/calicoctl.sh apply -f -\"\n stdin: \"{{ stdin is string | ternary(stdin, stdin | to_json) }}\"\n vars:\n stdin: >\n {\"apiVersion\": \"projectcalico.org/v3\",\n \"kind\": \"BGPPeer\",\n \"metadata\": {\n \"name\": \"{{ inventory_hostname }}-{{ item.name | default(item.router_id | replace(':', '-')) }}\"\n },\n \"spec\": {\n \"asNumber\": \"{{ item.as }}\",\n \"node\": \"{{ inventory_hostname }}\",\n \"peerIP\": \"{{ item.router_id }}\",\n {% if calico_version is version('3.26.0', '>=') and (item.filters | default([]) | length > 0) %}\n \"filters\": {{ item.filters }},\n {% endif %}\n {% if calico_version is version('3.23.0', '>=') and (item.numallowedlocalasnumbers | default(0) > 0) %}\n \"numAllowedLocalASNumbers\": {{ item.numallowedlocalasnumbers }},\n {% endif %}\n \"sourceAddress\": \"{{ item.sourceaddress | default('UseNodeIP') }}\"\n }}\n register: output\n retries: 4\n until: output.rc == 0\n delay: \"{{ retry_stagger | random + 3 }}\"\n with_items:\n - \"{{ peers | default([]) | selectattr('scope', 'undefined') | list | union(peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'node') | list ) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when:\n - ('k8s_cluster' in group_names)\n" }, { "path": "roles/network_plugin/calico/tasks/pre.yml", "content": "---\n- name: Slurp CNI config\n slurp:\n src: /etc/cni/net.d/10-calico.conflist\n register: calico_cni_config_slurp\n failed_when: false\n\n- name: Gather calico facts\n tags:\n - facts\n when: calico_cni_config_slurp.content is defined\n block:\n - name: Set fact calico_cni_config from slurped CNI config\n set_fact:\n calico_cni_config: \"{{ calico_cni_config_slurp['content'] | b64decode | from_json }}\"\n - name: Set fact calico_datastore to etcd if needed\n set_fact:\n calico_datastore: etcd\n when:\n - \"'plugins' in calico_cni_config\"\n - \"'etcd_endpoints' in calico_cni_config.plugins.0\"\n\n- name: Calico | Gather os specific variables\n include_vars: \"{{ item }}\"\n with_first_found:\n - files:\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml\"\n - \"{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml\"\n - \"{{ ansible_distribution | lower }}.yml\"\n - \"{{ ansible_os_family | lower }}-{{ ansible_architecture }}.yml\"\n - \"{{ ansible_os_family | lower }}.yml\"\n - defaults.yml\n paths:\n - ../vars\n skip: true\n" }, { "path": "roles/network_plugin/calico/tasks/repos.yml", "content": "---\n- name: Calico | Add wireguard yum repo\n when:\n - calico_wireguard_enabled\n block:\n\n - name: Calico | Add wireguard yum repo\n yum_repository:\n name: copr:copr.fedorainfracloud.org:jdoss:wireguard\n file: _copr:copr.fedorainfracloud.org:jdoss:wireguard\n description: Copr repo for wireguard owned by jdoss\n baseurl: \"{{ calico_wireguard_repo }}\"\n gpgcheck: true\n gpgkey: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/pubkey.gpg\n skip_if_unavailable: true\n enabled: true\n repo_gpgcheck: false\n when:\n - ansible_os_family in ['RedHat']\n - ansible_distribution not in ['Fedora']\n - ansible_facts['distribution_major_version'] | int < 9\n" }, { "path": "roles/network_plugin/calico/tasks/reset.yml", "content": "---\n- name: Reset | check vxlan.calico network device\n stat:\n path: /sys/class/net/vxlan.calico\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: vxlan\n\n- name: Reset | remove the network vxlan.calico device created by calico\n command: ip link del vxlan.calico\n when: vxlan.stat.exists\n\n- name: Reset | check dummy0 network device\n stat:\n path: /sys/class/net/dummy0\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: dummy0\n\n- name: Reset | remove the network device created by calico\n command: ip link del dummy0\n when: dummy0.stat.exists\n\n- name: Reset | get and remove remaining routes set by bird\n shell: set -o pipefail && ip route show proto bird | xargs -i bash -c \"ip route del {} proto bird \"\n args:\n executable: /bin/bash\n changed_when: false\n" }, { "path": "roles/network_plugin/calico/tasks/typha_certs.yml", "content": "---\n- name: Calico | Check if typha-server exists\n command: \"{{ kubectl }} -n kube-system get secret typha-server\"\n register: typha_server_secret\n changed_when: false\n failed_when: false\n\n- name: Calico | Ensure calico certs dir\n file:\n path: /etc/calico/certs\n state: directory\n mode: \"0755\"\n when: typha_server_secret.rc != 0\n\n- name: Calico | Copy ssl script for typha certs\n template:\n src: make-ssl-calico.sh.j2\n dest: \"{{ bin_dir }}/make-ssl-typha.sh\"\n mode: \"0755\"\n\n when: typha_server_secret.rc != 0\n\n- name: Calico | Copy ssl config for typha certs\n copy:\n src: openssl.conf\n dest: /etc/calico/certs/openssl.conf\n mode: \"0644\"\n when: typha_server_secret.rc != 0\n\n- name: Calico | Generate typha certs\n command: >-\n {{ bin_dir }}/make-ssl-typha.sh\n -f /etc/calico/certs/openssl.conf\n -c {{ kube_cert_dir }}\n -d /etc/calico/certs\n -s typha\n when: typha_server_secret.rc != 0\n\n- name: Calico | Create typha tls secrets\n command: >-\n {{ kubectl }} -n kube-system\n create secret tls {{ item.name }}\n --cert {{ item.cert }}\n --key {{ item.key }}\n with_items:\n - name: typha-server\n cert: /etc/calico/certs/typha-server.crt\n key: /etc/calico/certs/typha-server.key\n - name: typha-client\n cert: /etc/calico/certs/typha-client.crt\n key: /etc/calico/certs/typha-client.key\n when: typha_server_secret.rc != 0\n" }, { "path": "roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2", "content": "# This is a tech-preview manifest which installs the Calico API server. Note that this manifest is liable to change\n# or be removed in future releases without further warning.\n#\n# Namespace and namespace-scoped resources.\napiVersion: v1\nkind: Namespace\nmetadata:\n labels:\n name: calico-apiserver\n name: calico-apiserver\n" }, { "path": "roles/network_plugin/calico/templates/calico-apiserver.yml.j2", "content": "# Policy to ensure the API server isn't cut off. Can be modified, but ensure\n# that the main API server is always able to reach the Calico API server.\nkind: NetworkPolicy\napiVersion: networking.k8s.io/v1\nmetadata:\n name: allow-apiserver\n namespace: calico-apiserver\nspec:\n podSelector:\n matchLabels:\n apiserver: \"true\"\n ingress:\n - ports:\n - protocol: TCP\n port: 5443\n\n---\n\napiVersion: v1\nkind: Service\nmetadata:\n name: calico-api\n namespace: calico-apiserver\nspec:\n ports:\n - name: apiserver\n port: 443\n protocol: TCP\n targetPort: 5443\n selector:\n apiserver: \"true\"\n type: ClusterIP\n\n---\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n labels:\n apiserver: \"true\"\n k8s-app: calico-apiserver\n name: calico-apiserver\n namespace: calico-apiserver\nspec:\n replicas: 1\n selector:\n matchLabels:\n apiserver: \"true\"\n strategy:\n type: Recreate\n template:\n metadata:\n labels:\n apiserver: \"true\"\n k8s-app: calico-apiserver\n name: calico-apiserver\n namespace: calico-apiserver\n spec:\n containers:\n - args:\n - --secure-port=5443\n env:\n - name: DATASTORE_TYPE\n value: kubernetes\n image: {{ calico_apiserver_image_repo }}:{{ calico_apiserver_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n livenessProbe:\n httpGet:\n path: /version\n port: 5443\n scheme: HTTPS\n initialDelaySeconds: 90\n periodSeconds: 10\n name: calico-apiserver\n{% if calico_version is version('3.28.0', '>=') %}\n readinessProbe:\n httpGet:\n path: /readyz\n port: 5443\n scheme: HTTPS\n timeoutSeconds: 5\n periodSeconds: 60\n{% else %}\n readinessProbe:\n exec:\n command:\n - /code/filecheck\n failureThreshold: 5\n initialDelaySeconds: 5\n periodSeconds: 10\n{% endif %}\n securityContext:\n privileged: false\n runAsUser: 0\n volumeMounts:\n - mountPath: /code/apiserver.local.config/certificates\n name: calico-apiserver-certs\n dnsPolicy: ClusterFirst\n nodeSelector:\n kubernetes.io/os: linux\n restartPolicy: Always\n serviceAccount: calico-apiserver\n serviceAccountName: calico-apiserver\n tolerations:\n - effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n volumes:\n - name: calico-apiserver-certs\n secret:\n secretName: calico-apiserver-certs\n\n---\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-apiserver\n namespace: calico-apiserver\n\n---\n\n# Cluster-scoped resources below here.\napiVersion: apiregistration.k8s.io/v1\nkind: APIService\nmetadata:\n name: v3.projectcalico.org\nspec:\n group: projectcalico.org\n groupPriorityMinimum: 1500\n caBundle: {{ calico_apiserver_cabundle }}\n service:\n name: calico-api\n namespace: calico-apiserver\n port: 443\n version: v3\n versionPriority: 200\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: calico-crds\nrules:\n- apiGroups:\n - extensions\n - networking.k8s.io\n - \"\"\n resources:\n - networkpolicies\n - nodes\n - namespaces\n - pods\n - serviceaccounts\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - crd.projectcalico.org\n resources:\n - globalnetworkpolicies\n - networkpolicies\n - clusterinformations\n - hostendpoints\n - globalnetworksets\n - networksets\n - bgpconfigurations\n - bgppeers\n - bgpfilters\n - felixconfigurations\n - kubecontrollersconfigurations\n - ippools\n - ipamconfigs\n - ipreservations\n - ipamblocks\n - blockaffinities\n - caliconodestatuses\n - tiers\n - stagednetworkpolicies\n - stagedglobalnetworkpolicies\n - stagedkubernetesnetworkpolicies\n verbs:\n - get\n - list\n - watch\n - create\n - update\n - delete\n{% if calico_version is version('3.28.0', '>=') %}\n- apiGroups:\n - policy\n resourceNames:\n - calico-apiserver\n resources:\n - podsecuritypolicies\n verbs:\n - use\n{% endif %}\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: calico-extension-apiserver-auth-access\nrules:\n- apiGroups:\n - \"\"\n resourceNames:\n - extension-apiserver-authentication\n resources:\n - configmaps\n verbs:\n - list\n - watch\n - get\n- apiGroups:\n - rbac.authorization.k8s.io\n resources:\n - clusterroles\n - clusterrolebindings\n - roles\n - rolebindings\n verbs:\n - get\n - list\n - watch\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: calico-webhook-reader\nrules:\n- apiGroups:\n - admissionregistration.k8s.io\n resources:\n - mutatingwebhookconfigurations\n - validatingwebhookconfigurations\n - validatingadmissionpolicies # Required for Kubernetes 1.33+\n - validatingadmissionpolicybindings # Required for Kubernetes 1.33+\n verbs:\n - get\n - list\n - watch\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-apiserver-access-crds\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-crds\nsubjects:\n- kind: ServiceAccount\n name: calico-apiserver\n namespace: calico-apiserver\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-apiserver-delegate-auth\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: system:auth-delegator\nsubjects:\n- kind: ServiceAccount\n name: calico-apiserver\n namespace: calico-apiserver\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-apiserver-webhook-reader\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-webhook-reader\nsubjects:\n- kind: ServiceAccount\n name: calico-apiserver\n namespace: calico-apiserver\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-extension-apiserver-auth-access\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-extension-apiserver-auth-access\nsubjects:\n- kind: ServiceAccount\n name: calico-apiserver\n namespace: calico-apiserver\n" }, { "path": "roles/network_plugin/calico/templates/calico-config.yml.j2", "content": "kind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-config\n namespace: kube-system\ndata:\n{% if calico_datastore == \"etcd\" %}\n etcd_endpoints: \"{{ etcd_access_addresses }}\"\n etcd_ca: \"/calico-secrets/ca_cert.crt\"\n etcd_cert: \"/calico-secrets/cert.crt\"\n etcd_key: \"/calico-secrets/key.pem\"\n{% elif calico_datastore == \"kdd\" and typha_enabled %}\n # To enable Typha, set this to \"calico-typha\" *and* set a non-zero value for Typha replicas\n # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is\n # essential.\n typha_service_name: \"calico-typha\"\n{% endif %}\n{% if calico_network_backend == 'bird' %}\n cluster_type: \"kubespray,bgp\"\n calico_backend: \"bird\"\n{% else %}\n cluster_type: \"kubespray\"\n calico_backend: \"{{ calico_network_backend }}\"\n{% endif %}\n{% if inventory_hostname in groups['k8s_cluster'] and peer_with_router | default(false) %}\n as: \"{{ local_as | default(global_as_num) }}\"\n{% endif -%}\n # The CNI network configuration to install on each node. The special\n # values in this config will be automatically populated.\n cni_network_config: |-\n {\n \"name\": \"{{ calico_cni_name }}\",\n \"cniVersion\":\"0.3.1\",\n \"plugins\":[\n {\n {% if calico_datastore == \"kdd\" %}\n \"datastore_type\": \"kubernetes\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n {% endif %}\n \"type\": \"calico\",\n \"log_level\": \"info\",\n {% if calico_cni_log_file_path %}\n \"log_file_path\": \"{{ calico_cni_log_file_path }}\",\n {% endif %}\n {% if calico_datastore == \"etcd\" %}\n \"etcd_endpoints\": \"{{ etcd_access_addresses }}\",\n \"etcd_cert_file\": \"{{ calico_cert_dir }}/cert.crt\",\n \"etcd_key_file\": \"{{ calico_cert_dir }}/key.pem\",\n \"etcd_ca_cert_file\": \"{{ calico_cert_dir }}/ca_cert.crt\",\n {% endif %}\n {% if calico_ipam_host_local %}\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"usePodCidr\"\n },\n {% else %}\n \"ipam\": {\n \"type\": \"calico-ipam\",\n {% if ipv4_stack %}\n \"assign_ipv4\": \"true\"{{ ',' if (ipv6_stack and ipv4_stack) }}\n {% endif %}\n {% if ipv6_stack %}\n \"assign_ipv6\": \"true\"\n {% endif %}\n },\n {% endif %}\n {% if calico_allow_ip_forwarding %}\n \"container_settings\": {\n \"allow_ip_forwarding\": true\n },\n {% endif %}\n {% if (calico_feature_control is defined) and (calico_feature_control | length > 0) %}\n \"feature_control\": {\n {% for fc in calico_feature_control -%}\n {% set fcval = calico_feature_control[fc] -%}\n \"{{ fc }}\": {{ (fcval | string | lower) if (fcval == true or fcval == false) else \"\\\"\" + fcval + \"\\\"\" }}{{ \",\" if not loop.last else \"\" }}\n {% endfor -%}\n {{- \"\" }}\n },\n {% endif %}\n {% if enable_network_policy %}\n \"policy\": {\n \"type\": \"k8s\"\n },\n {% endif %}\n {% if calico_mtu is defined and calico_mtu is number %}\n \"mtu\": {{ calico_mtu }},\n {% endif %}\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n }\n },\n {\n \"type\":\"portmap\",\n \"capabilities\": {\n \"portMappings\": true\n }\n },\n {\n \"type\":\"bandwidth\",\n \"capabilities\": {\n \"bandwidth\": true\n }\n }\n ]\n }\n" }, { "path": "roles/network_plugin/calico/templates/calico-cr.yml.j2", "content": "---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-cni-plugin\nrules:\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n verbs:\n - get\n - apiGroups: [\"\"]\n resources:\n - pods/status\n verbs:\n - patch\n - apiGroups: [\"\"]\n resources:\n - nodes/status\n verbs:\n - update\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n - clusterinformations\n - ippools\n - ipreservations\n - ipamconfigs\n verbs:\n - get\n - list\n - create\n - update\n - delete\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-node\n namespace: kube-system\nrules:\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n - configmaps\n verbs:\n - get\n # EndpointSlices are used for Service-based network policy rule\n # enforcement.\n - apiGroups: [\"discovery.k8s.io\"]\n resources:\n - endpointslices\n verbs:\n - watch\n - list\n - apiGroups: [\"\"]\n resources:\n - endpoints\n - services\n verbs:\n - watch\n - list\n{% if calico_datastore == \"kdd\" %}\n # Used to discover Typhas.\n - get\n{% endif %}\n - apiGroups: [\"\"]\n resources:\n - nodes/status\n verbs:\n # Needed for clearing NodeNetworkUnavailable flag.\n - patch\n{% if calico_datastore == \"kdd\" %}\n # Calico stores some configuration information in node annotations.\n - update\n # Watch for changes to Kubernetes NetworkPolicies.\n - apiGroups: [\"networking.k8s.io\"]\n resources:\n - networkpolicies\n verbs:\n - watch\n - list\n # Watch for changes to Kubernetes AdminNetworkPolicies.\n - apiGroups: [\"policy.networking.k8s.io\"]\n resources:\n - adminnetworkpolicies\n - baselineadminnetworkpolicies\n verbs:\n - watch\n - list\n # Used by Calico for policy information.\n - apiGroups: [\"\"]\n resources:\n - pods\n - namespaces\n - serviceaccounts\n verbs:\n - list\n - watch\n # The CNI plugin patches pods/status.\n - apiGroups: [\"\"]\n resources:\n - pods/status\n verbs:\n - patch\n # Calico monitors various CRDs for config.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - globalfelixconfigs\n - felixconfigurations\n - bgppeers\n - bgpfilters\n - globalbgpconfigs\n - bgpconfigurations\n - ippools\n - ipreservations\n - ipamblocks\n - globalnetworkpolicies\n - stagedglobalnetworkpolicies\n - networkpolicies\n - stagednetworkpolicies\n - stagedkubernetesnetworkpolicies\n - globalnetworksets\n - networksets\n - clusterinformations\n - hostendpoints\n - blockaffinities\n - caliconodestatuses\n - tiers\n verbs:\n - get\n - list\n - watch\n # Calico creates some tiers on startup.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - tiers\n verbs:\n - create\n # Calico must create and update some CRDs on startup.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n - felixconfigurations\n - clusterinformations\n verbs:\n - create\n - update\n # Calico must update some CRDs.\n - apiGroups: [ \"crd.projectcalico.org\" ]\n resources:\n - caliconodestatuses\n verbs:\n - update\n # Calico stores some configuration information on the node.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n # These permissions are only required for upgrade from v2.6, and can\n # be removed after upgrade or on fresh installations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - bgpconfigurations\n - bgppeers\n verbs:\n - create\n - update\n # These permissions are required for Calico CNI to perform IPAM allocations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ipamconfigs\n verbs:\n - get\n - create\n # Block affinities must also be watchable by confd for route aggregation.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n verbs:\n - watch\n # The Calico IPAM migration needs to get daemonsets. These permissions can be\n # removed if not upgrading from an installation using host-local IPAM.\n - apiGroups: [\"apps\"]\n resources:\n - daemonsets\n verbs:\n - get\n{% endif %}\n # Used for creating service account tokens to be used by the CNI plugin\n - apiGroups: [\"\"]\n resources:\n - serviceaccounts/token\n resourceNames:\n - calico-cni-plugin\n verbs:\n - create\n{% if calico_version is version('3.29.0', '>=') %}\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-tier-getter\nrules:\n - apiGroups:\n - \"projectcalico.org\"\n resources:\n - \"tiers\"\n verbs:\n - \"get\"\n{% endif %}\n" }, { "path": "roles/network_plugin/calico/templates/calico-crb.yml.j2", "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-node\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-node\nsubjects:\n- kind: ServiceAccount\n name: calico-node\n namespace: kube-system\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-cni-plugin\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-cni-plugin\nsubjects:\n- kind: ServiceAccount\n name: calico-cni-plugin\n namespace: kube-system\n{% if calico_version is version('3.29.0', '>=') %}\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-tier-getter\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-tier-getter\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: system:kube-controller-manager\n{% endif %}\n" }, { "path": "roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2", "content": "apiVersion: crd.projectcalico.org/v1\nkind: IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: {{ calico_ipam_autoallocateblocks }}\n strictAffinity: {{ calico_ipam_strictaffinity }}\n maxBlocksPerHost: {{ calico_ipam_maxblocksperhost }}\n" }, { "path": "roles/network_plugin/calico/templates/calico-node-sa.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-node\n namespace: kube-system\n\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-cni-plugin\n namespace: kube-system\n" }, { "path": "roles/network_plugin/calico/templates/calico-node.yml.j2", "content": "---\n# This manifest installs the calico/node container, as well\n# as the Calico CNI plugins and network config on\n# each control plane and worker node in a Kubernetes cluster.\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: calico-node\n namespace: kube-system\n labels:\n k8s-app: calico-node\nspec:\n selector:\n matchLabels:\n k8s-app: calico-node\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n template:\n metadata:\n labels:\n k8s-app: calico-node\n annotations:\n{% if calico_datastore == \"etcd\" %}\n kubespray.etcd-cert/serial: \"{{ etcd_client_cert_serial }}\"\n{% endif %}\n{% if calico_felix_prometheusmetricsenabled %}\n prometheus.io/scrape: 'true'\n prometheus.io/port: \"{{ calico_felix_prometheusmetricsport }}\"\n{% endif %}\n spec:\n nodeSelector:\n {{ calico_ds_nodeselector }}\n priorityClassName: system-node-critical\n hostNetwork: true\n serviceAccountName: calico-node\n tolerations:\n # Make sure calico-node gets scheduled on all nodes.\n - effect: NoSchedule\n operator: Exists\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - effect: NoExecute\n operator: Exists\n # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a \"force\n # deletion\": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.\n terminationGracePeriodSeconds: 0\n initContainers:\n{% if calico_datastore == \"kdd\" and not calico_ipam_host_local %}\n # This container performs upgrade from host-local IPAM to calico-ipam.\n # It can be deleted if this is a fresh installation, or if you have already\n # upgraded to use calico-ipam.\n - name: upgrade-ipam\n image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [\"/opt/cni/bin/calico-ipam\", \"-upgrade\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n volumeMounts:\n - mountPath: /var/lib/cni/networks\n name: host-local-net-dir\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n securityContext:\n privileged: true\n{% endif %}\n # This container installs the Calico CNI binaries\n # and CNI network config file on each node.\n - name: install-cni\n image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [\"/opt/cni/bin/install\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # The CNI network config to install on each node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: cni_network_config\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: \"10-calico.conflist\"\n{% if calico_mtu is defined %}\n # CNI MTU Config variable\n - name: CNI_MTU\n value: \"{{ calico_veth_mtu | default(calico_mtu) }}\"\n{% endif %}\n # Prevents the container from sleeping forever.\n - name: SLEEP\n value: \"false\"\n{% if calico_datastore == \"etcd\" %}\n - name: ETCD_ENDPOINTS\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: etcd_endpoints\n{% endif %}\n{% if calico_datastore == \"kdd\" %}\n # Set the hostname based on the k8s node name.\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n{% endif %}\n volumeMounts:\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n securityContext:\n privileged: true\n # This init container mounts the necessary filesystems needed by the BPF data plane\n # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed\n # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode.\n - name: \"mount-bpffs\"\n image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [\"calico-node\", \"-init\", \"-best-effort\"]\n volumeMounts:\n - mountPath: /sys/fs\n name: sys-fs\n # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host\n # so that it outlives the init container.\n mountPropagation: Bidirectional\n - mountPath: /var/run/calico\n name: var-run-calico\n # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host\n # so that it outlives the init container.\n mountPropagation: Bidirectional\n # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary,\n # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly.\n - mountPath: /nodeproc\n name: nodeproc\n readOnly: true\n securityContext:\n privileged: true\n containers:\n # Runs calico/node container on each Kubernetes node. This\n # container programs network policy and routes on each\n # host.\n - name: calico-node\n image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # The location of the Calico etcd cluster.\n{% if calico_datastore == \"etcd\" %}\n - name: ETCD_ENDPOINTS\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: etcd_endpoints\n # Location of the CA certificate for etcd.\n - name: ETCD_CA_CERT_FILE\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: etcd_ca\n # Location of the client key for etcd.\n - name: ETCD_KEY_FILE\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: etcd_key\n # Location of the client certificate for etcd.\n - name: ETCD_CERT_FILE\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: etcd_cert\n{% elif calico_datastore == \"kdd\" %}\n # Use Kubernetes API as the backing datastore.\n - name: DATASTORE_TYPE\n value: \"kubernetes\"\n{% if typha_enabled %}\n # Typha support: controlled by the ConfigMap.\n - name: FELIX_TYPHAK8SSERVICENAME\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: typha_service_name\n{% if typha_secure %}\n - name: FELIX_TYPHACN\n value: typha-server\n - name: FELIX_TYPHACAFILE\n value: /etc/typha-ca/ca.crt\n - name: FELIX_TYPHACERTFILE\n value: /etc/typha-client/typha-client.crt\n - name: FELIX_TYPHAKEYFILE\n value: /etc/typha-client/typha-client.key\n{% endif %}\n{% endif %}\n # Wait for the datastore.\n - name: WAIT_FOR_DATASTORE\n value: \"true\"\n{% endif %}\n{% if calico_network_backend == 'vxlan' %}\n - name: FELIX_VXLANVNI\n value: \"{{ calico_vxlan_vni }}\"\n - name: FELIX_VXLANPORT\n value: \"{{ calico_vxlan_port }}\"\n{% endif %}\n # Choose the backend to use.\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n # Cluster type to identify the deployment type\n - name: CLUSTER_TYPE\n value: \"k8s,bgp\"\n # Set noderef for node controller.\n - name: CALICO_K8S_NODE_REF\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # Disable file logging so `kubectl logs` works.\n - name: CALICO_DISABLE_FILE_LOGGING\n value: \"true\"\n # Set Felix endpoint to host default action to ACCEPT.\n - name: FELIX_DEFAULTENDPOINTTOHOSTACTION\n value: \"{{ calico_endpoint_to_host_action | default('RETURN') }}\"\n - name: FELIX_HEALTHHOST\n value: \"{{ calico_healthhost }}\"\n{% if kube_proxy_mode == 'ipvs' and kube_apiserver_node_port_range is defined %}\n - name: FELIX_KUBENODEPORTRANGES\n value: \"{{ kube_apiserver_node_port_range.split('-')[0] }}:{{ kube_apiserver_node_port_range.split('-')[1] }}\"\n{% endif %}\n - name: FELIX_IPTABLESBACKEND\n value: \"{{ calico_iptables_backend }}\"\n - name: FELIX_IPTABLESLOCKTIMEOUTSECS\n value: \"{{ calico_iptables_lock_timeout_secs }}\"\n # The default IPv4 pool to create on startup if none exists. Pod IPs will be\n # chosen from this range. Changing this value after installation will have\n # no effect. This should fall within `--cluster-cidr`.\n # - name: CALICO_IPV4POOL_CIDR\n # value: \"192.168.0.0/16\"\n - name: CALICO_IPV4POOL_IPIP\n value: \"{{ calico_ipv4pool_ipip }}\"\n # Enable or Disable VXLAN on the default IP pool.\n - name: CALICO_IPV4POOL_VXLAN\n value: \"Never\"\n - name: FELIX_IPV6SUPPORT\n value: \"{{ ipv6_stack | default(false) }}\"\n # Set Felix logging to \"info\"\n - name: FELIX_LOGSEVERITYSCREEN\n value: \"{{ calico_loglevel }}\"\n # Set Calico startup logging to \"error\"\n - name: CALICO_STARTUP_LOGLEVEL\n value: \"{{ calico_node_startup_loglevel }}\"\n # Enable or disable usage report\n - name: FELIX_USAGEREPORTINGENABLED\n value: \"{{ calico_usage_reporting }}\"\n{% if calico_version is version('3.29.0', '>=') %}\n - name: FELIX_NFTABLESMODE\n value: \"{{ calico_nftable_mode }}\"\n{% endif %}\n # Set MTU for tunnel device used if ipip is enabled\n{% if calico_mtu is defined %}\n # Set MTU for tunnel device used if ipip is enabled\n - name: FELIX_IPINIPMTU\n value: \"{{ calico_veth_mtu | default(calico_mtu) }}\"\n # Set MTU for the VXLAN tunnel device.\n - name: FELIX_VXLANMTU\n value: \"{{ calico_veth_mtu | default(calico_mtu) }}\"\n # Set MTU for the Wireguard tunnel device.\n - name: FELIX_WIREGUARDMTU\n value: \"{{ calico_veth_mtu | default(calico_mtu) }}\"\n{% endif %}\n - name: FELIX_CHAININSERTMODE\n value: \"{{ calico_felix_chaininsertmode }}\"\n - name: FELIX_PROMETHEUSMETRICSENABLED\n value: \"{{ calico_felix_prometheusmetricsenabled }}\"\n - name: FELIX_PROMETHEUSMETRICSPORT\n value: \"{{ calico_felix_prometheusmetricsport }}\"\n - name: FELIX_PROMETHEUSGOMETRICSENABLED\n value: \"{{ calico_felix_prometheusgometricsenabled }}\"\n - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED\n value: \"{{ calico_felix_prometheusprocessmetricsenabled }}\"\n{% if calico_ip_auto_method is defined %}\n - name: IP_AUTODETECTION_METHOD\n value: \"{{ calico_ip_auto_method }}\"\n{% else %}\n - name: NODEIP\n valueFrom:\n fieldRef:\n fieldPath: status.hostIP\n - name: IP_AUTODETECTION_METHOD\n value: \"can-reach=$(NODEIP)\"\n{% endif %}\n{% if ipv4_stack %}\n - name: IP\n value: \"autodetect\"\n{% else %}\n - name: IP\n value: none\n{% endif %}\n{% if ipv6_stack %}\n - name: IP6\n value: autodetect\n{% endif %}\n{% if calico_ip6_auto_method is defined and ipv6_stack %}\n - name: IP6_AUTODETECTION_METHOD\n value: \"{{ calico_ip6_auto_method }}\"\n{% endif %}\n{% if calico_felix_mtu_iface_pattern is defined %}\n - name: FELIX_MTUIFACEPATTERN\n value: \"{{ calico_felix_mtu_iface_pattern }}\"\n{% endif %}\n{% if calico_use_default_route_src_ipaddr | default(false) %}\n - name: FELIX_DEVICEROUTESOURCEADDRESS\n valueFrom:\n fieldRef:\n fieldPath: status.hostIP\n{% endif %}\n - name: NODENAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: FELIX_HEALTHENABLED\n value: \"true\"\n - name: FELIX_IGNORELOOSERPF\n value: \"{{ calico_node_ignorelooserpf }}\"\n - name: CALICO_MANAGE_CNI\n value: \"true\"\n{% if calico_ipam_host_local %}\n - name: USE_POD_CIDR\n value: \"true\"\n{% endif %}\n{% if calico_node_extra_envs is defined %}\n{% for key in calico_node_extra_envs %}\n - name: {{ key }}\n value: \"{{ calico_node_extra_envs[key] }}\"\n{% endfor %}\n{% endif %}\n securityContext:\n privileged: true\n resources:\n limits:\n{% if calico_node_cpu_limit != \"0\" %}\n cpu: {{ calico_node_cpu_limit }}\n{% endif %}\n memory: {{ calico_node_memory_limit }}\n requests:\n cpu: {{ calico_node_cpu_requests }}\n memory: {{ calico_node_memory_requests }}\n lifecycle:\n preStop:\n exec:\n command:\n - /bin/calico-node\n - -shutdown\n livenessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-live\n{% if calico_network_backend == \"bird\" %}\n - -bird-live\n{% endif %}\n periodSeconds: 10\n initialDelaySeconds: 10\n timeoutSeconds: {{ calico_node_livenessprobe_timeout | default(10) }}\n failureThreshold: 6\n readinessProbe:\n exec:\n command:\n - /bin/calico-node\n{% if calico_network_backend == \"bird\" %}\n - -bird-ready\n{% endif %}\n - -felix-ready\n periodSeconds: 10\n timeoutSeconds: {{ calico_node_readinessprobe_timeout | default(10) }}\n failureThreshold: 6\n volumeMounts:\n - mountPath: /lib/modules\n name: lib-modules\n readOnly: true\n - mountPath: /var/run/calico\n name: var-run-calico\n readOnly: false\n - mountPath: /var/lib/calico\n name: var-lib-calico\n readOnly: false\n{% if calico_datastore == \"etcd\" %}\n - mountPath: /calico-secrets\n name: etcd-certs\n readOnly: true\n{% endif %}\n - name: xtables-lock\n mountPath: /run/xtables.lock\n readOnly: false\n # For maintaining CNI plugin API credentials.\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n readOnly: false\n{% if typha_secure %}\n - name: typha-client\n mountPath: /etc/typha-client\n readOnly: true\n - name: typha-cacert\n subPath: ca.crt\n mountPath: /etc/typha-ca/ca.crt\n readOnly: true\n{% endif %}\n - name: policysync\n mountPath: /var/run/nodeagent\n # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the\n # parent directory.\n - name: bpffs\n mountPath: /sys/fs/bpf\n - name: cni-log-dir\n mountPath: /var/log/calico/cni\n readOnly: true\n volumes:\n # Used by calico/node.\n - name: lib-modules\n hostPath:\n path: /lib/modules\n - name: var-run-calico\n hostPath:\n path: /var/run/calico\n type: DirectoryOrCreate\n - name: var-lib-calico\n hostPath:\n path: /var/lib/calico\n type: DirectoryOrCreate\n # Used to install CNI.\n - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n type: DirectoryOrCreate\n{% if calico_datastore == \"etcd\" %}\n # Mount in the etcd TLS secrets.\n - name: etcd-certs\n hostPath:\n path: \"{{ calico_cert_dir }}\"\n{% endif %}\n # Mount the global iptables lock file, used by calico/node\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n{% if calico_datastore == \"kdd\" and not calico_ipam_host_local %}\n # Mount in the directory for host-local IPAM allocations. This is\n # used when upgrading from host-local to calico-ipam, and can be removed\n # if not using the upgrade-ipam init container.\n - name: host-local-net-dir\n hostPath:\n path: /var/lib/cni/networks\n{% endif %}\n{% if typha_enabled and typha_secure %}\n - name: typha-client\n secret:\n secretName: typha-client\n items:\n - key: tls.crt\n path: typha-client.crt\n - key: tls.key\n path: typha-client.key\n - name: typha-cacert\n hostPath:\n path: \"/etc/kubernetes/ssl/\"\n{% endif %}\n - name: sys-fs\n hostPath:\n path: /sys/fs/\n type: DirectoryOrCreate\n - name: bpffs\n hostPath:\n path: /sys/fs/bpf\n type: Directory\n # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs.\n - name: nodeproc\n hostPath:\n path: /proc\n # Used to access CNI logs.\n - name: cni-log-dir\n hostPath:\n path: /var/log/calico/cni\n # Used to create per-pod Unix Domain Sockets\n - name: policysync\n hostPath:\n type: DirectoryOrCreate\n path: /var/run/nodeagent\n" }, { "path": "roles/network_plugin/calico/templates/calico-typha.yml.j2", "content": "# This manifest creates a Service, which will be backed by Calico's Typha daemon.\n# Typha sits in between Felix and the API server, reducing Calico's load on the API server.\n\napiVersion: v1\nkind: Service\nmetadata:\n name: calico-typha\n namespace: kube-system\n labels:\n k8s-app: calico-typha\nspec:\n ports:\n - port: 5473\n protocol: TCP\n targetPort: calico-typha\n name: calico-typha\n{% if typha_prometheusmetricsenabled %}\n - port: {{ typha_prometheusmetricsport }}\n protocol: TCP\n targetPort: http-metrics\n name: metrics\n{% endif %}\n selector:\n k8s-app: calico-typha\n\n---\n\n# This manifest creates a Deployment of Typha to back the above service.\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: calico-typha\n namespace: kube-system\n labels:\n k8s-app: calico-typha\nspec:\n # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the\n # typha_service_name variable in the calico-config ConfigMap above.\n #\n # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential\n # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In\n # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.\n replicas: {{ typha_replicas }}\n revisionHistoryLimit: 2\n selector:\n matchLabels:\n k8s-app: calico-typha\n template:\n metadata:\n labels:\n k8s-app: calico-typha\n annotations:\n cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'\n{% if typha_prometheusmetricsenabled %}\n prometheus.io/scrape: 'true'\n prometheus.io/port: \"{{ typha_prometheusmetricsport }}\"\n{% endif %}\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - key: node-role.kubernetes.io/control-plane\n operator: Exists\n effect: NoSchedule\n # Since Calico can't network a pod until Typha is up, we need to run Typha itself\n # as a host-networked pod.\n serviceAccountName: calico-node\n priorityClassName: system-cluster-critical\n # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573\n securityContext:\n fsGroup: 65534\n containers:\n - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n name: calico-typha\n ports:\n - containerPort: 5473\n name: calico-typha\n protocol: TCP\n{% if typha_prometheusmetricsenabled %}\n - containerPort: {{ typha_prometheusmetricsport }}\n name: http-metrics\n protocol: TCP\n{% endif %}\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Enable \"info\" logging by default. Can be set to \"debug\" to increase verbosity.\n - name: TYPHA_LOGSEVERITYSCREEN\n value: \"info\"\n # Disable logging to file and syslog since those don't make sense in Kubernetes.\n - name: TYPHA_LOGFILEPATH\n value: \"none\"\n - name: TYPHA_LOGSEVERITYSYS\n value: \"none\"\n # Monitor the Kubernetes API to find the number of running instances and rebalance\n # connections.\n - name: TYPHA_CONNECTIONREBALANCINGMODE\n value: \"kubernetes\"\n - name: TYPHA_DATASTORETYPE\n value: \"kubernetes\"\n - name: TYPHA_HEALTHENABLED\n value: \"true\"\n - name: TYPHA_MAXCONNECTIONSLOWERLIMIT\n value: \"{{ typha_max_connections_lower_limit }}\"\n{% if typha_secure %}\n - name: TYPHA_CAFILE\n value: /etc/ca/ca.crt\n - name: TYPHA_CLIENTCN\n value: typha-client\n - name: TYPHA_SERVERCERTFILE\n value: /etc/typha/server_certificate.pem\n - name: TYPHA_SERVERKEYFILE\n value: /etc/typha/server_key.pem\n{% endif %}\n{% if typha_prometheusmetricsenabled %}\n # Since Typha is host-networked,\n # this opens a port on the host, which may need to be secured.\n - name: TYPHA_PROMETHEUSMETRICSENABLED\n value: \"true\"\n - name: TYPHA_PROMETHEUSMETRICSPORT\n value: \"{{ typha_prometheusmetricsport }}\"\n{% endif %}\n{% if calico_ipam_host_local %}\n - name: USE_POD_CIDR\n value: \"true\"\n{% endif %}\n{% if typha_secure %}\n volumeMounts:\n - mountPath: /etc/typha\n name: typha-server\n readOnly: true\n - mountPath: /etc/ca/ca.crt\n subPath: ca.crt\n name: cacert\n readOnly: true\n{% endif %}\n livenessProbe:\n httpGet:\n path: /liveness\n port: 9098\n host: localhost\n periodSeconds: 30\n initialDelaySeconds: 30\n readinessProbe:\n httpGet:\n path: /readiness\n port: 9098\n host: localhost\n periodSeconds: 10\n{% if typha_secure %}\n volumes:\n - name: typha-server\n secret:\n secretName: typha-server\n items:\n - key: tls.crt\n path: server_certificate.pem\n - key: tls.key\n path: server_key.pem\n - name: cacert\n hostPath:\n path: \"{{ kube_cert_dir }}\"\n{% endif %}\n\n---\n\n# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict\n\napiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: calico-typha\n namespace: kube-system\n labels:\n k8s-app: calico-typha\nspec:\n maxUnavailable: 1\n selector:\n matchLabels:\n k8s-app: calico-typha\n" }, { "path": "roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2", "content": "#!/bin/bash\nETCD_ENDPOINTS={{ etcd_access_addresses }} \\\nETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \\\nETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \\\nETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \\\n{{ bin_dir }}/calicoctl --allow-version-mismatch \"$@\"\n" }, { "path": "roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2", "content": "#!/bin/bash\nDATASTORE_TYPE=kubernetes \\\n{% if inventory_hostname in groups['kube_control_plane'] %}\nKUBECONFIG=/etc/kubernetes/admin.conf \\\n{% else %}\nKUBECONFIG=/etc/cni/net.d/calico-kubeconfig \\\n{% endif %}\n{{ bin_dir }}/calicoctl --allow-version-mismatch \"$@\"\n" }, { "path": "roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2", "content": "---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n namespace: kube-system\n name: kubernetes-services-endpoint\ndata:\n{% if calico_bpf_enabled or loadbalancer_apiserver_localhost %}\n KUBERNETES_SERVICE_HOST: \"{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}\"\n KUBERNETES_SERVICE_PORT: \"{{ kube_apiserver_global_endpoint | urlsplit('port') }}\"\n{% endif %}\n" }, { "path": "roles/network_plugin/calico/templates/make-ssl-calico.sh.j2", "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\nusage()\n{\n cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) -f [-d ]\n -h | --help : Show this message\n -f | --config : Openssl configuration file\n -d | --ssldir : Directory where the certificates will be installed\n -c | --cadir : Directory where the existing CA is located\n -s | --service : Service for the ca\n\n ex :\n $(basename $0) -f openssl.conf -d /srv/ssl\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n case \"$1\" in\n -h | --help) usage; exit 0;;\n -f | --config) CONFIG=${2}; shift 2;;\n -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n -c | --cadir) CADIR=\"${2}\"; shift 2;;\n -s | --service) SERVICE=\"${2}\"; shift 2;;\n *)\n usage\n echo \"ERROR : Unknown option\"\n exit 3\n ;;\n esac\ndone\n\nif [ -z ${CONFIG} ]; then\n echo \"ERROR: the openssl configuration file is missing. option -f\"\n exit 1\nfi\nif [ -z ${SSLDIR} ]; then\n SSLDIR=\"/etc/calico/certs\"\nfi\n\ntmpdir=$(mktemp -d /tmp/calico_${SERVICE}_certs.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p ${SSLDIR} ${CADIR}\n\n# Root CA\nif [ -e \"$CADIR/ca.key\" ]; then\n # Reuse existing CA\n cp $CADIR/{ca.crt,ca.key} .\nelse\n openssl genrsa -out ca.key {{certificates_key_size}} > /dev/null 2>&1\n openssl req -x509 -new -nodes -key ca.key -days {{certificates_duration}} -out ca.crt -subj \"/CN=calico-${SERVICE}-ca\" > /dev/null 2>&1\nfi\n\nif [ $SERVICE == \"typha\" ]; then\n # Typha server\n openssl genrsa -out typha-server.key {{certificates_key_size}} > /dev/null 2>&1\n openssl req -new -key typha-server.key -out typha-server.csr -subj \"/CN=typha-server\" -config ${CONFIG} > /dev/null 2>&1\n openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1\n\n # Typha client\n openssl genrsa -out typha-client.key {{certificates_key_size}} > /dev/null 2>&1\n openssl req -new -key typha-client.key -out typha-client.csr -subj \"/CN=typha-client\" -config ${CONFIG} > /dev/null 2>&1\n openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1\n\nelif [ $SERVICE == \"apiserver\" ]; then\n # calico-apiserver\n openssl genrsa -out apiserver.key {{certificates_key_size}} > /dev/null 2>&1\n openssl req -new -key apiserver.key -out apiserver.csr -subj \"/CN=calico-apiserver\" -config ${CONFIG} > /dev/null 2>&1\n openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out apiserver.crt -days {{certificates_duration}} -extensions ssl_client_apiserver -extfile ${CONFIG} > /dev/null 2>&1\nelse\n echo \"ERROR: the openssl configuration file is missing. option -s\"\n exit 1\nfi\n\n# Install certs\nif [ -e \"$CADIR/ca.key\" ]; then\n # No pass existing CA\n rm -f ca.crt ca.key\nfi\n\nmv {*.crt,*.key} ${SSLDIR}/\n" }, { "path": "roles/network_plugin/calico/vars/amazon.yml", "content": "---\ncalico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-7-$basearch/\ncalico_wireguard_packages:\n - wireguard-dkms\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico/vars/centos-9.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico/vars/debian.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard\n" }, { "path": "roles/network_plugin/calico/vars/fedora.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico/vars/opensuse.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico/vars/redhat-9.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico/vars/redhat.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard-dkms\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico/vars/rocky-9.yml", "content": "---\ncalico_wireguard_packages:\n - wireguard-tools\n" }, { "path": "roles/network_plugin/calico_defaults/defaults/main.yml", "content": "---\n# the default value of name\ncalico_cni_name: k8s-pod-network\n\n# Enables Internet connectivity from containers\nnat_outgoing: true\nnat_outgoing_ipv6: false\n\n# add default ippool name\ncalico_pool_name: \"default-pool\"\ncalico_ipv4pool_ipip: \"Off\"\n\n# Change encapsulation mode, by default we enable vxlan which is the most mature and well tested mode\ncalico_ipip_mode: Never # valid values are 'Always', 'Never' and 'CrossSubnet'\ncalico_vxlan_mode: Always # valid values are 'Always', 'Never' and 'CrossSubnet'\n\n# add default ippool blockSize\ncalico_pool_blocksize: 26\n\n# Calico doesn't support ipip tunneling for the IPv6.\ncalico_ipip_mode_ipv6: Never\ncalico_vxlan_mode_ipv6: Always\n\n# add default ipv6 ippool blockSize\ncalico_pool_blocksize_ipv6: 122\n\n# Calico network backend can be 'bird', 'vxlan' and 'none'\ncalico_network_backend: vxlan\n\ncalico_cert_dir: /etc/calico/certs\n\n# Global as_num (/calico/bgp/v1/global/as_num)\nglobal_as_num: \"64512\"\n\n# You can set MTU value here. If left undefined or empty, it will\n# not be specified in calico CNI config, so Calico will use built-in\n# defaults. The value should be a number, not a string.\n# calico_mtu: 1500\n\n# Advertise Service External IPs\ncalico_advertise_service_external_ips: []\n\n# Advertise Service LoadBalancer IPs\ncalico_advertise_service_loadbalancer_ips: []\n\n# Calico eBPF support\ncalico_bpf_enabled: false\ncalico_bpf_log_level: \"\"\n# Valid option for service mode: Tunnel (default), DSR=Direct Server Return\ncalico_bpf_service_mode: Tunnel\n\n# Calico floatingIPs support\n# Valid option for floatingIPs: Disabled (default), Enabled\ncalico_felix_floating_ips: Disabled\n\n# Limits for apps\ncalico_node_memory_limit: 500M\ncalico_node_cpu_limit: \"0\"\ncalico_node_memory_requests: 64M\ncalico_node_cpu_requests: 150m\ncalico_felix_chaininsertmode: Insert\n\n# Calico daemonset nodeselector\ncalico_ds_nodeselector: \"kubernetes.io/os: linux\"\n\n# Virtual network ID to use for VXLAN traffic. A value of 0 means “use the kernel default”.\ncalico_vxlan_vni: 4096\n\n# Port to use for VXLAN traffic. A value of 0 means “use the kernel default”.\ncalico_vxlan_port: 4789\n\n# Enable Prometheus Metrics endpoint for felix\ncalico_felix_prometheusmetricsenabled: false\ncalico_felix_prometheusmetricsport: 9091\ncalico_felix_prometheusgometricsenabled: true\ncalico_felix_prometheusprocessmetricsenabled: true\n\n# Set the agent log level. Can be debug, warning, info or fatal\ncalico_loglevel: info\ncalico_node_startup_loglevel: error\n\n# Set log path for calico CNI plugin. Set to false to disable logging to disk.\ncalico_cni_log_file_path: /var/log/calico/cni/cni.log\n\n# Enable or disable usage report to 'usage.projectcalico.org'\ncalico_usage_reporting: false\n\n# Should calico ignore kernel's RPF check setting,\n# see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198\ncalico_node_ignorelooserpf: false\n\n# Define address on which Felix will respond to health requests\ncalico_healthhost: \"localhost\"\n\n# Configure time in seconds that calico will wait for the iptables lock\ncalico_iptables_lock_timeout_secs: 10\n\n# Choose Calico iptables backend: \"Legacy\", \"Auto\" or \"NFT\" (FELIX_IPTABLESBACKEND)\ncalico_iptables_backend: \"Auto\"\n\n# Calico NFTable Mode Support (tech preview 3.29)\n# Valid option: Disabled (default), Enabled\ncalico_nftable_mode: \"Disabled\"\n\n# Calico Wireguard support\ncalico_wireguard_enabled: false\ncalico_wireguard_packages: []\ncalico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-{{ ansible_distribution_major_version }}-$basearch/\n\n# If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of:\n# * can-reach=DESTINATION\n# * interface=INTERFACE-REGEX\n# see https://projectcalico.docs.tigera.io/reference/node/configuration#ip-autodetection-methods\n# calico_ip_auto_method: \"interface=eth.*\"\n# calico_ip6_auto_method: \"interface=eth.*\"\n\n# Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection.\n# see https://projectcalico.docs.tigera.io/reference/felix/configuration\n# calico_felix_mtu_iface_pattern: \"^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)\"\n\ncalico_baremetal_nodename: \"{{ kube_override_hostname | default(inventory_hostname) }}\"\n\nkube_etcd_cacert_file: ca.pem\nkube_etcd_cert_file: node-{{ inventory_hostname }}.pem\nkube_etcd_key_file: node-{{ inventory_hostname }}-key.pem\n\n# Choose data store type for calico: \"etcd\" or \"kdd\" (kubernetes datastore)\n# The default value for calico_datastore is set in role kubespray-default\n\n# Use typha (only with kdd)\ntypha_enabled: false\ntypha_prometheusmetricsenabled: false\ntypha_prometheusmetricsport: 9093\n\n# Scaling typha: 1 replica per 100 nodes is adequate\n# Number of typha replicas\ntypha_replicas: 1\n\n# Set max typha connections\ntypha_max_connections_lower_limit: 300\n\n# Generate certifcates for typha<->calico-node communication\ntypha_secure: false\n\ncalico_feature_control: {}\n\n# Calico default BGP port\ncalico_bgp_listen_port: 179\n\n# Calico FelixConfiguration options\ncalico_felix_reporting_interval: 0s\ncalico_felix_log_severity_screen: Info\n\n# Calico container settings\ncalico_allow_ip_forwarding: false\n\n# Calico IPAM strictAffinity\ncalico_ipam_strictaffinity: false\n\n# Calico IPAM autoAllocateBlocks\ncalico_ipam_autoallocateblocks: true\n\n# Calico IPAM maxBlocksPerHost, default 0\ncalico_ipam_maxblocksperhost: 0\n\n# Calico host local IPAM (use node .spec.podCIDR)\n\ncalico_ipam_host_local: false\n\n# Calico apiserver (only with kdd)\ncalico_apiserver_enabled: false\n\n# Calico feature detect override\ncalico_feature_detect_override: \"\"\n\n# Calico kubeconfig wait timeout in seconds\ncalico_kubeconfig_wait_timeout: 300\n" }, { "path": "roles/network_plugin/cilium/defaults/main.yml", "content": "---\ncilium_min_version_required: \"1.15\"\n\n# Log-level\ncilium_debug: false\n\ncilium_mtu: \"0\"\ncilium_enable_ipv4: \"{{ ipv4_stack }}\"\ncilium_enable_ipv6: \"{{ ipv6_stack }}\"\n\n# Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/\ncilium_l2announcements: false\n\n# Cilium agent health port\ncilium_agent_health_port: \"9879\"\n\n# Etcd SSL dirs\ncilium_cert_dir: /etc/cilium/certs\nkube_etcd_cacert_file: ca.pem\nkube_etcd_cert_file: node-{{ inventory_hostname }}.pem\nkube_etcd_key_file: node-{{ inventory_hostname }}-key.pem\n\n# Limits for apps\ncilium_memory_limit: 500M\ncilium_cpu_limit: 500m\ncilium_memory_requests: 64M\ncilium_cpu_requests: 100m\n\n# Overlay Network Mode\ncilium_tunnel_mode: vxlan\n\n# LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode\ncilium_loadbalancer_mode: snat\n\n# -- Configure Loadbalancer IP Pools\ncilium_loadbalancer_ip_pools: []\n\n# Optional features\ncilium_enable_prometheus: false\n# Enable if you want to make use of hostPort mappings\ncilium_enable_portmap: false\n# Monitor aggregation level (none/low/medium/maximum)\ncilium_monitor_aggregation: medium\n# Kube Proxy Replacement mode (true/false)\ncilium_kube_proxy_replacement: false\n\n# If not defined `cilium_dns_proxy_enable_transparent_mode`, it will following the Cilium behavior.\n# When Cilium is configured to replace kube-proxy, it automatically enables dnsProxy, which will conflict with nodelocaldns.\n# You can set `false` avoid conflict with nodelocaldns.\n# https://github.com/cilium/cilium/issues/33144\n# cilium_dns_proxy_enable_transparent_mode:\n\n# If upgrading from Cilium < 1.5, you may want to override some of these options\n# to prevent service disruptions. See also:\n# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action\ncilium_preallocate_bpf_maps: false\n\n# Auto direct nodes routes can be used to advertise pods routes in your cluster\n# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`).\n# This works only if you have a L2 connectivity between all your nodes.\n# You wil also have to specify the variable `cilium_native_routing_cidr` to\n# make this work. Please refer to the cilium documentation for more\n# information about this kind of setups.\ncilium_auto_direct_node_routes: false\n\n# Allows to explicitly specify the IPv4 CIDR for native routing.\n# When specified, Cilium assumes networking for this CIDR is preconfigured and\n# hands traffic destined for that range to the Linux network stack without\n# applying any SNAT.\n# Generally speaking, specifying a native routing CIDR implies that Cilium can\n# depend on the underlying networking stack to route packets to their\n# destination. To offer a concrete example, if Cilium is configured to use\n# direct routing and the Kubernetes CIDR is included in the native routing CIDR,\n# the user must configure the routes to reach pods, either manually or by\n# setting the auto-direct-node-routes flag.\ncilium_native_routing_cidr: \"\"\n\n# Allows to explicitly specify the IPv6 CIDR for native routing.\ncilium_native_routing_cidr_ipv6: \"\"\n\n# Enable transparent network encryption.\ncilium_encryption_enabled: false\n\n# Encryption method. Can be either ipsec or wireguard.\n# Only effective when `cilium_encryption_enabled` is set to true.\ncilium_encryption_type: \"ipsec\"\n\n# Enable encryption for pure node to node traffic.\n# This option is only effective when `cilium_encryption_type` is set to `wireguard`.\ncilium_encryption_node_encryption: false\n\n# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.\n# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,\n# it will fallback on the wireguard-go user-space implementation of WireGuard.\n# This option is only effective when `cilium_encryption_type` is set to `wireguard`.\ncilium_wireguard_userspace_fallback: false\n\n# Enable Bandwidth Manager\n# Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation.\n# Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies.\n# In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods.\n# Bandwidth Manager requires a v5.1.x or more recent Linux kernel.\ncilium_enable_bandwidth_manager: false\ncilium_enable_bandwidth_manager_bbr: false\n\n# IP Masquerade Agent\n# https://docs.cilium.io/en/stable/concepts/networking/masquerading/\n# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded\ncilium_ip_masq_agent_enable: false\n\n### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded\ncilium_non_masquerade_cidrs:\n - 10.0.0.0/8\n - 172.16.0.0/12\n - 192.168.0.0/16\n - 100.64.0.0/10\n - 192.0.0.0/24\n - 192.0.2.0/24\n - 192.88.99.0/24\n - 198.18.0.0/15\n - 198.51.100.0/24\n - 203.0.113.0/24\n - 240.0.0.0/4\n### Indicates whether to masquerade traffic to the link local prefix.\n### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.\ncilium_masq_link_local: false\ncilium_masq_link_local_ipv6: false\n### A time interval at which the agent attempts to reload config from disk\ncilium_ip_masq_resync_interval: 60s\n\n# Hubble\n### Enable Hubble without install\ncilium_enable_hubble: false\n### Enable Hubble-ui\ncilium_enable_hubble_ui: \"{{ cilium_enable_hubble }}\"\n### Enable Hubble Metrics (deprecated)\ncilium_enable_hubble_metrics: false\n### if cilium_enable_hubble_metrics: true\ncilium_hubble_metrics: []\n# - dns\n# - drop\n# - tcp\n# - flow\n# - icmp\n# - http\n### Enable Hubble install\ncilium_hubble_install: false\n### Enable auto generate certs if cilium_hubble_install: true\ncilium_hubble_tls_generate: false\n\ncilium_hubble_export_file_max_backups: \"5\"\ncilium_hubble_export_file_max_size_mb: \"10\"\n\ncilium_hubble_export_dynamic_enabled: false\ncilium_hubble_export_dynamic_config_content:\n - name: all\n fieldMask: []\n includeFilters: []\n excludeFilters: []\n filePath: \"/var/run/cilium/hubble/events.log\"\n\n# Override the DNS suffix that Hubble-Relay uses to resolve its peer service.\n# It defaults to the inventory's `dns_domain`.\ncilium_hubble_peer_service_cluster_domain: \"{{ dns_domain }}\"\n\n### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535\n### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095)\n# cilium_hubble_event_buffer_capacity: 4095\n### Buffer size of the channel to receive monitor events.\n# cilium_hubble_event_queue_size: 50\n\ncilium_gateway_api_enabled: false\n\n# The default IP address management mode is \"Cluster Scope\".\n# https://docs.cilium.io/en/stable/concepts/networking/ipam/\ncilium_ipam_mode: cluster-pool\n\n# Cluster Pod CIDRs use the kube_pods_subnet value by default.\n# If your node network is in the same range you will lose connectivity to other nodes.\n# Defaults to kube_pods_subnet if not set.\n# cilium_pool_cidr: 10.233.64.0/18\n\n# When cilium_enable_ipv6 is used, you need to set the IPV6 value. Defaults to kube_pods_subnet_ipv6 if not set.\n# cilium_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n\n# When cilium IPAM uses the \"Cluster Scope\" mode, it will pre-allocate a segment of IP to each node,\n# schedule the Pod to this node, and then allocate IP from here. cilium_pool_mask_size Specifies\n# the size allocated from cluster Pod CIDR to node.ipam.podCIDRs\n# Defaults to kube_network_node_prefix if not set.\n# cilium_pool_mask_size: \"24\"\n\n# cilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cluster Pod IPV6 CIDR\n# Defaults to kube_network_node_prefix_ipv6 if not set.\n# cilium_pool_mask_size_ipv6: \"120\"\n\n\n# Extra arguments for the Cilium agent\ncilium_agent_custom_args: [] # deprecated\ncilium_agent_extra_args: []\n\n# For adding and mounting extra volumes to the cilium agent\ncilium_agent_extra_volumes: []\ncilium_agent_extra_volume_mounts: []\n\ncilium_agent_extra_env_vars: []\n\ncilium_operator_replicas: 2\n\n# The address at which the cillium operator bind health check api\ncilium_operator_api_serve_addr: \"127.0.0.1:9234\"\n\n## A dictionary of extra config variables to add to cilium-config, formatted like:\n## cilium_config_extra_vars:\n## var1: \"value1\"\n## var2: \"value2\"\ncilium_config_extra_vars: {}\n\n# For adding and mounting extra volumes to the cilium operator\ncilium_operator_extra_volumes: []\ncilium_operator_extra_volume_mounts: []\n\n# Extra arguments for the Cilium Operator\ncilium_operator_custom_args: [] # deprecated\ncilium_operator_extra_args: []\n\n# Tolerations of the cilium operator\ncilium_operator_tolerations:\n - operator: \"Exists\"\n\n# Unique ID of the cluster. Must be unique across all connected\n# clusters and in the range of 1 to 255. Only required for Cluster Mesh,\n# may be 0 if Cluster Mesh is not used.\ncilium_cluster_id: 0\n# Name of the cluster. Only relevant when building a mesh of clusters.\n# The \"default\" name cannot be used if the Cluster ID is different from 0.\ncilium_cluster_name: default\n\n# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.\n# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.\n# Available for Cilium v1.10 and up.\ncilium_cni_exclusive: true\n\n# Configure the log file for CNI logging with retention policy of 7 days.\n# Disable CNI file logging by setting this field to empty explicitly.\n# Available for Cilium v1.12 and up.\ncilium_cni_log_file: \"/var/run/cilium/cilium-cni.log\"\n\n# -- Configure cgroup related configuration\n# -- Enable auto mount of cgroup2 filesystem.\n# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at\n# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.\n# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted\n# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the\n# volume will be mounted inside the cilium agent pod at the same path.\n# Available for Cilium v1.11 and up\ncilium_cgroup_auto_mount: true\n# -- Configure cgroup root where cgroup2 filesystem is mounted on the host\ncilium_cgroup_host_root: \"/run/cilium/cgroupv2\"\n\n# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic\n# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.\ncilium_bpf_map_dynamic_size_ratio: \"0.0025\"\n\n# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.\n# Available for Cilium v1.10 and up\ncilium_enable_ipv4_masquerade: true\n# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.\n# Available for Cilium v1.10 and up\ncilium_enable_ipv6_masquerade: true\n\n# -- Enable native IP masquerade support in eBPF\ncilium_enable_bpf_masquerade: false\n\n# -- Configure whether direct routing mode should route traffic via\n# host stack (true) or directly and more efficiently out of BPF (false) if\n# the kernel supports it. The latter has the implication that it will also\n# bypass netfilter in the host namespace.\ncilium_enable_host_legacy_routing: false\n\n# -- Enable use of the remote node identity.\n# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity\ncilium_enable_remote_node_identity: true\n\n# -- Enable the use of well-known identities.\ncilium_enable_well_known_identities: false\n\n# The monitor aggregation flags determine which TCP flags which, upon the\n# first observation, cause monitor notifications to be generated.\n#\n# Only effective when monitor aggregation is set to \"medium\" or higher.\ncilium_monitor_aggregation_flags: \"all\"\n\n# -- Enable BGP Control Plane\ncilium_enable_bgp_control_plane: false\n\n# -- Configure BGP Instances (New bgpv2 API v1.16+)\ncilium_bgp_cluster_configs: []\n\n# -- Configure BGP Peers (New bgpv2 API v1.16+)\ncilium_bgp_peer_configs: []\n\n# -- Configure BGP Advertisements (New bgpv2 API v1.16+)\ncilium_bgp_advertisements: []\n\n# -- Configure BGP Node Config Overrides (New bgpv2 API v1.16+)\ncilium_bgp_node_config_overrides: []\n\n# -- Configure BGP Peers (Legacy < v1.16)\ncilium_bgp_peering_policies: []\n\n# -- Whether to enable CNP status updates.\ncilium_disable_cnp_status_updates: true\n\n# Configure how long to wait for the Cilium DaemonSet to be ready again\ncilium_rolling_restart_wait_retries_count: 30\ncilium_rolling_restart_wait_retries_delay_seconds: 10\n\n# Cilium changed the default metrics exporter ports in 1.12\ncilium_agent_scrape_port: \"9962\"\ncilium_operator_scrape_port: \"9963\"\ncilium_hubble_scrape_port: \"9965\"\n\n# Cilium certgen args for generate certificate for hubble mTLS\ncilium_certgen_args:\n cilium-namespace: kube-system\n ca-reuse-secret: true\n ca-secret-name: hubble-ca-secret\n ca-generate: true\n ca-validity-duration: 94608000s\n hubble-server-cert-generate: true\n hubble-server-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io'\n hubble-server-cert-validity-duration: 94608000s\n hubble-server-cert-secret-name: hubble-server-certs\n hubble-relay-client-cert-generate: true\n hubble-relay-client-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io'\n hubble-relay-client-cert-validity-duration: 94608000s\n hubble-relay-client-cert-secret-name: hubble-relay-client-certs\n hubble-relay-server-cert-generate: false\n\ncilium_enable_host_firewall: false\ncilium_policy_audit_mode: false\n\n# Cilium extra install flags\ncilium_install_extra_flags: \"\"\n\n# Cilium extra values, use any values from cilium Helm Chart\n# ref: https://docs.cilium.io/en/stable/helm-reference/\ncilium_extra_values: {}\n" }, { "path": "roles/network_plugin/cilium/tasks/apply.yml", "content": "---\n- name: Check if Cilium Helm release exists (via cilium version)\n command: \"{{ bin_dir }}/cilium version\"\n register: cilium_release_info\n when: inventory_hostname == groups['kube_control_plane'][0]\n failed_when: false\n changed_when: false\n\n- name: Set action to install or upgrade\n set_fact:\n cilium_action: \"{{ 'install' if ('release: not found' in cilium_release_info.stderr | default('') or 'release: not found' in cilium_release_info.stdout | default('')) else 'upgrade' }}\"\n\n- name: Cilium | Install\n environment: \"{{ proxy_env }}\"\n command: \"{{ bin_dir }}/cilium {{ cilium_action }} --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml -f {{ kube_config_dir }}/cilium-extra-values.yaml {{ cilium_install_extra_flags }}\"\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cilium | Wait for pods to run\n command: \"{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'\" # noqa literal-compare\n register: pods_not_ready\n until: pods_not_ready.stdout.find(\"cilium\")==-1\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cilium | Wait for CiliumLoadBalancerIPPool CRD to be present\n command: \"{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumloadbalancerippools.cilium.io\"\n register: cillium_lbippool_crd_ready\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)\n\n- name: Cilium | Create CiliumLoadBalancerIPPool manifests\n template:\n src: \"{{ item.name }}/{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0\n - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)\n\n- name: Cilium | Apply CiliumLoadBalancerIPPool from cilium_loadbalancer_ip_pools\n kube:\n name: \"{{ item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n state: \"latest\"\n loop:\n - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0\n - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)\n\n- name: Cilium | Wait for CiliumBGPPeeringPolicy CRD to be present\n command: \"{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeeringpolicies.cilium.io\"\n register: cillium_bgpppolicy_crd_ready\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)\n\n- name: Cilium | Create CiliumBGPPeeringPolicy manifests\n template:\n src: \"{{ item.name }}/{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0\n - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)\n\n- name: Cilium | Apply CiliumBGPPeeringPolicy from cilium_bgp_peering_policies\n kube:\n name: \"{{ item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n state: \"latest\"\n loop:\n - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0\n - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)\n\n- name: Cilium | Wait for CiliumBGPClusterConfig CRD to be present\n command: \"{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpclusterconfigs.cilium.io\"\n register: cillium_bgpcconfig_crd_ready\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)\n\n- name: Cilium | Create CiliumBGPClusterConfig manifests\n template:\n src: \"{{ item.name }}/{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0\n - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)\n\n- name: Cilium | Apply CiliumBGPClusterConfig from cilium_bgp_cluster_configs\n kube:\n name: \"{{ item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n state: \"latest\"\n loop:\n - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0\n - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)\n\n- name: Cilium | Wait for CiliumBGPPeerConfig CRD to be present\n command: \"{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeerconfigs.cilium.io\"\n register: cillium_bgppconfig_crd_ready\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)\n\n- name: Cilium | Create CiliumBGPPeerConfig manifests\n template:\n src: \"{{ item.name }}/{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0\n - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)\n\n- name: Cilium | Apply CiliumBGPPeerConfig from cilium_bgp_peer_configs\n kube:\n name: \"{{ item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n state: \"latest\"\n loop:\n - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0\n - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)\n\n- name: Cilium | Wait for CiliumBGPAdvertisement CRD to be present\n command: \"{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpadvertisements.cilium.io\"\n register: cillium_bgpadvert_crd_ready\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)\n\n- name: Cilium | Create CiliumBGPAdvertisement manifests\n template:\n src: \"{{ item.name }}/{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0\n - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)\n\n- name: Cilium | Apply CiliumBGPAdvertisement from cilium_bgp_advertisements\n kube:\n name: \"{{ item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n state: \"latest\"\n loop:\n - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0\n - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)\n\n- name: Cilium | Wait for CiliumBGPNodeConfigOverride CRD to be present\n command: \"{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpnodeconfigoverrides.cilium.io\"\n register: cilium_bgp_node_config_crd_ready\n retries: \"{{ cilium_rolling_restart_wait_retries_count | int }}\"\n delay: \"{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}\"\n failed_when: false\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)\n\n- name: Cilium | Create CiliumBGPNodeConfigOverride manifests\n template:\n src: \"{{ item.name }}/{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0\n - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0)\n\n- name: Cilium | Apply CiliumBGPNodeConfigOverride from cilium_bgp_node_config_overrides\n kube:\n name: \"{{ item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}\"\n state: \"latest\"\n loop:\n - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride}\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0\n - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0)\n" }, { "path": "roles/network_plugin/cilium/tasks/check.yml", "content": "---\n- name: Cilium | Check Cilium encryption `cilium_ipsec_key` for ipsec\n assert:\n that:\n - \"cilium_ipsec_key is defined\"\n msg: \"cilium_ipsec_key should be defined to enable encryption using ipsec\"\n when:\n - cilium_encryption_enabled\n - cilium_encryption_type == \"ipsec\"\n - cilium_tunnel_mode in ['vxlan']\n\n# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`\n- name: Stop if `cilium_ipsec_enabled` is defined and `cilium_encryption_type` is not `ipsec`\n assert:\n that: cilium_encryption_type == 'ipsec'\n msg: >\n It is not possible to use `cilium_ipsec_enabled` when `cilium_encryption_type` is set to {{ cilium_encryption_type }}.\n when:\n - cilium_ipsec_enabled is defined\n - cilium_ipsec_enabled\n - kube_network_plugin == 'cilium' or cilium_deploy_additionally\n\n- name: Stop if kernel version is too low for Cilium Wireguard encryption\n assert:\n that: ansible_kernel.split('-')[0] is version('5.6.0', '>=')\n when:\n - kube_network_plugin == 'cilium' or cilium_deploy_additionally\n - cilium_encryption_enabled\n - cilium_encryption_type == \"wireguard\"\n - not ignore_assert_errors\n\n- name: Stop if bad Cilium identity allocation mode\n assert:\n that: cilium_identity_allocation_mode in ['crd', 'kvstore']\n msg: \"cilium_identity_allocation_mode must be either 'crd' or 'kvstore'\"\n\n- name: Stop if bad Cilium Cluster ID\n assert:\n that:\n - cilium_cluster_id <= 255\n - cilium_cluster_id >= 0\n msg: \"'cilium_cluster_id' must be between 1 and 255\"\n when: cilium_cluster_id is defined\n\n- name: Stop if bad encryption type\n assert:\n that: cilium_encryption_type in ['ipsec', 'wireguard']\n msg: \"cilium_encryption_type must be either 'ipsec' or 'wireguard'\"\n when: cilium_encryption_enabled\n\n- name: Stop if cilium_version is < {{ cilium_min_version_required }}\n assert:\n that: cilium_version is version(cilium_min_version_required, '>=')\n msg: \"cilium_version is too low. Minimum version {{ cilium_min_version_required }}\"\n\n# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`\n- name: Set `cilium_encryption_type` to \"ipsec\" and if `cilium_ipsec_enabled` is true\n set_fact:\n cilium_encryption_type: ipsec\n cilium_encryption_enabled: true\n when:\n - cilium_ipsec_enabled is defined\n - cilium_ipsec_enabled\n\n- name: Stop if cilium_hubble_event_buffer_capacity is not a power of 2 minus 1 and is not between 1 and 65535\n assert:\n that: \"cilium_hubble_event_buffer_capacity in [1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, 65535]\"\n msg: \"Error: cilium_hubble_event_buffer_capacity:{{ cilium_hubble_event_buffer_capacity }} is not a power of 2 minus 1 and it should be between 1 and 65535.\"\n when: cilium_hubble_event_buffer_capacity is defined\n" }, { "path": "roles/network_plugin/cilium/tasks/install.yml", "content": "---\n- name: Cilium | Ensure BPFFS mounted\n ansible.posix.mount:\n fstype: bpf\n path: /sys/fs/bpf\n src: bpffs\n state: mounted\n\n- name: Cilium | Create Cilium certs directory\n file:\n dest: \"{{ cilium_cert_dir }}\"\n state: directory\n mode: \"0750\"\n owner: root\n group: root\n when:\n - cilium_identity_allocation_mode == \"kvstore\"\n\n- name: Cilium | Link etcd certificates for cilium\n file:\n src: \"{{ etcd_cert_dir }}/{{ item.s }}\"\n dest: \"{{ cilium_cert_dir }}/{{ item.d }}\"\n mode: \"0644\"\n state: hard\n force: true\n loop:\n - {s: \"{{ kube_etcd_cacert_file }}\", d: \"ca_cert.crt\"}\n - {s: \"{{ kube_etcd_cert_file }}\", d: \"cert.crt\"}\n - {s: \"{{ kube_etcd_key_file }}\", d: \"key.pem\"}\n when:\n - cilium_identity_allocation_mode == \"kvstore\"\n\n- name: Cilium | Render values\n template:\n src: values.yaml.j2\n dest: \"{{ kube_config_dir }}/cilium-values.yaml\"\n mode: \"0644\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cilium | Copy extra values\n copy:\n content: \"{{ cilium_extra_values | to_nice_yaml(indent=2) }}\"\n dest: \"{{ kube_config_dir }}/cilium-extra-values.yaml\"\n mode: \"0644\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Cilium | Copy Ciliumcli binary from download dir\n copy:\n src: \"{{ local_release_dir }}/cilium\"\n dest: \"{{ bin_dir }}/cilium\"\n mode: \"0755\"\n remote_src: true\n" }, { "path": "roles/network_plugin/cilium/tasks/main.yml", "content": "---\n- name: Cilium check\n import_tasks: check.yml\n\n- name: Cilium install\n include_tasks: install.yml\n\n- name: Cilium apply\n include_tasks: apply.yml\n" }, { "path": "roles/network_plugin/cilium/tasks/reset.yml", "content": "---\n- name: Reset | check and remove devices if still present\n include_tasks: reset_iface.yml\n vars:\n iface: \"{{ item }}\"\n loop:\n - cilium_host\n - cilium_net\n - cilium_vxlan\n" }, { "path": "roles/network_plugin/cilium/tasks/reset_iface.yml", "content": "---\n- name: \"Reset | check if network device {{ iface }} is present\"\n stat:\n path: \"/sys/class/net/{{ iface }}\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: device_remains\n\n- name: \"Reset | remove network device {{ iface }}\"\n command: \"ip link del {{ iface }}\"\n when: device_remains.stat.exists\n" }, { "path": "roles/network_plugin/cilium/templates/cilium/cilium-bgp-advertisement.yml.j2", "content": "{% for cilium_bgp_advertisement in cilium_bgp_advertisements %}\n---\napiVersion: \"cilium.io/v2\"\nkind: CiliumBGPAdvertisement\nmetadata:\n name: \"{{ cilium_bgp_advertisement.name }}\"\n{% if cilium_bgp_advertisement.labels %}\n labels: {{ cilium_bgp_advertisement.labels | to_yaml }}\n{% endif %}\nspec:\n {{ cilium_bgp_advertisement.spec | to_yaml | indent(4) }}\n{% endfor %}\n" }, { "path": "roles/network_plugin/cilium/templates/cilium/cilium-bgp-cluster-config.yml.j2", "content": "{% for cilium_bgp_cluster_config in cilium_bgp_cluster_configs %}\n---\napiVersion: \"cilium.io/v2\"\nkind: CiliumBGPClusterConfig\nmetadata:\n name: \"{{ cilium_bgp_cluster_config.name }}\"\nspec:\n {{ cilium_bgp_cluster_config.spec | to_yaml | indent(2) }}\n{% endfor %}\n" }, { "path": "roles/network_plugin/cilium/templates/cilium/cilium-bgp-node-config-override.yml.j2", "content": "{% for cilium_bgp_node_config_override in cilium_bgp_node_config_overrides %}\n---\napiVersion: \"cilium.io/v2\"\nkind: CiliumBGPNodeConfigOverride\nmetadata:\n name: \"{{ cilium_bgp_node_config_override.name }}\"\nspec:\n {{ cilium_bgp_node_config_override.spec | to_yaml | indent(2) }}\n{% endfor %}\n" }, { "path": "roles/network_plugin/cilium/templates/cilium/cilium-bgp-peer-config.yml.j2", "content": "{% for cilium_bgp_peer_config in cilium_bgp_peer_configs %}\n---\napiVersion: \"cilium.io/v2\"\nkind: CiliumBGPPeerConfig\nmetadata:\n name: \"{{ cilium_bgp_peer_config.name }}\"\nspec:\n {{ cilium_bgp_peer_config.spec | to_yaml | indent(2) }}\n{% endfor %}\n" }, { "path": "roles/network_plugin/cilium/templates/cilium/cilium-bgp-peering-policy.yml.j2", "content": "{% for cilium_bgp_peering_policy in cilium_bgp_peering_policies %}\n---\napiVersion: \"cilium.io/v2alpha1\"\nkind: CiliumBGPPeeringPolicy\nmetadata:\n name: \"{{ cilium_bgp_peering_policy.name }}\"\nspec:\n {{ cilium_bgp_peering_policy.spec | to_yaml | indent(2) }}\n{% endfor %}\n" }, { "path": "roles/network_plugin/cilium/templates/cilium/cilium-loadbalancer-ip-pool.yml.j2", "content": "{% for cilium_loadbalancer_ip_pool in cilium_loadbalancer_ip_pools %}\n---\napiVersion: \"cilium.io/v2\"\nkind: CiliumLoadBalancerIPPool\nmetadata:\n name: \"{{ cilium_loadbalancer_ip_pool.name }}\"\nspec:\n blocks:\n{% for cblock in cilium_loadbalancer_ip_pool.cidrs | default([]) %}\n - cidr: \"{{ cblock }}\"\n{% endfor %}\n{% for rblock in cilium_loadbalancer_ip_pool.ranges | default([]) %}\n - start: \"{{ rblock.start }}\"\n stop: \"{{ rblock.stop | default(rblock.start) }}\"\n{% endfor %}\n{% endfor %}\n" }, { "path": "roles/network_plugin/cilium/templates/values.yaml.j2", "content": "#jinja2: trim_blocks: True, lstrip_blocks: True\nMTU: {{ cilium_mtu }}\ndebug:\n enabled: {{ cilium_debug | to_json }}\n\nimage:\n repository: {{ cilium_image_repo }}\n tag: {{ cilium_image_tag }}\n\nk8sServiceHost: \"{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}\"\nk8sServicePort: \"{{ kube_apiserver_global_endpoint | urlsplit('port') }}\"\n\nipv4:\n enabled: {{ cilium_enable_ipv4 | to_json }}\nipv6:\n enabled: {{ cilium_enable_ipv6 | to_json }}\n\nl2announcements:\n enabled: {{ cilium_l2announcements | to_json }}\n\nbgpControlPlane:\n enabled: {{ cilium_enable_bgp_control_plane | to_json }}\n\nhealthPort: {{ cilium_agent_health_port }}\n\nidentityAllocationMode: {{ cilium_identity_allocation_mode }}\n\ntunnelProtocol: {{ cilium_tunnel_mode }}\n\nloadBalancer:\n mode: {{ cilium_loadbalancer_mode }}\n\nkubeProxyReplacement: {{ cilium_kube_proxy_replacement | to_json }}\n\n{% if cilium_dns_proxy_enable_transparent_mode is defined %}\ndnsProxy:\n enableTransparentMode: {{ cilium_dns_proxy_enable_transparent_mode | to_json }}\n{% endif %}\n\nextraVolumes:\n {{ cilium_agent_extra_volumes | to_nice_yaml(indent=2) | indent(2) }}\n\nextraVolumeMounts:\n {{ cilium_agent_extra_volume_mounts | to_nice_yaml(indent=2) | indent(2) }}\n\nextraArgs:\n {{ cilium_agent_extra_args | to_nice_yaml(indent=2) | indent(2) }}\n\nbpf:\n masquerade: {{ cilium_enable_bpf_masquerade | to_json }}\n hostLegacyRouting: {{ cilium_enable_host_legacy_routing | to_json }}\n monitorAggregation: {{ cilium_monitor_aggregation }}\n preallocateMaps: {{ cilium_preallocate_bpf_maps | to_json }}\n mapDynamicSizeRatio: {{ cilium_bpf_map_dynamic_size_ratio }}\n\ncni:\n exclusive: {{ cilium_cni_exclusive | to_json }}\n logFile: {{ cilium_cni_log_file }}\n{% if cilium_enable_portmap %}\n chainingMode: portmap\n{% endif %}\n\nautoDirectNodeRoutes: {{ cilium_auto_direct_node_routes | to_json }}\n\nipv4NativeRoutingCIDR: \"{{ cilium_native_routing_cidr }}\"\nipv6NativeRoutingCIDR: \"{{ cilium_native_routing_cidr_ipv6 }}\"\n\nencryption:\n enabled: {{ cilium_encryption_enabled | to_json }}\n{% if cilium_encryption_enabled %}\n type: {{ cilium_encryption_type }}\n{% if cilium_encryption_type == 'wireguard' %}\n nodeEncryption: {{ cilium_encryption_node_encryption | to_json }}\n{% endif %}\n{% endif %}\n\nbandwidthManager:\n enabled: {{ cilium_enable_bandwidth_manager | to_json }}\n bbr: {{ cilium_enable_bandwidth_manager_bbr | to_json }}\n\nipMasqAgent:\n enabled: {{ cilium_ip_masq_agent_enable | to_json }}\n{% if cilium_ip_masq_agent_enable %}\n config:\n nonMasqueradeCIDRs: {{ cilium_non_masquerade_cidrs }}\n masqLinkLocal: {{ cilium_masq_link_local | to_json }}\n masqLinkLocalIPv6: {{ cilium_masq_link_local_ipv6 | to_json }}\n # cilium_ip_masq_resync_interval\n{% endif %}\n\nhubble:\n peerService:\n clusterDomain: {{ cilium_hubble_peer_service_cluster_domain }}\n enabled: {{ cilium_enable_hubble | to_json }}\n relay:\n enabled: {{ cilium_enable_hubble | to_json }}\n image:\n repository: {{ cilium_hubble_relay_image_repo }}\n tag: {{ cilium_hubble_relay_image_tag }}\n ui:\n enabled: {{ cilium_enable_hubble_ui | to_json }}\n backend:\n image:\n repository: {{ cilium_hubble_ui_backend_image_repo }}\n tag: {{ cilium_hubble_ui_backend_image_tag }}\n frontend:\n image:\n repository: {{ cilium_hubble_ui_image_repo }}\n tag: {{ cilium_hubble_ui_image_tag }}\n metrics:\n enabled: {{ cilium_hubble_metrics | to_json }}\n export:\n{% if cilium_version is version('1.18.0', '>=') %}\n static:\n fileMaxBackups: {{ cilium_hubble_export_file_max_backups }}\n fileMaxSizeMb: {{ cilium_hubble_export_file_max_size_mb }}\n{% else %}\n fileMaxBackups: {{ cilium_hubble_export_file_max_backups }}\n fileMaxSizeMb: {{ cilium_hubble_export_file_max_size_mb }}\n{% endif %}\n dynamic:\n enabled: {{ cilium_hubble_export_dynamic_enabled | to_json }}\n config:\n content:\n {{ cilium_hubble_export_dynamic_config_content | to_nice_yaml(indent=10) | indent(10) }}\n\ngatewayAPI:\n enabled: {{ cilium_gateway_api_enabled | to_json }}\n\nipam:\n mode: {{ cilium_ipam_mode }}\n operator:\n clusterPoolIPv4PodCIDRList:\n - {{ cilium_pool_cidr | default(kube_pods_subnet) }}\n clusterPoolIPv4MaskSize: {{ cilium_pool_mask_size | default(kube_network_node_prefix) }}\n\n clusterPoolIPv6PodCIDRList:\n - {{ cilium_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}\n clusterPoolIPv6MaskSize: {{ cilium_pool_mask_size_ipv6 | default(kube_network_node_prefix_ipv6) }}\n\ncgroup:\n autoMount:\n enabled: {{ cilium_cgroup_auto_mount | to_json }}\n hostRoot: {{ cilium_cgroup_host_root }}\n\nresources:\n limits:\n memory: \"{{ cilium_memory_limit }}\"\n cpu: \"{{ cilium_cpu_limit }}\"\n requests:\n memory: \"{{ cilium_memory_requests }}\"\n cpu: \"{{ cilium_cpu_requests }}\"\n\noperator:\n image:\n repository: {{ cilium_operator_image_repo }}\n tag: {{ cilium_operator_image_tag }}\n replicas: {{ cilium_operator_replicas }}\n extraArgs:\n {{ cilium_operator_extra_args | to_nice_yaml(indent=2) | indent(4) }}\n extraVolumes:\n {{ cilium_operator_extra_volumes | to_nice_yaml(indent=2) | indent(4) }}\n extraVolumeMounts:\n {{ cilium_operator_extra_volume_mounts | to_nice_yaml(indent=2) | indent(4) }}\n tolerations:\n {{ cilium_operator_tolerations | to_nice_yaml(indent=2) | indent(4) }}\n\ncluster:\n id: {{ cilium_cluster_id }}\n name: {{ cilium_cluster_name }}\n\nenableIPv4Masquerade: {{ cilium_enable_ipv4_masquerade | to_json }}\nenableIPv6Masquerade: {{ cilium_enable_ipv6_masquerade | to_json }}\n\nhostFirewall:\n enabled: {{ cilium_enable_host_firewall | to_json }}\n\npolicyAuditMode: {{ cilium_policy_audit_mode | to_json }}\n\ncertgen:\n image:\n repository: {{ cilium_hubble_certgen_image_repo }}\n tag: {{ cilium_hubble_certgen_image_tag }}\n\nenvoy:\n image:\n repository: {{ cilium_hubble_envoy_image_repo }}\n tag: {{ cilium_hubble_envoy_image_tag }}\n\nextraConfig:\n {{ cilium_config_extra_vars | to_yaml | indent(2) }}\n" }, { "path": "roles/network_plugin/cni/defaults/main.yml", "content": "---\ncni_bin_owner: \"{{ kube_owner }}\"\n" }, { "path": "roles/network_plugin/cni/tasks/main.yml", "content": "---\n- name: CNI | make sure /opt/cni/bin exists\n file:\n path: /opt/cni/bin\n state: directory\n mode: \"0755\"\n owner: \"{{ cni_bin_owner }}\"\n recurse: true\n\n- name: CNI | Copy cni plugins\n unarchive:\n src: \"{{ downloads.cni.dest }}\"\n dest: \"/opt/cni/bin\"\n mode: \"0755\"\n owner: \"{{ cni_bin_owner }}\"\n remote_src: true\n" }, { "path": "roles/network_plugin/custom_cni/defaults/main.yml", "content": "---\n\ncustom_cni_manifests: []\n\ncustom_cni_chart_namespace: kube-system\ncustom_cni_chart_release_name: \"\"\ncustom_cni_chart_repository_name: \"\"\ncustom_cni_chart_repository_url: \"\"\ncustom_cni_chart_ref: \"\"\ncustom_cni_chart_version: \"\"\ncustom_cni_chart_values: {}\n" }, { "path": "roles/network_plugin/custom_cni/meta/main.yml", "content": "---\ndependencies:\n - role: helm-apps\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n - custom_cni_chart_release_name | length > 0\n environment:\n http_proxy: \"{{ http_proxy | default('') }}\"\n https_proxy: \"{{ https_proxy | default('') }}\"\n release_common_opts: {}\n releases:\n - name: \"{{ custom_cni_chart_release_name }}\"\n namespace: \"{{ custom_cni_chart_namespace }}\"\n chart_ref: \"{{ custom_cni_chart_ref }}\"\n chart_version: \"{{ custom_cni_chart_version }}\"\n wait: true\n create_namespace: true\n values: \"{{ custom_cni_chart_values }}\"\n repositories:\n - name: \"{{ custom_cni_chart_repository_name }}\"\n url: \"{{ custom_cni_chart_repository_url }}\"\n" }, { "path": "roles/network_plugin/custom_cni/tasks/main.yml", "content": "---\n- name: Custom CNI | Manifest deployment\n when: not custom_cni_chart_release_name | length > 0\n block:\n - name: Custom CNI | Check Custom CNI Manifests\n assert:\n that:\n - \"custom_cni_manifests | length > 0\"\n msg: \"custom_cni_manifests should not be empty\"\n\n - name: Custom CNI | Copy Custom manifests\n template:\n src: \"{{ item }}\"\n dest: \"{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}\"\n mode: \"0644\"\n loop: \"{{ custom_cni_manifests }}\"\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n run_once: true\n\n - name: Custom CNI | Start Resources\n kube:\n namespace: \"kube-system\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}\"\n state: \"latest\"\n wait: true\n loop: \"{{ custom_cni_manifests }}\"\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n run_once: true\n" }, { "path": "roles/network_plugin/flannel/defaults/main.yml", "content": "---\n# Flannel public IP\n# The address that flannel should advertise as how to access the system\n# Disabled until https://github.com/coreos/flannel/issues/712 is fixed\n# flannel_public_ip: \"{{ main_access_ip }}\"\n\n## interface that should be used for flannel operations\n## This is actually an inventory cluster-level item\n# flannel_interface:\n\n## Select interface that should be used for flannel operations by regexp on Name or IP\n## This is actually an inventory cluster-level item\n## example: select interface with ip from net 10.0.0.0/23\n## single quote and escape backslashes\n# flannel_interface_regexp: '10\\\\.0\\\\.[0-2]\\\\.\\\\d{1,3}'\n\n# You can choose what type of flannel backend to use\n# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md\nflannel_backend_type: \"vxlan\"\nflannel_vxlan_vni: 1\nflannel_vxlan_port: 8472\nflannel_vxlan_direct_routing: false\n\n# Limits for apps\nflannel_memory_limit: 500M\nflannel_cpu_limit: 300m\nflannel_memory_requests: 64M\nflannel_cpu_requests: 150m\n" }, { "path": "roles/network_plugin/flannel/meta/main.yml", "content": "---\ndependencies:\n - role: network_plugin/cni\n" }, { "path": "roles/network_plugin/flannel/tasks/main.yml", "content": "---\n\n- name: Flannel | Stop if kernel version is too low for Flannel Wireguard encryption\n assert:\n that: ansible_kernel.split('-')[0] is version('5.6.0', '>=')\n when:\n - kube_network_plugin == 'flannel'\n - flannel_backend_type == 'wireguard'\n - not ignore_assert_errors\n\n- name: Flannel | Create Flannel manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: flannel, file: cni-flannel-rbac.yml, type: sa}\n - {name: kube-flannel, file: cni-flannel.yml, type: ds}\n register: flannel_node_manifests\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Flannel | Start Resources\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"kube-system\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ flannel_node_manifests.results }}\"\n when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped\n\n- name: Flannel | Wait for flannel subnet.env file presence\n wait_for:\n path: /run/flannel/subnet.env\n delay: 5\n timeout: 600\n" }, { "path": "roles/network_plugin/flannel/tasks/reset.yml", "content": "---\n- name: Reset | check cni network device\n stat:\n path: /sys/class/net/cni0\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: cni\n\n- name: Reset | remove the network device created by the flannel\n command: ip link del cni0\n when: cni.stat.exists\n\n- name: Reset | check flannel network device\n stat:\n path: /sys/class/net/flannel.1\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: flannel\n\n- name: Reset | remove the network device created by the flannel\n command: ip link del flannel.1\n when: flannel.stat.exists\n" }, { "path": "roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: flannel\n namespace: kube-system\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: flannel\nrules:\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - get\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - nodes/status\n verbs:\n - patch\n- apiGroups:\n - \"networking.k8s.io\"\n resources:\n - clustercidrs\n verbs:\n - list\n - watch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: flannel\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: flannel\nsubjects:\n- kind: ServiceAccount\n name: flannel\n namespace: kube-system\n" }, { "path": "roles/network_plugin/flannel/templates/cni-flannel.yml.j2", "content": "---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: kube-flannel-cfg\n namespace: kube-system\n labels:\n tier: node\n app: flannel\ndata:\n cni-conf.json: |\n {\n \"name\": \"cbr0\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"flannel\",\n \"delegate\": {\n \"hairpinMode\": true,\n \"isDefaultGateway\": true\n }\n },\n {\n \"type\": \"portmap\",\n \"capabilities\": {\n \"portMappings\": true\n }\n }\n ]\n }\n net-conf.json: |\n {\n{% if ipv4_stack %}\n \"Network\": \"{{ kube_pods_subnet }}\",\n \"EnableIPv4\": true,\n{% endif %}\n{% if ipv6_stack %}\n \"EnableIPv6\": true,\n \"IPv6Network\": \"{{ kube_pods_subnet_ipv6 }}\",\n{% endif %}\n \"Backend\": {\n \"Type\": \"{{ flannel_backend_type }}\"{% if flannel_backend_type == \"vxlan\" %},\n \"VNI\": {{ flannel_vxlan_vni }},\n \"Port\": {{ flannel_vxlan_port }},\n \"DirectRouting\": {{ flannel_vxlan_direct_routing | to_json }}\n{% endif %}\n }\n }\n{% for arch in ['amd64', 'arm64', 'arm', 'ppc64le', 's390x'] %}\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n{% if arch == 'amd64' %}\n name: kube-flannel\n{% else %}\n name: kube-flannel-ds-{{ arch }}\n{% endif %}\n namespace: kube-system\n labels:\n tier: node\n app: flannel\nspec:\n selector:\n matchLabels:\n app: flannel\n template:\n metadata:\n labels:\n tier: node\n app: flannel\n spec:\n priorityClassName: system-node-critical\n serviceAccountName: flannel\n containers:\n - name: kube-flannel\n image: {{ flannel_image_repo }}:{{ flannel_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n resources:\n limits:\n cpu: {{ flannel_cpu_limit }}\n memory: {{ flannel_memory_limit }}\n requests:\n cpu: {{ flannel_cpu_requests }}\n memory: {{ flannel_memory_requests }}\n command: [ \"/opt/bin/flanneld\", \"--ip-masq\", \"--kube-subnet-mgr\"{% if flannel_interface is defined %}, \"--iface={{ flannel_interface }}\"{% endif %}{% if flannel_interface_regexp is defined %}, \"--iface-regex={{ flannel_interface_regexp }}\"{% endif %} ]\n securityContext:\n privileged: false\n capabilities:\n add: [\"NET_ADMIN\", \"NET_RAW\"]\n env:\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n - name: EVENT_QUEUE_DEPTH\n value: \"5000\"\n volumeMounts:\n - name: run\n mountPath: /run/flannel\n - name: flannel-cfg\n mountPath: /etc/kube-flannel/\n - name: xtables-lock\n mountPath: /run/xtables.lock\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: kubernetes.io/os\n operator: In\n values:\n - linux\n - key: kubernetes.io/arch\n operator: In\n values:\n - {{ arch }}\n initContainers:\n - name: install-cni-plugin\n image: {{ flannel_init_image_repo }}:{{ flannel_init_image_tag }}\n command:\n - cp\n args:\n - -f\n - /flannel\n - /opt/cni/bin/flannel\n volumeMounts:\n - name: cni-plugin\n mountPath: /opt/cni/bin\n - name: install-cni\n image: {{ flannel_image_repo }}:{{ flannel_image_tag }}\n command:\n - cp\n args:\n - -f\n - /etc/kube-flannel/cni-conf.json\n - /etc/cni/net.d/10-flannel.conflist\n volumeMounts:\n - name: cni\n mountPath: /etc/cni/net.d\n - name: flannel-cfg\n mountPath: /etc/kube-flannel/\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n tolerations:\n - operator: Exists\n volumes:\n - name: run\n hostPath:\n path: /run/flannel\n - name: cni\n hostPath:\n path: /etc/cni/net.d\n - name: flannel-cfg\n configMap:\n name: kube-flannel-cfg\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n - name: cni-plugin\n hostPath:\n path: /opt/cni/bin\n updateStrategy:\n rollingUpdate:\n maxUnavailable: {{ serial | default('20%') }}\n type: RollingUpdate\n{% endfor %}\n" }, { "path": "roles/network_plugin/kube-ovn/defaults/main.yml", "content": "---\nkube_ovn_db_cpu_request: 500m\nkube_ovn_db_memory_request: 200Mi\nkube_ovn_db_cpu_limit: 3000m\nkube_ovn_db_memory_limit: 3000Mi\nkube_ovn_node_cpu_request: 200m\nkube_ovn_node_memory_request: 200Mi\nkube_ovn_node_cpu_limit: 1000m\nkube_ovn_node_memory_limit: 800Mi\nkube_ovn_cni_server_cpu_request: 200m\nkube_ovn_cni_server_memory_request: 200Mi\nkube_ovn_cni_server_cpu_limit: 1000m\nkube_ovn_cni_server_memory_limit: 1Gi\nkube_ovn_controller_cpu_request: 200m\nkube_ovn_controller_memory_request: 200Mi\nkube_ovn_controller_cpu_limit: 1000m\nkube_ovn_controller_memory_limit: 1Gi\nkube_ovn_pinger_cpu_request: 100m\nkube_ovn_pinger_memory_request: 200Mi\nkube_ovn_pinger_cpu_limit: 200m\nkube_ovn_pinger_memory_limit: 400Mi\nkube_ovn_monitor_memory_request: 200Mi\nkube_ovn_monitor_cpu_request: 200m\nkube_ovn_monitor_memory_limit: 200Mi\nkube_ovn_monitor_cpu_limit: 200m\nkube_ovn_dpdk_node_cpu_request: 1000m\nkube_ovn_dpdk_node_memory_request: 2Gi\nkube_ovn_dpdk_node_cpu_limit: 1000m\nkube_ovn_dpdk_node_memory_limit: 2Gi\n\nkube_ovn_central_hosts: \"{{ groups['kube_control_plane'] }}\"\nkube_ovn_central_replics: \"{{ kube_ovn_central_hosts | length }}\"\nkube_ovn_controller_replics: \"{{ kube_ovn_central_hosts | length }}\"\nkube_ovn_central_ips: |-\n {% for item in kube_ovn_central_hosts -%}\n {{ hostvars[item]['main_ip'] }}{% if not loop.last %},{% endif %}\n {%- endfor %}\n\nkube_ovn_ic_enable: false\nkube_ovn_ic_autoroute: true\nkube_ovn_ic_dbhost: \"127.0.0.1\"\nkube_ovn_ic_zone: \"kubernetes\"\n\n# geneve or vlan\nkube_ovn_network_type: geneve\n\n# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module\nkube_ovn_tunnel_type: geneve\n\n## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use.\n# kube_ovn_iface: eth1\n## The MTU used by pod iface in overlay networks (default iface MTU - 100)\n# kube_ovn_mtu: 1333\n\n## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port.\nkube_ovn_hw_offload: false\n# traffic mirror\nkube_ovn_traffic_mirror: false\n\n# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112\n# kube_ovn_default_interface_name: eth0\n\nkube_ovn_external_address: 8.8.8.8\nkube_ovn_external_address_ipv6: 2400:3200::1\nkube_ovn_external_address_merged: >-\n {%- if ipv4_stack and ipv6_stack -%}\n {{ kube_ovn_external_address }},{{ kube_ovn_external_address_ipv6 }}\n {%- elif ipv4_stack -%}\n {{ kube_ovn_external_address }}\n {%- else -%}\n {{ kube_ovn_external_address_ipv6 }}\n {%- endif -%}\n\nkube_ovn_external_dns: alauda.cn\n\n# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0\nkube_ovn_default_gateway_check: true\nkube_ovn_default_logical_gateway: false\n\n# u2o_interconnection\nkube_ovn_u2o_interconnection: false\n\n# kube_ovn_default_exclude_ips: 10.16.0.1\nkube_ovn_node_switch_cidr: 100.64.0.0/16\nkube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64\nkube_ovn_node_switch_cidr_merged: >-\n {%- if ipv4_stack and ipv6_stack -%}\n {{ kube_ovn_node_switch_cidr }},{{ kube_ovn_node_switch_cidr_ipv6 }}\n {%- elif ipv4_stack -%}\n {{ kube_ovn_node_switch_cidr }}\n {%- else -%}\n {{ kube_ovn_node_switch_cidr_ipv6 }}\n {%- endif -%}\n\n## vlan config, set default interface name and vlan id\n# kube_ovn_default_interface_name: eth0\nkube_ovn_default_vlan_id: 100\nkube_ovn_vlan_name: product\n\n## pod nic type, support: veth-pair or internal-port\nkube_ovn_pod_nic_type: veth_pair\n\n## Enable load balancer\nkube_ovn_enable_lb: true\n\n## Enable network policy support\nkube_ovn_enable_np: true\n\n## Enable external vpc support\nkube_ovn_enable_external_vpc: true\n\n## Enable checksum\nkube_ovn_encap_checksum: true\n\n## enable ssl\nkube_ovn_enable_ssl: false\n\n## dpdk\nkube_ovn_dpdk_enabled: false\nkube_ovn_dpdk_tunnel_iface: br-phy\n\n## bind local ip\nkube_ovn_bind_local_ip_enabled: true\n\n## eip snat\nkube_ovn_eip_snat_enabled: true\n\n# ls dnat mod dl dst\nkube_ovn_ls_dnat_mod_dl_dst: true\n\n## keep vm ip\nkube_ovn_keep_vm_ip: true\n\n## cni config priority, default: 01\nkube_ovn_cni_config_priority: '01'\n" }, { "path": "roles/network_plugin/kube-ovn/tasks/main.yml", "content": "---\n- name: Kube-OVN | Label ovn-db node\n command: \"{{ kubectl }} label --overwrite node {{ item }} kube-ovn/role=master\"\n loop: \"{{ kube_ovn_central_hosts }}\"\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Kube-OVN | Create Kube-OVN manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: kube-ovn-crd, file: cni-kube-ovn-crd.yml}\n - {name: ovn, file: cni-ovn.yml}\n - {name: kube-ovn, file: cni-kube-ovn.yml}\n register: kube_ovn_node_manifests\n\n- name: Kube-OVN | Start Resources\n kube:\n name: \"{{ item.item.name }}\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n with_items: \"{{ kube_ovn_node_manifests.results }}\"\n when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped\n" }, { "path": "roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2", "content": "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: vpc-dnses.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: vpc-dnses\n singular: vpc-dns\n shortNames:\n - vpc-dns\n kind: VpcDns\n listKind: VpcDnsList\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - jsonPath: .status.active\n name: Active\n type: boolean\n - jsonPath: .spec.vpc\n name: Vpc\n type: string\n - jsonPath: .spec.subnet\n name: Subnet\n type: string\n name: v1\n served: true\n storage: true\n subresources:\n status: {}\n schema:\n openAPIV3Schema:\n type: object\n properties:\n spec:\n type: object\n properties:\n vpc:\n type: string\n subnet:\n type: string\n replicas:\n type: integer\n minimum: 1\n maximum: 3\n status:\n type: object\n properties:\n active:\n type: boolean\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: switch-lb-rules.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: switch-lb-rules\n singular: switch-lb-rule\n shortNames:\n - slr\n kind: SwitchLBRule\n listKind: SwitchLBRuleList\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.vip\n name: vip\n type: string\n - jsonPath: .status.ports\n name: port(s)\n type: string\n - jsonPath: .status.service\n name: service\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: age\n type: date\n name: v1\n served: true\n storage: true\n subresources:\n status: {}\n schema:\n openAPIV3Schema:\n type: object\n properties:\n spec:\n type: object\n properties:\n namespace:\n type: string\n vip:\n type: string\n sessionAffinity:\n type: string\n ports:\n items:\n properties:\n name:\n type: string\n port:\n type: integer\n minimum: 1\n maximum: 65535\n protocol:\n type: string\n targetPort:\n type: integer\n minimum: 1\n maximum: 65535\n type: object\n type: array\n selector:\n items:\n type: string\n type: array\n endpoints:\n items:\n type: string\n type: array\n status:\n type: object\n properties:\n ports:\n type: string\n service:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: vpc-nat-gateways.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: vpc-nat-gateways\n singular: vpc-nat-gateway\n shortNames:\n - vpc-nat-gw\n kind: VpcNatGateway\n listKind: VpcNatGatewayList\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.vpc\n name: Vpc\n type: string\n - jsonPath: .spec.subnet\n name: Subnet\n type: string\n - jsonPath: .spec.lanIp\n name: LanIP\n type: string\n name: v1\n served: true\n storage: true\n subresources:\n status: {}\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n externalSubnets:\n items:\n type: string\n type: array\n selector:\n type: array\n items:\n type: string\n qosPolicy:\n type: string\n tolerations:\n type: array\n items:\n type: object\n properties:\n key:\n type: string\n operator:\n type: string\n enum:\n - Equal\n - Exists\n value:\n type: string\n effect:\n type: string\n enum:\n - NoExecute\n - NoSchedule\n - PreferNoSchedule\n tolerationSeconds:\n type: integer\n affinity:\n properties:\n nodeAffinity:\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n preference:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchFields:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n type: object\n weight:\n format: int32\n type: integer\n required:\n - preference\n - weight\n type: object\n type: array\n requiredDuringSchedulingIgnoredDuringExecution:\n properties:\n nodeSelectorTerms:\n items:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchFields:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n type: object\n type: array\n required:\n - nodeSelectorTerms\n type: object\n type: object\n podAffinity:\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n podAffinityTerm:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n weight:\n format: int32\n type: integer\n required:\n - podAffinityTerm\n - weight\n type: object\n type: array\n requiredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n type: array\n type: object\n podAntiAffinity:\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n podAffinityTerm:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n weight:\n format: int32\n type: integer\n required:\n - podAffinityTerm\n - weight\n type: object\n type: array\n requiredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n type: array\n type: object\n type: object\n spec:\n type: object\n properties:\n lanIp:\n type: string\n subnet:\n type: string\n externalSubnets:\n items:\n type: string\n type: array\n vpc:\n type: string\n selector:\n type: array\n items:\n type: string\n qosPolicy:\n type: string\n tolerations:\n type: array\n items:\n type: object\n properties:\n key:\n type: string\n operator:\n type: string\n enum:\n - Equal\n - Exists\n value:\n type: string\n effect:\n type: string\n enum:\n - NoExecute\n - NoSchedule\n - PreferNoSchedule\n tolerationSeconds:\n type: integer\n affinity:\n properties:\n nodeAffinity:\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n preference:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchFields:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n type: object\n weight:\n format: int32\n type: integer\n required:\n - preference\n - weight\n type: object\n type: array\n requiredDuringSchedulingIgnoredDuringExecution:\n properties:\n nodeSelectorTerms:\n items:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchFields:\n items:\n properties:\n key:\n type: string\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n type: object\n type: array\n required:\n - nodeSelectorTerms\n type: object\n type: object\n podAffinity:\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n podAffinityTerm:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n weight:\n format: int32\n type: integer\n required:\n - podAffinityTerm\n - weight\n type: object\n type: array\n requiredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n type: array\n type: object\n podAntiAffinity:\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n podAffinityTerm:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n weight:\n format: int32\n type: integer\n required:\n - podAffinityTerm\n - weight\n type: object\n type: array\n requiredDuringSchedulingIgnoredDuringExecution:\n items:\n properties:\n labelSelector:\n properties:\n matchExpressions:\n items:\n properties:\n key:\n type: string\n x-kubernetes-patch-strategy: merge\n x-kubernetes-patch-merge-key: key\n operator:\n type: string\n values:\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n type: object\n type: object\n namespaces:\n items:\n type: string\n type: array\n topologyKey:\n type: string\n required:\n - topologyKey\n type: object\n type: array\n type: object\n type: object\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: iptables-eips.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: iptables-eips\n singular: iptables-eip\n shortNames:\n - eip\n kind: IptablesEIP\n listKind: IptablesEIPList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.ip\n name: IP\n type: string\n - jsonPath: .spec.macAddress\n name: Mac\n type: string\n - jsonPath: .status.nat\n name: Nat\n type: string\n - jsonPath: .spec.natGwDp\n name: NatGwDp\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n ip:\n type: string\n nat:\n type: string\n redo:\n type: string\n qosPolicy:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n v4ip:\n type: string\n v6ip:\n type: string\n macAddress:\n type: string\n natGwDp:\n type: string\n qosPolicy:\n type: string\n externalSubnet:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: iptables-fip-rules.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: iptables-fip-rules\n singular: iptables-fip-rule\n shortNames:\n - fip\n kind: IptablesFIPRule\n listKind: IptablesFIPRuleList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .spec.eip\n name: Eip\n type: string\n - jsonPath: .status.v4ip\n name: V4ip\n type: string\n - jsonPath: .spec.internalIp\n name: InternalIp\n type: string\n - jsonPath: .status.v6ip\n name: V6ip\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n - jsonPath: .status.natGwDp\n name: NatGwDp\n type: string\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n v4ip:\n type: string\n v6ip:\n type: string\n natGwDp:\n type: string\n redo:\n type: string\n internalIp:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n eip:\n type: string\n internalIp:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: iptables-dnat-rules.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: iptables-dnat-rules\n singular: iptables-dnat-rule\n shortNames:\n - dnat\n kind: IptablesDnatRule\n listKind: IptablesDnatRuleList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .spec.eip\n name: Eip\n type: string\n - jsonPath: .spec.protocol\n name: Protocol\n type: string\n - jsonPath: .status.v4ip\n name: V4ip\n type: string\n - jsonPath: .status.v6ip\n name: V6ip\n type: string\n - jsonPath: .spec.internalIp\n name: InternalIp\n type: string\n - jsonPath: .spec.externalPort\n name: ExternalPort\n type: string\n - jsonPath: .spec.internalPort\n name: InternalPort\n type: string\n - jsonPath: .status.natGwDp\n name: NatGwDp\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n v4ip:\n type: string\n v6ip:\n type: string\n natGwDp:\n type: string\n redo:\n type: string\n protocol:\n type: string\n internalIp:\n type: string\n internalPort:\n type: string\n externalPort:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n eip:\n type: string\n externalPort:\n type: string\n protocol:\n type: string\n internalIp:\n type: string\n internalPort:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: iptables-snat-rules.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: iptables-snat-rules\n singular: iptables-snat-rule\n shortNames:\n - snat\n kind: IptablesSnatRule\n listKind: IptablesSnatRuleList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .spec.eip\n name: EIP\n type: string\n - jsonPath: .status.v4ip\n name: V4ip\n type: string\n - jsonPath: .status.v6ip\n name: V6ip\n type: string\n - jsonPath: .spec.internalCIDR\n name: InternalCIDR\n type: string\n - jsonPath: .status.natGwDp\n name: NatGwDp\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n v4ip:\n type: string\n v6ip:\n type: string\n natGwDp:\n type: string\n redo:\n type: string\n internalCIDR:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n eip:\n type: string\n internalCIDR:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ovn-eips.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: ovn-eips\n singular: ovn-eip\n shortNames:\n - oeip\n kind: OvnEip\n listKind: OvnEipList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.v4Ip\n name: V4IP\n type: string\n - jsonPath: .status.v6Ip\n name: V6IP\n type: string\n - jsonPath: .status.macAddress\n name: Mac\n type: string\n - jsonPath: .status.type\n name: Type\n type: string\n - jsonPath: .status.nat\n name: Nat\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n type:\n type: string\n nat:\n type: string\n ready:\n type: boolean\n v4Ip:\n type: string\n v6Ip:\n type: string\n macAddress:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n externalSubnet:\n type: string\n type:\n type: string\n v4Ip:\n type: string\n v6Ip:\n type: string\n macAddress:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ovn-fips.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: ovn-fips\n singular: ovn-fip\n shortNames:\n - ofip\n kind: OvnFip\n listKind: OvnFipList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.vpc\n name: Vpc\n type: string\n - jsonPath: .status.v4Eip\n name: V4Eip\n type: string\n - jsonPath: .status.v4Ip\n name: V4Ip\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n - jsonPath: .spec.ipType\n name: IpType\n type: string\n - jsonPath: .spec.ipName\n name: IpName\n type: string\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n v4Eip:\n type: string\n v4Ip:\n type: string\n vpc:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n ovnEip:\n type: string\n ipType:\n type: string\n ipName:\n type: string\n vpc:\n type: string\n v4Ip:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ovn-snat-rules.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: ovn-snat-rules\n singular: ovn-snat-rule\n shortNames:\n - osnat\n kind: OvnSnatRule\n listKind: OvnSnatRuleList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.vpc\n name: Vpc\n type: string\n - jsonPath: .status.v4Eip\n name: V4Eip\n type: string\n - jsonPath: .status.v4IpCidr\n name: V4IpCidr\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n v4Eip:\n type: string\n v4IpCidr:\n type: string\n vpc:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n ovnEip:\n type: string\n vpcSubnet:\n type: string\n ipName:\n type: string\n vpc:\n type: string\n v4IpCidr:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ovn-dnat-rules.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: ovn-dnat-rules\n singular: ovn-dnat-rule\n shortNames:\n - odnat\n kind: OvnDnatRule\n listKind: OvnDnatRuleList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .status.vpc\n name: Vpc\n type: string\n - jsonPath: .spec.ovnEip\n name: Eip\n type: string\n - jsonPath: .status.protocol\n name: Protocol\n type: string\n - jsonPath: .status.v4Eip\n name: V4Eip\n type: string\n - jsonPath: .status.v4Ip\n name: V4Ip\n type: string\n - jsonPath: .status.internalPort\n name: InternalPort\n type: string\n - jsonPath: .status.externalPort\n name: ExternalPort\n type: string\n - jsonPath: .spec.ipName\n name: IpName\n type: string\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n ready:\n type: boolean\n v4Eip:\n type: string\n v4Ip:\n type: string\n vpc:\n type: string\n externalPort:\n type: string\n internalPort:\n type: string\n protocol:\n type: string\n ipName:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n ovnEip:\n type: string\n ipType:\n type: string\n ipName:\n type: string\n externalPort:\n type: string\n internalPort:\n type: string\n protocol:\n type: string\n vpc:\n type: string\n v4Ip:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: vpcs.kubeovn.io\nspec:\n group: kubeovn.io\n versions:\n - additionalPrinterColumns:\n - jsonPath: .status.enableExternal\n name: EnableExternal\n type: boolean\n - jsonPath: .status.enableBfd\n name: EnableBfd\n type: boolean\n - jsonPath: .status.standby\n name: Standby\n type: boolean\n - jsonPath: .status.subnets\n name: Subnets\n type: string\n - jsonPath: .status.extraExternalSubnets\n name: ExtraExternalSubnets\n type: string\n - jsonPath: .spec.namespaces\n name: Namespaces\n type: string\n name: v1\n schema:\n openAPIV3Schema:\n properties:\n spec:\n properties:\n enableExternal:\n type: boolean\n enableBfd:\n type: boolean\n namespaces:\n items:\n type: string\n type: array\n extraExternalSubnets:\n items:\n type: string\n type: array\n staticRoutes:\n items:\n properties:\n policy:\n type: string\n cidr:\n type: string\n nextHopIP:\n type: string\n ecmpMode:\n type: string\n bfdId:\n type: string\n routeTable:\n type: string\n type: object\n type: array\n policyRoutes:\n items:\n properties:\n priority:\n type: integer\n action:\n type: string\n match:\n type: string\n nextHopIP:\n type: string\n type: object\n type: array\n vpcPeerings:\n items:\n properties:\n remoteVpc:\n type: string\n localConnectIP:\n type: string\n type: object\n type: array\n type: object\n status:\n properties:\n conditions:\n items:\n properties:\n lastTransitionTime:\n type: string\n lastUpdateTime:\n type: string\n message:\n type: string\n reason:\n type: string\n status:\n type: string\n type:\n type: string\n type: object\n type: array\n default:\n type: boolean\n defaultLogicalSwitch:\n type: string\n router:\n type: string\n standby:\n type: boolean\n enableExternal:\n type: boolean\n enableBfd:\n type: boolean\n subnets:\n items:\n type: string\n type: array\n extraExternalSubnets:\n items:\n type: string\n type: array\n vpcPeerings:\n items:\n type: string\n type: array\n tcpLoadBalancer:\n type: string\n tcpSessionLoadBalancer:\n type: string\n udpLoadBalancer:\n type: string\n udpSessionLoadBalancer:\n type: string\n sctpLoadBalancer:\n type: string\n sctpSessionLoadBalancer:\n type: string\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n names:\n kind: Vpc\n listKind: VpcList\n plural: vpcs\n shortNames:\n - vpc\n singular: vpc\n scope: Cluster\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ips.kubeovn.io\nspec:\n group: kubeovn.io\n versions:\n - name: v1\n served: true\n storage: true\n additionalPrinterColumns:\n - name: V4IP\n type: string\n jsonPath: .spec.v4IpAddress\n - name: V6IP\n type: string\n jsonPath: .spec.v6IpAddress\n - name: Mac\n type: string\n jsonPath: .spec.macAddress\n - name: Node\n type: string\n jsonPath: .spec.nodeName\n - name: Subnet\n type: string\n jsonPath: .spec.subnet\n schema:\n openAPIV3Schema:\n type: object\n properties:\n spec:\n type: object\n properties:\n podName:\n type: string\n namespace:\n type: string\n subnet:\n type: string\n attachSubnets:\n type: array\n items:\n type: string\n nodeName:\n type: string\n ipAddress:\n type: string\n v4IpAddress:\n type: string\n v6IpAddress:\n type: string\n attachIps:\n type: array\n items:\n type: string\n macAddress:\n type: string\n attachMacs:\n type: array\n items:\n type: string\n containerID:\n type: string\n podType:\n type: string\n scope: Cluster\n names:\n plural: ips\n singular: ip\n kind: IP\n shortNames:\n - ip\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: vips.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: vips\n singular: vip\n shortNames:\n - vip\n kind: Vip\n listKind: VipList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n additionalPrinterColumns:\n - name: V4IP\n type: string\n jsonPath: .status.v4ip\n - name: V6IP\n type: string\n jsonPath: .status.v6ip\n - name: Mac\n type: string\n jsonPath: .status.mac\n - name: PMac\n type: string\n jsonPath: .spec.parentMac\n - name: Subnet\n type: string\n jsonPath: .spec.subnet\n - jsonPath: .status.ready\n name: Ready\n type: boolean\n - jsonPath: .status.type\n name: Type\n type: string\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n type:\n type: string\n ready:\n type: boolean\n v4ip:\n type: string\n v6ip:\n type: string\n mac:\n type: string\n pv4ip:\n type: string\n pv6ip:\n type: string\n pmac:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n namespace:\n type: string\n subnet:\n type: string\n type:\n type: string\n attachSubnets:\n type: array\n items:\n type: string\n v4ip:\n type: string\n macAddress:\n type: string\n v6ip:\n type: string\n parentV4ip:\n type: string\n parentMac:\n type: string\n parentV6ip:\n type: string\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: subnets.kubeovn.io\nspec:\n group: kubeovn.io\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - name: Provider\n type: string\n jsonPath: .spec.provider\n - name: Vpc\n type: string\n jsonPath: .spec.vpc\n - name: Protocol\n type: string\n jsonPath: .spec.protocol\n - name: CIDR\n type: string\n jsonPath: .spec.cidrBlock\n - name: Private\n type: boolean\n jsonPath: .spec.private\n - name: NAT\n type: boolean\n jsonPath: .spec.natOutgoing\n - name: Default\n type: boolean\n jsonPath: .spec.default\n - name: GatewayType\n type: string\n jsonPath: .spec.gatewayType\n - name: V4Used\n type: number\n jsonPath: .status.v4usingIPs\n - name: V4Available\n type: number\n jsonPath: .status.v4availableIPs\n - name: V6Used\n type: number\n jsonPath: .status.v6usingIPs\n - name: V6Available\n type: number\n jsonPath: .status.v6availableIPs\n - name: ExcludeIPs\n type: string\n jsonPath: .spec.excludeIps\n - name: U2OInterconnectionIP\n type: string\n jsonPath: .status.u2oInterconnectionIP\n schema:\n openAPIV3Schema:\n type: object\n properties:\n metadata:\n type: object\n properties:\n name:\n type: string\n pattern: ^[^0-9]\n status:\n type: object\n properties:\n v4availableIPs:\n type: number\n v4usingIPs:\n type: number\n v6availableIPs:\n type: number\n v6usingIPs:\n type: number\n activateGateway:\n type: string\n dhcpV4OptionsUUID:\n type: string\n dhcpV6OptionsUUID:\n type: string\n u2oInterconnectionIP:\n type: string\n u2oInterconnectionVPC:\n type: string\n v4usingIPrange:\n type: string\n v4availableIPrange:\n type: string\n v6usingIPrange:\n type: string\n v6availableIPrange:\n type: string\n natOutgoingPolicyRules:\n type: array\n items:\n type: object\n properties:\n ruleID:\n type: string\n action:\n type: string\n enum:\n - nat\n - forward\n match:\n type: object\n properties:\n srcIPs:\n type: string\n dstIPs:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n vpc:\n type: string\n default:\n type: boolean\n protocol:\n type: string\n enum:\n - IPv4\n - IPv6\n - Dual\n cidrBlock:\n type: string\n namespaces:\n type: array\n items:\n type: string\n gateway:\n type: string\n provider:\n type: string\n excludeIps:\n type: array\n items:\n type: string\n vips:\n type: array\n items:\n type: string\n gatewayType:\n type: string\n allowSubnets:\n type: array\n items:\n type: string\n gatewayNode:\n type: string\n natOutgoing:\n type: boolean\n externalEgressGateway:\n type: string\n policyRoutingPriority:\n type: integer\n minimum: 1\n maximum: 32765\n policyRoutingTableID:\n type: integer\n minimum: 1\n maximum: 2147483647\n not:\n enum:\n - 252 # compat\n - 253 # default\n - 254 # main\n - 255 # local\n mtu:\n type: integer\n minimum: 68\n maximum: 65535\n private:\n type: boolean\n vlan:\n type: string\n logicalGateway:\n type: boolean\n disableGatewayCheck:\n type: boolean\n disableInterConnection:\n type: boolean\n enableDHCP:\n type: boolean\n dhcpV4Options:\n type: string\n dhcpV6Options:\n type: string\n enableIPv6RA:\n type: boolean\n ipv6RAConfigs:\n type: string\n acls:\n type: array\n items:\n type: object\n properties:\n direction:\n type: string\n enum:\n - from-lport\n - to-lport\n priority:\n type: integer\n minimum: 0\n maximum: 32767\n match:\n type: string\n action:\n type: string\n enum:\n - allow-related\n - allow-stateless\n - allow\n - drop\n - reject\n natOutgoingPolicyRules:\n type: array\n items:\n type: object\n properties:\n action:\n type: string\n enum:\n - nat\n - forward\n match:\n type: object\n properties:\n srcIPs:\n type: string\n dstIPs:\n type: string\n u2oInterconnection:\n type: boolean\n u2oInterconnectionIP:\n type: string\n enableLb:\n type: boolean\n enableEcmp:\n type: boolean\n enableMulticastSnoop:\n type: boolean\n routeTable:\n type: string\n scope: Cluster\n names:\n plural: subnets\n singular: subnet\n kind: Subnet\n shortNames:\n - subnet\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.kubeovn.io\nspec:\n group: kubeovn.io\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - name: Subnet\n type: string\n jsonPath: .spec.subnet\n - name: IPs\n type: string\n jsonPath: .spec.ips\n - name: V4Used\n type: number\n jsonPath: .status.v4UsingIPs\n - name: V4Available\n type: number\n jsonPath: .status.v4AvailableIPs\n - name: V6Used\n type: number\n jsonPath: .status.v6UsingIPs\n - name: V6Available\n type: number\n jsonPath: .status.v6AvailableIPs\n schema:\n openAPIV3Schema:\n type: object\n properties:\n spec:\n type: object\n properties:\n subnet:\n type: string\n x-kubernetes-validations:\n - rule: \"self == oldSelf\"\n message: \"This field is immutable.\"\n namespaces:\n type: array\n x-kubernetes-list-type: set\n items:\n type: string\n ips:\n type: array\n minItems: 1\n x-kubernetes-list-type: set\n items:\n type: string\n anyOf:\n - format: ipv4\n - format: ipv6\n - format: cidr\n - pattern: ^(?:(?:[01]?\\d{1,2}|2[0-4]\\d|25[0-5])\\.){3}(?:[01]?\\d{1,2}|2[0-4]\\d|25[0-5])\\.\\.(?:(?:[01]?\\d{1,2}|2[0-4]\\d|25[0-5])\\.){3}(?:[01]?\\d{1,2}|2[0-4]\\d|25[0-5])$\n - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))\\.\\.((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$\n required:\n - subnet\n - ips\n status:\n type: object\n properties:\n v4AvailableIPs:\n type: number\n v4UsingIPs:\n type: number\n v6AvailableIPs:\n type: number\n v6UsingIPs:\n type: number\n v4AvailableIPRange:\n type: string\n v4UsingIPRange:\n type: string\n v6AvailableIPRange:\n type: string\n v6UsingIPRange:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n scope: Cluster\n names:\n plural: ippools\n singular: ippool\n kind: IPPool\n shortNames:\n - ippool\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: vlans.kubeovn.io\nspec:\n group: kubeovn.io\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n schema:\n openAPIV3Schema:\n type: object\n properties:\n spec:\n type: object\n properties:\n id:\n type: integer\n minimum: 0\n maximum: 4095\n provider:\n type: string\n vlanId:\n type: integer\n description: Deprecated in favor of id\n providerInterfaceName:\n type: string\n description: Deprecated in favor of provider\n required:\n - provider\n status:\n type: object\n properties:\n subnets:\n type: array\n items:\n type: string\n additionalPrinterColumns:\n - name: ID\n type: string\n jsonPath: .spec.id\n - name: Provider\n type: string\n jsonPath: .spec.provider\n scope: Cluster\n names:\n plural: vlans\n singular: vlan\n kind: Vlan\n shortNames:\n - vlan\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: provider-networks.kubeovn.io\nspec:\n group: kubeovn.io\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n schema:\n openAPIV3Schema:\n type: object\n properties:\n metadata:\n type: object\n properties:\n name:\n type: string\n maxLength: 12\n not:\n enum:\n - int\n spec:\n type: object\n properties:\n defaultInterface:\n type: string\n maxLength: 15\n pattern: '^[^/\\s]+$'\n customInterfaces:\n type: array\n items:\n type: object\n properties:\n interface:\n type: string\n maxLength: 15\n pattern: '^[^/\\s]+$'\n nodes:\n type: array\n items:\n type: string\n exchangeLinkName:\n type: boolean\n excludeNodes:\n type: array\n items:\n type: string\n required:\n - defaultInterface\n status:\n type: object\n properties:\n ready:\n type: boolean\n readyNodes:\n type: array\n items:\n type: string\n notReadyNodes:\n type: array\n items:\n type: string\n vlans:\n type: array\n items:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n node:\n type: string\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n additionalPrinterColumns:\n - name: DefaultInterface\n type: string\n jsonPath: .spec.defaultInterface\n - name: Ready\n type: boolean\n jsonPath: .status.ready\n scope: Cluster\n names:\n plural: provider-networks\n singular: provider-network\n kind: ProviderNetwork\n listKind: ProviderNetworkList\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: security-groups.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: security-groups\n singular: security-group\n shortNames:\n - sg\n kind: SecurityGroup\n listKind: SecurityGroupList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n schema:\n openAPIV3Schema:\n type: object\n properties:\n spec:\n type: object\n properties:\n ingressRules:\n type: array\n items:\n type: object\n properties:\n ipVersion:\n type: string\n protocol:\n type: string\n priority:\n type: integer\n remoteType:\n type: string\n remoteAddress:\n type: string\n remoteSecurityGroup:\n type: string\n portRangeMin:\n type: integer\n portRangeMax:\n type: integer\n policy:\n type: string\n egressRules:\n type: array\n items:\n type: object\n properties:\n ipVersion:\n type: string\n protocol:\n type: string\n priority:\n type: integer\n remoteType:\n type: string\n remoteAddress:\n type: string\n remoteSecurityGroup:\n type: string\n portRangeMin:\n type: integer\n portRangeMax:\n type: integer\n policy:\n type: string\n allowSameGroupTraffic:\n type: boolean\n status:\n type: object\n properties:\n portGroup:\n type: string\n allowSameGroupTraffic:\n type: boolean\n ingressMd5:\n type: string\n egressMd5:\n type: string\n ingressLastSyncSuccess:\n type: boolean\n egressLastSyncSuccess:\n type: boolean\n subresources:\n status: {}\n conversion:\n strategy: None\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: qos-policies.kubeovn.io\nspec:\n group: kubeovn.io\n names:\n plural: qos-policies\n singular: qos-policy\n shortNames:\n - qos\n kind: QoSPolicy\n listKind: QoSPolicyList\n scope: Cluster\n versions:\n - name: v1\n served: true\n storage: true\n subresources:\n status: {}\n additionalPrinterColumns:\n - jsonPath: .spec.shared\n name: Shared\n type: string\n - jsonPath: .spec.bindingType\n name: BindingType\n type: string\n schema:\n openAPIV3Schema:\n type: object\n properties:\n status:\n type: object\n properties:\n shared:\n type: boolean\n bindingType:\n type: string\n bandwidthLimitRules:\n type: array\n items:\n type: object\n properties:\n name:\n type: string\n interface:\n type: string\n rateMax:\n type: string\n burstMax:\n type: string\n priority:\n type: integer\n direction:\n type: string\n matchType:\n type: string\n matchValue:\n type: string\n conditions:\n type: array\n items:\n type: object\n properties:\n type:\n type: string\n status:\n type: string\n reason:\n type: string\n message:\n type: string\n lastUpdateTime:\n type: string\n lastTransitionTime:\n type: string\n spec:\n type: object\n properties:\n shared:\n type: boolean\n bindingType:\n type: string\n bandwidthLimitRules:\n type: array\n items:\n type: object\n properties:\n name:\n type: string\n interface:\n type: string\n rateMax:\n type: string\n burstMax:\n type: string\n priority:\n type: integer\n direction:\n type: string\n matchType:\n type: string\n matchValue:\n type: string\n required:\n - name\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n" }, { "path": "roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2", "content": "---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: ovn-vpc-nat-config\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n kube-ovn vpc-nat common config\ndata:\n image: {{ kube_ovn_vpc_container_image_repo }}:{{ kube_ovn_vpc_container_image_tag }}\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: ovn-vpc-nat-gw-config\n namespace: kube-system\ndata:\n enable-vpc-nat-gw: \"true\"\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: kube-ovn-cni\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n annotations:\n rbac.authorization.k8s.io/system-only: \"true\"\n name: system:kube-ovn-cni\nrules:\n - apiGroups:\n - \"kubeovn.io\"\n resources:\n - subnets\n - vlans\n - provider-networks\n verbs:\n - get\n - list\n - watch\n - apiGroups:\n - \"\"\n - \"kubeovn.io\"\n resources:\n - ovn-eips\n - ovn-eips/status\n - nodes\n - pods\n - vlans\n verbs:\n - get\n - list\n - patch\n - watch\n - apiGroups:\n - \"kubeovn.io\"\n resources:\n - ips\n verbs:\n - get\n - update\n - apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n - update\n - apiGroups:\n - \"\"\n resources:\n - configmaps\n verbs:\n - get\n - list\n - watch\n - apiGroups:\n - authentication.k8s.io\n resources:\n - tokenreviews\n verbs:\n - create\n - apiGroups:\n - authorization.k8s.io\n resources:\n - subjectaccessreviews\n verbs:\n - create\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: kube-ovn-cni\nroleRef:\n name: system:kube-ovn-cni\n kind: ClusterRole\n apiGroup: rbac.authorization.k8s.io\nsubjects:\n - kind: ServiceAccount\n name: kube-ovn-cni\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: kube-ovn-cni\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: extension-apiserver-authentication-reader\nsubjects:\n - kind: ServiceAccount\n name: kube-ovn-cni\n namespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: kube-ovn-app\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n annotations:\n rbac.authorization.k8s.io/system-only: \"true\"\n name: system:kube-ovn-app\nrules:\n - apiGroups:\n - \"\"\n resources:\n - pods\n - nodes\n verbs:\n - get\n - list\n - apiGroups:\n - apps\n resources:\n - daemonsets\n verbs:\n - get\n - apiGroups:\n - authentication.k8s.io\n resources:\n - tokenreviews\n verbs:\n - create\n - apiGroups:\n - authorization.k8s.io\n resources:\n - subjectaccessreviews\n verbs:\n - create\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: kube-ovn-app\nroleRef:\n name: system:kube-ovn-app\n kind: ClusterRole\n apiGroup: rbac.authorization.k8s.io\nsubjects:\n - kind: ServiceAccount\n name: kube-ovn-app\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: kube-ovn-app\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: extension-apiserver-authentication-reader\nsubjects:\n - kind: ServiceAccount\n name: kube-ovn-app\n namespace: kube-system\n---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: kube-ovn-controller\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n kube-ovn controller\nspec:\n replicas: {{ kube_ovn_controller_replics }}\n selector:\n matchLabels:\n app: kube-ovn-controller\n strategy:\n rollingUpdate:\n maxSurge: 0%\n maxUnavailable: 100%\n type: RollingUpdate\n template:\n metadata:\n labels:\n app: kube-ovn-controller\n component: network\n type: infra\n spec:\n tolerations:\n - effect: NoSchedule\n operator: Exists\n - key: CriticalAddonsOnly\n operator: Exists\n affinity:\n nodeAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - preference:\n matchExpressions:\n - key: \"ovn.kubernetes.io/ic-gw\"\n operator: NotIn\n values:\n - \"true\"\n weight: 100\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n app: kube-ovn-controller\n topologyKey: kubernetes.io/hostname\n priorityClassName: system-cluster-critical\n serviceAccountName: ovn\n hostNetwork: true\n containers:\n - name: kube-ovn-controller\n image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - /kube-ovn/start-controller.sh\n - --default-cidr={{ kube_pods_subnets }}\n - --default-gateway={% if kube_ovn_default_gateway is defined %}{{ kube_ovn_default_gateway }}{% endif %}{{ '' }}\n - --default-gateway-check={{ kube_ovn_default_gateway_check | string }}\n - --default-logical-gateway={{ kube_ovn_default_logical_gateway | string }}\n - --default-u2o-interconnection={{ kube_ovn_u2o_interconnection }}\n - --default-exclude-ips={% if kube_ovn_default_exclude_ips is defined %}{{ kube_ovn_default_exclude_ips }}{% endif %}{{ '' }}\n - --node-switch-cidr={{ kube_ovn_node_switch_cidr_merged }}\n - --service-cluster-ip-range={{ kube_service_subnets }}\n - --network-type={{ kube_ovn_network_type }}\n - --default-interface-name={{ kube_ovn_default_interface_name | default('') }}\n - --default-vlan-id={{ kube_ovn_default_vlan_id }}\n - --ls-dnat-mod-dl-dst={{ kube_ovn_ls_dnat_mod_dl_dst }}\n - --pod-nic-type={{ kube_ovn_pod_nic_type }}\n - --enable-lb={{ kube_ovn_enable_lb | string }}\n - --enable-np={{ kube_ovn_enable_np | string }}\n - --enable-eip-snat={{ kube_ovn_eip_snat_enabled }}\n - --enable-external-vpc={{ kube_ovn_enable_external_vpc | string }}\n - --logtostderr=false\n - --alsologtostderr=true\n - --gc-interval=360\n - --inspect-interval=20\n - --log_file=/var/log/kube-ovn/kube-ovn-controller.log\n - --log_file_max_size=0\n - --enable-lb-svc=false\n - --keep-vm-ip={{ kube_ovn_keep_vm_ip }}\n securityContext:\n runAsUser: 0\n privileged: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n env:\n - name: ENABLE_SSL\n value: \"{{ kube_ovn_enable_ssl | lower }}\"\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: KUBE_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: OVN_DB_IPS\n value: \"{{ kube_ovn_central_ips }}\"\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: POD_IPS\n valueFrom:\n fieldRef:\n fieldPath: status.podIPs\n - name: ENABLE_BIND_LOCAL_IP\n value: \"{{ kube_ovn_bind_local_ip_enabled }}\"\n volumeMounts:\n - mountPath: /etc/localtime\n name: localtime\n - mountPath: /var/log/kube-ovn\n name: kube-ovn-log\n - mountPath: /var/log/ovn\n name: ovn-log\n - mountPath: /var/run/tls\n name: kube-ovn-tls\n readinessProbe:\n exec:\n command:\n - /kube-ovn/kube-ovn-healthcheck\n - --port=10660\n - --tls=false\n periodSeconds: 3\n timeoutSeconds: 45\n livenessProbe:\n exec:\n command:\n - /kube-ovn/kube-ovn-healthcheck\n - --port=10660\n - --tls=false\n initialDelaySeconds: 300\n periodSeconds: 7\n failureThreshold: 5\n timeoutSeconds: 45\n resources:\n requests:\n cpu: {{ kube_ovn_controller_cpu_request }}\n memory: {{ kube_ovn_controller_memory_request }}\n limits:\n cpu: {{ kube_ovn_controller_cpu_limit }}\n memory: {{ kube_ovn_controller_memory_limit }}\n nodeSelector:\n kubernetes.io/os: \"linux\"\n volumes:\n - name: localtime\n hostPath:\n path: /etc/localtime\n - name: kube-ovn-log\n hostPath:\n path: /var/log/kube-ovn\n - name: ovn-log\n hostPath:\n path: /var/log/ovn\n - name: kube-ovn-tls\n secret:\n optional: true\n secretName: kube-ovn-tls\n\n---\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: kube-ovn-cni\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n This daemon set launches the kube-ovn cni daemon.\nspec:\n selector:\n matchLabels:\n app: kube-ovn-cni\n template:\n metadata:\n labels:\n app: kube-ovn-cni\n component: network\n type: infra\n spec:\n tolerations:\n - effect: NoSchedule\n operator: Exists\n - effect: NoExecute\n operator: Exists\n - key: CriticalAddonsOnly\n operator: Exists\n priorityClassName: system-node-critical\n serviceAccountName: kube-ovn-cni\n hostNetwork: true\n hostPID: true\n initContainers:\n - name: install-cni\n image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [\"/kube-ovn/install-cni.sh\"]\n securityContext:\n runAsUser: 0\n privileged: true\n volumeMounts:\n - mountPath: /opt/cni/bin\n name: cni-bin\n - mountPath: /usr/local/bin\n name: local-bin\n containers:\n - name: cni-server\n image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command:\n - bash\n - /kube-ovn/start-cniserver.sh\n args:\n - --enable-mirror={{ kube_ovn_traffic_mirror | lower }}\n - --encap-checksum={{ kube_ovn_encap_checksum | lower }}\n - --service-cluster-ip-range={{ kube_service_subnets }}\n - --iface={{ kube_ovn_iface | default('') }}\n - --dpdk-tunnel-iface={{ kube_ovn_dpdk_tunnel_iface }}\n - --network-type={{ kube_ovn_network_type }}\n - --default-interface-name={{ kube_ovn_default_interface_name | default('') }}\n {% if kube_ovn_mtu is defined %}\n - --mtu={{ kube_ovn_mtu }}\n{% endif %}\n - --cni-conf-name={{ kube_ovn_cni_config_priority }}-kube-ovn.conflist\n - --logtostderr=false\n - --alsologtostderr=true\n - --log_file=/var/log/kube-ovn/kube-ovn-cni.log\n - --log_file_max_size=0\n securityContext:\n runAsUser: 0\n privileged: false\n capabilities:\n add:\n - NET_ADMIN\n - NET_BIND_SERVICE\n - NET_RAW\n - SYS_ADMIN\n env:\n - name: ENABLE_SSL\n value: \"{{ kube_ovn_enable_ssl | lower }}\"\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: MODULES\n value: kube_ovn_fastpath.ko\n - name: RPMS\n value: openvswitch-kmod\n - name: POD_IPS\n valueFrom:\n fieldRef:\n fieldPath: status.podIPs\n - name: ENABLE_BIND_LOCAL_IP\n value: \"{{ kube_ovn_bind_local_ip_enabled }}\"\n - name: DBUS_SYSTEM_BUS_ADDRESS\n value: \"unix:path=/host/var/run/dbus/system_bus_socket\"\n volumeMounts:\n - name: host-modules\n mountPath: /lib/modules\n readOnly: true\n - name: shared-dir\n mountPath: $KUBELET_DIR/pods\n - mountPath: /etc/openvswitch\n name: systemid\n readOnly: true\n - mountPath: /etc/cni/net.d\n name: cni-conf\n - mountPath: /run/openvswitch\n name: host-run-ovs\n mountPropagation: HostToContainer\n - mountPath: /run/ovn\n name: host-run-ovn\n - mountPath: /host/var/run/dbus\n name: host-dbus\n mountPropagation: HostToContainer\n - mountPath: /var/run/netns\n name: host-ns\n mountPropagation: HostToContainer\n - mountPath: /var/log/kube-ovn\n name: kube-ovn-log\n - mountPath: /var/log/openvswitch\n name: host-log-ovs\n - mountPath: /var/log/ovn\n name: host-log-ovn\n - mountPath: /etc/localtime\n name: localtime\n readOnly: true\n - mountPath: /tmp\n name: tmp\n livenessProbe:\n failureThreshold: 3\n initialDelaySeconds: 30\n periodSeconds: 7\n successThreshold: 1\n exec:\n command:\n - /kube-ovn/kube-ovn-healthcheck\n - --port=10665\n - --tls=false\n timeoutSeconds: 5\n readinessProbe:\n failureThreshold: 3\n periodSeconds: 7\n successThreshold: 1\n exec:\n command:\n - /kube-ovn/kube-ovn-healthcheck\n - --port=10665\n - --tls=false\n timeoutSeconds: 5\n resources:\n requests:\n cpu: {{ kube_ovn_cni_server_cpu_request }}\n memory: {{ kube_ovn_cni_server_memory_request }}\n limits:\n cpu: {{ kube_ovn_cni_server_cpu_limit }}\n memory: {{ kube_ovn_cni_server_memory_limit }}\n nodeSelector:\n kubernetes.io/os: \"linux\"\n volumes:\n - name: host-modules\n hostPath:\n path: /lib/modules\n - name: shared-dir\n hostPath:\n path: /var/lib/kubelet/pods\n - name: systemid\n hostPath:\n path: /etc/origin/openvswitch\n - name: host-run-ovs\n hostPath:\n path: /run/openvswitch\n - name: host-run-ovn\n hostPath:\n path: /run/ovn\n - name: cni-conf\n hostPath:\n path: /etc/cni/net.d\n - name: cni-bin\n hostPath:\n path: /opt/cni/bin\n - name: host-ns\n hostPath:\n path: /var/run/netns\n - name: host-dbus\n hostPath:\n path: /var/run/dbus\n - name: host-log-ovs\n hostPath:\n path: /var/log/openvswitch\n - name: kube-ovn-log\n hostPath:\n path: /var/log/kube-ovn\n - name: host-log-ovn\n hostPath:\n path: /var/log/ovn\n - name: localtime\n hostPath:\n path: /etc/localtime\n - name: tmp\n hostPath:\n path: /tmp\n - name: local-bin\n hostPath:\n path: /usr/local/bin\n---\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: kube-ovn-pinger\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n This daemon set launches the openvswitch daemon.\nspec:\n selector:\n matchLabels:\n app: kube-ovn-pinger\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n app: kube-ovn-pinger\n component: network\n type: infra\n spec:\n priorityClassName: system-node-critical\n serviceAccountName: ovn\n hostPID: true\n containers:\n - name: pinger\n image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}\n command:\n - /kube-ovn/kube-ovn-pinger\n args:\n - --external-address={{ kube_ovn_external_address_merged }}\n - --external-dns={{ kube_ovn_external_dns }}\n - --logtostderr=false\n - --alsologtostderr=true\n - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log\n - --log_file_max_size=0\n imagePullPolicy: {{ k8s_image_pull_policy }}\n securityContext:\n runAsUser: 0\n privileged: false\n env:\n - name: ENABLE_SSL\n value: \"{{ kube_ovn_enable_ssl | lower }}\"\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: HOST_IP\n valueFrom:\n fieldRef:\n fieldPath: status.hostIP\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n volumeMounts:\n - mountPath: /var/run/openvswitch\n name: host-run-ovs\n - mountPath: /var/run/ovn\n name: host-run-ovn\n - mountPath: /etc/openvswitch\n name: host-config-openvswitch\n - mountPath: /var/log/openvswitch\n name: host-log-ovs\n readOnly: true\n - mountPath: /var/log/ovn\n name: host-log-ovn\n readOnly: true\n - mountPath: /var/log/kube-ovn\n name: kube-ovn-log\n - mountPath: /etc/localtime\n name: localtime\n readOnly: true\n - mountPath: /var/run/tls\n name: kube-ovn-tls\n resources:\n requests:\n cpu: {{ kube_ovn_pinger_cpu_request }}\n memory: {{ kube_ovn_pinger_memory_request }}\n limits:\n cpu: {{ kube_ovn_pinger_cpu_limit }}\n memory: {{ kube_ovn_pinger_memory_limit }}\n nodeSelector:\n kubernetes.io/os: \"linux\"\n volumes:\n - name: host-run-ovs\n hostPath:\n path: /run/openvswitch\n - name: host-run-ovn\n hostPath:\n path: /run/ovn\n - name: host-config-openvswitch\n hostPath:\n path: /etc/origin/openvswitch\n - name: host-log-ovs\n hostPath:\n path: /var/log/openvswitch\n - name: kube-ovn-log\n hostPath:\n path: /var/log/kube-ovn\n - name: host-log-ovn\n hostPath:\n path: /var/log/ovn\n - name: localtime\n hostPath:\n path: /etc/localtime\n - name: kube-ovn-tls\n secret:\n optional: true\n secretName: kube-ovn-tls\n---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: kube-ovn-monitor\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n Metrics for OVN components: northd, nb and sb.\nspec:\n replicas: 1\n strategy:\n rollingUpdate:\n maxSurge: 1\n maxUnavailable: 1\n type: RollingUpdate\n selector:\n matchLabels:\n app: kube-ovn-monitor\n template:\n metadata:\n labels:\n app: kube-ovn-monitor\n component: network\n type: infra\n spec:\n tolerations:\n - effect: NoSchedule\n operator: Exists\n - key: CriticalAddonsOnly\n operator: Exists\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n app: kube-ovn-monitor\n topologyKey: kubernetes.io/hostname\n priorityClassName: system-cluster-critical\n serviceAccountName: ovn\n hostNetwork: true\n containers:\n - name: kube-ovn-monitor\n image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [\"/kube-ovn/start-ovn-monitor.sh\"]\n args:\n - --secure-serving=false\n - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log\n - --logtostderr=false\n - --alsologtostderr=true\n - --log_file_max_size=200\n securityContext:\n runAsUser: 0\n privileged: false\n env:\n - name: ENABLE_SSL\n value: \"{{ kube_ovn_enable_ssl | lower }}\"\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: POD_IPS\n valueFrom:\n fieldRef:\n fieldPath: status.podIPs\n - name: ENABLE_BIND_LOCAL_IP\n value: \"{{ kube_ovn_bind_local_ip_enabled }}\"\n resources:\n requests:\n cpu: {{ kube_ovn_monitor_cpu_request }}\n memory: {{ kube_ovn_monitor_memory_request }}\n limits:\n cpu: {{ kube_ovn_monitor_cpu_limit }}\n memory: {{ kube_ovn_monitor_memory_limit }}\n volumeMounts:\n - mountPath: /var/run/openvswitch\n name: host-run-ovs\n - mountPath: /var/run/ovn\n name: host-run-ovn\n - mountPath: /etc/openvswitch\n name: host-config-openvswitch\n - mountPath: /etc/ovn\n name: host-config-ovn\n - mountPath: /var/log/ovn\n name: host-log-ovn\n readOnly: true\n - mountPath: /etc/localtime\n name: localtime\n readOnly: true\n - mountPath: /var/run/tls\n name: kube-ovn-tls\n - mountPath: /var/log/kube-ovn\n name: kube-ovn-log\n livenessProbe:\n failureThreshold: 3\n initialDelaySeconds: 30\n periodSeconds: 7\n successThreshold: 1\n exec:\n command:\n - /kube-ovn/kube-ovn-healthcheck\n - --port=10661\n - --tls=false\n timeoutSeconds: 5\n readinessProbe:\n failureThreshold: 3\n initialDelaySeconds: 30\n periodSeconds: 7\n successThreshold: 1\n exec:\n command:\n - /kube-ovn/kube-ovn-healthcheck\n - --port=10661\n - --tls=false\n timeoutSeconds: 5\n nodeSelector:\n kubernetes.io/os: \"linux\"\n kube-ovn/role: \"master\"\n volumes:\n - name: host-run-ovs\n hostPath:\n path: /run/openvswitch\n - name: host-run-ovn\n hostPath:\n path: /run/ovn\n - name: host-config-openvswitch\n hostPath:\n path: /etc/origin/openvswitch\n - name: host-config-ovn\n hostPath:\n path: /etc/origin/ovn\n - name: host-log-ovs\n hostPath:\n path: /var/log/openvswitch\n - name: host-log-ovn\n hostPath:\n path: /var/log/ovn\n - name: localtime\n hostPath:\n path: /etc/localtime\n - name: kube-ovn-tls\n secret:\n optional: true\n secretName: kube-ovn-tls\n - name: kube-ovn-log\n hostPath:\n path: /var/log/kube-ovn\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: kube-ovn-monitor\n namespace: kube-system\n labels:\n app: kube-ovn-monitor\nspec:\n ports:\n - name: metrics\n port: 10661\n type: ClusterIP\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: kube-ovn-monitor\n sessionAffinity: None\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: kube-ovn-pinger\n namespace: kube-system\n labels:\n app: kube-ovn-pinger\nspec:\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: kube-ovn-pinger\n ports:\n - port: 8080\n name: metrics\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: kube-ovn-controller\n namespace: kube-system\n labels:\n app: kube-ovn-controller\nspec:\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: kube-ovn-controller\n ports:\n - port: 10660\n name: metrics\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: kube-ovn-cni\n namespace: kube-system\n labels:\n app: kube-ovn-cni\nspec:\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: kube-ovn-cni\n ports:\n - port: 10665\n name: metrics\n{% if kube_ovn_ic_enable %}\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: ovn-ic-config\n namespace: kube-system\ndata:\n enable-ic: \"{{ kube_ovn_ic_enable | lower }}\"\n az-name: \"{{ kube_ovn_ic_zone }}\"\n ic-db-host: \"{{ kube_ovn_ic_dbhost }}\"\n ic-nb-port: \"6645\"\n ic-sb-port: \"6646\"\n gw-nodes: \"{{ kube_ovn_central_hosts | join(',') }}\"\n auto-route: \"{{ kube_ovn_ic_autoroute | lower }}\"\n{% endif %}\n" }, { "path": "roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: ovn-ovs\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n annotations:\n rbac.authorization.k8s.io/system-only: \"true\"\n name: system:ovn-ovs\nrules:\n - apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - get\n - patch\n - apiGroups:\n - \"\"\n resources:\n - services\n - endpoints\n verbs:\n - get\n - apiGroups:\n - apps\n resources:\n - controllerrevisions\n verbs:\n - get\n - list\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: ovn-ovs\nroleRef:\n name: system:ovn-ovs\n kind: ClusterRole\n apiGroup: rbac.authorization.k8s.io\nsubjects:\n - kind: ServiceAccount\n name: ovn-ovs\n namespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: ovn\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n annotations:\n rbac.authorization.k8s.io/system-only: \"true\"\n name: system:ovn\nrules:\n - apiGroups:\n - \"kubeovn.io\"\n resources:\n - vpcs\n - vpcs/status\n - vpc-nat-gateways\n - vpc-nat-gateways/status\n - subnets\n - subnets/status\n - ippools\n - ippools/status\n - ips\n - vips\n - vips/status\n - vlans\n - vlans/status\n - provider-networks\n - provider-networks/status\n - security-groups\n - security-groups/status\n - iptables-eips\n - iptables-fip-rules\n - iptables-dnat-rules\n - iptables-snat-rules\n - iptables-eips/status\n - iptables-fip-rules/status\n - iptables-dnat-rules/status\n - iptables-snat-rules/status\n - ovn-eips\n - ovn-fips\n - ovn-snat-rules\n - ovn-eips/status\n - ovn-fips/status\n - ovn-snat-rules/status\n - ovn-dnat-rules\n - ovn-dnat-rules/status\n - switch-lb-rules\n - switch-lb-rules/status\n - vpc-dnses\n - vpc-dnses/status\n - qos-policies\n - qos-policies/status\n verbs:\n - \"*\"\n - apiGroups:\n - \"\"\n resources:\n - pods\n - namespaces\n verbs:\n - get\n - list\n - patch\n - watch\n - apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - get\n - list\n - patch\n - update\n - watch\n - apiGroups:\n - \"\"\n resources:\n - pods/exec\n verbs:\n - create\n - apiGroups:\n - \"k8s.cni.cncf.io\"\n resources:\n - network-attachment-definitions\n verbs:\n - get\n - apiGroups:\n - \"\"\n - networking.k8s.io\n resources:\n - networkpolicies\n - configmaps\n verbs:\n - get\n - list\n - watch\n - apiGroups:\n - apps\n resources:\n - daemonsets\n verbs:\n - get\n - apiGroups:\n - \"\"\n resources:\n - services\n - services/status\n verbs:\n - get\n - list\n - update\n - create\n - delete\n - watch\n - apiGroups:\n - \"\"\n resources:\n - endpoints\n verbs:\n - create\n - update\n - get\n - list\n - watch\n - apiGroups:\n - apps\n resources:\n - statefulsets\n - deployments\n - deployments/scale\n verbs:\n - get\n - list\n - create\n - delete\n - update\n - apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n - update\n - apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs:\n - \"*\"\n - apiGroups:\n - \"kubevirt.io\"\n resources:\n - virtualmachines\n - virtualmachineinstances\n verbs:\n - get\n - list\n - apiGroups:\n - authentication.k8s.io\n resources:\n - tokenreviews\n verbs:\n - create\n - apiGroups:\n - authorization.k8s.io\n resources:\n - subjectaccessreviews\n verbs:\n - create\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: ovn\nroleRef:\n name: system:ovn\n kind: ClusterRole\n apiGroup: rbac.authorization.k8s.io\nsubjects:\n - kind: ServiceAccount\n name: ovn\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: ovn\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: extension-apiserver-authentication-reader\nsubjects:\n - kind: ServiceAccount\n name: ovn\n namespace: kube-system\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: ovn-nb\n namespace: kube-system\nspec:\n ports:\n - name: ovn-nb\n protocol: TCP\n port: 6641\n targetPort: 6641\n type: ClusterIP\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: ovn-central\n ovn-nb-leader: \"true\"\n sessionAffinity: None\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: ovn-sb\n namespace: kube-system\nspec:\n ports:\n - name: ovn-sb\n protocol: TCP\n port: 6642\n targetPort: 6642\n type: ClusterIP\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: ovn-central\n ovn-sb-leader: \"true\"\n sessionAffinity: None\n---\nkind: Service\napiVersion: v1\nmetadata:\n name: ovn-northd\n namespace: kube-system\nspec:\n ports:\n - name: ovn-northd\n protocol: TCP\n port: 6643\n targetPort: 6643\n type: ClusterIP\n{% if ipv6_stack %}\n ipFamilyPolicy: PreferDualStack\n{% endif %}\n selector:\n app: ovn-central\n ovn-northd-leader: \"true\"\n sessionAffinity: None\n---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: ovn-central\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n OVN components: northd, nb and sb.\nspec:\n replicas: {{ kube_ovn_central_replics }}\n strategy:\n rollingUpdate:\n maxSurge: 0\n maxUnavailable: 1\n type: RollingUpdate\n selector:\n matchLabels:\n app: ovn-central\n template:\n metadata:\n labels:\n app: ovn-central\n component: network\n type: infra\n spec:\n tolerations:\n - effect: NoSchedule\n operator: Exists\n - effect: NoExecute\n operator: Exists\n - key: CriticalAddonsOnly\n operator: Exists\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n app: ovn-central\n topologyKey: kubernetes.io/hostname\n priorityClassName: system-cluster-critical\n serviceAccountName: ovn-ovs\n hostNetwork: true\n containers:\n - name: ovn-central\n image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [\"/kube-ovn/start-db.sh\"]\n securityContext:\n capabilities:\n add:\n - NET_BIND_SERVICE\n - SYS_NICE\n env:\n - name: ENABLE_SSL\n value: \"{{ kube_ovn_enable_ssl | lower }}\"\n - name: NODE_IPS\n value: \"{{ kube_ovn_central_ips }}\"\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n - name: POD_IPS\n valueFrom:\n fieldRef:\n fieldPath: status.podIPs\n - name: ENABLE_BIND_LOCAL_IP\n value: \"{{ kube_ovn_bind_local_ip_enabled }}\"\n - name: PROBE_INTERVAL\n value: \"180000\"\n - name: OVN_NORTHD_PROBE_INTERVAL\n value: \"5000\"\n - name: OVN_LEADER_PROBE_INTERVAL\n value: \"5\"\n resources:\n requests:\n cpu: {{ kube_ovn_db_cpu_request }}\n memory: {{ kube_ovn_db_memory_request }}\n limits:\n cpu: {{ kube_ovn_db_cpu_limit }}\n memory: {{ kube_ovn_db_memory_limit }}\n volumeMounts:\n - mountPath: /var/run/openvswitch\n name: host-run-ovs\n - mountPath: /var/run/ovn\n name: host-run-ovn\n - mountPath: /sys\n name: host-sys\n readOnly: true\n - mountPath: /etc/openvswitch\n name: host-config-openvswitch\n - mountPath: /etc/ovn\n name: host-config-ovn\n - mountPath: /var/log/openvswitch\n name: host-log-ovs\n - mountPath: /var/log/ovn\n name: host-log-ovn\n - mountPath: /etc/localtime\n name: localtime\n - mountPath: /var/run/tls\n name: kube-ovn-tls\n readinessProbe:\n exec:\n command:\n - bash\n - /kube-ovn/ovn-healthcheck.sh\n periodSeconds: 15\n timeoutSeconds: 45\n livenessProbe:\n exec:\n command:\n - bash\n - /kube-ovn/ovn-healthcheck.sh\n initialDelaySeconds: 30\n periodSeconds: 15\n failureThreshold: 5\n timeoutSeconds: 45\n nodeSelector:\n kubernetes.io/os: \"linux\"\n kube-ovn/role: \"master\"\n volumes:\n - name: host-run-ovs\n hostPath:\n path: /run/openvswitch\n - name: host-run-ovn\n hostPath:\n path: /run/ovn\n - name: host-sys\n hostPath:\n path: /sys\n - name: host-config-openvswitch\n hostPath:\n path: /etc/origin/openvswitch\n - name: host-config-ovn\n hostPath:\n path: /etc/origin/ovn\n - name: host-log-ovs\n hostPath:\n path: /var/log/openvswitch\n - name: host-log-ovn\n hostPath:\n path: /var/log/ovn\n - name: localtime\n hostPath:\n path: /etc/localtime\n - name: kube-ovn-tls\n secret:\n optional: true\n secretName: kube-ovn-tls\n---\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: ovs-ovn\n namespace: kube-system\n annotations:\n kubernetes.io/description: |\n This daemon set launches the openvswitch daemon.\nspec:\n selector:\n matchLabels:\n app: ovs\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 1\n maxUnavailable: 0\n template:\n metadata:\n labels:\n app: ovs\n component: network\n type: infra\n spec:\n tolerations:\n - effect: NoSchedule\n operator: Exists\n - effect: NoExecute\n operator: Exists\n - key: CriticalAddonsOnly\n operator: Exists\n priorityClassName: system-node-critical\n serviceAccountName: ovn-ovs\n hostNetwork: true\n hostPID: true\n containers:\n - name: openvswitch\n image: {% if kube_ovn_dpdk_enabled %}{{ kube_ovn_dpdk_container_image_repo }}:{{ kube_ovn_dpdk_container_image_tag }}{% else %}{{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}{% endif %}\n\n imagePullPolicy: {{ k8s_image_pull_policy }}\n command: [{% if kube_ovn_dpdk_enabled %}\"/kube-ovn/start-ovs-dpdk.sh\"{% else %}\"/kube-ovn/start-ovs.sh\"{% endif %}]\n securityContext:\n runAsUser: 0\n privileged: false\n capabilities:\n add:\n - NET_ADMIN\n - NET_BIND_SERVICE\n - SYS_MODULE\n - SYS_NICE\n env:\n - name: ENABLE_SSL\n value: \"{{ kube_ovn_enable_ssl | lower }}\"\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n fieldPath: metadata.namespace\n{% if not kube_ovn_dpdk_enabled %}\n - name: HW_OFFLOAD\n value: \"{{ kube_ovn_hw_offload | string | lower }}\"\n - name: TUNNEL_TYPE\n value: \"{{ kube_ovn_tunnel_type }}\"\n{% endif %}\n - name: KUBE_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: OVN_DB_IPS\n value: \"{{ kube_ovn_central_ips }}\"\n volumeMounts:\n - mountPath: /var/run/netns\n name: host-ns\n mountPropagation: HostToContainer\n - mountPath: /lib/modules\n name: host-modules\n readOnly: true\n - mountPath: /var/run/openvswitch\n name: host-run-ovs\n - mountPath: /var/run/ovn\n name: host-run-ovn\n - mountPath: /sys\n name: host-sys\n readOnly: true\n - mountPath: /etc/cni/net.d\n name: cni-conf\n - mountPath: /etc/openvswitch\n name: host-config-openvswitch\n - mountPath: /etc/ovn\n name: host-config-ovn\n - mountPath: /var/log/openvswitch\n name: host-log-ovs\n - mountPath: /var/log/ovn\n name: host-log-ovn\n{% if kube_ovn_dpdk_enabled %}\n - mountPath: /opt/ovs-config\n name: host-config-ovs\n - mountPath: /dev/hugepages\n name: hugepage\n{% endif %}\n - mountPath: /etc/localtime\n name: localtime\n - mountPath: /var/run/tls\n name: kube-ovn-tls\n - mountPath: /var/run/containerd\n name: cruntime\n readOnly: true\n readinessProbe:\n exec:\n command:\n - bash\n{% if kube_ovn_dpdk_enabled %}\n - /kube-ovn/ovs-dpdk-healthcheck.sh\n{% else %}\n - /kube-ovn/ovs-healthcheck.sh\n{% endif %}\n periodSeconds: 5\n timeoutSeconds: 45\n livenessProbe:\n exec:\n command:\n - bash\n{% if kube_ovn_dpdk_enabled %}\n - /kube-ovn/ovs-dpdk-healthcheck.sh\n{% else %}\n - /kube-ovn/ovs-healthcheck.sh\n{% endif %}\n initialDelaySeconds: 60\n periodSeconds: 5\n failureThreshold: 5\n timeoutSeconds: 45\n resources:\n{% if kube_ovn_dpdk_enabled %}\n requests:\n cpu: {{ kube_ovn_dpdk_node_cpu_request }}\n memory: {{ kube_ovn_dpdk_node_memory_request }}\n limits:\n cpu: {{ kube_ovn_dpdk_node_cpu_limit }}\n memory: {{ kube_ovn_dpdk_node_memory_limit }}\n hugepages-1Gi: 1Gi\n{% else %}\n requests:\n cpu: {{ kube_ovn_node_cpu_request }}\n memory: {{ kube_ovn_node_memory_request }}\n limits:\n cpu: {{ kube_ovn_node_cpu_limit }}\n memory: {{ kube_ovn_node_memory_limit }}\n{% endif %}\n nodeSelector:\n kubernetes.io/os: \"linux\"\n volumes:\n - name: host-modules\n hostPath:\n path: /lib/modules\n - name: host-run-ovs\n hostPath:\n path: /run/openvswitch\n - name: host-run-ovn\n hostPath:\n path: /run/ovn\n - name: host-sys\n hostPath:\n path: /sys\n - name: host-ns\n hostPath:\n path: /var/run/netns\n - name: cni-conf\n hostPath:\n path: /etc/cni/net.d\n - name: host-config-openvswitch\n hostPath:\n path: /etc/origin/openvswitch\n - name: host-config-ovn\n hostPath:\n path: /etc/origin/ovn\n - name: host-log-ovs\n hostPath:\n path: /var/log/openvswitch\n - name: host-log-ovn\n hostPath:\n path: /var/log/ovn\n{% if kube_ovn_dpdk_enabled %}\n - name: host-config-ovs\n hostPath:\n path: /opt/ovs-config\n type: DirectoryOrCreate\n - name: hugepage\n emptyDir:\n medium: HugePages\n{% endif %}\n - name: localtime\n hostPath:\n path: /etc/localtime\n - name: cruntime\n hostPath:\n path: /var/run/containerd\n - name: kube-ovn-tls\n secret:\n optional: true\n secretName: kube-ovn-tls\n" }, { "path": "roles/network_plugin/kube-router/defaults/main.yml", "content": "---\n# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP\nkube_router_run_router: true\n\n# Enables Network Policy -- sets up iptables to provide ingress firewall for pods\nkube_router_run_firewall: true\n\n# Enables Service Proxy -- sets up IPVS for Kubernetes Services\n# see docs/kube-router.md \"Caveats\" section\nkube_router_run_service_proxy: false\n\n# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.\nkube_router_advertise_cluster_ip: false\n\n# Add External IP of service to the RIB so that it gets advertised to the BGP peers.\nkube_router_advertise_external_ip: false\n\n# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.\nkube_router_advertise_loadbalancer_ip: false\n\n# Enables BGP graceful restarts\nkube_router_bgp_graceful_restart: true\n\n# Adjust manifest of kube-router daemonset template with DSR needed changes\nkube_router_enable_dsr: false\n\n# Array of arbitrary extra arguments to kube-router, see\n# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md\nkube_router_extra_args: []\n\n# ASN number of the cluster, used when communicating with external BGP routers\nkube_router_cluster_asn: ~\n\n# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.\nkube_router_peer_router_asns: ~\n\n# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.\nkube_router_peer_router_ips: ~\n\n# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.\nkube_router_peer_router_ports: ~\n\n# Setups node CNI to allow hairpin mode, requires node reboots, see\n# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode\nkube_router_support_hairpin_mode: false\n\n# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc.\nkube_router_dns_policy: ClusterFirstWithHostNet\n\n# Adds annotations to kubernetes nodes for advanced configuration of BGP Peers.\n# https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md\n\n# Array of annotations for master\nkube_router_annotations_master: []\n\n# Array of annotations for every node\nkube_router_annotations_node: []\n\n# Array of common annotations for every node\nkube_router_annotations_all: []\n\n# Enables scraping kube-router metrics with Prometheus\nkube_router_enable_metrics: false\n\n# Path to serve Prometheus metrics on\nkube_router_metrics_path: /metrics\n\n# Prometheus metrics port to use\nkube_router_metrics_port: 9255\n" }, { "path": "roles/network_plugin/kube-router/handlers/main.yml", "content": "---\n- name: Kube-router | delete kube-router docker containers\n shell: \"set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f\"\n args:\n executable: /bin/bash\n register: docker_kube_router_remove\n until: docker_kube_router_remove is succeeded\n retries: 5\n when: container_manager in [\"docker\"]\n listen: Reset_kube_router\n\n- name: Kube-router | delete kube-router crio/containerd containers\n shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c \"{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %\"'\n args:\n executable: /bin/bash\n register: crictl_kube_router_remove\n until: crictl_kube_router_remove is succeeded\n retries: 5\n when: container_manager in [\"crio\", \"containerd\"]\n listen: Reset_kube_router\n" }, { "path": "roles/network_plugin/kube-router/meta/main.yml", "content": "---\ndependencies:\n - role: network_plugin/cni\n" }, { "path": "roles/network_plugin/kube-router/tasks/annotate.yml", "content": "---\n- name: Kube-router | Add annotations on kube_control_plane\n command: \"{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}\"\n with_items:\n - \"{{ kube_router_annotations_master }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when: kube_router_annotations_master is defined and 'kube_control_plane' in group_names\n\n- name: Kube-router | Add annotations on kube_node\n command: \"{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}\"\n with_items:\n - \"{{ kube_router_annotations_node }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when: kube_router_annotations_node is defined and 'kube_node' in group_names\n\n- name: Kube-router | Add common annotations on all servers\n command: \"{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}\"\n with_items:\n - \"{{ kube_router_annotations_all }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when: kube_router_annotations_all is defined and 'k8s_cluster' in group_names\n" }, { "path": "roles/network_plugin/kube-router/tasks/main.yml", "content": "---\n- name: Kube-router | Create annotations\n import_tasks: annotate.yml\n tags: annotate\n\n- name: Kube-router | Create config directory\n file:\n path: /var/lib/kube-router\n state: directory\n owner: \"{{ kube_owner }}\"\n recurse: true\n mode: \"0755\"\n\n- name: Kube-router | Create kubeconfig\n template:\n src: kubeconfig.yml.j2\n dest: /var/lib/kube-router/kubeconfig\n mode: \"0644\"\n owner: \"{{ kube_owner }}\"\n notify:\n - Reset_kube_router\n\n- name: Kube-router | Slurp cni config\n slurp:\n src: /etc/cni/net.d/10-kuberouter.conflist\n register: cni_config_slurp\n ignore_errors: true # noqa ignore-errors\n\n- name: Kube-router | Set cni_config variable\n set_fact:\n cni_config: \"{{ cni_config_slurp.content | b64decode | from_json }}\"\n when:\n - not cni_config_slurp.failed\n\n- name: Kube-router | Set host_subnet variable\n when:\n - cni_config is defined\n - cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | length > 0\n set_fact:\n host_subnet: \"{{ cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | first }}\"\n\n- name: Kube-router | Create cni config\n template:\n src: cni-conf.json.j2\n dest: /etc/cni/net.d/10-kuberouter.conflist\n mode: \"0644\"\n owner: \"{{ kube_owner }}\"\n notify:\n - Reset_kube_router\n\n- name: Kube-router | Delete old configuration\n file:\n path: /etc/cni/net.d/10-kuberouter.conf\n state: absent\n\n- name: Kube-router | Create manifest\n template:\n src: kube-router.yml.j2\n dest: \"{{ kube_config_dir }}/kube-router.yml\"\n mode: \"0644\"\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n run_once: true\n\n- name: Kube-router | Start Resources\n kube:\n name: \"kube-router\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n filename: \"{{ kube_config_dir }}/kube-router.yml\"\n resource: \"ds\"\n namespace: \"kube-system\"\n state: \"latest\"\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n run_once: true\n\n- name: Kube-router | Wait for kube-router pods to be ready\n command: \"{{ kubectl }} -n kube-system get pods -l k8s-app=kube-router -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'\" # noqa ignore-errors\n register: pods_not_ready\n until: pods_not_ready.stdout.find(\"kube-router\")==-1\n retries: 30\n delay: 10\n ignore_errors: true\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n run_once: true\n changed_when: false\n" }, { "path": "roles/network_plugin/kube-router/tasks/reset.yml", "content": "---\n- name: Reset | check kube-dummy-if network device\n stat:\n path: /sys/class/net/kube-dummy-if\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kube_dummy_if\n\n- name: Reset | remove the network device created by kube-router\n command: ip link del kube-dummy-if\n when: kube_dummy_if.stat.exists\n\n- name: Check kube-bridge exists\n stat:\n path: /sys/class/net/kube-bridge\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kube_bridge_if\n\n- name: Reset | donw the network bridge create by kube-router\n command: ip link set kube-bridge down\n when: kube_bridge_if.stat.exists\n\n- name: Reset | remove the network bridge create by kube-router\n command: ip link del kube-bridge\n when: kube_bridge_if.stat.exists\n" }, { "path": "roles/network_plugin/kube-router/templates/cni-conf.json.j2", "content": "{\n \"cniVersion\":\"0.3.0\",\n \"name\":\"kubernetes\",\n \"plugins\":[\n {\n \"name\":\"kubernetes\",\n \"type\":\"bridge\",\n \"bridge\":\"kube-bridge\",\n \"isDefaultGateway\":true,\n{% if kube_router_support_hairpin_mode %}\n \"hairpinMode\":true,\n{% endif %}\n \"ipam\":{\n{% if host_subnet is defined %}\n \"subnet\": \"{{ host_subnet }}\",\n{% endif %}\n \"type\":\"host-local\"\n }\n },\n {\n \"type\":\"portmap\",\n \"capabilities\":{\n \"portMappings\":true\n }\n }\n ]\n}\n" }, { "path": "roles/network_plugin/kube-router/templates/kube-router.yml.j2", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n labels:\n k8s-app: kube-router\n tier: node\n name: kube-router\n namespace: kube-system\nspec:\n minReadySeconds: 3\n updateStrategy:\n rollingUpdate:\n maxUnavailable: 1\n type: RollingUpdate\n selector:\n matchLabels:\n k8s-app: kube-router\n tier: node\n template:\n metadata:\n labels:\n k8s-app: kube-router\n tier: node\n annotations:\n{% if kube_router_enable_metrics %}\n prometheus.io/path: {{ kube_router_metrics_path }}\n prometheus.io/port: \"{{ kube_router_metrics_port }}\"\n prometheus.io/scrape: \"true\"\n{% endif %}\n spec:\n priorityClassName: system-node-critical\n serviceAccountName: kube-router\n containers:\n - name: kube-router\n image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }}\n imagePullPolicy: {{ k8s_image_pull_policy }}\n args:\n - --run-router={{ kube_router_run_router | bool }}\n - --run-firewall={{ kube_router_run_firewall | bool }}\n - --run-service-proxy={{ kube_router_run_service_proxy | bool }}\n - --kubeconfig=/var/lib/kube-router/kubeconfig\n - --bgp-graceful-restart={{ kube_router_bgp_graceful_restart }}\n{% if kube_router_advertise_cluster_ip %}\n - --advertise-cluster-ip\n{% endif %}\n{% if kube_router_advertise_external_ip %}\n - --advertise-external-ip\n{% endif %}\n{% if kube_router_advertise_loadbalancer_ip %}\n - --advertise-loadbalancer-ip\n{% endif %}\n{% if kube_router_cluster_asn %}\n - --cluster-asn={{ kube_router_cluster_asn }}\n{% endif %}\n{% if kube_router_peer_router_asns %}\n - --peer-router-asns={{ kube_router_peer_router_asns }}\n{% endif %}\n{% if kube_router_peer_router_ips %}\n - --peer-router-ips={{ kube_router_peer_router_ips }}\n{% endif %}\n{% if kube_router_peer_router_ports %}\n - --peer-router-ports={{ kube_router_peer_router_ports }}\n{% endif %}\n{% if kube_router_enable_metrics %}\n - --metrics-path={{ kube_router_metrics_path }}\n - --metrics-port={{ kube_router_metrics_port }}\n{% endif %}\n{% if kube_router_enable_dsr %}\n{% if container_manager == \"docker\" %}\n - --runtime-endpoint=unix:///var/run/docker.sock\n{% endif %}\n{% if container_manager == \"containerd\" %}\n{% endif %}\n - --runtime-endpoint=unix:///run/containerd/containerd.sock\n{% endif %}\n{% for arg in kube_router_extra_args %}\n - \"{{ arg }}\"\n{% endfor %}\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: KUBE_ROUTER_CNI_CONF_FILE\n value: /etc/cni/net.d/10-kuberouter.conflist\n livenessProbe:\n httpGet:\n path: /healthz\n port: 20244\n initialDelaySeconds: 10\n periodSeconds: 3\n resources:\n requests:\n cpu: 250m\n memory: 250Mi\n securityContext:\n privileged: true\n volumeMounts:\n{% if kube_router_enable_dsr %}\n{% if container_manager == \"docker\" %}\n - name: docker-socket\n mountPath: /var/run/docker.sock\n readOnly: true\n{% endif %}\n{% if container_manager == \"containerd\" %}\n - name: containerd-socket\n mountPath: /run/containerd/containerd.sock\n readOnly: true\n{% endif %}\n{% endif %}\n - name: lib-modules\n mountPath: /lib/modules\n readOnly: true\n - name: cni-conf-dir\n mountPath: /etc/cni/net.d\n - name: kubeconfig\n mountPath: /var/lib/kube-router\n readOnly: true\n - name: xtables-lock\n mountPath: /run/xtables.lock\n readOnly: false\n{% if kube_router_enable_metrics %}\n ports:\n - containerPort: {{ kube_router_metrics_port }}\n hostPort: {{ kube_router_metrics_port }}\n name: metrics\n protocol: TCP\n{% endif %}\n hostNetwork: true\n dnsPolicy: {{ kube_router_dns_policy }}\n{% if kube_router_enable_dsr %}\n hostIPC: true\n hostPID: true\n{% endif %}\n tolerations:\n - operator: Exists\n volumes:\n{% if kube_router_enable_dsr %}\n{% if container_manager == \"docker\" %}\n - name: docker-socket\n hostPath:\n path: /var/run/docker.sock\n type: Socket\n{% endif %}\n{% if container_manager == \"containerd\" %}\n - name: containerd-socket\n hostPath:\n path: /run/containerd/containerd.sock\n type: Socket\n{% endif %}\n{% endif %}\n - name: lib-modules\n hostPath:\n path: /lib/modules\n - name: cni-conf-dir\n hostPath:\n path: /etc/cni/net.d\n - name: kubeconfig\n hostPath:\n path: /var/lib/kube-router\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: kube-router\n namespace: kube-system\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: kube-router\n namespace: kube-system\nrules:\n - apiGroups:\n - \"\"\n resources:\n - namespaces\n - pods\n - services\n - nodes\n - endpoints\n verbs:\n - list\n - get\n - watch\n - apiGroups:\n - \"networking.k8s.io\"\n resources:\n - networkpolicies\n verbs:\n - list\n - get\n - watch\n - apiGroups:\n - extensions\n resources:\n - networkpolicies\n verbs:\n - get\n - list\n - watch\n - apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - get\n - list\n - watch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: kube-router\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: kube-router\nsubjects:\n- kind: ServiceAccount\n name: kube-router\n namespace: kube-system\n" }, { "path": "roles/network_plugin/kube-router/templates/kubeconfig.yml.j2", "content": "apiVersion: v1\nkind: Config\nclusterCIDR: {{ kube_pods_subnets }}\nclusters:\n- name: cluster\n cluster:\n certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n server: {{ kube_apiserver_endpoint }}\nusers:\n- name: kube-router\n user:\n tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\ncontexts:\n- context:\n cluster: cluster\n user: kube-router\n name: kube-router-context\ncurrent-context: kube-router-context\n" }, { "path": "roles/network_plugin/macvlan/defaults/main.yml", "content": "---\nmacvlan_interface: eth0\nenable_nat_default_gateway: true\n\n# sysctl_file_path to add sysctl conf to\nsysctl_file_path: \"/etc/sysctl.d/99-sysctl.conf\"\n" }, { "path": "roles/network_plugin/macvlan/files/ifdown-local", "content": "#!/bin/bash\n\nPOSTDOWNNAME=\"/etc/sysconfig/network-scripts/post-down-$1\"\nif [ -x $POSTDOWNNAME ]; then\n exec $POSTDOWNNAME\nfi\n" }, { "path": "roles/network_plugin/macvlan/files/ifdown-macvlan", "content": "#!/bin/bash\n#\n# initscripts-macvlan\n# Copyright (C) 2014 Lars Kellogg-Stedman\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\n. /etc/init.d/functions\n\ncd /etc/sysconfig/network-scripts\n. ./network-functions\n\n[ -f ../network ] && . ../network\n\nCONFIG=${1}\n\nneed_config ${CONFIG}\n\nsource_config\n\nOTHERSCRIPT=\"/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}\"\n\nif [ ! -x ${OTHERSCRIPT} ]; then\n\tOTHERSCRIPT=\"/etc/sysconfig/network-scripts/ifdown-eth\"\nfi\n\n${OTHERSCRIPT} ${CONFIG}\n\nip link del ${DEVICE} type ${TYPE:-macvlan}\n" }, { "path": "roles/network_plugin/macvlan/files/ifup-local", "content": "#!/bin/bash\n\nPOSTUPNAME=\"/etc/sysconfig/network-scripts/post-up-$1\"\nif [ -x $POSTUPNAME ]; then\n exec $POSTUPNAME\nfi\n" }, { "path": "roles/network_plugin/macvlan/files/ifup-macvlan", "content": "#!/bin/bash\n#\n# initscripts-macvlan\n# Copyright (C) 2014 Lars Kellogg-Stedman\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\n. /etc/init.d/functions\n\ncd /etc/sysconfig/network-scripts\n. ./network-functions\n\n[ -f ../network ] && . ../network\n\nCONFIG=${1}\n\nneed_config ${CONFIG}\n\nsource_config\n\nOTHERSCRIPT=\"/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}\"\n\nif [ ! -x ${OTHERSCRIPT} ]; then\n\tOTHERSCRIPT=\"/etc/sysconfig/network-scripts/ifup-eth\"\nfi\n\nip link add \\\n\tlink ${MACVLAN_PARENT} \\\n\tname ${DEVICE} \\\n\ttype ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private}\n\n${OTHERSCRIPT} ${CONFIG}\n" }, { "path": "roles/network_plugin/macvlan/handlers/main.yml", "content": "---\n- name: Macvlan | reload network\n service:\n # noqa: jinja[spacing]\n name: >-\n {% if ansible_os_family == \"RedHat\" -%}\n network\n {%- elif ansible_distribution == \"Ubuntu\" and ansible_distribution_release == \"bionic\" -%}\n systemd-networkd\n {%- elif ansible_os_family == \"Debian\" -%}\n networking\n {%- endif %}\n state: restarted\n when: not ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"] and kube_network_plugin not in ['calico']\n listen: Macvlan | restart network\n" }, { "path": "roles/network_plugin/macvlan/meta/main.yml", "content": "---\ndependencies:\n - role: network_plugin/cni\n" }, { "path": "roles/network_plugin/macvlan/tasks/main.yml", "content": "---\n- name: Macvlan | Retrieve Pod Cidr\n command: \"{{ kubectl }} get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'\"\n changed_when: false\n register: node_pod_cidr_cmd\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Macvlan | set node_pod_cidr\n set_fact:\n node_pod_cidr: \"{{ node_pod_cidr_cmd.stdout }}\"\n\n- name: Macvlan | Retrieve default gateway network interface\n become: false\n raw: ip -4 route list 0/0 | sed 's/.*dev \\([[:alnum:]]*\\).*/\\1/'\n changed_when: false\n register: node_default_gateway_interface_cmd\n\n- name: Macvlan | set node_default_gateway_interface\n set_fact:\n node_default_gateway_interface: \"{{ node_default_gateway_interface_cmd.stdout | trim }}\"\n\n- name: Macvlan | Install network gateway interface on debian\n template:\n src: debian-network-macvlan.cfg.j2\n dest: /etc/network/interfaces.d/60-mac0.cfg\n mode: \"0644\"\n notify: Macvlan | restart network\n when: ansible_os_family in [\"Debian\"]\n\n- name: Install macvlan config on RH distros\n when: ansible_os_family == \"RedHat\"\n block:\n - name: Macvlan | Install macvlan script on centos\n copy:\n src: \"{{ item }}\"\n dest: /etc/sysconfig/network-scripts/\n owner: root\n group: root\n mode: \"0755\"\n with_fileglob:\n - files/*\n\n - name: Macvlan | Install post-up script on centos\n copy:\n src: \"files/ifup-local\"\n dest: /sbin/\n owner: root\n group: root\n mode: \"0755\"\n when: enable_nat_default_gateway\n\n - name: Macvlan | Install network gateway interface on centos\n template:\n src: \"{{ item.src }}.j2\"\n dest: \"/etc/sysconfig/network-scripts/{{ item.dst }}\"\n mode: \"0644\"\n with_items:\n - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 }\n - {src: centos-routes-macvlan.cfg, dst: route-mac0 }\n - {src: centos-postup-macvlan.cfg, dst: post-up-mac0 }\n notify: Macvlan | restart network\n\n- name: Install macvlan config on Flatcar\n when: ansible_os_family in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n block:\n - name: Macvlan | Install service nat via gateway on Flatcar Container Linux\n template:\n src: coreos-service-nat_ouside.j2\n dest: /etc/systemd/system/enable_nat_ouside.service\n mode: \"0644\"\n when: enable_nat_default_gateway\n\n - name: Macvlan | Enable service nat via gateway on Flatcar Container Linux\n command: \"{{ item }}\"\n with_items:\n - systemctl daemon-reload\n - systemctl enable enable_nat_ouside.service\n when: enable_nat_default_gateway\n\n - name: Macvlan | Install network gateway interface on Flatcar Container Linux\n template:\n src: \"{{ item.src }}.j2\"\n dest: \"/etc/systemd/network/{{ item.dst }}\"\n mode: \"0644\"\n with_items:\n - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev }\n - {src: coreos-interface-macvlan.cfg, dst: output.network }\n - {src: coreos-network-macvlan.cfg, dst: macvlan.network }\n notify: Macvlan | restart network\n\n- name: Macvlan | Install cni definition for Macvlan\n template:\n src: 10-macvlan.conf.j2\n dest: /etc/cni/net.d/10-macvlan.conf\n mode: \"0644\"\n\n- name: Macvlan | Install loopback definition for Macvlan\n template:\n src: 99-loopback.conf.j2\n dest: /etc/cni/net.d/99-loopback.conf\n mode: \"0644\"\n\n- name: Enable net.ipv4.conf.all.arp_notify in sysctl\n ansible.posix.sysctl:\n name: net.ipv4.conf.all.arp_notify\n value: 1\n sysctl_set: true\n sysctl_file: \"{{ sysctl_file_path }}\"\n state: present\n reload: true\n ignoreerrors: \"{{ sysctl_ignore_unknown_keys }}\"\n" }, { "path": "roles/network_plugin/macvlan/templates/10-macvlan.conf.j2", "content": "{\n \"cniVersion\": \"0.4.0\",\n \"name\": \"mynet\",\n \"type\": \"macvlan\",\n \"master\": \"{{ macvlan_interface }}\",\n \"hairpinMode\": true,\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"{{ node_pod_cidr }}\",\n \"routes\": [\n { \"dst\": \"0.0.0.0/0\" }\n ],\n \"gateway\": \"{{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}\"\n }\n}\n" }, { "path": "roles/network_plugin/macvlan/templates/99-loopback.conf.j2", "content": "{\n \"cniVersion\": \"0.2.0\",\n \"name\": \"lo\",\n \"type\": \"loopback\"\n}\n" }, { "path": "roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2", "content": "DEVICE=mac0\nDEVICETYPE=macvlan\nTYPE=macvlan\nBOOTPROTO=none\nONBOOT=yes\nNM_CONTROLLED=no\n\nMACVLAN_PARENT={{ macvlan_interface }}\nMACVLAN_MODE=bridge\n\nIPADDR={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}\nNETMASK={{ node_pod_cidr|ansible.utils.ipaddr('netmask') }}\nNETWORK={{ node_pod_cidr|ansible.utils.ipaddr('network') }}\n" }, { "path": "roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2", "content": "{% if enable_nat_default_gateway %}\niptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE\n{% endif %}\n" }, { "path": "roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2", "content": "{% if enable_nat_default_gateway %}\niptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE\n{% endif %}\n" }, { "path": "roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2", "content": "{% for host in groups['kube_node'] %}\n{% if hostvars[host]['access_ip'] is defined %}\n{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}\n{{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}\n{% endif %}\n{% endif %}\n{% endfor %}\n" }, { "path": "roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2", "content": "[NetDev]\nName=mac0\nKind=macvlan\n\n[MACVLAN]\nMode=bridge\n" }, { "path": "roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2", "content": "[Match]\nName={{ macvlan_interface }}\n\n[Network]\nMACVLAN=mac0\nDHCP=yes\n" }, { "path": "roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2", "content": "[Match]\nName=mac0\n\n[Network]\nAddress={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}/{{ node_pod_cidr|ansible.utils.ipaddr('prefix') }}\n\n{% for host in groups['kube_node'] %}\n{% if hostvars[host]['access_ip'] is defined %}\n{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}\n[Route]\nGateway={{ hostvars[host]['access_ip'] }}\nDestination={{ hostvars[host]['node_pod_cidr'] }}\nGatewayOnlink=yes\n\n{% endif %}\n{% endif %}\n{% endfor %}\n" }, { "path": "roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2", "content": "[Service]\nType=oneshot\nExecStart=/bin/bash -c \"iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE\"\n\n[Install]\nWantedBy=sys-subsystem-net-devices-mac0.device\n" }, { "path": "roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2", "content": "auto mac0\niface mac0 inet static\n address {{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}\n network {{ node_pod_cidr|ansible.utils.ipaddr('network') }}\n netmask {{ node_pod_cidr|ansible.utils.ipaddr('netmask') }}\n broadcast {{ node_pod_cidr|ansible.utils.ipaddr('broadcast') }}\n pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge\n{% for host in groups['kube_node'] %}\n{% if hostvars[host]['access_ip'] is defined %}\n{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}\n post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}\n{% endif %}\n{% endif %}\n{% endfor %}\n{% if enable_nat_default_gateway %}\n post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE\n{% endif %}\n{% for host in groups['kube_node'] %}\n{% if hostvars[host]['access_ip'] is defined %}\n{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}\n post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}\n{% endif %}\n{% endif %}\n{% endfor %}\n post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE\n post-down ip link delete mac0\n" }, { "path": "roles/network_plugin/multus/defaults/main.yml", "content": "---\nmultus_conf_file: \"auto\"\nmultus_cni_conf_dir_host: \"/etc/cni/net.d\"\nmultus_cni_bin_dir_host: \"/opt/cni/bin\"\nmultus_cni_run_dir_host: \"/run\"\nmultus_cni_conf_dir: \"{{ ('/host', multus_cni_conf_dir_host) | join }}\"\nmultus_cni_bin_dir: \"{{ ('/host', multus_cni_bin_dir_host) | join }}\"\nmultus_cni_run_dir: \"{{ ('/host', multus_cni_run_dir_host) | join }}\"\nmultus_kubeconfig_file_host: \"{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}\"\nmultus_namespace_isolation: false\n" }, { "path": "roles/network_plugin/multus/files/multus-clusterrole.yml", "content": "---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: multus\nrules:\n - apiGroups: [\"k8s.cni.cncf.io\"]\n resources:\n - '*'\n verbs:\n - '*'\n - apiGroups:\n - \"\"\n resources:\n - pods\n - pods/status\n verbs:\n - get\n - update\n - apiGroups:\n - \"\"\n - events.k8s.io\n resources:\n - events\n verbs:\n - create\n - patch\n - update\n" }, { "path": "roles/network_plugin/multus/files/multus-clusterrolebinding.yml", "content": "---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: multus\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: multus\nsubjects:\n- kind: ServiceAccount\n name: multus\n namespace: kube-system\n" }, { "path": "roles/network_plugin/multus/files/multus-crd.yml", "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: network-attachment-definitions.k8s.cni.cncf.io\nspec:\n group: k8s.cni.cncf.io\n scope: Namespaced\n names:\n plural: network-attachment-definitions\n singular: network-attachment-definition\n kind: NetworkAttachmentDefinition\n shortNames:\n - net-attach-def\n versions:\n - name: v1\n served: true\n storage: true\n schema:\n openAPIV3Schema:\n description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing\n Working Group to express the intent for attaching pods to one or more logical or physical\n networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'\n type: object\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this represen\n tation of an object. Servers should convert recognized schemas to the\n latest internal value, and may reject unrecognized values. More info:\n https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'\n type: object\n properties:\n config:\n description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'\n type: string\n" }, { "path": "roles/network_plugin/multus/files/multus-serviceaccount.yml", "content": "---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: multus\n namespace: kube-system\n" }, { "path": "roles/network_plugin/multus/meta/main.yml", "content": "---\ndependencies:\n - role: network_plugin/cni\n" }, { "path": "roles/network_plugin/multus/tasks/main.yml", "content": "---\n- name: Multus | Copy manifest files\n copy:\n src: \"{{ item.file }}\"\n dest: \"{{ kube_config_dir }}\"\n mode: \"0644\"\n with_items:\n - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition}\n - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount}\n - {name: multus-clusterrole, file: multus-clusterrole.yml, type: clusterrole}\n - {name: multus-clusterrolebinding, file: multus-clusterrolebinding.yml, type: clusterrolebinding}\n register: multus_manifest_1\n when: inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Multus | Check container engine type\n set_fact:\n container_manager_types: \"{{ ansible_play_hosts_all | map('extract', hostvars, ['container_manager']) | list | unique }}\"\n\n- name: Multus | Copy manifest templates\n template:\n src: multus-daemonset.yml.j2\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: multus-daemonset-containerd, file: multus-daemonset-containerd.yml, type: daemonset, engine: containerd }\n - {name: multus-daemonset-docker, file: multus-daemonset-docker.yml, type: daemonset, engine: docker }\n - {name: multus-daemonset-crio, file: multus-daemonset-crio.yml, type: daemonset, engine: crio }\n register: multus_manifest_2\n vars:\n host_query: \"*|[?container_manager=='{{ container_manager }}']|[0].inventory_hostname\"\n vars_from_node: \"{{ hostvars | json_query(host_query) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when:\n - item.engine in container_manager_types\n - hostvars[inventory_hostname].container_manager == item.engine\n - inventory_hostname == vars_from_node\n\n- name: Multus | Start resources\n kube:\n name: \"{{ item.item.name }}\"\n namespace: \"kube-system\"\n kubectl: \"{{ bin_dir }}/kubectl\"\n resource: \"{{ item.item.type }}\"\n filename: \"{{ kube_config_dir }}/{{ item.item.file }}\"\n state: \"latest\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n with_items: \"{{ (multus_manifest_1.results | default([])) + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2') | map('default', []) | list | json_query('[].results')) }}\"\n loop_control:\n label: \"{{ item.item.name if item != None else 'skipped' }}\"\n vars:\n multus_nodes_list: \"{{ groups['k8s_cluster'] if ansible_play_batch | length == ansible_play_hosts_all | length else ansible_play_batch }}\"\n when:\n - not item is skipped\n" }, { "path": "roles/network_plugin/multus/templates/multus-daemonset.yml.j2", "content": "---\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n{% if container_manager_types | length >= 2 %}\n name: kube-multus-{{ container_manager }}-{{ image_arch }}\n{% else %}\n name: kube-multus-ds-{{ image_arch }}\n{% endif %}\n namespace: kube-system\n labels:\n tier: node\n app: multus\nspec:\n selector:\n matchLabels:\n tier: node\n app: multus\n template:\n metadata:\n labels:\n tier: node\n app: multus\n spec:\n hostNetwork: true\n dnsPolicy: ClusterFirstWithHostNet\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/arch: {{ image_arch }}\n{% if container_manager_types | length >= 2 %}\n kubespray.io/container_manager: {{ container_manager }}\n{% endif %}\n tolerations:\n - operator: Exists\n serviceAccountName: multus\n initContainers:\n - name: install-multus-binary\n image: {{ multus_image_repo }}:{{ multus_image_tag }}\n command: [\"/install_multus\"]\n args:\n - \"--type\"\n - \"thin\"\n resources:\n requests:\n cpu: \"10m\"\n memory: \"15Mi\"\n securityContext:\n privileged: true\n terminationMessagePolicy: FallbackToLogsOnError\n volumeMounts:\n - name: cnibin\n mountPath: {{ multus_cni_bin_dir }}\n mountPropagation: Bidirectional\n containers:\n - name: kube-multus\n image: {{ multus_image_repo }}:{{ multus_image_tag }}\n command: [\"/thin_entrypoint\"]\n args:\n - \"--cni-conf-dir={{ multus_cni_conf_dir }}\"\n - \"--multus-autoconfig-dir={{ multus_cni_conf_dir }}\"\n - \"--cni-bin-dir={{ multus_cni_bin_dir }}\"\n - \"--multus-conf-file={{ multus_conf_file }}\"\n - \"--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}\"\n - \"--namespace-isolation={{ multus_namespace_isolation | string | lower }}\"\n resources:\n requests:\n cpu: \"100m\"\n memory: \"90Mi\"\n limits:\n cpu: \"100m\"\n memory: \"90Mi\"\n securityContext:\n privileged: true\n{% if container_manager == 'crio' %}\n capabilities:\n add: [\"SYS_ADMIN\"]\n{% endif %}\n terminationMessagePolicy: FallbackToLogsOnError\n volumeMounts:\n{% if container_manager == 'crio' %}\n - name: run\n mountPath: {{ multus_cni_run_dir }}\n mountPropagation: HostToContainer\n{% endif %}\n - name: cni\n mountPath: {{ multus_cni_conf_dir }}\n - name: cnibin\n mountPath: {{ multus_cni_bin_dir }}\n volumes:\n{% if container_manager == 'crio' %}\n - name: run\n hostPath:\n path: {{ multus_cni_run_dir_host }}\n{% endif %}\n - name: cni\n hostPath:\n path: {{ multus_cni_conf_dir_host }}\n - name: cnibin\n hostPath:\n path: {{ multus_cni_bin_dir_host }}\n" }, { "path": "roles/network_plugin/ovn4nfv/tasks/main.yml", "content": "---\n- name: Ovn4nfv | Label control-plane node\n command: >-\n {{ kubectl }} label --overwrite node {{ groups['kube_control_plane'] | first }} ovn4nfv-k8s-plugin=ovn-control-plane\n when:\n - inventory_hostname == groups['kube_control_plane'][0]\n\n- name: Ovn4nfv | Create ovn4nfv-k8s manifests\n template:\n src: \"{{ item.file }}.j2\"\n dest: \"{{ kube_config_dir }}/{{ item.file }}\"\n mode: \"0644\"\n with_items:\n - {name: ovn-daemonset, file: ovn-daemonset.yml}\n - {name: ovn4nfv-k8s-plugin, file: ovn4nfv-k8s-plugin.yml}\n register: ovn4nfv_node_manifests\n" }, { "path": "roles/network_plugin/tasks/main.yml", "content": "---\n- name: Container Network Interface plugin\n include_role:\n name: network_plugin/cni\n when: kube_network_plugin != 'none'\n\n- name: Network plugin\n include_role:\n name: \"network_plugin/{{ kube_network_plugin }}\"\n apply:\n tags:\n - \"{{ kube_network_plugin }}\"\n - network\n when:\n - kube_network_plugin != 'none'\n tags:\n - cilium\n - calico\n - flannel\n - macvlan\n - kube-ovn\n - kube-router\n - custom_cni\n\n- name: Cilium additional\n include_role:\n name: network_plugin/cilium\n apply:\n tags:\n - cilium\n - network\n when:\n - kube_network_plugin != 'cilium'\n - cilium_deploy_additionally\n tags:\n - cilium\n\n- name: Multus\n include_role:\n name: network_plugin/multus\n apply:\n tags:\n - multus\n - network\n when: kube_network_plugin_multus\n tags:\n - multus\n" }, { "path": "roles/recover_control_plane/control-plane/defaults/main.yml", "content": "---\nbin_dir: /usr/local/bin\n" }, { "path": "roles/recover_control_plane/control-plane/tasks/main.yml", "content": "---\n- name: Wait for apiserver\n command: \"{{ kubectl }} get nodes\"\n environment:\n KUBECONFIG: \"{{ ansible_env.HOME | default('/root') }}/.kube/config\"\n register: apiserver_is_ready\n until: apiserver_is_ready.rc == 0\n retries: 6\n delay: 10\n changed_when: false\n when: groups['broken_kube_control_plane']\n\n- name: Delete broken kube_control_plane nodes from cluster\n command: \"{{ kubectl }} delete node {{ item }}\"\n environment:\n KUBECONFIG: \"{{ ansible_env.HOME | default('/root') }}/.kube/config\"\n with_items: \"{{ groups['broken_kube_control_plane'] }}\"\n register: delete_broken_kube_control_plane_nodes\n failed_when: false\n when: groups['broken_kube_control_plane']\n\n- name: Fail if unable to delete broken kube_control_plane nodes from cluster\n fail:\n msg: \"Unable to delete broken kube_control_plane node: {{ item.item }}\"\n loop: \"{{ delete_broken_kube_control_plane_nodes.results }}\"\n changed_when: false\n when:\n - groups['broken_kube_control_plane']\n - \"item.rc != 0 and not 'NotFound' in item.stderr\"\n" }, { "path": "roles/recover_control_plane/etcd/tasks/main.yml", "content": "---\n- name: Get etcd endpoint health\n command: \"{{ bin_dir }}/etcdctl endpoint health\"\n register: etcd_endpoint_health\n ignore_errors: true # noqa ignore-errors\n changed_when: false\n check_mode: false\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n when:\n - groups['broken_etcd']\n\n- name: Set healthy fact\n set_fact:\n healthy: \"{{ etcd_endpoint_health.stderr is match('Error: unhealthy cluster') }}\"\n when:\n - groups['broken_etcd']\n\n- name: Set has_quorum fact\n set_fact:\n has_quorum: \"{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}\"\n when:\n - groups['broken_etcd']\n\n- name: Recover lost etcd quorum\n include_tasks: recover_lost_quorum.yml\n when:\n - groups['broken_etcd']\n - not has_quorum\n\n- name: Remove etcd data dir\n file:\n path: \"{{ etcd_data_dir }}\"\n state: absent\n delegate_to: \"{{ item }}\"\n with_items: \"{{ groups['broken_etcd'] }}\"\n ignore_errors: true # noqa ignore-errors\n ignore_unreachable: true\n when:\n - groups['broken_etcd']\n - has_quorum\n\n- name: Delete old certificates\n shell: \"rm {{ etcd_cert_dir }}/*{{ item }}*\"\n with_items: \"{{ groups['broken_etcd'] }}\"\n register: delete_old_cerificates\n ignore_errors: true\n when: groups['broken_etcd']\n\n- name: Fail if unable to delete old certificates\n fail:\n msg: \"Unable to delete old certificates for: {{ item.item }}\"\n loop: \"{{ delete_old_cerificates.results }}\"\n changed_when: false\n when:\n - groups['broken_etcd']\n - \"item.rc != 0 and not 'No such file or directory' in item.stderr\"\n\n- name: Get etcd cluster members\n command: \"{{ bin_dir }}/etcdctl member list\"\n register: member_list\n changed_when: false\n check_mode: false\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n when:\n - groups['broken_etcd']\n - not healthy\n - has_quorum\n\n- name: Remove broken cluster members\n command: \"{{ bin_dir }}/etcdctl member remove {{ item[1].replace(' ', '').split(',')[0] }}\"\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n with_nested:\n - \"{{ groups['broken_etcd'] }}\"\n - \"{{ member_list.stdout_lines }}\"\n when:\n - groups['broken_etcd']\n - not healthy\n - has_quorum\n - hostvars[item[0]]['etcd_member_name'] == item[1].replace(' ', '').split(',')[2]\n" }, { "path": "roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml", "content": "---\n- name: Save etcd snapshot\n command: \"{{ bin_dir }}/etcdctl snapshot save /tmp/snapshot.db\"\n environment:\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses.split(',') | first }}\"\n ETCDCTL_API: \"3\"\n when: etcd_snapshot is not defined\n\n- name: Transfer etcd snapshot to host\n copy:\n src: \"{{ etcd_snapshot }}\"\n dest: /tmp/snapshot.db\n mode: \"0640\"\n when: etcd_snapshot is defined\n\n- name: Stop etcd\n systemd_service:\n name: etcd\n state: stopped\n\n- name: Remove etcd data-dir\n file:\n path: \"{{ etcd_data_dir }}\"\n state: absent\n\n- name: Restore etcd snapshot # noqa command-instead-of-shell\n shell: \"{{ bin_dir }}/etcdctl snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}\"\n environment:\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n ETCDCTL_API: \"3\"\n\n- name: Remove etcd snapshot\n file:\n path: /tmp/snapshot.db\n state: absent\n\n- name: Change etcd data-dir owner\n file:\n path: \"{{ etcd_data_dir }}\"\n owner: etcd\n group: etcd\n recurse: true\n\n- name: Reconfigure etcd\n replace:\n path: /etc/etcd.env\n regexp: \"^(ETCD_INITIAL_CLUSTER=).*\"\n replace: '\\1{{ etcd_member_name }}={{ etcd_peer_url }}'\n\n- name: Start etcd\n systemd_service:\n name: etcd\n state: started\n" }, { "path": "roles/recover_control_plane/post-recover/tasks/main.yml", "content": "---\n# TODO: Figure out why kubeadm does not fix this\n- name: Set etcd-servers fact\n set_fact:\n # noqa: jinja[spacing]\n etcd_servers: >-\n {% for host in groups['etcd'] -%}\n {% if not loop.last -%}\n https://{{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:2379,\n {%- endif -%}\n {%- if loop.last -%}\n https://{{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:2379\n {%- endif -%}\n {%- endfor -%}\n\n- name: Update apiserver etcd-servers list\n replace:\n path: /etc/kubernetes/manifests/kube-apiserver.yaml\n regexp: \"(etcd-servers=).*\"\n replace: \"\\\\1{{ etcd_servers }}\"\n" }, { "path": "roles/remove-node/post-remove/defaults/main.yml", "content": "---\ndelete_node_retries: 10\ndelete_node_delay_seconds: 3\n" }, { "path": "roles/remove-node/post-remove/tasks/main.yml", "content": "---\n- name: Remove-node | Delete node\n command: \"{{ kubectl }} delete node {{ kube_override_hostname }}\"\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n when:\n - groups['kube_control_plane'] | length > 0\n # ignore servers that are not nodes\n - ('k8s_cluster' in group_names) and kube_override_hostname in nodes.stdout_lines\n retries: \"{{ delete_node_retries }}\"\n # Sometimes the api-server can have a short window of indisponibility when we delete a control plane node\n delay: \"{{ delete_node_delay_seconds }}\"\n register: result\n until: result is not failed\n" }, { "path": "roles/remove-node/pre-remove/tasks/main.yml", "content": "---\n- name: Warn for usage of deprecated role\n fail:\n msg: remove-node/pre-remove is deprecated, switch to remove_node/pre_remove\n ignore_errors: true # noqa ignore-errors\n run_once: true\n\n- name: Compat for direct role import\n import_role:\n name: remove_node/pre_remove\n" }, { "path": "roles/remove-node/remove-etcd-node/tasks/main.yml", "content": "---\n- name: Remove etcd member from cluster\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ kube_cert_dir + '/etcd/server.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '.pem' }}\"\n ETCDCTL_KEY: \"{{ kube_cert_dir + '/etcd/server.key' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '-key.pem' }}\"\n ETCDCTL_CACERT: \"{{ kube_cert_dir + '/etcd/ca.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/ca.pem' }}\"\n ETCDCTL_ENDPOINTS: \"https://127.0.0.1:2379\"\n delegate_to: \"{{ groups['etcd'] | first }}\"\n block:\n - name: Lookup members infos\n command: \"{{ bin_dir }}/etcdctl member list -w json\"\n register: etcd_members\n changed_when: false\n check_mode: false\n tags:\n - facts\n - name: Remove member from cluster\n command:\n argv:\n - \"{{ bin_dir }}/etcdctl\"\n - member\n - remove\n - \"{{ '%x' | format(etcd_removed_nodes[0].ID) }}\"\n vars:\n etcd_removed_nodes: \"{{ (etcd_members.stdout | from_json).members | selectattr('peerURLs.0', '==', etcd_peer_url) }}\"\n # This should always have at most one member, since the etcd_peer_url should be unique in the etcd cluster\n when: etcd_removed_nodes != []\n register: etcd_removal_output\n changed_when: \"'Removed member' in etcd_removal_output.stdout\"\n" }, { "path": "roles/remove_node/pre_remove/defaults/main.yml", "content": "---\nallow_ungraceful_removal: false\ndrain_grace_period: 300\ndrain_timeout: 360s\ndrain_retries: 3\ndrain_retry_delay_seconds: 10\n" }, { "path": "roles/remove_node/pre_remove/tasks/main.yml", "content": "---\n- name: Remove-node | List nodes\n command: >-\n {{ kubectl }} get nodes -o go-template={% raw %}'{{ range .items }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}'{% endraw %}\n register: nodes\n when:\n - groups['kube_control_plane'] | length > 0\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n changed_when: false\n run_once: true\n\n- name: Remove-node | Drain node except daemonsets resource\n command: >-\n {{ kubectl }} drain\n --force\n --ignore-daemonsets\n --grace-period {{ drain_grace_period }}\n --timeout {{ drain_timeout }}\n --delete-emptydir-data {{ kube_override_hostname }}\n async: \"{{ (drain_timeout | regex_replace('s$', '') | int) + 120 }}\"\n poll: 15\n when:\n - groups['kube_control_plane'] | length > 0\n # ignore servers that are not nodes\n - kube_override_hostname in nodes.stdout_lines\n register: result\n failed_when: result.rc != 0 and not allow_ungraceful_removal\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n until: result.rc == 0 or allow_ungraceful_removal\n retries: \"{{ drain_retries }}\"\n delay: \"{{ drain_retry_delay_seconds }}\"\n\n- name: Remove-node | Wait until Volumes will be detached from the node\n command: >-\n {{ kubectl }} get volumeattachments -o go-template={% raw %}'{{ range .items }}{{ .spec.nodeName }}{{ \"\\n\" }}{{ end }}'{% endraw %}\n register: nodes_with_volumes\n delegate_to: \"{{ groups['kube_control_plane'] | first }}\"\n changed_when: false\n until: not (kube_override_hostname in nodes_with_volumes.stdout_lines)\n retries: 3\n delay: \"{{ drain_grace_period }}\"\n when:\n - groups['kube_control_plane'] | length > 0\n - not allow_ungraceful_removal\n - kube_override_hostname in nodes.stdout_lines\n" }, { "path": "roles/reset/defaults/main.yml", "content": "---\nflush_iptables: true\nreset_restart_network: true\n\n# crictl stop container grace period\ncri_stop_containers_grace_period: 0\n" }, { "path": "roles/reset/tasks/main.yml", "content": "---\n- name: Reset | stop services\n service:\n name: \"{{ item }}\"\n state: stopped\n enabled: false\n with_items:\n - kubelet.service\n - cri-dockerd.service\n - cri-dockerd.socket\n failed_when: false\n tags:\n - services\n\n- name: Reset | remove services\n file:\n path: \"/etc/systemd/system/{{ item }}\"\n state: absent\n with_items:\n - kubelet.service\n - cri-dockerd.service\n - cri-dockerd.socket\n - calico-node.service\n - containerd.service.d/http-proxy.conf\n - crio.service.d/http-proxy.conf\n - k8s-certs-renew.service\n - k8s-certs-renew.timer\n register: services_removed\n tags:\n - services\n - containerd\n - crio\n\n- name: Reset | Remove Docker\n include_role:\n name: container-engine/docker\n tasks_from: reset\n when: container_manager == 'docker'\n tags:\n - docker\n\n- name: Reset | systemctl daemon-reload # noqa no-handler\n systemd_service:\n daemon_reload: true\n when: services_removed.changed\n\n- name: Reset | check if crictl is present\n stat:\n path: \"{{ bin_dir }}/crictl\"\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: crictl\n\n- name: Reset | stop all cri containers\n shell: \"set -o pipefail && {{ bin_dir }}/crictl ps -q | xargs -r {{ bin_dir }}/crictl -t 60s stop -t {{ cri_stop_containers_grace_period }}\"\n args:\n executable: /bin/bash\n register: remove_all_cri_containers\n retries: 5\n until: remove_all_cri_containers.rc == 0\n delay: 5\n tags:\n - crio\n - containerd\n when:\n - crictl.stat.exists\n - container_manager in [\"crio\", \"containerd\"]\n - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined\n ignore_errors: true # noqa ignore-errors\n\n- name: Reset | force remove all cri containers\n command: \"{{ bin_dir }}/crictl rm -a -f\"\n register: remove_all_cri_containers\n retries: 5\n until: remove_all_cri_containers.rc == 0\n delay: 5\n tags:\n - crio\n - containerd\n when:\n - crictl.stat.exists\n - container_manager in [\"crio\", \"containerd\"]\n - deploy_container_engine\n - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined\n ignore_errors: true # noqa ignore-errors\n\n- name: Reset | stop and disable crio service\n service:\n name: crio\n state: stopped\n enabled: false\n failed_when: false\n tags: [ crio ]\n when: container_manager == \"crio\"\n\n- name: Reset | forcefully wipe CRI-O's container and image storage\n command: \"crio wipe -f\"\n failed_when: false\n tags: [ crio ]\n when: container_manager == \"crio\"\n\n- name: Reset | stop all cri pods\n shell: \"set -o pipefail && {{ bin_dir }}/crictl pods -q | xargs -r {{ bin_dir }}/crictl -t 60s stopp\"\n args:\n executable: /bin/bash\n register: remove_all_cri_containers\n retries: 5\n until: remove_all_cri_containers.rc == 0\n delay: 5\n tags: [ containerd ]\n when:\n - crictl.stat.exists\n - container_manager == \"containerd\"\n - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined\n ignore_errors: true # noqa ignore-errors\n\n- name: Reset | force remove all cri pods\n block:\n - name: Reset | force remove all cri pods\n command: \"{{ bin_dir }}/crictl rmp -a -f\"\n register: remove_all_cri_containers\n retries: 5\n until: remove_all_cri_containers.rc == 0\n delay: 5\n tags: [ containerd ]\n when:\n - crictl.stat.exists\n - container_manager == \"containerd\"\n - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined\n\n rescue:\n - name: Reset | force remove all cri pods (rescue)\n shell: \"ip netns list | cut -d' ' -f 1 | xargs -n1 ip netns delete && {{ bin_dir }}/crictl rmp -a -f\"\n ignore_errors: true # noqa ignore-errors\n changed_when: true\n\n- name: Reset | stop containerd and etcd services\n service:\n name: \"{{ item }}\"\n state: stopped\n enabled: false\n with_items:\n - containerd.service\n - etcd.service\n - etcd-events.service\n failed_when: false\n tags:\n - services\n\n- name: Reset | remove containerd and etcd services\n file:\n path: \"/etc/systemd/system/{{ item }}\"\n state: absent\n with_items:\n - containerd.service\n - etcd.service\n - etcd-events.service\n register: services_removed_secondary\n tags:\n - services\n - containerd\n\n- name: Reset | systemctl daemon-reload # noqa no-handler\n systemd_service:\n daemon_reload: true\n when: services_removed_secondary.changed\n\n- name: Reset | gather mounted kubelet dirs\n shell: set -o pipefail && mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac\n args:\n executable: /bin/bash\n check_mode: false\n register: mounted_dirs\n failed_when: false\n changed_when: false\n tags:\n - mounts\n\n- name: Reset | unmount kubelet dirs\n command: umount -f {{ item }}\n with_items: \"{{ mounted_dirs.stdout_lines }}\"\n register: umount_dir\n when: mounted_dirs\n retries: 4\n until: umount_dir.rc == 0\n delay: 5\n tags:\n - mounts\n\n- name: Set IPv4 iptables default policies to ACCEPT\n iptables:\n chain: \"{{ item }}\"\n policy: ACCEPT\n with_items:\n - INPUT\n - FORWARD\n - OUTPUT\n when: flush_iptables | bool and ipv4_stack\n tags:\n - iptables\n\n- name: Flush iptables\n iptables:\n table: \"{{ item }}\"\n flush: true\n with_items:\n - filter\n - nat\n - mangle\n - raw\n when: flush_iptables | bool and ipv4_stack\n tags:\n - iptables\n\n- name: Delete IPv4 user-defined chains # noqa command-instead-of-module\n command: iptables -X\n when: flush_iptables | bool and ipv4_stack\n tags:\n - iptables\n\n- name: Set IPv6 ip6tables default policies to ACCEPT\n iptables:\n chain: \"{{ item }}\"\n policy: ACCEPT\n ip_version: ipv6\n with_items:\n - INPUT\n - FORWARD\n - OUTPUT\n when: flush_iptables | bool and ipv6_stack\n tags:\n - ip6tables\n\n- name: Flush ip6tables\n iptables:\n table: \"{{ item }}\"\n flush: true\n ip_version: ipv6\n with_items:\n - filter\n - nat\n - mangle\n - raw\n when: flush_iptables | bool and ipv6_stack\n tags:\n - ip6tables\n\n- name: Delete IPv6 user-defined chains # noqa command-instead-of-module\n command: ip6tables -X\n when: flush_iptables | bool and ipv6_stack\n tags:\n - ip6tables\n\n- name: Clear IPVS virtual server table\n command: \"ipvsadm -C\"\n ignore_errors: true # noqa ignore-errors\n when:\n - kube_proxy_mode == 'ipvs' and 'k8s_cluster' in group_names\n\n- name: Reset | check kube-ipvs0 network device\n stat:\n path: /sys/class/net/kube-ipvs0\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: kube_ipvs0\n\n- name: Reset | Remove kube-ipvs0\n command: \"ip link del kube-ipvs0\"\n when:\n - kube_proxy_mode == 'ipvs'\n - kube_ipvs0.stat.exists\n\n- name: Reset | check nodelocaldns network device\n stat:\n path: /sys/class/net/nodelocaldns\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: nodelocaldns_device\n\n- name: Reset | Remove nodelocaldns\n command: \"ip link del nodelocaldns\"\n when:\n - enable_nodelocaldns | default(false) | bool\n - nodelocaldns_device.stat.exists\n\n- name: Reset | Check whether /var/lib/kubelet directory exists\n stat:\n path: /var/lib/kubelet\n get_attributes: false\n get_checksum: false\n get_mime: false\n register: var_lib_kubelet_directory\n\n- name: Reset | Find files/dirs with immutable flag in /var/lib/kubelet\n command: lsattr -laR /var/lib/kubelet/\n become: true\n register: var_lib_kubelet_files_dirs_w_attrs\n changed_when: false\n no_log: true\n when: var_lib_kubelet_directory.stat.exists\n\n- name: Reset | Remove immutable flag from files/dirs in /var/lib/kubelet\n file:\n path: \"{{ filedir_path }}\"\n state: touch\n attributes: \"-i\"\n mode: \"0644\"\n loop: \"{{ var_lib_kubelet_files_dirs_w_attrs.stdout_lines | select('search', 'Immutable') | list }}\"\n loop_control:\n loop_var: file_dir_line\n label: \"{{ filedir_path }}\"\n vars:\n filedir_path: \"{{ file_dir_line.split(' ')[0] }}\"\n when: var_lib_kubelet_directory.stat.exists\n\n- name: Reset | delete some files and directories\n file:\n path: \"{{ item }}\"\n state: absent\n with_items:\n - \"{{ kube_config_dir }}\"\n - /var/lib/kubelet\n - \"{{ containerd_storage_dir }}\"\n - \"{{ ansible_env.HOME | default('/root') }}/.kube\"\n - \"{{ ansible_env.HOME | default('/root') }}/.helm\"\n - \"{{ ansible_env.HOME | default('/root') }}/.config/helm\"\n - \"{{ ansible_env.HOME | default('/root') }}/.cache/helm\"\n - \"{{ ansible_env.HOME | default('/root') }}/.local/share/helm\"\n - \"{{ etcd_data_dir }}\"\n - \"{{ etcd_events_data_dir }}\"\n - \"{{ etcd_config_dir }}\"\n - /var/log/calico\n - /var/log/openvswitch\n - /var/log/ovn\n - /var/log/kube-ovn\n - /var/log/containers\n - /etc/cni\n - /etc/nerdctl\n - \"{{ nginx_config_dir }}\"\n - /etc/systemd/resolved.conf.d/kubespray.conf\n - /etc/etcd.env\n - /etc/calico\n - /etc/NetworkManager/conf.d/calico.conf\n - /etc/NetworkManager/conf.d/dns.conf\n - /etc/NetworkManager/conf.d/k8s.conf\n - /opt/cni\n - /etc/dhcp/dhclient.d/zdnsupdate.sh\n - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate\n - /run/flannel\n - /etc/flannel\n - /run/kubernetes\n - /usr/local/share/ca-certificates/etcd-ca.crt\n - /usr/local/share/ca-certificates/kube-ca.crt\n - /etc/ssl/certs/etcd-ca.pem\n - /etc/ssl/certs/kube-ca.pem\n - /etc/pki/ca-trust/source/anchors/etcd-ca.crt\n - /etc/pki/ca-trust/source/anchors/kube-ca.crt\n - /var/log/pods/\n - \"{{ bin_dir }}/kubelet\"\n - \"{{ bin_dir }}/cri-dockerd\"\n - \"{{ bin_dir }}/etcd-scripts\"\n - \"{{ bin_dir }}/etcd\"\n - \"{{ bin_dir }}/etcd-events\"\n - \"{{ bin_dir }}/etcdctl\"\n - \"{{ bin_dir }}/etcdctl.sh\"\n - \"{{ bin_dir }}/kubernetes-scripts\"\n - \"{{ bin_dir }}/kubectl\"\n - \"{{ bin_dir }}/kubeadm\"\n - \"{{ bin_dir }}/helm\"\n - \"{{ bin_dir }}/calicoctl\"\n - \"{{ bin_dir }}/calicoctl.sh\"\n - \"{{ bin_dir }}/calico-upgrade\"\n - \"{{ bin_dir }}/crictl\"\n - \"{{ bin_dir }}/nerdctl\"\n - \"{{ bin_dir }}/netctl\"\n - \"{{ bin_dir }}/k8s-certs-renew.sh\"\n - /var/lib/cni\n - /etc/openvswitch\n - /run/openvswitch\n - /var/lib/kube-router\n - /var/lib/calico\n - /etc/cilium\n - /run/calico\n - /etc/bash_completion.d/kubectl.sh\n - /etc/bash_completion.d/crictl\n - /etc/bash_completion.d/nerdctl\n - /etc/modules-load.d/kube_proxy-ipvs.conf\n - /etc/modules-load.d/kubespray-br_netfilter.conf\n - /etc/modules-load.d/kubespray-kata-containers.conf\n - /usr/libexec/kubernetes\n - /etc/origin/openvswitch\n - /etc/origin/ovn\n - \"{{ sysctl_file_path }}\"\n - /etc/crictl.yaml\n ignore_errors: true # noqa ignore-errors\n tags:\n - files\n\n- name: Reset | remove containerd binary files\n file:\n path: \"{{ containerd_bin_dir }}/{{ item }}\"\n state: absent\n with_items:\n - containerd\n - containerd-shim\n - containerd-shim-runc-v1\n - containerd-shim-runc-v2\n - containerd-stress\n - crictl\n - critest\n - ctd-decoder\n - ctr\n - runc\n ignore_errors: true # noqa ignore-errors\n when: container_manager == 'containerd'\n tags:\n - files\n\n- name: Reset | remove dns settings from dhclient.conf\n blockinfile:\n path: \"{{ item }}\"\n state: absent\n marker: \"# Ansible entries {mark}\"\n failed_when: false\n with_items:\n - /etc/dhclient.conf\n - /etc/dhcp/dhclient.conf\n tags:\n - files\n - dns\n\n- name: Reset | include file with reset tasks specific to the network_plugin if exists\n include_role:\n name: \"network_plugin/{{ kube_network_plugin }}\"\n tasks_from: reset\n when:\n - kube_network_plugin in ['flannel', 'cilium', 'kube-router', 'calico']\n tags:\n - network\n\n- name: Reset | Restart network\n when:\n - ansible_os_family not in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n - reset_restart_network | bool\n tags:\n - services\n - network\n block:\n - name: Gather active network services\n systemd:\n name: \"{{ item }}\"\n loop:\n - NetworkManager\n - systemd-networkd\n - networking\n - network\n register: service_status\n changed_when: false\n ignore_errors: true\n\n - name: Restart active network services\n systemd:\n name: \"{{ item }}\"\n state: restarted\n loop: \"{{ service_status.results | selectattr('status.ActiveState', '==', 'active') | map(attribute='item') }}\"\n" }, { "path": "roles/system_packages/defaults/main.yml", "content": "---\n# number of times package install task should be retried\npkg_install_retries: 4\npkg_install_timeout: \"{{ 5 * 60 }}\"\nyum_repo_dir: /etc/yum.repos.d\n" }, { "path": "roles/system_packages/tasks/main.yml", "content": "---\n- name: Gather OS information\n setup:\n gather_subset:\n - distribution\n - pkg_mgr\n\n- name: Update package management cache (zypper) - SUSE\n command: zypper -n --gpg-auto-import-keys ref\n register: make_cache_output\n until: make_cache_output is succeeded\n retries: 4\n delay: \"{{ retry_stagger | random + 3 }}\"\n when:\n - ansible_pkg_mgr == 'zypper'\n tags: bootstrap_os\n\n- name: Remove legacy docker repo file\n file:\n path: \"{{ yum_repo_dir }}/docker.repo\"\n state: absent\n when:\n - ansible_os_family == \"RedHat\"\n - not is_fedora_coreos\n\n- name: Install epel-release on RHEL derivatives\n package:\n name: epel-release\n state: present\n when:\n - ansible_os_family == \"RedHat\"\n - not is_fedora_coreos\n - epel_enabled | bool\n tags:\n - bootstrap_os\n\n# Remove this after ansible-core >= 2.19.0\n# See https://github.com/kubernetes-sigs/kubespray/pull/12138#issuecomment-3019304574\n- name: Install python3-libdnf5 on Fedora >= 41\n raw: >\n dnf install --assumeyes python3-libdnf5\n become: true\n retries: \"{{ pkg_install_retries }}\"\n when:\n - ansible_distribution == \"Fedora\"\n - ansible_distribution_major_version | int >= 41\n\n- name: Manage packages\n package:\n name: \"{{ item.packages }}\"\n state: \"{{ item.state }}\"\n update_cache: \"{{ true if ansible_pkg_mgr in ['zypper', 'apt', 'dnf'] else omit }}\"\n cache_valid_time: \"{{ 86400 if ansible_pkg_mgr == 'apt' else omit }}\" # 24h\n register: pkgs_task_result\n until: pkgs_task_result is succeeded\n retries: \"{{ pkg_install_retries }}\"\n delay: \"{{ retry_stagger | random + 3 }}\"\n when:\n - ansible_os_family not in [\"Flatcar\", \"Flatcar Container Linux by Kinvolk\"]\n - not is_fedora_coreos\n - item.packages != []\n loop:\n - packages: \"{{ pkgs_to_remove | dict2items | selectattr('value', 'ansible.builtin.all') | map(attribute='key') }}\"\n state: \"absent\"\n action_label: \"remove\"\n - packages: \"{{ pkgs | dict2items | selectattr('value', 'ansible.builtin.all') | map(attribute='key') }}\"\n state: \"present\"\n action_label: \"install\"\n loop_control:\n label: \"{{ item.action_label }}\"\n tags:\n - bootstrap_os\n timeout: \"{{ pkg_install_timeout }}\"\n" }, { "path": "roles/system_packages/vars/main.yml", "content": "---\npkgs_to_remove:\n systemd-timesyncd:\n - \"{{ ntp_enabled }}\"\n - \"{{ ntp_package == 'ntp' }}\"\n - \"{{ ansible_os_family == 'Debian' }}\"\npkgs:\n apparmor:\n - \"{{ ansible_os_family == 'Debian' }}\"\n apparmor-parser:\n - \"{{ ansible_os_family == 'Suse' }}\"\n apt-transport-https:\n - \"{{ ansible_os_family == 'Debian' }}\"\n aufs-tools:\n - \"{{ ansible_os_family == 'Debian' }}\"\n - \"{{ ansible_distribution_major_version == '10' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n bash-completion: []\n chrony:\n - \"{{ ntp_enabled }}\"\n - \"{{ ntp_package == 'chrony' }}\"\n conntrack:\n - \"{{ ansible_os_family in ['Debian', 'RedHat'] }}\"\n - \"{{ ansible_distribution != 'openEuler' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n conntrack-tools:\n - \"{{ ansible_os_family == 'Suse' or ansible_distribution in ['Amazon', 'openEuler'] }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n container-selinux:\n - \"{{ ansible_os_family == 'RedHat' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n containers-basic:\n - \"{{ ansible_os_family == 'ClearLinux' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n curl: []\n device-mapper:\n - \"{{ ansible_os_family == 'Suse' or ansible_distribution == 'openEuler' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n device-mapper-libs:\n - \"{{ ansible_os_family == 'RedHat' }}\"\n - \"{{ ansible_distribution != 'openEuler' }}\"\n e2fsprogs: []\n ebtables: []\n gnupg:\n - \"{{ ansible_distribution == 'Debian' }}\"\n - \"{{ ansible_distribution_major_version in ['11', '12'] }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n iproute:\n - \"{{ ansible_os_family == 'RedHat' }}\"\n iproute2:\n - \"{{ ansible_os_family != 'RedHat' }}\"\n ipset:\n - \"{{ kube_proxy_mode != 'ipvs' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n iptables:\n - \"{{ ansible_os_family in ['Debian', 'RedHat', 'Suse'] }}\"\n iputils:\n - \"{{ not ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk', 'Debian'] }}\"\n - \"{{ main_access_ip is defined }}\"\n - \"{{ ping_access_ip }}\"\n - \"{{ not is_fedora_coreos }}\"\n iputils-ping:\n - \"{{ ansible_os_family == 'Debian' }}\"\n - \"{{ main_access_ip is defined }}\"\n - \"{{ ping_access_ip }}\"\n ipvsadm:\n - \"{{ kube_proxy_mode == 'ipvs' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n libseccomp:\n - \"{{ ansible_os_family == 'RedHat' }}\"\n libseccomp2:\n - \"{{ ansible_os_family in ['Debian', 'Suse'] }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n libselinux-python:\n - \"{{ ansible_distribution == 'Amazon' }}\"\n libselinux-python3:\n - \"{{ ansible_distribution == 'Fedora' }}\"\n mergerfs:\n - \"{{ ansible_distribution == 'Debian' }}\"\n - \"{{ ansible_distribution_major_version == '12' }}\"\n nftables:\n - \"{{ kube_proxy_mode == 'nftables' }}\"\n - \"{{ 'k8s_cluster' in group_names }}\"\n nss:\n - \"{{ ansible_os_family == 'RedHat' }}\"\n ntp:\n - \"{{ ntp_enabled }}\"\n - \"{{ ntp_package == 'ntp' }}\"\n ntpsec:\n - \"{{ ntp_enabled }}\"\n - \"{{ ntp_package == 'ntpsec' }}\"\n openssl: []\n python-apt:\n - \"{{ ansible_os_family == 'Debian' }}\"\n - \"{{ ansible_distribution_major_version == '10' }}\"\n python-cryptography:\n - \"{{ ansible_os_family == 'Suse' and ansible_distribution_version is version('15.4', '<') }}\"\n python3-apt:\n - \"{{ ansible_os_family == 'Debian' }}\"\n - \"{{ ansible_distribution_major_version != '10' }}\"\n python3-cryptography:\n - \"{{ ansible_os_family == 'Suse' and ansible_distribution_version is version('15.4', '>=') }}\"\n python3-libselinux:\n - \"{{ ansible_distribution in ['RedHat', 'CentOS'] }}\"\n rsync: []\n socat: []\n software-properties-common:\n - \"{{ ansible_os_family == 'Debian' }}\"\n - \"{{ ansible_distribution_major_version != '13' }}\"\n tar: []\n unzip: []\n xfsprogs: []\n" }, { "path": "roles/upgrade/post-upgrade/defaults/main.yml", "content": "---\n# how long to wait for cilium after upgrade before uncordoning\nupgrade_post_cilium_wait_timeout: 120s\nupgrade_node_post_upgrade_confirm: false\nupgrade_node_post_upgrade_pause_seconds: 0\n" }, { "path": "roles/upgrade/post-upgrade/tasks/main.yml", "content": "---\n- name: Wait for cilium\n when:\n - needs_cordoning | default(false)\n - kube_network_plugin == 'cilium'\n command: >\n {{ kubectl }}\n wait pod -n kube-system -l k8s-app=cilium\n --field-selector 'spec.nodeName=={{ kube_override_hostname | default(inventory_hostname) }}'\n --for=condition=Ready\n --timeout={{ upgrade_post_cilium_wait_timeout }}\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n\n- name: Confirm node uncordon\n pause:\n echo: true\n prompt: \"Ready to uncordon node {{ kube_override_hostname | default(inventory_hostname) }}?\"\n when:\n - upgrade_node_post_upgrade_confirm\n\n- name: Wait before uncordoning node\n pause:\n seconds: \"{{ upgrade_node_post_upgrade_pause_seconds }}\"\n when:\n - not upgrade_node_post_upgrade_confirm\n - upgrade_node_post_upgrade_pause_seconds != 0\n\n- name: Run post upgrade hooks before uncordon\n loop: \"{{ post_upgrade_hooks | default([]) }}\"\n ansible.builtin.include_tasks: \"{{ item }}\"\n\n- name: Uncordon node\n command: \"{{ kubectl }} uncordon {{ kube_override_hostname | default(inventory_hostname) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when:\n - needs_cordoning | default(false)\n" }, { "path": "roles/upgrade/pre-upgrade/defaults/main.yml", "content": "---\ndrain_grace_period: 300\ndrain_timeout: 360s\ndrain_pod_selector: \"\"\ndrain_nodes: true\ndrain_retries: 3\ndrain_retry_delay_seconds: 10\n\ndrain_fallback_enabled: false\ndrain_fallback_grace_period: 300\ndrain_fallback_timeout: 360s\ndrain_fallback_retries: 0\ndrain_fallback_retry_delay_seconds: 10\n\nupgrade_node_always_cordon: false\nupgrade_node_uncordon_after_drain_failure: true\nupgrade_node_fail_if_drain_fails: true\n\nupgrade_node_confirm: false\nupgrade_node_pause_seconds: 0\n" }, { "path": "roles/upgrade/pre-upgrade/tasks/main.yml", "content": "---\n# Wait for upgrade\n- name: Confirm node upgrade\n pause:\n echo: true\n prompt: \"Ready to upgrade node {{ kube_override_hostname | default(inventory_hostname) }}? (Press Enter to continue or Ctrl+C for other options)\"\n when:\n - upgrade_node_confirm\n\n- name: Wait before upgrade node\n pause:\n seconds: \"{{ upgrade_node_pause_seconds }}\"\n when:\n - not upgrade_node_confirm\n - upgrade_node_pause_seconds != 0\n\n# Node Ready: type = ready, status = True\n# Node NotReady: type = ready, status = Unknown\n- name: See if node is in ready state\n command: >\n {{ kubectl }} get node {{ kube_override_hostname | default(inventory_hostname) }}\n -o jsonpath='{ range .status.conditions[?(@.type == \"Ready\")].status }{ @ }{ end }'\n register: kubectl_node_ready\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n failed_when: false\n changed_when: false\n\n# SchedulingDisabled: unschedulable = true\n# else unschedulable key doesn't exist\n- name: See if node is schedulable\n command: >\n {{ kubectl }} get node {{ kube_override_hostname | default(inventory_hostname) }}\n -o jsonpath='{ .spec.unschedulable }'\n register: kubectl_node_unschedulable\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n failed_when: false\n changed_when: false\n\n- name: Set if node needs cordoning\n set_fact:\n needs_cordoning: \"{{ (kubectl_node_ready.stdout == 'True' and not kubectl_node_unschedulable.stdout) or upgrade_node_always_cordon }}\"\n\n- name: Node draining\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n when:\n - needs_cordoning\n block:\n - name: Cordon node\n command: \"{{ kubectl }} cordon {{ kube_override_hostname | default(inventory_hostname) }}\"\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n changed_when: true\n\n - name: Drain node\n command: >-\n {{ kubectl }} drain\n --force\n --ignore-daemonsets\n --grace-period {{ drain_grace_period }}\n --timeout {{ drain_timeout }}\n --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }}\n {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %}\n async: \"{{ (drain_timeout | regex_replace('s$', '') | int) + 120 }}\"\n poll: 15\n when: drain_nodes\n register: result\n failed_when:\n - result.rc != 0\n - not drain_fallback_enabled\n until: result.rc == 0\n retries: \"{{ drain_retries }}\"\n delay: \"{{ drain_retry_delay_seconds }}\"\n\n - name: Drain node - fallback with disabled eviction\n when:\n - drain_nodes\n - drain_fallback_enabled\n - result.rc != 0\n command: >-\n {{ kubectl }} drain\n --force\n --ignore-daemonsets\n --grace-period {{ drain_fallback_grace_period }}\n --timeout {{ drain_fallback_timeout }}\n --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }}\n {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %}\n --disable-eviction\n async: \"{{ (drain_fallback_timeout | regex_replace('s$', '') | int) + 120 }}\"\n poll: 15\n register: drain_fallback_result\n until: drain_fallback_result.rc == 0\n retries: \"{{ drain_fallback_retries }}\"\n delay: \"{{ drain_fallback_retry_delay_seconds }}\"\n changed_when: drain_fallback_result.rc == 0\n\n rescue:\n - name: Set node back to schedulable\n command: \"{{ kubectl }} uncordon {{ kube_override_hostname | default(inventory_hostname) }}\"\n when: upgrade_node_uncordon_after_drain_failure\n - name: Fail after rescue\n fail:\n msg: \"Failed to drain node {{ kube_override_hostname | default(inventory_hostname) }}\"\n when: upgrade_node_fail_if_drain_fails\n" }, { "path": "roles/upgrade/system-upgrade/tasks/apt.yml", "content": "---\n- name: APT Dist-Upgrade\n apt:\n update_cache: true\n upgrade: dist\n autoremove: true\n dpkg_options: force-confold,force-confdef\n register: apt_upgrade\n\n- name: Reboot after APT Dist-Upgrade # noqa no-handler\n when:\n - apt_upgrade.changed or system_upgrade_reboot == 'always'\n - system_upgrade_reboot != 'never'\n reboot:\n" }, { "path": "roles/upgrade/system-upgrade/tasks/main.yml", "content": "---\n- name: APT upgrade\n when:\n - system_upgrade\n - ansible_os_family == \"Debian\"\n include_tasks: apt.yml\n tags:\n - system-upgrade-apt\n\n- name: YUM upgrade\n when:\n - system_upgrade\n - ansible_os_family == \"RedHat\"\n - not is_fedora_coreos\n include_tasks: yum.yml\n tags:\n - system-upgrade-yum\n" }, { "path": "roles/upgrade/system-upgrade/tasks/yum.yml", "content": "---\n- name: YUM upgrade all packages # noqa package-latest\n yum:\n name: '*'\n state: latest\n register: yum_upgrade\n\n- name: Reboot after YUM upgrade # noqa no-handler\n when:\n - yum_upgrade.changed or system_upgrade_reboot == 'always'\n - system_upgrade_reboot != 'never'\n reboot:\n" }, { "path": "roles/validate_inventory/meta/main.yml", "content": "---\ndependencies:\n - role: kubespray_defaults\n" }, { "path": "roles/validate_inventory/tasks/main.yml", "content": "---\n# This should only contains check of:\n# - the inventory\n# - extra variables\n# - ansible commandline\n# -> nothing depending on facts or similar cluster state\n# Checks depending on current state (of the nodes or the cluster)\n# should be in roles/kubernetes/preinstall/tasks/0040-verify-settings.yml\n- name: Fail if removed variables are used\n vars:\n # Always remove items from this list after the release in comments\n removed_vars:\n - kubelet_static_pod_path # 2.31.0\n removed_vars_found: \"{{ query('varnames', '^' + (removed_vars | join('|')) + '$') }}\"\n assert:\n that: removed_vars_found | length == 0\n fail_msg: \"Removed variables present: {{ removed_vars_found | join(', ') }}\"\n run_once: true\n\n- name: Stop if kube_control_plane group is empty\n assert:\n that: groups.get( 'kube_control_plane' )\n run_once: true\n when: not ignore_assert_errors\n\n- name: Stop if etcd group is empty in external etcd mode\n assert:\n that: groups.get('etcd') or etcd_deployment_type == 'kubeadm'\n fail_msg: \"Group 'etcd' cannot be empty in external etcd mode\"\n run_once: true\n when:\n - not ignore_assert_errors\n\n- name: Warn if `kube_network_plugin` is `none`\n debug:\n msg: |\n \"WARNING! => `kube_network_plugin` is set to `none`. The network configuration will be skipped.\n The cluster won't be ready to use, we recommend to select one of the available plugins\"\n when:\n - kube_network_plugin == 'none'\n\n- name: Stop if unsupported version of Kubernetes\n assert:\n that: kube_version is version(kube_version_min_required, '>=')\n msg: \"The current release of Kubespray only support newer version of Kubernetes than {{ kube_version_min_required }} - You are trying to apply {{ kube_version }}\"\n when: not ignore_assert_errors\n\n- name: \"Stop if known booleans are set as strings (Use JSON format on CLI: -e \\\"{'key': true }\\\")\"\n assert:\n that:\n - download_run_once | type_debug == 'bool'\n - download_always_pull | type_debug == 'bool'\n - helm_enabled | type_debug == 'bool'\n - openstack_lbaas_enabled | type_debug == 'bool'\n run_once: true\n when: not ignore_assert_errors\n\n- name: Stop if even number of etcd hosts\n assert:\n that: groups.get('etcd', groups.kube_control_plane) | length is not divisibleby 2\n run_once: true\n when:\n - not ignore_assert_errors\n\n# This assertion will fail on the safe side: One can indeed schedule more pods\n# on a node than the CIDR-range has space for when additional pods use the host\n# network namespace. It is impossible to ascertain the number of such pods at\n# provisioning time, so to establish a guarantee, we factor these out.\n# NOTICE: the check blatantly ignores the inet6-case\n- name: Guarantee that enough network address space is available for all pods\n assert:\n that: \"{{ (kubelet_max_pods | default(110)) | int <= (2 ** (32 - kube_network_node_prefix | int)) - 2 }}\"\n msg: \"Do not schedule more pods on a node than inet addresses are available.\"\n when:\n - not ignore_assert_errors\n - ('k8s_cluster' in group_names)\n - kube_network_plugin not in ['calico', 'none']\n - ipv4_stack | bool\n\n- name: Check cloud_provider value\n assert:\n that: cloud_provider == 'external'\n when:\n - cloud_provider\n - not ignore_assert_errors\n\n- name: Check external_cloud_provider value\n assert:\n that: external_cloud_provider in ['hcloud', 'huaweicloud', 'oci', 'openstack', 'vsphere', 'manual']\n when:\n - cloud_provider == 'external'\n - not ignore_assert_errors\n\n- name: \"Check that kube_service_addresses is a network range\"\n assert:\n that:\n - kube_service_addresses | ansible.utils.ipaddr('net')\n msg: \"kube_service_addresses = '{{ kube_service_addresses }}' is not a valid network range\"\n run_once: true\n when: ipv4_stack | bool\n\n- name: \"Check that kube_pods_subnet is a network range\"\n assert:\n that:\n - kube_pods_subnet | ansible.utils.ipaddr('net')\n msg: \"kube_pods_subnet = '{{ kube_pods_subnet }}' is not a valid network range\"\n run_once: true\n when: ipv4_stack | bool\n\n- name: \"Check that kube_pods_subnet does not collide with kube_service_addresses\"\n assert:\n that:\n - kube_pods_subnet | ansible.utils.ipaddr(kube_service_addresses) | string == 'None'\n msg: \"kube_pods_subnet cannot be the same network segment as kube_service_addresses\"\n run_once: true\n when: ipv4_stack | bool\n\n- name: \"Check that ipv4 IP range is enough for the nodes\"\n assert:\n that:\n - 2 ** (kube_network_node_prefix - kube_pods_subnet | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length\n msg: \"Not enough ipv4 IPs are available for the desired node count.\"\n when:\n - ipv4_stack | bool\n - kube_network_plugin != 'calico'\n run_once: true\n\n- name: \"Check that kube_service_addresses_ipv6 is a network range\"\n assert:\n that:\n - kube_service_addresses_ipv6 | ansible.utils.ipaddr('net')\n msg: \"kube_service_addresses_ipv6 = '{{ kube_service_addresses_ipv6 }}' is not a valid network range\"\n run_once: true\n when: ipv6_stack | bool\n\n- name: \"Check that kube_pods_subnet_ipv6 is a network range\"\n assert:\n that:\n - kube_pods_subnet_ipv6 | ansible.utils.ipaddr('net')\n msg: \"kube_pods_subnet_ipv6 = '{{ kube_pods_subnet_ipv6 }}' is not a valid network range\"\n run_once: true\n when: ipv6_stack | bool\n\n- name: \"Check that kube_pods_subnet_ipv6 does not collide with kube_service_addresses_ipv6\"\n assert:\n that:\n - kube_pods_subnet_ipv6 | ansible.utils.ipaddr(kube_service_addresses_ipv6) | string == 'None'\n msg: \"kube_pods_subnet_ipv6 cannot be the same network segment as kube_service_addresses_ipv6\"\n run_once: true\n when: ipv6_stack | bool\n\n- name: \"Check that ipv6 IP range is enough for the nodes\"\n assert:\n that:\n - 2 ** (kube_network_node_prefix_ipv6 - kube_pods_subnet_ipv6 | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length\n msg: \"Not enough ipv6 IPs are available for the desired node count.\"\n when:\n - ipv6_stack | bool\n - kube_network_plugin != 'calico'\n run_once: true\n\n- name: Stop if unsupported options selected\n assert:\n that:\n - kube_network_plugin in ['calico', 'flannel', 'cloud', 'cilium', 'cni', 'kube-ovn', 'kube-router', 'macvlan', 'custom_cni', 'none']\n - dns_mode in ['coredns', 'coredns_dual', 'manual', 'none']\n - kube_proxy_mode in ['iptables', 'ipvs', 'nftables']\n - cert_management in ['script', 'none']\n - resolvconf_mode in ['docker_dns', 'host_resolvconf', 'none']\n - etcd_deployment_type in ['host', 'docker', 'kubeadm']\n - etcd_deployment_type in ['host', 'kubeadm'] or container_manager == 'docker'\n - container_manager in ['docker', 'crio', 'containerd']\n msg: The selected choice is not supported\n run_once: true\n\n- name: Warn if `enable_dual_stack_networks` is set\n debug:\n msg: \"WARNING! => `enable_dual_stack_networks` deprecation. Please switch to using ipv4_stack and ipv6_stack.\"\n when:\n - enable_dual_stack_networks is defined\n\n- name: Stop if download_localhost is enabled but download_run_once is not\n assert:\n that: download_run_once\n msg: \"download_localhost requires enable download_run_once\"\n when: download_localhost\n\n- name: Stop if kata_containers_enabled is enabled when container_manager is docker\n assert:\n that: container_manager != 'docker'\n msg: \"kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details\"\n when: kata_containers_enabled\n\n- name: Stop if gvisor_enabled is enabled when container_manager is not containerd\n assert:\n that: container_manager == 'containerd'\n msg: \"gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details\"\n when: gvisor_enabled\n\n- name: Ensure minimum containerd version\n assert:\n that: containerd_version is version(containerd_min_version_required, '>=')\n msg: \"containerd_version is too low. Minimum version {{ containerd_min_version_required }}\"\n run_once: true\n when:\n - containerd_version not in ['latest', 'edge', 'stable']\n - container_manager == 'containerd'\n\n- name: Stop if auto_renew_certificates is enabled when certificates are managed externally (kube_external_ca_mode is true)\n assert:\n that: not auto_renew_certificates\n msg: \"Variable auto_renew_certificates must be disabled when CA are managed externally: kube_external_ca_mode = true\"\n when:\n - kube_external_ca_mode\n - not ignore_assert_errors\n\n- name: Download_file | Check if requested Kubernetes are supported\n assert:\n that:\n - kube_version in kubeadm_checksums[image_arch]\n - kube_version in kubelet_checksums[image_arch]\n - kube_version in kubectl_checksums[image_arch]\n msg: >-\n Kubernetes v{{ kube_version }} is not supported for {{ image_arch }}.\n Please check roles/kubespray_defaults/vars/main/checksums.yml for supported versions.\n" }, { "path": "roles/win_nodes/kubernetes_patch/defaults/main.yml", "content": "---\n\nkubernetes_user_manifests_path: \"{{ ansible_env.HOME }}/kube-manifests\"\nkube_proxy_nodeselector: \"kubernetes.io/os\"\n" }, { "path": "roles/win_nodes/kubernetes_patch/tasks/main.yml", "content": "---\n\n- name: Ensure that user manifests directory exists\n file:\n path: \"{{ kubernetes_user_manifests_path }}/kubernetes\"\n state: directory\n recurse: true\n tags: [init, cni]\n\n- name: Apply kube-proxy nodeselector\n tags: init\n when:\n - kube_proxy_deployed\n block:\n # Due to https://github.com/kubernetes/kubernetes/issues/58212 we cannot rely on exit code for \"kubectl patch\"\n - name: Check current nodeselector for kube-proxy daemonset\n command: >-\n {{ kubectl }}\n get ds kube-proxy --namespace=kube-system\n -o jsonpath={.spec.template.spec.nodeSelector.{{ kube_proxy_nodeselector | regex_replace('\\.', '\\\\.') }}}\n register: current_kube_proxy_state\n retries: 60\n delay: 5\n until: current_kube_proxy_state is succeeded\n changed_when: false\n\n - name: Apply nodeselector patch for kube-proxy daemonset\n command: >\n {{ kubectl }}\n patch ds kube-proxy --namespace=kube-system --type=strategic -p\n '{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"{{ kube_proxy_nodeselector }}\":\"linux\"} }}}}'\n register: patch_kube_proxy_state\n when: current_kube_proxy_state.stdout | trim | lower != \"linux\"\n\n - debug: # noqa name[missing]\n msg: \"{{ patch_kube_proxy_state.stdout_lines }}\"\n when: patch_kube_proxy_state is not skipped\n\n - debug: # noqa name[missing]\n msg: \"{{ patch_kube_proxy_state.stderr_lines }}\"\n when: patch_kube_proxy_state is not skipped\n" }, { "path": "scale.yml", "content": "---\n- name: Scale the cluster\n ansible.builtin.import_playbook: playbooks/scale.yml\n" }, { "path": "scripts/Dockerfile.j2", "content": "# syntax=docker/dockerfile:1\n\n# Use immutable image tags rather than mutable tags (like ubuntu:24.04)\nFROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b\n\n# Some tools like yamllint need this\n# Pip needs this as well at the moment to install ansible\n# (and potentially other packages)\n# See: https://github.com/pypa/pip/issues/10219\nENV LANG=C.UTF-8 \\\n DEBIAN_FRONTEND=noninteractive \\\n PYTHONDONTWRITEBYTECODE=1\n\nWORKDIR /kubespray\n\n# hadolint ignore=DL3008\nRUN --mount=type=cache,target=/var/cache/apt,sharing=locked \\\n apt-get update -q \\\n && apt-get install -yq --no-install-recommends \\\n curl \\\n python3 \\\n python3-pip \\\n sshpass \\\n vim \\\n rsync \\\n openssh-client \\\n && apt-get clean \\\n && rm -rf /var/lib/apt/lists/* /var/log/*\n\nRUN --mount=type=bind,source=requirements.txt,target=requirements.txt \\\n --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \\\n pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt \\\n && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \\;\n\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nRUN OS_ARCHITECTURE=$(dpkg --print-architecture) \\\n && curl -L \"https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl\" -o /usr/local/bin/kubectl \\\n && echo \"$(curl -L \"https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256\")\" /usr/local/bin/kubectl | sha256sum --check \\\n && chmod a+x /usr/local/bin/kubectl\n\nCOPY *.yml ./\nCOPY *.cfg ./\nCOPY roles ./roles\nCOPY contrib ./contrib\nCOPY inventory ./inventory\nCOPY library ./library\nCOPY extra_playbooks ./extra_playbooks\nCOPY playbooks ./playbooks\nCOPY plugins ./plugins\n" }, { "path": "scripts/assert-sorted-checksums.yml", "content": "#!/usr/bin/env ansible-playbook\n---\n- name: Verify correct structure of Kubespray variables\n hosts: localhost\n connection: local\n gather_facts: false\n vars:\n fallback_ip: 'bypass tasks in kubespray_defaults'\n _keys: \"{{ query('ansible.builtin.varnames', '^.+_checksums$') }}\"\n _values: \"{{ query('ansible.builtin.vars', *_keys) | map('dict2items') }}\"\n _components_archs_values: \"{{ _keys | zip(_values) | community.general.dict | dict2items | subelements('value') }}\"\n _minimal_data_needed: \"{{ _components_archs_values | map(attribute='0.key') | zip(_components_archs_values | map(attribute='1')) }}\"\n roles:\n - kubespray_defaults\n tasks:\n - name: Check all versions are strings\n assert:\n that: \"{{ item.1.value | reject('string') == [] }}\"\n quiet: true\n loop: \"{{ _minimal_data_needed }}\"\n loop_control:\n label: \"{{ item.0 }}:{{ item.1.key }}\"\n - name: Check all checksums are sorted by version\n vars:\n actual: \"{{ item.1.value.keys() | map('string') | reverse}}\"\n sorted: \"{{ item.1.value.keys() | map('string') | community.general.version_sort }}\"\n assert:\n that: actual == sorted\n quiet: true\n msg: \"{{ actual | ansible.utils.fact_diff(sorted) }}\"\n loop: \"{{ _minimal_data_needed }}\"\n loop_control:\n label: \"{{ item.0 }}:{{ item.1.key }}\"\n when:\n - item.1.value is not string\n - (item.1.value | dict2items)[0].value is string or\n (item.1.value | dict2items)[0].value is number\n # only do list, the others are checksums with a different structure\n - name: Include the packages list variable\n include_vars: ../roles/system_packages/vars/main.yml\n\n - name: Verify that the packages list is sorted\n loop:\n - pkgs_to_remove\n - pkgs\n vars:\n pkgs_lists: \"{{ lookup('vars', item).keys() | list }}\"\n ansible_distribution: irrelevant\n ansible_distribution_major_version: irrelevant\n ansible_distribution_minor_version: irrelevant\n ansible_distribution_version: 1.0\n ansible_os_family: irrelevant\n assert:\n that: \"pkgs_lists | sort == pkgs_lists\"\n fail_msg: \"{{ item }} is not sorted: {{ pkgs_lists | ansible.utils.fact_diff(pkgs_lists | sort) }}\"\n" }, { "path": "scripts/collect-info.yaml", "content": "---\n- name: Collect debug info\n hosts: all\n become: true\n gather_facts: false\n\n vars:\n docker_bin_dir: /usr/bin\n bin_dir: /usr/local/bin\n ansible_ssh_pipelining: true\n etcd_cert_dir: /etc/ssl/etcd/ssl\n kube_network_plugin: calico\n archive_dirname: collect-info\n commands:\n - name: timedate_info\n cmd: timedatectl status\n - name: kernel_info\n cmd: uname -r\n - name: docker_info\n cmd: \"{{ docker_bin_dir }}/docker info\"\n - name: ip_info\n cmd: ip -4 -o a\n - name: route_info\n cmd: ip ro\n - name: proc_info\n cmd: ps auxf | grep -v ]$\n - name: systemctl_failed_info\n cmd: systemctl --state=failed --no-pager\n - name: k8s_info\n cmd: \"{{ bin_dir }}/kubectl get all --all-namespaces -o wide\"\n - name: errors_info\n cmd: journalctl -p err --no-pager\n - name: etcd_info\n cmd: \"{{ bin_dir }}/etcdctl endpoint --cluster health\"\n - name: calico_info\n cmd: \"{{ bin_dir }}/calicoctl node status\"\n when: '{{ kube_network_plugin == \"calico\" }}'\n - name: calico_workload_info\n cmd: \"{{ bin_dir }}/calicoctl get workloadEndpoint -o wide\"\n when: '{{ kube_network_plugin == \"calico\" }}'\n - name: calico_pool_info\n cmd: \"{{ bin_dir }}/calicoctl get ippool -o wide\"\n when: '{{ kube_network_plugin == \"calico\" }}'\n - name: kube_describe_all\n cmd: \"{{ bin_dir }}/kubectl describe all --all-namespaces\"\n - name: kube_describe_nodes\n cmd: \"{{ bin_dir }}/kubectl describe nodes\"\n - name: kubelet_logs\n cmd: journalctl -u kubelet --no-pager\n - name: coredns_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=coredns -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done\"\n - name: apiserver_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-apiserver -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done\"\n - name: controller_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-controller-manager -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done\"\n - name: scheduler_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-scheduler -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done\"\n - name: proxy_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=kube-proxy -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done\"\n - name: nginx_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=kube-nginx -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done\"\n - name: flannel_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l app=flannel -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system flannel-container; done\"\n when: '{{ kube_network_plugin == \"flannel\" }}'\n - name: canal_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=canal-node -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system flannel; done\"\n when: '{{ kube_network_plugin == \"canal\" }}'\n - name: calico_policy_logs\n cmd: \"for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=calico-kube-controllers -o jsonpath={.items..metadata.name}`;\n do {{ bin_dir }}/kubectl logs ${i} -n kube-system ; done\"\n when: '{{ kube_network_plugin in [\"canal\", \"calico\"] }}'\n - name: helm_show_releases_history\n cmd: \"for i in `{{ bin_dir }}/helm list -q`; do {{ bin_dir }}/helm history ${i} --col-width=0; done\"\n when: \"{{ helm_enabled | default(true) }}\"\n\n logs:\n - /var/log/syslog\n - /var/log/daemon.log\n - /var/log/kern.log\n - /var/log/dpkg.log\n - /var/log/apt/history.log\n - /var/log/yum.log\n - /var/log/messages\n - /var/log/dmesg\n\n environment:\n ETCDCTL_API: \"3\"\n ETCDCTL_CERT: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem\"\n ETCDCTL_KEY: \"{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem\"\n ETCDCTL_CACERT: \"{{ etcd_cert_dir }}/ca.pem\"\n ETCDCTL_ENDPOINTS: \"{{ etcd_access_addresses }}\"\n\n tasks:\n - name: Set etcd_access_addresses\n set_fact:\n etcd_access_addresses: |-\n {% for item in groups['etcd'] -%}\n https://{{ item }}:2379{% if not loop.last %},{% endif %}\n {%- endfor %}\n when: \"'etcd' in groups\"\n\n - name: Storing commands output\n shell: \"{{ item.cmd }} &> {{ item.name }}\"\n failed_when: false\n with_items: \"{{ commands }}\"\n when: item.when | default(True)\n no_log: true\n\n - name: Fetch results\n fetch:\n src: \"{{ item.name }}\"\n dest: \"/tmp/{{ archive_dirname }}/commands\"\n with_items: \"{{ commands }}\"\n when: item.when | default(True)\n failed_when: false\n\n - name: Fetch logs\n fetch:\n src: \"{{ item }}\"\n dest: \"/tmp/{{ archive_dirname }}/logs\"\n with_items: \"{{ logs }}\"\n failed_when: false\n\n - name: Pack results and logs\n community.general.archive:\n path: \"/tmp/{{ archive_dirname }}\"\n dest: \"{{ dir | default('.') }}/logs.tar.gz\"\n remove: true\n mode: \"0640\"\n delegate_to: localhost\n connection: local\n become: false\n run_once: true\n\n - name: Clean up collected command outputs\n file:\n path: \"{{ item.name }}\"\n state: absent\n with_items: \"{{ commands }}\"\n" }, { "path": "scripts/component_hash_update/pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools >= 61.0\",\n \"setuptools_scm >= 8.0\",\n]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"kubespray_component_hash_update\"\nversion = \"1.0.1\"\ndependencies = [\n \"more_itertools\",\n \"ruamel.yaml\",\n \"requests\",\n \"packaging\",\n]\n\nrequires-python = \">= 3.10\"\n\nauthors = [\n { name = \"Craig Rodrigues\", email = \"rodrigc@crodrigues.org\" },\n { name = \"Simon Wessel\" },\n { name = \"Max Gautier\", email = \"mg@max.gautier.name\" },\n]\nmaintainers = [\n { name = \"The Kubespray maintainers\" },\n]\n\ndescription = \"Download or compute hashes for new versions of components deployed by Kubespray\"\n\nclassifiers = [\n \"License :: OSI Approved :: Apache-2.0\",\n]\n\n[project.scripts]\nupdate-hashes = \"component_hash_update.download:main\"\n" }, { "path": "scripts/component_hash_update/src/component_hash_update/__init__.py", "content": "" }, { "path": "scripts/component_hash_update/src/component_hash_update/components.py", "content": "\"\"\"\nStatic download metadata for components updated by the update-hashes command.\n\"\"\"\n\ninfos = {\n \"calicoctl_binary\": {\n \"url\": \"https://github.com/projectcalico/calico/releases/download/v{version}/SHA256SUMS\",\n \"graphql_id\": \"R_kgDOA87D0g\",\n },\n \"calico_crds\": {\n \"url\": \"https://github.com/projectcalico/calico/raw/v{version}/manifests/crds.yaml\",\n \"graphql_id\": \"R_kgDOA87D0g\",\n \"binary\": True,\n },\n \"ciliumcli_binary\": {\n \"url\": \"https://github.com/cilium/cilium-cli/releases/download/v{version}/cilium-{os}-{arch}.tar.gz.sha256sum\",\n \"graphql_id\": \"R_kgDOE0nmLg\",\n },\n \"cni_binary\": {\n \"url\": \"https://github.com/containernetworking/plugins/releases/download/v{version}/cni-plugins-{os}-{arch}-v{version}.tgz.sha256\",\n \"graphql_id\": \"R_kgDOBQqEpg\",\n },\n \"containerd_archive\": {\n \"url\": \"https://github.com/containerd/containerd/releases/download/v{version}/containerd-{version}-{os}-{arch}.tar.gz.sha256sum\",\n \"graphql_id\": \"R_kgDOAr9FWA\",\n },\n \"containerd_static_archive\": {\n \"url\": \"https://github.com/containerd/containerd/releases/download/v{version}/containerd-static-{version}-{os}-{arch}.tar.gz.sha256sum\",\n \"graphql_id\": \"R_kgDOAr9FWA\",\n },\n \"cri_dockerd_archive\": {\n \"binary\": True,\n \"url\": \"https://github.com/Mirantis/cri-dockerd/releases/download/v{version}/cri-dockerd-{version}.{arch}.tgz\",\n \"graphql_id\": \"R_kgDOEvvLcQ\",\n },\n \"crictl\": {\n \"url\": \"https://github.com/kubernetes-sigs/cri-tools/releases/download/v{version}/crictl-v{version}-{os}-{arch}.tar.gz.sha256\",\n \"graphql_id\": \"R_kgDOBMdURA\",\n },\n \"crio_archive\": {\n \"url\": \"https://storage.googleapis.com/cri-o/artifacts/cri-o.{arch}.v{version}.tar.gz.sha256sum\",\n \"graphql_id\": \"R_kgDOBAr5pg\",\n },\n \"crun\": {\n \"url\": \"https://github.com/containers/crun/releases/download/{version}/crun-{version}-linux-{arch}\",\n \"binary\": True,\n \"graphql_id\": \"R_kgDOBip3vA\",\n },\n \"etcd_binary\": {\n \"url\": \"https://github.com/etcd-io/etcd/releases/download/v{version}/SHA256SUMS\",\n \"graphql_id\": \"R_kgDOAKtHtg\",\n },\n \"gvisor_containerd_shim_binary\": {\n \"url\": \"https://storage.googleapis.com/gvisor/releases/release/{version}/{alt_arch}/containerd-shim-runsc-v1.sha512\",\n \"hashtype\": \"sha512\",\n \"tags\": True,\n \"graphql_id\": \"R_kgDOB9IlXg\",\n },\n \"gvisor_runsc_binary\": {\n \"url\": \"https://storage.googleapis.com/gvisor/releases/release/{version}/{alt_arch}/runsc.sha512\",\n \"hashtype\": \"sha512\",\n \"tags\": True,\n \"graphql_id\": \"R_kgDOB9IlXg\",\n },\n \"kata_containers_binary\": {\n \"url\": \"https://github.com/kata-containers/kata-containers/releases/download/{version}/kata-static-{version}-{arch}.tar.xz\",\n \"binary\": True,\n \"graphql_id\": \"R_kgDOBsJsHQ\",\n },\n \"kubeadm\": {\n \"url\": \"https://dl.k8s.io/release/v{version}/bin/linux/{arch}/kubeadm.sha256\",\n \"graphql_id\": \"R_kgDOAToIkg\",\n },\n \"kubectl\": {\n \"url\": \"https://dl.k8s.io/release/v{version}/bin/linux/{arch}/kubectl.sha256\",\n \"graphql_id\": \"R_kgDOAToIkg\",\n },\n \"kubelet\": {\n \"url\": \"https://dl.k8s.io/release/v{version}/bin/linux/{arch}/kubelet.sha256\",\n \"graphql_id\": \"R_kgDOAToIkg\",\n },\n \"nerdctl_archive\": {\n \"url\": \"https://github.com/containerd/nerdctl/releases/download/v{version}/SHA256SUMS\",\n \"graphql_id\": \"R_kgDOEvuRnQ\",\n },\n \"runc\": {\n \"url\": \"https://github.com/opencontainers/runc/releases/download/v{version}/runc.sha256sum\",\n \"graphql_id\": \"R_kgDOAjP4QQ\",\n },\n \"skopeo_binary\": {\n \"url\": \"https://github.com/lework/skopeo-binary/releases/download/v{version}/skopeo-{os}-{arch}.sha256\",\n \"graphql_id\": \"R_kgDOHQ6J9w\",\n },\n \"youki\": {\n \"url\": \"https://github.com/youki-dev/youki/releases/download/v{version}/youki-{version}-{alt_arch}-gnu.tar.gz\",\n \"binary\": True,\n \"graphql_id\": \"R_kgDOFPvgPg\",\n },\n \"yq\": {\n \"url\": \"https://github.com/mikefarah/yq/releases/download/v{version}/checksums-bsd\", # see https://github.com/mikefarah/yq/pull/1691 for why we use this url\n \"graphql_id\": \"R_kgDOApOQGQ\",\n },\n \"argocd_install\": {\n \"url\": \"https://raw.githubusercontent.com/argoproj/argo-cd/v{version}/manifests/install.yaml\",\n \"graphql_id\": \"R_kgDOBzS60g\",\n \"binary\": True,\n \"hashtype\": \"sha256\",\n },\n \"gateway_api_standard_crds\": {\n \"url\": \"https://github.com/kubernetes-sigs/gateway-api/releases/download/v{version}/standard-install.yaml\",\n \"graphql_id\": \"R_kgDODQ6RZw\",\n \"binary\": True,\n },\n \"gateway_api_experimental_crds\": {\n \"url\": \"https://github.com/kubernetes-sigs/gateway-api/releases/download/v{version}/experimental-install.yaml\",\n \"graphql_id\": \"R_kgDODQ6RZw\",\n \"binary\": True,\n },\n \"prometheus_operator_crds\": {\n \"url\": \"https://github.com/prometheus-operator/prometheus-operator/releases/download/v{version}/stripped-down-crds.yaml\",\n \"graphql_id\": \"R_kgDOBBxPpw\",\n \"binary\": True,\n },\n}\n" }, { "path": "scripts/component_hash_update/src/component_hash_update/download.py", "content": "#!/usr/bin/env python3\n\n# After a new version of Kubernetes has been released,\n# run this script to update roles/kubespray_defaults/defaults/main/download.yml\n# with new hashes.\n\nimport sys\nimport os\nimport logging\nimport subprocess\n\nfrom itertools import groupby, chain\nfrom more_itertools import partition\nfrom functools import cache\nimport argparse\nimport requests\nimport hashlib\nfrom datetime import datetime\nfrom ruamel.yaml import YAML\nfrom packaging.version import Version, InvalidVersion\nfrom importlib.resources import files\nfrom pathlib import Path\n\nfrom typing import Optional, Any\n\nfrom . import components\n\nCHECKSUMS_YML = Path(\"roles/kubespray_defaults/vars/main/checksums.yml\")\n\nlogger = logging.getLogger(__name__)\n\n\ndef open_yaml(file: Path):\n yaml = YAML()\n yaml.explicit_start = True\n yaml.preserve_quotes = True\n yaml.width = 4096\n\n with open(file, \"r\") as checksums_yml:\n data = yaml.load(checksums_yml)\n\n return data, yaml\n\n\narch_alt_name = {\n \"amd64\": \"x86_64\",\n \"arm64\": \"aarch64\",\n \"ppc64le\": None,\n \"arm\": None,\n \"no_arch\": None,\n}\n\n# TODO: downloads not supported\n# helm_archive: PGP signatures\n\n# TODO:\n# different verification methods (gpg, cosign) ( needs download role changes) (or verify the sig in this script and only use the checksum in the playbook)\n# perf improvements (async)\n\n\ndef download_hash(downloads: {str: {str: Any}}) -> None:\n # Handle file with multiples hashes, with various formats.\n # the lambda is expected to produce a dictionary of hashes indexed by arch name\n download_hash_extract = {\n \"calicoctl_binary\": lambda hashes: {\n line.split(\"-\")[-1]: line.split()[0]\n for line in hashes.strip().split(\"\\n\")\n if line.count(\"-\") == 2 and line.split(\"-\")[-2] == \"linux\"\n },\n \"etcd_binary\": lambda hashes: {\n line.split(\"-\")[-1].removesuffix(\".tar.gz\"): line.split()[0]\n for line in hashes.strip().split(\"\\n\")\n if line.split(\"-\")[-2] == \"linux\"\n },\n \"nerdctl_archive\": lambda hashes: {\n line.split()[1].removesuffix(\".tar.gz\").split(\"-\")[3]: line.split()[0]\n for line in hashes.strip().split(\"\\n\")\n if [x for x in line.split(\" \") if x][1].split(\"-\")[2] == \"linux\"\n },\n \"runc\": lambda hashes: {\n parts[1].split(\".\")[1]: parts[0]\n for parts in (line.split() for line in hashes.split(\"\\n\")[3:9])\n },\n \"yq\": lambda rhashes_bsd: {\n pair[0].split(\"_\")[-1]: pair[1]\n # pair = (yq__, )\n for pair in (\n (line.split()[1][1:-1], line.split()[3])\n for line in rhashes_bsd.splitlines()\n if line.startswith(\"SHA256\")\n )\n if pair[0].startswith(\"yq\")\n and pair[0].split(\"_\")[1] == \"linux\"\n and not pair[0].endswith(\".tar.gz\")\n },\n }\n\n checksums_file = (\n Path(\n subprocess.Popen(\n [\"git\", \"rev-parse\", \"--show-toplevel\"], stdout=subprocess.PIPE\n )\n .communicate()[0]\n .rstrip()\n .decode(\"utf-8\")\n )\n / CHECKSUMS_YML\n )\n logger.info(\"Opening checksums file %s...\", checksums_file)\n data, yaml = open_yaml(checksums_file)\n s = requests.Session()\n\n @cache\n def _get_hash_by_arch(download: str, version: str) -> {str: str}:\n\n hash_file = s.get(\n downloads[download][\"url\"].format(\n version=version,\n os=\"linux\",\n ),\n allow_redirects=True,\n )\n hash_file.raise_for_status()\n return download_hash_extract[download](hash_file.content.decode())\n\n releases, tags = map(\n dict, partition(lambda r: r[1].get(\"tags\", False), downloads.items())\n )\n unique_release_ids = list(dict.fromkeys(\n r[\"graphql_id\"] for r in releases.values()\n ))\n unique_tag_ids = list(dict.fromkeys(\n t[\"graphql_id\"] for t in tags.values()\n ))\n response = s.post(\n \"https://api.github.com/graphql\",\n json={\n \"query\": files(__package__).joinpath(\"list_releases.graphql\").read_text(),\n \"variables\": {\n \"with_releases\": unique_release_ids,\n \"with_tags\": unique_tag_ids,\n },\n },\n headers={\n \"Authorization\": f\"Bearer {os.environ['API_KEY']}\",\n },\n )\n if \"x-ratelimit-used\" in response.headers._store:\n logger.info(\n \"Github graphQL API ratelimit status: used %s of %s. Next reset at %s\",\n response.headers[\"X-RateLimit-Used\"],\n response.headers[\"X-RateLimit-Limit\"],\n datetime.fromtimestamp(int(response.headers[\"X-RateLimit-Reset\"])),\n )\n response.raise_for_status()\n\n def valid_version(possible_version: str) -> Optional[Version]:\n try:\n return Version(possible_version)\n except InvalidVersion:\n return None\n\n resp_data = response.json()[\"data\"]\n release_versions_by_id = {\n gql_id: {\n v\n for r in repo[\"releases\"][\"nodes\"]\n if not r[\"isPrerelease\"]\n and (v := valid_version(r[\"tagName\"])) is not None\n }\n for gql_id, repo in zip(unique_release_ids, resp_data[\"with_releases\"])\n }\n tag_versions_by_id = {\n gql_id: {\n v\n for t in repo[\"refs\"][\"nodes\"]\n if (v := valid_version(t[\"name\"].removeprefix(\"release-\")))\n is not None\n }\n for gql_id, repo in zip(unique_tag_ids, resp_data[\"with_tags\"])\n }\n github_versions = {}\n for name, info in releases.items():\n github_versions[name] = release_versions_by_id[info[\"graphql_id\"]]\n for name, info in tags.items():\n github_versions[name] = tag_versions_by_id[info[\"graphql_id\"]]\n\n components_supported_arch = {\n component.removesuffix(\"_checksums\"): [a for a in archs.keys()]\n for component, archs in data.items()\n }\n new_versions = {\n c: {\n v\n for v in github_versions[c]\n if any(\n v > version\n and (\n (v.major, v.minor) == (version.major, version.minor)\n or c.startswith(\"gvisor\")\n )\n for version in [\n max(minors)\n for _, minors in groupby(cur_v, lambda v: (v.minor, v.major))\n ]\n )\n # only get:\n # - patch versions (no minor or major bump) (exception for gvisor which does not have a major.minor.patch scheme\n # - newer ones (don't get old patch version)\n }\n - set(cur_v)\n for component, archs in data.items()\n if (c := component.removesuffix(\"_checksums\")) in downloads.keys()\n # this is only to bound cur_v in the scope\n and (\n cur_v := sorted(\n Version(str(k)) for k in next(archs.values().__iter__()).keys()\n )\n )\n }\n\n hash_set_to_0 = {\n c: {\n Version(str(v))\n for v, h in chain.from_iterable(a.items() for a in archs.values())\n if h == 0\n }\n for component, archs in data.items()\n if (c := component.removesuffix(\"_checksums\")) in downloads.keys()\n }\n\n def get_hash(component: str, version: Version, arch: str):\n if component in download_hash_extract:\n hashes = _get_hash_by_arch(component, version)\n return hashes[arch]\n else:\n hash_file = s.get(\n downloads[component][\"url\"].format(\n version=version,\n os=\"linux\",\n arch=arch,\n alt_arch=arch_alt_name[arch],\n ),\n allow_redirects=True,\n )\n hash_file.raise_for_status()\n if downloads[component].get(\"binary\", False):\n return hashlib.new(\n downloads[component].get(\"hashtype\", \"sha256\"), hash_file.content\n ).hexdigest()\n return hash_file.content.decode().split()[0]\n\n for component, versions in chain(new_versions.items(), hash_set_to_0.items()):\n c = component + \"_checksums\"\n for arch in components_supported_arch[component]:\n for version in versions:\n data[c][arch][\n str(version)\n ] = f\"{downloads[component].get('hashtype', 'sha256')}:{get_hash(component, version, arch)}\"\n\n data[c] = {\n arch: {\n v: versions[v]\n for v in sorted(\n versions.keys(), key=lambda v: Version(str(v)), reverse=True\n )\n }\n for arch, versions in data[c].items()\n }\n\n with open(checksums_file, \"w\") as checksums_yml:\n yaml.dump(data, checksums_yml)\n logger.info(\"Updated %s\", checksums_file)\n\n\ndef main():\n\n logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n parser = argparse.ArgumentParser(\n description=f\"Add new patch versions hashes in {CHECKSUMS_YML}\",\n formatter_class=argparse.RawTextHelpFormatter,\n epilog=f\"\"\"\n This script only lookup new patch versions relative to those already existing\n in the data in {CHECKSUMS_YML},\n which means it won't add new major or minor versions.\n In order to add one of these, edit {CHECKSUMS_YML}\n by hand, adding the new versions with a patch number of 0 (or the lowest relevant patch versions)\n and a hash value of 0.\n ; then run this script.\n\n Note that the script will try to add the versions on all\n architecture keys already present for a given download target.\n\n EXAMPLES:\n\n crictl_checksums:\n ...\n amd64:\n + 1.30.0: 0\n 1.29.0: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb\n 1.28.0: 8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508\"\"\",\n )\n\n # Workaround for https://github.com/python/cpython/issues/53834#issuecomment-2060825835\n # Fixed in python 3.14\n class Choices(tuple):\n\n def __init__(self, _iterable=None, default=None):\n self.default = default or []\n\n def __contains__(self, item):\n return super().__contains__(item) or item == self.default\n\n choices = Choices(components.infos.keys(), default=list(components.infos.keys()))\n\n parser.add_argument(\n \"only\",\n nargs=\"*\",\n choices=choices,\n help=\"if provided, only obtain hashes for these compoments\",\n default=choices.default,\n )\n parser.add_argument(\n \"-e\",\n \"--exclude\",\n action=\"append\",\n choices=components.infos.keys(),\n help=\"do not obtain hashes for this component\",\n default=[],\n )\n\n args = parser.parse_args()\n download_hash(\n {k: components.infos[k] for k in (set(args.only) - set(args.exclude))}\n )\n" }, { "path": "scripts/component_hash_update/src/component_hash_update/list_releases.graphql", "content": "query($with_releases: [ID!]!, $with_tags: [ID!]!) {\n with_releases: nodes(ids: $with_releases) {\n\n ... on Repository {\n releases(first: 100) {\n nodes {\n tagName\n isPrerelease\n }\n }\n }\n }\n\n with_tags: nodes(ids: $with_tags) {\n\n ... on Repository {\n refs(refPrefix: \"refs/tags/\", last: 25) {\n nodes {\n name\n }\n }\n }\n }\n}\n" }, { "path": "scripts/galaxy_version.py", "content": "#!/usr/bin/env python\n\nimport subprocess\nimport ruamel.yaml\nimport os\n\nlast_tag = (\n subprocess.Popen(\n [\"git\", \"describe\", \"--tags\", \"--abbrev=0\"], stdout=subprocess.PIPE\n )\n .communicate()[0]\n .rstrip()\n .decode(\"utf-8\")\n .removeprefix(\"v\")\n .split(\".\")\n)\n# Use CI provided base ref if available, else use HEAD to guess\ngit_branch = os.getenv(\n \"GITHUB_BASE_REF\",\n (\n subprocess.Popen(\n [\"git\", \"rev-parse\", \"--abbrev-ref\", \"HEAD\"], stdout=subprocess.PIPE\n )\n .communicate()[0]\n .rstrip()\n .decode(\"utf-8\")\n ),\n)\nif git_branch.startswith(\"release\"):\n version_comp_index = 2\nelse:\n version_comp_index = 1\n\nlast_tag[version_comp_index] = str(int(last_tag[version_comp_index]) + 1)\nnew_tag = \".\".join(last_tag)\n\nyaml = ruamel.yaml.YAML()\nyaml.indent(mapping=2, sequence=4, offset=2)\nyaml.explicit_start = True\n\nwith open(\n \"galaxy.yml\",\n) as galaxy_yml:\n config = yaml.load(galaxy_yml)\n\nconfig[\"version\"] = new_tag\n\nwith open(\"galaxy.yml\", \"w\") as galaxy_yml:\n yaml.dump(config, galaxy_yml)\n" }, { "path": "scripts/gen_docs_sidebar.sh", "content": "#!/usr/bin/env bash\n\n# Generate documentation\n# This script generates a list of all the markdown files in the docs folder\n# and prints them in a markdown list format.\n# The script will print the name of the folder and the files inside it.\n# The script will also convert the folder and file names to a more human-readable format.\n# The script will ignore any files that are not markdown files.\n# Usage: bash scripts/gen_docs_sidebar.sh > docs/_sidebar.md\n\nexport LANG=C\n{\necho \"* [Readme](/)\"\n\nfor folder in $(find docs/*/ | sort -f); do\n # Check if it is a directory\n if [ -d \"$folder\" ]; then\n subdir=$(basename \"$folder\")\n subdir=${subdir//_/ } # Replace \"_\" with empty string\n subdir=$(echo \"$subdir\" | awk '{for(i=1;i<=NF;i++)sub(/./,toupper(substr($i,1,1)),$i)}1') # Convert first letter of each word to uppercase\n if [ -n \"$(find \"$folder\" -name '*.md' -type f)\" ]; then\n echo \"* $subdir\"\n fi\n for file in $(find docs/\"$(basename \"$folder\")\"/*.md | sort -f); do\n if [ -f \"$file\" ]; then\n FILE=$(basename \"$file\" .md)\n FILE=${FILE//_/ } # Replace \"_\" with empty string\n FILE=$(echo \"$FILE\" | awk '{for(i=1;i<=NF;i++)sub(/./,toupper(substr($i,1,1)),$i)}1') # Convert first letter of each word to uppercase\n echo \" * [$FILE](/$file)\"\n fi\n done\n fi\ndone\n} > docs/_sidebar.md\n" }, { "path": "scripts/get_node_ids.sh", "content": "#!/bin/sh\ngh api graphql -H \"X-Github-Next-Global-ID: 1\" -f query='{\n calicoctl_binary: repository(owner: \"projectcalico\", name: \"calico\") {\n id\n }\n ciliumcli_binary: repository(owner: \"cilium\", name: \"cilium-cli\") {\n id\n }\n crictl: repository(owner: \"kubernetes-sigs\", name: \"cri-tools\") {\n id\n }\n crio_archive: repository(owner: \"cri-o\", name: \"cri-o\") {\n id\n }\n etcd_binary: repository(owner: \"etcd-io\", name: \"etcd\") {\n id\n }\n kubectl: repository(owner: \"kubernetes\", name: \"kubernetes\") {\n id\n }\n nerdctl_archive: repository(owner: \"containerd\", name: \"nerdctl\") {\n id\n }\n runc: repository(owner: \"opencontainers\", name: \"runc\") {\n id\n }\n skopeo_binary: repository(owner: \"lework\", name: \"skopeo-binary\") {\n id\n }\n yq: repository(owner: \"mikefarah\", name: \"yq\") {\n id\n }\n youki: repository(owner: \"youki-dev\", name: \"youki\") {\n id\n }\n kubernetes: repository(owner: \"kubernetes\", name: \"kubernetes\") {\n id\n }\n cri_dockerd: repository(owner: \"Mirantis\", name: \"cri-dockerd\") {\n id\n }\n kata: repository(owner: \"kata-containers\", name: \"kata-containers\") {\n id\n }\n crun: repository(owner: \"containers\", name: \"crun\") {\n id\n }\n gvisor: repository(owner: \"google\", name: \"gvisor\") {\n id\n }\n argocd: repository(owner: \"argoproj\", name: \"argo-cd\") {\n id\n }\n\n}'\n" }, { "path": "scripts/gitlab-runner.sh", "content": "#!/bin/sh\n\ndocker run -d --name gitlab-runner --restart always -v /srv/gitlab-runner/cache:/srv/gitlab-runner/cache -v /srv/gitlab-runner/config:/etc/gitlab-runner -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:v1.10.0\n\n#\n#/srv/gitlab-runner/config# cat config.toml\n#concurrent = 10\n#check_interval = 1\n\n#[[runners]]\n# name = \"2edf3d71fe19\"\n# url = \"https://gitlab.com\"\n# token = \"THE TOKEN-CHANGEME\"\n# executor = \"docker\"\n# [runners.docker]\n# tls_verify = false\n# image = \"docker:latest\"\n# privileged = true\n# disable_cache = false\n# cache_dir = \"/srv/gitlab-runner/cache\"\n# volumes = [\"/var/run/docker.sock:/var/run/docker.sock\", \"/srv/gitlab-runner/cache:/cache:rw\"]\n# [runners.cache]\n" }, { "path": "scripts/openstack-cleanup/.gitignore", "content": "openrc\n" }, { "path": "scripts/openstack-cleanup/README.md", "content": "# openstack-cleanup\n\nTool to deletes openstack servers older than a specific age (default 4h).\n\nUseful to cleanup orphan servers that are left behind when CI is manually cancelled or fails unexpectedly.\n\n## Installation\n\n```shell\npip install -r requirements.txt\npython main.py --help\n```\n\n## Usage\n\n```console\n$ python main.py\nThis will delete VMs... (ctrl+c to cancel)\nWill delete server example1\nWill delete server example2\n```\n" }, { "path": "scripts/openstack-cleanup/main.py", "content": "#!/usr/bin/env python\nimport argparse\nimport openstack\nimport logging\nimport datetime\nimport time\n\nlog = logging.getLogger(__name__)\n\nparser = argparse.ArgumentParser(description=\"Cleanup OpenStack resources\")\n\nparser.add_argument(\n \"--hours\",\n type=int,\n default=4,\n help=\"Age (in hours) of VMs to cleanup (default: 4h)\",\n)\nparser.add_argument(\"--dry-run\", action=\"store_true\", help=\"Do not delete anything\")\n\nargs = parser.parse_args()\n\noldest_allowed = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(\n hours=args.hours\n)\n\n\ndef main():\n logging.basicConfig(level=logging.INFO)\n if args.dry_run:\n log.info(\"Running in dry-run mode\")\n\n conn = openstack.connect()\n\n log.info(\"Deleting servers...\")\n map_if_old(conn.compute.delete_server, conn.compute.servers())\n\n log.info(\"Deleting ports...\")\n try:\n map_if_old(conn.network.delete_port, conn.network.ports())\n except openstack.exceptions.ConflictException as ex:\n # Need to find subnet-id which should be removed from a router\n for sn in conn.network.subnets():\n try:\n fn_if_old(conn.network.delete_subnet, sn)\n except openstack.exceptions.ConflictException:\n for r in conn.network.routers():\n log.info(\"Deleting subnet %s from router %s\", sn, r)\n try:\n conn.network.remove_interface_from_router(r, subnet_id=sn.id)\n except Exception as ex:\n log.error(\"Failed to delete subnet from router\", exc_info=True)\n\n for ip in conn.network.ips():\n fn_if_old(conn.network.delete_ip, ip)\n\n # After removing unnecessary subnet from router, retry to delete ports\n map_if_old(conn.network.delete_port, conn.network.ports())\n\n log.info(\"Deleting security groups...\")\n try:\n map_if_old(conn.network.delete_security_group, conn.network.security_groups())\n except openstack.exceptions.ConflictException as ex:\n # Need to delete port when security groups is in used\n map_if_old(conn.network.delete_port, conn.network.ports())\n map_if_old(conn.network.delete_security_group, conn.network.security_groups())\n\n log.info(\"Deleting Subnets...\")\n map_if_old(conn.network.delete_subnet, conn.network.subnets())\n\n log.info(\"Deleting networks...\")\n for n in conn.network.networks():\n if not n.is_router_external:\n fn_if_old(conn.network.delete_network, n)\n\n log.info(\"Deleting keypairs...\")\n map_if_old(\n conn.compute.delete_keypair,\n (conn.compute.get_keypair(x.name) for x in conn.compute.keypairs()),\n # LIST API for keypairs does not give us a created_at (WTF)\n )\n\n\n# runs the given fn to all elements of the that are older than allowed\ndef map_if_old(fn, items):\n for item in items:\n fn_if_old(fn, item)\n\n\n# run the given fn function only if the passed item is older than allowed\ndef fn_if_old(fn, item):\n created_at = datetime.datetime.fromisoformat(item.created_at)\n if created_at.tzinfo is None:\n created_at = created_at.replace(tzinfo=datetime.timezone.utc)\n # Handle TZ unaware object by assuming UTC\n # Can't compare to oldest_allowed otherwise\n if item.name == \"default\": # skip default security group\n return\n if created_at < oldest_allowed:\n log.info(\"Will delete %s %s)\", item.name, item.id)\n if not args.dry_run:\n fn(item)\n\n\nif __name__ == \"__main__\":\n # execute only if run as a script\n main()\n" }, { "path": "scripts/openstack-cleanup/requirements.txt", "content": "openstacksdk>=0.43.0\nsix\n" }, { "path": "scripts/pipeline.Dockerfile.j2", "content": "# Use immutable image tags rather than mutable tags (like ubuntu:24.04)\nFROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b\n# Some tools like yamllint need this\n# Pip needs this as well at the moment to install ansible\n# (and potentially other packages)\n# See: https://github.com/pypa/pip/issues/10219\nENV VAGRANT_VERSION=2.4.1 \\\n VAGRANT_DEFAULT_PROVIDER=libvirt \\\n VAGRANT_ANSIBLE_TAGS=facts \\\n LANG=C.UTF-8 \\\n DEBIAN_FRONTEND=noninteractive \\\n PYTHONDONTWRITEBYTECODE=1\n\nRUN apt update -q \\\n && apt install -yq \\\n libssl-dev \\\n python3-dev \\\n python3-pip \\\n sshpass \\\n apt-transport-https \\\n jq \\\n moreutils \\\n libvirt-dev \\\n openssh-client \\\n rsync \\\n git \\\n ca-certificates \\\n curl \\\n gnupg2 \\\n unzip \\\n libvirt-clients \\\n qemu-utils \\\n qemu-kvm \\\n dnsmasq \\\n && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \\\n && echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \\\n $(. /etc/os-release && echo \"${UBUNTU_CODENAME:-$VERSION_CODENAME}\") stable\" | tee /etc/apt/sources.list.d/docker.list \\\n && apt update -q \\\n && apt install --no-install-recommends -yq docker-ce \\\n && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*\n\nWORKDIR /kubespray\nADD ./requirements.txt /kubespray/requirements.txt\nADD ./tests/requirements.txt /kubespray/tests/requirements.txt\n\nRUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \\\n && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \\\n && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \\\n && curl -L https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \\\n && echo $(curl -L https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \\\n && chmod a+x /usr/local/bin/kubectl \\\n # Install Vagrant\n && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \\\n && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \\\n && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \\\n && vagrant plugin install vagrant-libvirt \\\n # Install Kubernetes collections\n && pip install --break-system-packages --no-compile --no-cache-dir kubernetes \\\n && ansible-galaxy collection install kubernetes.core\n" }, { "path": "scripts/propagate_ansible_variables.yml", "content": "#!/usr/bin/env ansible-playbook\n---\n- name: Update README.md versions\n hosts: localhost\n connection: local\n gather_facts: false\n vars:\n fallback_ip: 'bypass tasks in kubespray_defaults'\n roles:\n - kubespray_defaults\n tasks:\n - name: Include versions not in kubespray_defaults\n include_vars: \"{{ item }}\"\n loop:\n - ../roles/container-engine/docker/defaults/main.yml\n - ../roles/kubernetes/node/defaults/main.yml\n - ../roles/kubernetes-apps/argocd/defaults/main.yml\n - name: Render versions in README.md\n blockinfile:\n marker: ''\n block: \"\\n{{ lookup('ansible.builtin.template', 'readme_versions.md.j2') }}\\n\\n\"\n path: ../README.md\n - name: Render Dockerfiles\n template:\n src: \"{{ item }}.j2\"\n dest: \"../{{ item }}\"\n mode: \"0644\"\n loop:\n - 'pipeline.Dockerfile'\n - 'Dockerfile'\n" }, { "path": "scripts/readme_versions.md.j2", "content": "- Core\n - [kubernetes](https://github.com/kubernetes/kubernetes) {{ kube_version }}\n - [etcd](https://github.com/etcd-io/etcd) {{ etcd_version }}\n - [docker](https://www.docker.com/) {{ docker_version }}\n - [containerd](https://containerd.io/) {{ containerd_version }}\n - [cri-o](http://cri-o.io/) {{ crio_version }} (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)\n- Network Plugin\n - [cni-plugins](https://github.com/containernetworking/plugins) {{ cni_version }}\n - [calico](https://github.com/projectcalico/calico) {{ calico_version }}\n - [cilium](https://github.com/cilium/cilium) {{ cilium_version }}\n - [flannel](https://github.com/flannel-io/flannel) {{ flannel_version }}\n - [kube-ovn](https://github.com/alauda/kube-ovn) {{ kube_ovn_version }}\n - [kube-router](https://github.com/cloudnativelabs/kube-router) {{ kube_router_version }}\n - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) {{ multus_version }}\n - [kube-vip](https://github.com/kube-vip/kube-vip) {{ kube_vip_version }}\n- Application\n - [cert-manager](https://github.com/jetstack/cert-manager) {{ cert_manager_version }}\n - [coredns](https://github.com/coredns/coredns) {{ coredns_version }}\n - [argocd](https://argoproj.github.io/) {{ argocd_version }}\n - [helm](https://helm.sh/) {{ helm_version }}\n - [metallb](https://metallb.universe.tf/) {{ metallb_version }}\n - [registry](https://github.com/distribution/distribution) {{ registry_version }}\n- Storage Plugin\n - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) {{ aws_ebs_csi_plugin_version }}\n - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) {{ azure_csi_plugin_version }}\n - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) {{ cinder_csi_plugin_version }}\n - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) {{ gcp_pd_csi_plugin_version }}\n - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) {{ local_path_provisioner_version }}\n - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) {{ local_volume_provisioner_version }}\n - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) {{ node_feature_discovery_version }}\n" }, { "path": "test-infra/image-builder/Makefile", "content": "deploy:\n\tansible-playbook -i hosts.ini -e docker_password=$(docker_password) cluster.yml\n" }, { "path": "test-infra/image-builder/OWNERS", "content": "# See the OWNERS docs at https://go.k8s.io/owners\n\napprovers:\n - yankay\n - ant31\nreviewers:\n - yankay\n - ant31\n" }, { "path": "test-infra/image-builder/README.md", "content": "# KubeVirt Image Builder\n\nBuild and push KubeVirt VM disk images to quay.io for Kubespray CI testing.\n\n## How It Works\n\nThe Ansible playbook downloads upstream cloud images, converts them to qcow2, resizes (+8G), wraps each in a Docker image based on `kubevirt/registry-disk-v1alpha`, and pushes to `quay.io/kubespray/vm-:`.\n\n## Prerequisites\n\n- Docker, `qemu-img`, Ansible\n- Push access to [quay.io/kubespray](https://quay.io/organization/kubespray) (robot account `kubespray+buildvmimages`)\n\n## Image Definitions\n\nAll OS images are defined in [`roles/kubevirt-images/defaults/main.yml`](roles/kubevirt-images/defaults/main.yml).\n\nEach entry specifies:\n\n| Field | Description |\n|-------|-------------|\n| `filename` | Downloaded file name |\n| `url` | Upstream cloud image URL |\n| `checksum` | Checksum for download verification |\n| `converted` | `true` if the source is already qcow2, `false` if conversion is needed |\n| `tag` | Docker image tag (usually `latest`) |\n\n## Usage\n\n### Build and push all images\n\n```bash\ncd test-infra/image-builder/\nmake docker_password=\n```\n\n### Add a new OS image\n\n1. Add a new entry to `roles/kubevirt-images/defaults/main.yml`:\n\n ```yaml\n new-os-name:\n filename: cloud-image-file.qcow2\n url: https://example.com/cloud-image-file.qcow2\n checksum: sha256:\n converted: true\n tag: \"latest\"\n ```\n\n2. Build and push the image:\n\n ```bash\n make docker_password=\n ```\n\n3. Submit a PR with the `defaults/main.yml` change so CI can use the new image.\n See [#12379](https://github.com/kubernetes-sigs/kubespray/pull/12379) for an example.\n" }, { "path": "test-infra/image-builder/cluster.yml", "content": "---\n- name: Build kubevirt images\n hosts: image-builder\n gather_facts: false\n roles:\n - kubevirt-images\n" }, { "path": "test-infra/image-builder/hosts.ini", "content": "image-builder-1 ansible_ssh_host=xxx.xxx.xxx.xxx\n\n[image-builder]\nimage-builder-1\n" }, { "path": "test-infra/image-builder/roles/kubevirt-images/defaults/main.yml", "content": "---\nimages_dir: /images/base\n\ndocker_user: kubespray+buildvmimages\ndocker_host: quay.io\nregistry: quay.io/kubespray\n\nimages:\n ubuntu-2004:\n filename: focal-server-cloudimg-amd64.img\n url: https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64-disk-kvm.img\n checksum: sha256:8faf1f5a27c956ad0c49dac3114a355fbaf1b2d21709e10a18e67213fbb95b81\n converted: false\n tag: \"latest\"\n\n ubuntu-2204:\n filename: jammy-server-cloudimg-amd64.img\n url: https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img\n checksum: sha256:d3f3f446bf35b2e58b82c10c8fa65525264efe5b0e398238f00ab670f49528ab\n converted: false\n tag: \"latest\"\n\n ubuntu-2404:\n filename: noble-server-cloudimg-amd64.img\n url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img\n checksum: sha256:0cf56a2b23b430c350311dbcb9221b64823a5f7a401b5cf6ab4821f2ffdabe76\n converted: false\n tag: \"latest\"\n\n fedora-37:\n filename: Fedora-Cloud-Base-37-1.7.x86_64.qcow2\n url: https://download.fedoraproject.org/pub/fedora/linux/releases/37/Cloud/x86_64/images/Fedora-Cloud-Base-37-1.7.x86_64.qcow2\n checksum: sha256:b5b9bec91eee65489a5745f6ee620573b23337cbb1eb4501ce200b157a01f3a0\n converted: true\n tag: \"latest\"\n\n fedora-38:\n filename: Fedora-Cloud-Base-38-1.6.x86_64.qcow2\n url: https://download.fedoraproject.org/pub/fedora/linux/releases/38/Cloud/x86_64/images/Fedora-Cloud-Base-38-1.6.x86_64.qcow2\n checksum: sha256:d334670401ff3d5b4129fcc662cf64f5a6e568228af59076cc449a4945318482\n converted: true\n tag: \"latest\"\n\n fedora-39:\n filename: Fedora-Cloud-Base-39-1.5.x86_64.qcow2\n url: https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2\n checksum: sha256:ab5be5058c5c839528a7d6373934e0ce5ad6c8f80bd71ed3390032027da52f37\n converted: true\n tag: \"latest\"\n\n fedora-40:\n filename: Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2\n url: https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2\n checksum: sha256:ac58f3c35b73272d5986fa6d3bc44fd246b45df4c334e99a07b3bbd00684adee\n converted: true\n tag: \"latest\"\n\n fedora-41:\n filename: Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2\n url: https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2\n checksum: sha256:6205ae0c524b4d1816dbd3573ce29b5c44ed26c9fbc874fbe48c41c89dd0bac2\n converted: true\n tag: \"latest\"\n\n fedora-42:\n filename: Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2\n url: https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2\n checksum: sha256:e401a4db2e5e04d1967b6729774faa96da629bcf3ba90b67d8d9cce9906bec0f\n converted: true\n tag: \"latest\"\n\n fedora-coreos:\n filename: fedora-coreos-32.20200601.3.0-openstack.x86_64.qcow2.xz\n url: https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200601.3.0/x86_64/fedora-coreos-32.20200601.3.0-openstack.x86_64.qcow2.xz\n checksum: sha256:fe78c348189d745eb5f6f80ff9eb2af67da8e84880d264f4301faaf7c2a72646\n converted: true\n tag: \"latest\"\n\n centos-8:\n filename: CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2\n url: http://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2\n checksum: sha256:7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0\n converted: true\n tag: \"latest\"\n\n almalinux-8:\n filename: AlmaLinux-8-GenericCloud-latest.x86_64.qcow2\n url: https://repo.almalinux.org/almalinux/8.9/cloud/x86_64/images/AlmaLinux-8-GenericCloud-8.9-20231128.x86_64.qcow2\n checksum: sha256:a1686bc537bce699b512e3233666f5b8f69ed797ff1ce0af52c17fdc52942621\n converted: true\n tag: \"latest\"\n\n almalinux-9:\n filename: AlmaLinux-9-GenericCloud-9.5-20241120.x86_64.qcow2\n url: https://repo.almalinux.org/almalinux/9.5/cloud/x86_64/images/AlmaLinux-9-GenericCloud-9.5-20241120.x86_64.qcow2\n checksum: sha256:abddf01589d46c841f718cec239392924a03b34c4fe84929af5d543c50e37e37\n converted: true\n tag: \"latest\"\n\n rockylinux-8:\n filename: Rocky-8-GenericCloud-8.6-20220515.x86_64.qcow2\n url: https://download.rockylinux.org/pub/rocky/8.6/images/Rocky-8-GenericCloud-8.6-20220515.x86_64.qcow2\n checksum: sha256:77e79f487c70f6bfa5655d8084e02cb8d31900a2c2a22b2334c3401b40a1231c\n converted: true\n tag: \"latest\"\n\n rockylinux-9:\n filename: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2\n url: https://download.rockylinux.org/pub/rocky/9.5/images/x86_64/Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2\n checksum: sha256:069493fdc807300a22176540e9171fcff2227a92b40a7985a0c1c9e21aeebf57\n converted: true\n tag: \"latest\"\n\n rockylinux-10:\n filename: Rocky-10-GenericCloud-Base-10.0-20250609.1.x86_64.qcow2\n url: https://download.rockylinux.org/pub/rocky/10.0/images/x86_64/Rocky-10-GenericCloud-Base-10.0-20250609.1.x86_64.qcow2\n checksum: sha256:20e771c654724e002c32fb92a05fdfdd7ac878c192f50e2fc21f53e8f098b8f9\n converted: true\n tag: \"latest\"\n\n # rockylinux-10-extra:\n # default cloud image doesn't included `kernel-modules-extra`. How to build RockyLinux 10 + `kernel-module-extra` with dib\n # https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093\n\n debian-10:\n filename: debian-10-openstack-amd64.qcow2\n url: https://cdimage.debian.org/cdimage/openstack/current-10/debian-10-openstack-amd64.qcow2\n checksum: sha512:296ad8345cb49e52464a0cb8bf4365eb0b9e4220c47ebdd73d134d51effc756d5554aee15027fffd038fef4ad5fa984c94208bce60572d58b2ab26f74bb2a5de\n converted: true\n tag: \"latest\"\n\n debian-11:\n filename: debian-11-generic-amd64-20210814-734.qcow2\n url: https://cdimage.debian.org/cdimage/cloud/bullseye/20210814-734/debian-11-generic-amd64-20210814-734.qcow2\n checksum: sha512:ed680265ce925e3e02336b052bb476883e2d3b023f7b7d39d064d58ba5f1856869f75dca637c27c0303b731d082ff23a7e45ea2e3f9bcb8f3c4ce0c24332885d\n converted: true\n tag: \"latest\"\n\n debian-12:\n filename: debian-12-generic-amd64-20230612-1409.qcow2\n url: https://cdimage.debian.org/cdimage/cloud/bookworm/20230612-1409/debian-12-generic-amd64-20230612-1409.qcow2\n checksum: sha512:61358292dbec302446a272d5011019091ca78e3fe8878b2d67d31b32e0661306c53a72f793f109394daf937a3db7b2db34422d504e07fdbb300a7bf87109fcf1\n converted: true\n tag: \"latest\"\n\n debian-13:\n filename: debian-13-generic-amd64-20250806-2196.qcow2\n url: https://cdimage.debian.org/cdimage/cloud/trixie/20250806-2196/debian-13-generic-amd64-20250806-2196.qcow2\n checksum: sha512:a7dfe434afc40afb0a791c777f3edba6b1a5c4b7315a61073fe5e34752d3bc5fd44ff67ef054eb4263687a97f7ce4896bf5bad5f216ef8b9d4a84541759e743d\n converted: true\n tag: \"latest\"\n\n oracle-7:\n filename: oracle-linux-76.qcow2\n url: https://storage.googleapis.com/born-images/oracle76/oracle-linux-76.qcow2\n checksum: sha256:f396c03e907fa2a0c94d6807b9f62622f23ee3499df4456ae2a15da381fbdca5\n converted: true\n tag: \"latest\"\n\n opensuse-leap-15:\n filename: openSUSE-Leap-15.3.x86_64-1.0.1-NoCloud-Build2.63.qcow2\n url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.3/images/openSUSE-Leap-15.3.x86_64-1.0.1-NoCloud-Build2.63.qcow2\n checksum: sha256:289248945e2d058551c71c1bdcb31a361cefe7136c7fd88a09b524eedfaf5215\n converted: true\n tag: \"latest\"\n\n opensuse-leap-15-6:\n filename: openSUSE-Leap-15.6.x86_64-1.0.1-NoCloud-Build1.177.qcow2\n url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.6/images/openSUSE-Leap-15.6.x86_64-1.0.1-NoCloud-Build1.177.qcow2\n checksum: sha256:9ecd197b34faf1b43627946d0c26e38b5c3058207d1c86c4784b8f765c3289f3\n converted: true\n tag: \"latest\"\n\n openeuler-2203:\n filename: openEuler-22.03-LTS-SP4-x86_64.qcow2.xz\n url: https://mirrors.ocf.berkeley.edu/openeuler/openEuler-22.03-LTS-SP4/virtual_machine_img/x86_64/openEuler-22.03-LTS-SP4-x86_64.qcow2.xz\n checksum: sha256:2dc9c9d73d172af0443d1a17786ee2e387006b9912fad24b5497ef103ecd7afb\n converted: true\n tag: \"latest\"\n\n openeuler-2403:\n filename: openEuler-24.03-LTS-x86_64.qcow2.xz\n url: https://mirrors.ocf.berkeley.edu/openeuler/openEuler-24.03-LTS/virtual_machine_img/x86_64/openEuler-24.03-LTS-x86_64.qcow2.xz\n checksum: sha256:c6af522d36d659b66da668cc4eb86b032a9cff05a95a0e37505a63e70ed585dc\n converted: true\n tag: \"latest\"\n\n flatcar-4081:\n filename: flatcar_production_kubevirt_image.qcow2\n url: https://stable.release.flatcar-linux.net/amd64-usr/4081.2.1/flatcar_production_kubevirt_image.qcow2\n checksum: sha512:6999ef068380c9842e4caf7afc2a1c66d4d03309f7bfa2f5f500757c36d1f935961f5662cc69376aa3d701e4c2d264f4356d4daadbb68e55becb710067e22c5d\n converted: true\n tag: latest\n" }, { "path": "test-infra/image-builder/roles/kubevirt-images/tasks/main.yml", "content": "---\n\n- name: Create image directory\n file:\n state: directory\n path: \"{{ images_dir }}\"\n mode: \"0755\"\n\n- name: Download images files\n get_url:\n url: \"{{ item.value.url }}\"\n dest: \"{{ images_dir }}/{{ item.value.filename }}\"\n checksum: \"{{ item.value.checksum }}\"\n mode: \"0644\"\n loop: \"{{ images | dict2items }}\"\n\n- name: Unxz compressed images\n command: unxz --force {{ images_dir }}/{{ item.value.filename }}\n loop: \"{{ images | dict2items }}\"\n when:\n - item.value.filename.endswith('.xz')\n\n- name: Convert images which is not in qcow2 format\n command: qemu-img convert -O qcow2 {{ images_dir }}/{{ item.value.filename.rstrip('.xz') }} {{ images_dir }}/{{ item.key }}.qcow2\n loop: \"{{ images | dict2items }}\"\n when:\n - not (item.value.converted | bool)\n\n- name: Make sure all images are ending with qcow2\n command: cp {{ images_dir }}/{{ item.value.filename.rstrip('.xz') }} {{ images_dir }}/{{ item.key }}.qcow2\n loop: \"{{ images | dict2items }}\"\n when:\n - item.value.converted | bool\n\n- name: Resize images\n command: qemu-img resize {{ images_dir }}/{{ item.key }}.qcow2 +8G\n loop: \"{{ images | dict2items }}\"\n\n# STEP 2: Include the images inside a container\n- name: Template default Dockerfile\n template:\n src: Dockerfile\n dest: \"{{ images_dir }}/Dockerfile\"\n mode: \"0644\"\n\n- name: Create docker images for each OS\n command: docker build -t {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} --build-arg cloud_image=\"{{ item.key }}.qcow2\" {{ images_dir }}\n loop: \"{{ images | dict2items }}\"\n\n- name: Docker login\n command: docker login -u=\"{{ docker_user }}\" -p=\"{{ docker_password }}\" \"{{ docker_host }}\"\n\n- name: Docker push image\n command: docker push {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }}\n loop: \"{{ images | dict2items }}\"\n\n- name: Docker logout\n command: docker logout -u=\"{{ docker_user }}\" \"{{ docker_host }}\"\n" }, { "path": "test-infra/image-builder/roles/kubevirt-images/templates/Dockerfile", "content": "FROM kubevirt/registry-disk-v1alpha\n\nARG cloud_image\n\nLABEL org.opencontainers.image.authors=\"The Kubespray Project \"\n\nCOPY $cloud_image /disk\n" }, { "path": "test-infra/vagrant-docker/Dockerfile", "content": "# Docker image published at quay.io/kubespray/vagrant\n\nARG KUBESPRAY_VERSION\nFROM quay.io/kubespray/kubespray:${KUBESPRAY_VERSION}\n\nENV VAGRANT_VERSION=2.3.7\nENV VAGRANT_DEFAULT_PROVIDER=libvirt\nENV VAGRANT_ANSIBLE_TAGS=facts\n\nRUN apt-get update && apt-get install -y wget libvirt-dev openssh-client rsync git build-essential\n\n# Install Vagrant\nRUN wget https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_amd64.deb && \\\n dpkg -i vagrant_${VAGRANT_VERSION}-1_amd64.deb && \\\n rm vagrant_${VAGRANT_VERSION}-1_amd64.deb && \\\n vagrant plugin install vagrant-libvirt\n" }, { "path": "test-infra/vagrant-docker/README.md", "content": "# vagrant docker image\n\nThis image is used for the vagrant CI jobs. It is using the libvirt driver.\n\n## Usage\n\n```console\n$ docker run --net host --rm -it -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock quay.io/kubespray/vagrant\n$ vagrant up\nBringing machine 'k8s-1' up with 'libvirt' provider...\nBringing machine 'k8s-2' up with 'libvirt' provider...\nBringing machine 'k8s-3' up with 'libvirt' provider...\n[...]\n```\n\n## Cache\n\nYou can set `/root/kubespray_cache` as a volume to keep cache between runs.\n\n## Building\n\n```shell\n./build.sh v2.12.5\n```\n" }, { "path": "test-infra/vagrant-docker/build.sh", "content": "#!/bin/sh\nset -euo pipefail\n\nif [ \"$#\" -ne 1 ]; then\n echo \"Usage: $0 tag\" >&2\n exit 1\nfi\n\nVERSION=\"$1\"\nIMG=\"quay.io/kubespray/vagrant:${VERSION}\"\n\ndocker build . --build-arg \"KUBESPRAY_VERSION=${VERSION}\" --tag \"$IMG\"\ndocker push \"$IMG\"\n" }, { "path": "tests/Makefile", "content": "create-tf:\n\t./scripts/create-tf.sh\n\ndelete-tf:\n\t./scripts/delete-tf.sh\n\n$(ANSIBLE_INVENTORY):\n\tmkdir $@\n\ncreate-packet: | $(ANSIBLE_INVENTORY)\n\tansible-playbook cloud_playbooks/create-kubevirt.yml -c local \\\n\t\t-e @\"files/${CI_JOB_NAME}.yml\"\n\ndelete-packet: ;\n\ncreate-vagrant: | $(ANSIBLE_INVENTORY)\n\tvagrant up\n\tcp $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $|\n\ndelete-vagrant:\n\tvagrant destroy -f\n" }, { "path": "tests/ansible.cfg", "content": "[ssh_connection]\npipelining=True\nansible_ssh_common_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100\nretries=2\n[defaults]\nforks = 20\nhost_key_checking=False\ngathering = smart\nfact_caching = jsonfile\nfact_caching_connection = /tmp\nstdout_callback = default\ndisplay_skipped_hosts = no\nlibrary = ./library:../library\ncallbacks_enabled = profile_tasks\njinja2_extensions = jinja2.ext.do\nroles_path = ../roles\n" }, { "path": "tests/cloud_playbooks/create-kubevirt.yml", "content": "---\n- name: Provision Packet VMs\n hosts: localhost\n gather_facts: false\n become: true\n tasks:\n - name: Create Kubevirt VMs\n import_role:\n name: packet-ci\n - name: Update inventory for Molecule\n meta: refresh_inventory\n\n- name: Wait until SSH is available\n hosts: all\n become: false\n gather_facts: false\n\n tasks:\n # Check ssh access without relying on python - this is an horrible hack\n # but wait_for_connection does not work without python\n # and 'until' is incompatible with unreachable errors\n # https://github.com/ansible/ansible/issues/78358\n - name: Wait until SSH is available\n command: >\n ssh -i \"{{ lookup('env', 'ANSIBLE_PRIVATE_KEY_FILE') }}\"\n -o StrictHostKeyChecking=no\n -o UserKnownHostsFile=/dev/null\n -o ConnectTimeout=3 \"{{ lookup('env', 'ANSIBLE_REMOTE_USER') }}@{{ ansible_host }}\"\n register: ssh_command\n delay: 0\n until: ssh_command.rc != 255\n retries: 60\n delegate_to: localhost\n" }, { "path": "tests/cloud_playbooks/roles/packet-ci/defaults/main.yml", "content": "---\n\n# VM sizing\nvm_cpu_cores: 2\nvm_cpu_sockets: 1\nvm_cpu_threads: 2\nvm_memory: 4096\nreleases_disk_size: 2Gi\n\n# Request/Limit allocation settings\ncpu_allocation_ratio: 0.25\nmemory_allocation_ratio: 1\n\n# Deployment mode\nmode: default\n\nnode_groups:\n- 'all'\ncluster_layout: \"{{ molecule_yml.platforms | d(scenarios[mode]) }}\"\n" }, { "path": "tests/cloud_playbooks/roles/packet-ci/tasks/main.yml", "content": "---\n- name: Generate SSH keypair\n community.crypto.openssh_keypair:\n size: 2048\n path: \"{{ lookup('env', 'ANSIBLE_PRIVATE_KEY_FILE') }}\"\n mode: '400'\n register: ssh_key\n\n- name: Start vms for CI job\n kubernetes.core.k8s:\n definition: \"{{ lookup('template', 'vm.yml.j2', template_vars=item) }}\"\n loop: \"{{ cluster_layout }}\"\n loop_control:\n index_var: index\n\n- name: Wait for vms to have IP addresses\n kubernetes.core.k8s_info:\n api_version: kubevirt.io/v1\n kind: VirtualMachineInstance\n label_selectors:\n - \"ci_job_id={{ ci_job_id }}\"\n namespace: \"{{ pod_namespace }}\"\n register: vmis\n until: vmis.resources\n | map(attribute='status.interfaces.0')\n | rejectattr('ipAddress', 'defined') == []\n retries: 30\n delay: 10\n\n- name: Massage VirtualMachineInstance data into an Ansible inventory structure\n vars:\n ips: \"{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}\"\n names: \"{{ vmis.resources | map(attribute='metadata.annotations.inventory_name') }}\"\n _groups: \"{{ (vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ','))}}\"\n vm_hosts: \"{{ ips | zip(_groups, names)\n | map('zip', ['ansible_host', 'ansible_groups', 'inventory_name'])\n | map('map', 'reverse') | map('community.general.dict') }}\"\n loop: \"{{ vm_hosts | map(attribute='ansible_groups') | flatten | unique }}\"\n set_fact:\n ci_inventory: \"{{ ci_inventory|d({}) | combine({\n item: {\n 'hosts': vm_hosts | selectattr('ansible_groups', 'contains', item)\n | rekey_on_member('inventory_name')\n }\n })\n }}\"\n\n- name: Create inventory for CI tests\n copy:\n content: \"{{ ci_inventory | to_yaml }}\"\n dest: \"{{ ansible_inventory_sources[0] }}/ci_inventory.yml\"\n mode: \"0644\"\n" }, { "path": "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2", "content": "---\napiVersion: kubevirt.io/v1\nkind: VirtualMachineInstance\nmetadata:\n generateName: {{ cloud_image }}-\n namespace: {{ pod_namespace }}\n annotations:\n kubespray.com/ci.template-path: \"tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2\"\n ansible_groups: \"{{ node_groups | join(',') }}\"\n inventory_name: \"{{ name | d(cloud_image ~ '-' ~ index) }}\"\n # This does not use a dns prefix because dots are hard to escape with map(attribute=) in Jinja\n labels:\n kubevirt.io/os: {{ cloud_image }}\n kubevirt.io/size: small\n ci_job_id: \"{{ ci_job_id }}\"\n ci_job_name: \"{{ lookup('ansible.builtin.env', 'CI_JOB_NAME_SLUG') }}\"\n ci_pipeline_id: \"{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}\"\n ci_pr_id: \"{{ lookup('ansible.builtin.env', 'PR_ID') }}\"\n # leverage the Kubernetes GC for resources cleanup\n ownerReferences:\n - apiVersion: v1\n kind: Pod\n name: \"{{ pod_name }}\"\n uid: \"{{ pod_uid }}\"\nspec:\n domain:\n devices:\n blockMultiQueue: true\n disks:\n - disk:\n bus: virtio\n name: containervolume\n cache: writethrough\n - disk:\n bus: virtio\n name: cloudinitvolume\n - disk:\n bus: virtio\n name: releases\n serial: '2825A83CBDC8A32D5E'\n interfaces:\n - name: default\n bridge: {}\n cpu:\n cores: {{ vm_cpu_cores }}\n sockets: {{ vm_cpu_sockets }}\n threads: {{ vm_cpu_threads }}\n resources:\n requests:\n memory: \"{{ vm_memory * memory_allocation_ratio }}Mi\"\n cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}\n limits:\n memory: \"{{ vm_memory }}Mi\"\n cpu: {{ vm_cpu_cores }}\n networks:\n - name: default\n pod: {}\n terminationGracePeriodSeconds: 0\n volumes:\n - name: containervolume\n containerDisk:\n image: quay.io/kubespray/vm-{{ cloud_image }}\n - name: cloudinitvolume\n cloudInit{{ 'ConfigDrive' if cloud_image.startswith('flatcar') else 'NoCloud' }}:\n userDataBase64: '{{ ((ignition_config | to_json) if cloud_image.startswith('flatcar') else cloudinit_config) | b64encode }}'\n - name: releases\n emptyDisk:\n capacity: '{{ releases_disk_size }}'\n" }, { "path": "tests/cloud_playbooks/roles/packet-ci/vars/main.yml", "content": "---\n# This is a list of nodes with groups for each scenario/cluster layouts\nscenarios:\n separate:\n - node_groups: ['kube_control_plane']\n - node_groups: ['kube_node']\n - node_groups: ['etcd']\n ha:\n - node_groups: ['kube_control_plane', 'etcd']\n - node_groups: ['kube_control_plane', 'etcd']\n - node_groups: ['kube_node', 'etcd']\n default:\n - node_groups: ['kube_control_plane', 'etcd']\n - node_groups: ['kube_node']\n all-in-one:\n - node_groups: ['kube_control_plane', 'etcd', 'kube_node']\n ha-recover:\n - node_groups: ['kube_control_plane', 'etcd']\n - node_groups: ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd']\n - node_groups: ['kube_node', 'etcd']\n ha-recover-noquorum:\n - node_groups: ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd']\n - node_groups: ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd']\n - node_groups: ['kube_node', 'etcd']\n node-etcd-client:\n - node_groups: ['kube_node', 'kube_control_plane', 'etcd']\n - node_groups: ['kube_node', 'etcd']\n - node_groups: ['kube_node', 'etcd']\n - node_groups: ['kube_node']\n\n# Get pod metadata / CI vars from environment\n\nci_job_id: \"{{ lookup('ansible.builtin.env', 'CI_JOB_ID', default=undefined) }}\"\npod_name: \"{{ lookup('ansible.builtin.env', 'POD_NAME', default=undefined) }}\"\npod_uid: \"{{ lookup('ansible.builtin.env', 'POD_UID', default=undefined) }}\"\npod_namespace: \"{{ lookup('ansible.builtin.env', 'POD_NAMESPACE', default=undefined) }}\"\n\ncloudinit_config: |\n #cloud-config\n users:\n - name: {{ lookup('env', 'ANSIBLE_REMOTE_USER') }}\n sudo: ALL=(ALL) NOPASSWD:ALL\n shell: /bin/bash\n lock_passwd: False\n ssh_authorized_keys:\n - {{ ssh_key.public_key }}\n fs_setup:\n - device: '/dev/disk/by-id/virtio-2825A83CBDC8A32D5E'\n filesystem: 'ext4'\n partition: 'none'\n mounts:\n - ['/dev/disk/by-id/virtio-2825A83CBDC8A32D5E', '/tmp/releases']\n runcmd:\n - chmod 777 /tmp/releases\n\nignition_config:\n ignition:\n version: \"3.2.0\"\n passwd:\n users:\n - name: \"{{ lookup('env', 'ANSIBLE_REMOTE_USER') }}\"\n groups:\n - sudo\n - wheel\n sshAuthorizedKeys:\n - \"{{ ssh_key.public_key }}\"\n storage:\n filesystems:\n - device: '/dev/disk/by-id/virtio-2825A83CBDC8A32D5E'\n format: ext4\n path: /tmp/releases\n wipeFilesystem: true\n directories:\n - path: /tmp/releases\n # ignition require a integer, so using the octal notation is easier\n # than noting it in decimal form\n # yamllint disable-line rule:octal-values\n mode: 0777\n" }, { "path": "tests/common_vars.yml", "content": "---\n# Kubespray settings for tests\ndns_min_replicas: 1\nunsafe_show_logs: true\n\nbin_dir: \"{{ '/opt/bin' if ansible_os_family == 'Flatcar' else '/usr/local/bin' }}\"\n\n# Registry mirrors settings\ndocker_registry_mirrors:\n - \"https://mirror.gcr.io\"\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - host: https://registry-1.docker.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\ncrio_registries:\n - prefix: docker.io\n insecure: false\n blocked: false\n unqualified: false\n location: registry-1.docker.io\n mirrors:\n - location: mirror.gcr.io\n insecure: false\n\nnginx_image_repo: \"{{ quay_image_repo }}/kubespray/nginx\"\n\nflannel_image_repo: \"{{ quay_image_repo }}/kubespray/flannel\"\nflannel_init_image_repo: \"{{ quay_image_repo }}/kubespray/flannel-cni-plugin\"\n\nlocal_release_dir: \"{{ '/tmp/releases' if inventory_hostname != 'localhost' else (lookup('env', 'PWD') + '/downloads') }}\"\n\nhydrophone_version: \"0.7.0\"\nhydrophone_arch: \"x86_64\"\nhydrophone_checksum: \"sha256:15a6c09962f9bd4a1587af068b5edef1072327a77012d8fbb84992c7c87c0475\"\nhydrophone_parallel: 1\nhydrophone_path: \"{{ bin_dir }}/hydrophone\"\n" }, { "path": "tests/files/almalinux9-calico-ha-ebpf.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\nmode: ha\nvm_memory: 3072\n\n# Kubespray settings\ncalico_bpf_enabled: true\nloadbalancer_apiserver_localhost: true\nauto_renew_certificates: true\n" }, { "path": "tests/files/almalinux9-calico-nodelocaldns-secondary.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\nvm_memory: 3072\n\n# Kubespray settings\nenable_nodelocaldns_secondary: true\nloadbalancer_apiserver_type: haproxy\n" }, { "path": "tests/files/almalinux9-calico-remove-node", "content": "REMOVE_NODE_CHECK=true\nREMOVE_NODE_NAME=instance-3\n" }, { "path": "tests/files/almalinux9-calico-remove-node.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\nmode: ha\n\n# Kubespray settings\nauto_renew_certificates: true\n" }, { "path": "tests/files/almalinux9-calico.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\nvm_memory: 3072\n\n# Kubespray settings\nmetrics_server_enabled: true\nloadbalancer_apiserver_type: haproxy\nlocal_path_provisioner_enabled: true\n\nkube_proxy_mode: nftables\n\n# NTP management\nntp_enabled: true\nntp_package: chrony\nntp_timezone: Etc/UTC\nntp_manage_config: true\nntp_tinker_panic: true\nntp_force_sync_immediately: true\n\n# Scheduler plugins\nscheduler_plugins_enabled: true\n" }, { "path": "tests/files/almalinux9-crio.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\n\n# Kubespray settings\ncontainer_manager: crio\nauto_renew_certificates: true\n" }, { "path": "tests/files/almalinux9-docker.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\nvm_memory: 3072\n\n# Use docker\ncontainer_manager: docker\netcd_deployment_type: docker\nresolvconf_mode: docker_dns\n" }, { "path": "tests/files/almalinux9-kube-ovn.yml", "content": "---\n# Instance settings\ncloud_image: almalinux-9\nvm_memory: 3072\n\n# Kubespray settings\nkube_network_plugin: kube-ovn\n" }, { "path": "tests/files/amazon-linux-2-all-in-one.yml", "content": "---\n# Instance settings\ncloud_image: amazon-linux-2\nmode: all-in-one\n\n# Workaround for RHEL8: kernel version 4.18 is lower than Kubernetes system verification.\nkubeadm_ignore_preflight_errors:\n - SystemVerification\n" }, { "path": "tests/files/custom_cni/README.md", "content": "# Custom CNI manifest generation\n\nAs an example we are using Cilium for testing the network_plugins/custom_cni.\n\nTo update the generated manifests to the latest version do the following:\n\n```sh\nhelm repo add cilium https://helm.cilium.io/\nhelm repo update\nhelm template cilium/cilium -n kube-system -f values.yaml > cilium.yaml\n```\n" }, { "path": "tests/files/custom_cni/cilium.yaml", "content": "---\n# Source: cilium/templates/cilium-secrets-namespace.yaml\napiVersion: v1\nkind: Namespace\nmetadata:\n name: \"cilium-secrets\"\n labels:\n app.kubernetes.io/part-of: cilium\n annotations:\n---\n# Source: cilium/templates/cilium-agent/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: \"cilium\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-envoy/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: \"cilium-envoy\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-operator/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: \"cilium-operator\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cilium-config\n namespace: kube-system\ndata:\n\n # Identity allocation mode selects how identities are shared between cilium\n # nodes by setting how they are stored. The options are \"crd\", \"kvstore\" or\n # \"doublewrite-readkvstore\" / \"doublewrite-readcrd\".\n # - \"crd\" stores identities in kubernetes as CRDs (custom resource definition).\n # These can be queried with:\n # kubectl get ciliumid\n # - \"kvstore\" stores identities in an etcd kvstore, that is\n # configured below. Cilium versions before 1.6 supported only the kvstore\n # backend. Upgrades from these older cilium versions should continue using\n # the kvstore by commenting out the identity-allocation-mode below, or\n # setting it to \"kvstore\".\n # - \"doublewrite\" modes store identities in both the kvstore and CRDs. This is useful\n # for seamless migrations from the kvstore mode to the crd mode. Consult the\n # documentation for more information on how to perform the migration.\n identity-allocation-mode: crd\n\n identity-heartbeat-timeout: \"30m0s\"\n identity-gc-interval: \"15m0s\"\n cilium-endpoint-gc-interval: \"5m0s\"\n nodes-gc-interval: \"5m0s\"\n\n # If you want to run cilium in debug mode change this value to true\n debug: \"false\"\n debug-verbose: \"\"\n metrics-sampling-interval: \"5m\"\n # The agent can be put into the following three policy enforcement modes\n # default, always and never.\n # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes\n enable-policy: \"default\"\n policy-cidr-match-mode: \"\"\n # If you want metrics enabled in cilium-operator, set the port for\n # which the Cilium Operator will have their metrics exposed.\n # NOTE that this will open the port on the nodes where Cilium operator pod\n # is scheduled.\n operator-prometheus-serve-addr: \":9963\"\n enable-metrics: \"true\"\n enable-policy-secrets-sync: \"true\"\n policy-secrets-only-from-secrets-namespace: \"true\"\n policy-secrets-namespace: \"cilium-secrets\"\n\n # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4\n # address.\n enable-ipv4: \"true\"\n\n # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6\n # address.\n enable-ipv6: \"false\"\n # Users who wish to specify their own custom CNI configuration file must set\n # custom-cni-conf to \"true\", otherwise Cilium may overwrite the configuration.\n custom-cni-conf: \"false\"\n enable-bpf-clock-probe: \"false\"\n # If you want cilium monitor to aggregate tracing for packets, set this level\n # to \"low\", \"medium\", or \"maximum\". The higher the level, the less packets\n # that will be seen in monitor output.\n monitor-aggregation: medium\n\n # The monitor aggregation interval governs the typical time between monitor\n # notification events for each allowed connection.\n #\n # Only effective when monitor aggregation is set to \"medium\" or higher.\n monitor-aggregation-interval: \"5s\"\n\n # The monitor aggregation flags determine which TCP flags which, upon the\n # first observation, cause monitor notifications to be generated.\n #\n # Only effective when monitor aggregation is set to \"medium\" or higher.\n monitor-aggregation-flags: all\n # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic\n # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.\n bpf-map-dynamic-size-ratio: \"0.0025\"\n # bpf-policy-map-max specifies the maximum number of entries in endpoint\n # policy map (per endpoint)\n bpf-policy-map-max: \"16384\"\n # bpf-policy-stats-map-max specifies the maximum number of entries in global\n # policy stats map\n bpf-policy-stats-map-max: \"65536\"\n # bpf-lb-map-max specifies the maximum number of entries in bpf lb service,\n # backend and affinity maps.\n bpf-lb-map-max: \"65536\"\n bpf-lb-external-clusterip: \"false\"\n bpf-lb-source-range-all-types: \"false\"\n bpf-lb-algorithm-annotation: \"false\"\n bpf-lb-mode-annotation: \"false\"\n\n bpf-distributed-lru: \"false\"\n bpf-events-drop-enabled: \"true\"\n bpf-events-policy-verdict-enabled: \"true\"\n bpf-events-trace-enabled: \"true\"\n\n # Pre-allocation of map entries allows per-packet latency to be reduced, at\n # the expense of up-front memory allocation for the entries in the maps. The\n # default value below will minimize memory usage in the default installation;\n # users who are sensitive to latency may consider setting this to \"true\".\n #\n # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore\n # this option and behave as though it is set to \"true\".\n #\n # If this value is modified, then during the next Cilium startup the restore\n # of existing endpoints and tracking of ongoing connections may be disrupted.\n # As a result, reply packets may be dropped and the load-balancing decisions\n # for established connections may change.\n #\n # If this option is set to \"false\" during an upgrade from 1.3 or earlier to\n # 1.4 or later, then it may cause one-time disruptions during the upgrade.\n preallocate-bpf-maps: \"false\"\n\n # Name of the cluster. Only relevant when building a mesh of clusters.\n cluster-name: \"default\"\n # Unique ID of the cluster. Must be unique across all conneted clusters and\n # in the range of 1 and 255. Only relevant when building a mesh of clusters.\n cluster-id: \"0\"\n\n # Encapsulation mode for communication between nodes\n # Possible values:\n # - disabled\n # - vxlan (default)\n # - geneve\n\n routing-mode: \"tunnel\"\n tunnel-protocol: \"vxlan\"\n tunnel-source-port-range: \"0-0\"\n service-no-backend-response: \"reject\"\n\n\n # Enables L7 proxy for L7 policy enforcement and visibility\n enable-l7-proxy: \"true\"\n enable-ipv4-masquerade: \"true\"\n enable-ipv4-big-tcp: \"false\"\n enable-ipv6-big-tcp: \"false\"\n enable-ipv6-masquerade: \"true\"\n enable-tcx: \"true\"\n datapath-mode: \"veth\"\n enable-masquerade-to-route-source: \"false\"\n\n enable-xt-socket-fallback: \"true\"\n install-no-conntrack-iptables-rules: \"false\"\n iptables-random-fully: \"false\"\n\n auto-direct-node-routes: \"false\"\n direct-routing-skip-unreachable: \"false\"\n\n\n\n kube-proxy-replacement: \"false\"\n bpf-lb-sock: \"false\"\n enable-node-port: \"false\"\n nodeport-addresses: \"\"\n enable-health-check-nodeport: \"true\"\n enable-health-check-loadbalancer-ip: \"false\"\n node-port-bind-protection: \"true\"\n enable-auto-protect-node-port-range: \"true\"\n bpf-lb-acceleration: \"disabled\"\n enable-svc-source-range-check: \"true\"\n enable-l2-neigh-discovery: \"false\"\n k8s-require-ipv4-pod-cidr: \"false\"\n k8s-require-ipv6-pod-cidr: \"false\"\n enable-k8s-networkpolicy: \"true\"\n enable-endpoint-lockdown-on-policy-overflow: \"false\"\n # Tell the agent to generate and write a CNI configuration file\n write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist\n cni-exclusive: \"true\"\n cni-log-file: \"/var/run/cilium/cilium-cni.log\"\n enable-endpoint-health-checking: \"true\"\n enable-health-checking: \"true\"\n health-check-icmp-failure-threshold: \"3\"\n enable-well-known-identities: \"false\"\n enable-node-selector-labels: \"false\"\n synchronize-k8s-nodes: \"true\"\n operator-api-serve-addr: \"127.0.0.1:9234\"\n\n enable-hubble: \"false\"\n ipam: \"cluster-pool\"\n ipam-cilium-node-update-rate: \"15s\"\n cluster-pool-ipv4-cidr: \"{{ kube_pods_subnet }}\"\n cluster-pool-ipv4-mask-size: \"24\"\n\n default-lb-service-ipam: \"lbipam\"\n egress-gateway-reconciliation-trigger-interval: \"1s\"\n enable-vtep: \"false\"\n vtep-endpoint: \"\"\n vtep-cidr: \"\"\n vtep-mask: \"\"\n vtep-mac: \"\"\n procfs: \"/host/proc\"\n bpf-root: \"/sys/fs/bpf\"\n cgroup-root: \"/run/cilium/cgroupv2\"\n\n identity-management-mode: \"agent\"\n enable-sctp: \"false\"\n remove-cilium-node-taints: \"true\"\n set-cilium-node-taints: \"true\"\n set-cilium-is-up-condition: \"true\"\n unmanaged-pod-watcher-interval: \"15\"\n # default DNS proxy to transparent mode in non-chaining modes\n dnsproxy-enable-transparent-mode: \"true\"\n dnsproxy-socket-linger-timeout: \"10\"\n tofqdns-dns-reject-response-code: \"refused\"\n tofqdns-enable-dns-compression: \"true\"\n tofqdns-endpoint-max-ip-per-hostname: \"1000\"\n tofqdns-idle-connection-grace-period: \"0s\"\n tofqdns-max-deferred-connection-deletes: \"10000\"\n tofqdns-proxy-response-max-delay: \"100ms\"\n tofqdns-preallocate-identities: \"true\"\n agent-not-ready-taint-key: \"node.cilium.io/agent-not-ready\"\n\n mesh-auth-enabled: \"true\"\n mesh-auth-queue-size: \"1024\"\n mesh-auth-rotated-identities-queue-size: \"1024\"\n mesh-auth-gc-interval: \"5m0s\"\n\n proxy-xff-num-trusted-hops-ingress: \"0\"\n proxy-xff-num-trusted-hops-egress: \"0\"\n proxy-connect-timeout: \"2\"\n proxy-initial-fetch-timeout: \"30\"\n proxy-max-requests-per-connection: \"0\"\n proxy-max-connection-duration-seconds: \"0\"\n proxy-idle-timeout-seconds: \"60\"\n proxy-max-concurrent-retries: \"128\"\n http-retry-count: \"3\"\n http-stream-idle-timeout: \"300\"\n\n external-envoy-proxy: \"true\"\n envoy-base-id: \"0\"\n envoy-access-log-buffer-size: \"4096\"\n envoy-keep-cap-netbindservice: \"false\"\n max-connected-clusters: \"255\"\n clustermesh-enable-endpoint-sync: \"false\"\n clustermesh-enable-mcs-api: \"false\"\n policy-default-local-cluster: \"false\"\n\n nat-map-stats-entries: \"32\"\n nat-map-stats-interval: \"30s\"\n enable-internal-traffic-policy: \"true\"\n enable-lb-ipam: \"true\"\n enable-non-default-deny-policies: \"true\"\n enable-source-ip-verification: \"true\"\n\n# Extra config allows adding arbitrary properties to the cilium config.\n# By putting it at the end of the ConfigMap, it's also possible to override existing properties.\n---\n# Source: cilium/templates/cilium-envoy/configmap.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cilium-envoy-config\n namespace: kube-system\ndata:\n # Keep the key name as bootstrap-config.json to avoid breaking changes\n bootstrap-config.json: |\n {\"admin\":{\"address\":{\"pipe\":{\"path\":\"/var/run/cilium/envoy/sockets/admin.sock\"}}},\"applicationLogConfig\":{\"logFormat\":{\"textFormat\":\"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v\"}},\"bootstrapExtensions\":[{\"name\":\"envoy.bootstrap.internal_listener\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener\"}}],\"dynamicResources\":{\"cdsConfig\":{\"apiConfigSource\":{\"apiType\":\"GRPC\",\"grpcServices\":[{\"envoyGrpc\":{\"clusterName\":\"xds-grpc-cilium\"}}],\"setNodeOnFirstMessageOnly\":true,\"transportApiVersion\":\"V3\"},\"initialFetchTimeout\":\"30s\",\"resourceApiVersion\":\"V3\"},\"ldsConfig\":{\"apiConfigSource\":{\"apiType\":\"GRPC\",\"grpcServices\":[{\"envoyGrpc\":{\"clusterName\":\"xds-grpc-cilium\"}}],\"setNodeOnFirstMessageOnly\":true,\"transportApiVersion\":\"V3\"},\"initialFetchTimeout\":\"30s\",\"resourceApiVersion\":\"V3\"}},\"node\":{\"cluster\":\"ingress-cluster\",\"id\":\"host~127.0.0.1~no-id~localdomain\"},\"overloadManager\":{\"resourceMonitors\":[{\"name\":\"envoy.resource_monitors.global_downstream_max_connections\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig\",\"max_active_downstream_connections\":\"50000\"}}]},\"staticResources\":{\"clusters\":[{\"circuitBreakers\":{\"thresholds\":[{\"maxRetries\":128}]},\"cleanupInterval\":\"2.500s\",\"connectTimeout\":\"2s\",\"lbPolicy\":\"CLUSTER_PROVIDED\",\"name\":\"ingress-cluster\",\"type\":\"ORIGINAL_DST\",\"typedExtensionProtocolOptions\":{\"envoy.extensions.upstreams.http.v3.HttpProtocolOptions\":{\"@type\":\"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions\",\"commonHttpProtocolOptions\":{\"idleTimeout\":\"60s\",\"maxConnectionDuration\":\"0s\",\"maxRequestsPerConnection\":0},\"useDownstreamProtocolConfig\":{}}}},{\"circuitBreakers\":{\"thresholds\":[{\"maxRetries\":128}]},\"cleanupInterval\":\"2.500s\",\"connectTimeout\":\"2s\",\"lbPolicy\":\"CLUSTER_PROVIDED\",\"name\":\"egress-cluster-tls\",\"transportSocket\":{\"name\":\"cilium.tls_wrapper\",\"typedConfig\":{\"@type\":\"type.googleapis.com/cilium.UpstreamTlsWrapperContext\"}},\"type\":\"ORIGINAL_DST\",\"typedExtensionProtocolOptions\":{\"envoy.extensions.upstreams.http.v3.HttpProtocolOptions\":{\"@type\":\"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions\",\"commonHttpProtocolOptions\":{\"idleTimeout\":\"60s\",\"maxConnectionDuration\":\"0s\",\"maxRequestsPerConnection\":0},\"upstreamHttpProtocolOptions\":{},\"useDownstreamProtocolConfig\":{}}}},{\"circuitBreakers\":{\"thresholds\":[{\"maxRetries\":128}]},\"cleanupInterval\":\"2.500s\",\"connectTimeout\":\"2s\",\"lbPolicy\":\"CLUSTER_PROVIDED\",\"name\":\"egress-cluster\",\"type\":\"ORIGINAL_DST\",\"typedExtensionProtocolOptions\":{\"envoy.extensions.upstreams.http.v3.HttpProtocolOptions\":{\"@type\":\"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions\",\"commonHttpProtocolOptions\":{\"idleTimeout\":\"60s\",\"maxConnectionDuration\":\"0s\",\"maxRequestsPerConnection\":0},\"useDownstreamProtocolConfig\":{}}}},{\"circuitBreakers\":{\"thresholds\":[{\"maxRetries\":128}]},\"cleanupInterval\":\"2.500s\",\"connectTimeout\":\"2s\",\"lbPolicy\":\"CLUSTER_PROVIDED\",\"name\":\"ingress-cluster-tls\",\"transportSocket\":{\"name\":\"cilium.tls_wrapper\",\"typedConfig\":{\"@type\":\"type.googleapis.com/cilium.UpstreamTlsWrapperContext\"}},\"type\":\"ORIGINAL_DST\",\"typedExtensionProtocolOptions\":{\"envoy.extensions.upstreams.http.v3.HttpProtocolOptions\":{\"@type\":\"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions\",\"commonHttpProtocolOptions\":{\"idleTimeout\":\"60s\",\"maxConnectionDuration\":\"0s\",\"maxRequestsPerConnection\":0},\"upstreamHttpProtocolOptions\":{},\"useDownstreamProtocolConfig\":{}}}},{\"connectTimeout\":\"2s\",\"loadAssignment\":{\"clusterName\":\"xds-grpc-cilium\",\"endpoints\":[{\"lbEndpoints\":[{\"endpoint\":{\"address\":{\"pipe\":{\"path\":\"/var/run/cilium/envoy/sockets/xds.sock\"}}}}]}]},\"name\":\"xds-grpc-cilium\",\"type\":\"STATIC\",\"typedExtensionProtocolOptions\":{\"envoy.extensions.upstreams.http.v3.HttpProtocolOptions\":{\"@type\":\"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions\",\"explicitHttpConfig\":{\"http2ProtocolOptions\":{}}}}},{\"connectTimeout\":\"2s\",\"loadAssignment\":{\"clusterName\":\"/envoy-admin\",\"endpoints\":[{\"lbEndpoints\":[{\"endpoint\":{\"address\":{\"pipe\":{\"path\":\"/var/run/cilium/envoy/sockets/admin.sock\"}}}}]}]},\"name\":\"/envoy-admin\",\"type\":\"STATIC\"}],\"listeners\":[{\"address\":{\"socketAddress\":{\"address\":\"0.0.0.0\",\"portValue\":9964}},\"filterChains\":[{\"filters\":[{\"name\":\"envoy.filters.network.http_connection_manager\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager\",\"httpFilters\":[{\"name\":\"envoy.filters.http.router\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router\"}}],\"internalAddressConfig\":{\"cidrRanges\":[{\"addressPrefix\":\"10.0.0.0\",\"prefixLen\":8},{\"addressPrefix\":\"172.16.0.0\",\"prefixLen\":12},{\"addressPrefix\":\"192.168.0.0\",\"prefixLen\":16},{\"addressPrefix\":\"127.0.0.1\",\"prefixLen\":32}]},\"routeConfig\":{\"virtualHosts\":[{\"domains\":[\"*\"],\"name\":\"prometheus_metrics_route\",\"routes\":[{\"match\":{\"prefix\":\"/metrics\"},\"name\":\"prometheus_metrics_route\",\"route\":{\"cluster\":\"/envoy-admin\",\"prefixRewrite\":\"/stats/prometheus\"}}]}]},\"statPrefix\":\"envoy-prometheus-metrics-listener\",\"streamIdleTimeout\":\"300s\"}}]}],\"name\":\"envoy-prometheus-metrics-listener\"},{\"address\":{\"socketAddress\":{\"address\":\"127.0.0.1\",\"portValue\":9878}},\"filterChains\":[{\"filters\":[{\"name\":\"envoy.filters.network.http_connection_manager\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager\",\"httpFilters\":[{\"name\":\"envoy.filters.http.router\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router\"}}],\"internalAddressConfig\":{\"cidrRanges\":[{\"addressPrefix\":\"10.0.0.0\",\"prefixLen\":8},{\"addressPrefix\":\"172.16.0.0\",\"prefixLen\":12},{\"addressPrefix\":\"192.168.0.0\",\"prefixLen\":16},{\"addressPrefix\":\"127.0.0.1\",\"prefixLen\":32}]},\"routeConfig\":{\"virtual_hosts\":[{\"domains\":[\"*\"],\"name\":\"health\",\"routes\":[{\"match\":{\"prefix\":\"/healthz\"},\"name\":\"health\",\"route\":{\"cluster\":\"/envoy-admin\",\"prefixRewrite\":\"/ready\"}}]}]},\"statPrefix\":\"envoy-health-listener\",\"streamIdleTimeout\":\"300s\"}}]}],\"name\":\"envoy-health-listener\"}]}}\n---\n# Source: cilium/templates/cilium-agent/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cilium\n labels:\n app.kubernetes.io/part-of: cilium\nrules:\n- apiGroups:\n - networking.k8s.io\n resources:\n - networkpolicies\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - namespaces\n - services\n - pods\n - endpoints\n - nodes\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - apiextensions.k8s.io\n resources:\n - customresourcedefinitions\n verbs:\n - list\n - watch\n # This is used when validating policies in preflight. This will need to stay\n # until we figure out how to avoid \"get\" inside the preflight, and then\n # should be removed ideally.\n - get\n- apiGroups:\n - cilium.io\n resources:\n - ciliumloadbalancerippools\n - ciliumbgppeeringpolicies\n - ciliumbgpnodeconfigs\n - ciliumbgpadvertisements\n - ciliumbgppeerconfigs\n - ciliumclusterwideenvoyconfigs\n - ciliumclusterwidenetworkpolicies\n - ciliumegressgatewaypolicies\n - ciliumendpoints\n - ciliumendpointslices\n - ciliumenvoyconfigs\n - ciliumidentities\n - ciliumlocalredirectpolicies\n - ciliumnetworkpolicies\n - ciliumnodes\n - ciliumnodeconfigs\n - ciliumcidrgroups\n - ciliuml2announcementpolicies\n - ciliumpodippools\n verbs:\n - list\n - watch\n- apiGroups:\n - cilium.io\n resources:\n - ciliumidentities\n - ciliumendpoints\n - ciliumnodes\n verbs:\n - create\n- apiGroups:\n - cilium.io\n # To synchronize garbage collection of such resources\n resources:\n - ciliumidentities\n verbs:\n - update\n- apiGroups:\n - cilium.io\n resources:\n - ciliumendpoints\n verbs:\n - delete\n - get\n- apiGroups:\n - cilium.io\n resources:\n - ciliumnodes\n - ciliumnodes/status\n verbs:\n - get\n - update\n- apiGroups:\n - cilium.io\n resources:\n - ciliumendpoints/status\n - ciliumendpoints\n - ciliuml2announcementpolicies/status\n - ciliumbgpnodeconfigs/status\n verbs:\n - patch\n---\n# Source: cilium/templates/cilium-operator/clusterrole.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cilium-operator\n labels:\n app.kubernetes.io/part-of: cilium\nrules:\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - get\n - list\n - watch\n # to automatically delete [core|kube]dns pods so that are starting to being\n # managed by Cilium\n - delete\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n resourceNames:\n - cilium-config\n verbs:\n # allow patching of the configmap to set annotations\n - patch\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n # To remove node taints\n - nodes\n # To set NetworkUnavailable false on startup\n - nodes/status\n verbs:\n - patch\n- apiGroups:\n - discovery.k8s.io\n resources:\n - endpointslices\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n # to perform LB IP allocation for BGP\n - services/status\n verbs:\n - update\n - patch\n- apiGroups:\n - \"\"\n resources:\n # to check apiserver connectivity\n - namespaces\n - secrets\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n # to perform the translation of a CNP that contains `ToGroup` to its endpoints\n - services\n - endpoints\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - cilium.io\n resources:\n - ciliumnetworkpolicies\n - ciliumclusterwidenetworkpolicies\n verbs:\n # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups'\n - create\n - update\n - deletecollection\n # To update the status of the CNPs and CCNPs\n - patch\n - get\n - list\n - watch\n- apiGroups:\n - cilium.io\n resources:\n - ciliumnetworkpolicies/status\n - ciliumclusterwidenetworkpolicies/status\n verbs:\n # Update the auto-generated CNPs and CCNPs status.\n - patch\n - update\n- apiGroups:\n - cilium.io\n resources:\n - ciliumendpoints\n - ciliumidentities\n verbs:\n # To perform garbage collection of such resources\n - delete\n - list\n - watch\n- apiGroups:\n - cilium.io\n resources:\n - ciliumidentities\n verbs:\n # To synchronize garbage collection of such resources\n - update\n- apiGroups:\n - cilium.io\n resources:\n - ciliumnodes\n verbs:\n - create\n - update\n - get\n - list\n - watch\n # To perform CiliumNode garbage collector\n - delete\n- apiGroups:\n - cilium.io\n resources:\n - ciliumnodes/status\n verbs:\n - update\n- apiGroups:\n - cilium.io\n resources:\n - ciliumendpointslices\n - ciliumenvoyconfigs\n - ciliumbgppeerconfigs\n - ciliumbgpadvertisements\n - ciliumbgpnodeconfigs\n verbs:\n - create\n - update\n - get\n - list\n - watch\n - delete\n - patch\n- apiGroups:\n - cilium.io\n resources:\n - ciliumbgpclusterconfigs/status\n - ciliumbgppeerconfigs/status\n verbs:\n - update\n- apiGroups:\n - apiextensions.k8s.io\n resources:\n - customresourcedefinitions\n verbs:\n - create\n - get\n - list\n - watch\n- apiGroups:\n - apiextensions.k8s.io\n resources:\n - customresourcedefinitions\n verbs:\n - update\n resourceNames:\n - ciliumloadbalancerippools.cilium.io\n - ciliumbgppeeringpolicies.cilium.io\n - ciliumbgpclusterconfigs.cilium.io\n - ciliumbgppeerconfigs.cilium.io\n - ciliumbgpadvertisements.cilium.io\n - ciliumbgpnodeconfigs.cilium.io\n - ciliumbgpnodeconfigoverrides.cilium.io\n - ciliumclusterwideenvoyconfigs.cilium.io\n - ciliumclusterwidenetworkpolicies.cilium.io\n - ciliumegressgatewaypolicies.cilium.io\n - ciliumendpoints.cilium.io\n - ciliumendpointslices.cilium.io\n - ciliumenvoyconfigs.cilium.io\n - ciliumidentities.cilium.io\n - ciliumlocalredirectpolicies.cilium.io\n - ciliumnetworkpolicies.cilium.io\n - ciliumnodes.cilium.io\n - ciliumnodeconfigs.cilium.io\n - ciliumcidrgroups.cilium.io\n - ciliuml2announcementpolicies.cilium.io\n - ciliumpodippools.cilium.io\n - ciliumgatewayclassconfigs.cilium.io\n- apiGroups:\n - cilium.io\n resources:\n - ciliumloadbalancerippools\n - ciliumpodippools\n - ciliumbgppeeringpolicies\n - ciliumbgpclusterconfigs\n - ciliumbgpnodeconfigoverrides\n - ciliumbgppeerconfigs\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - cilium.io\n resources:\n - ciliumpodippools\n verbs:\n - create\n- apiGroups:\n - cilium.io\n resources:\n - ciliumloadbalancerippools/status\n verbs:\n - patch\n# For cilium-operator running in HA mode.\n#\n# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election\n# between multiple running instances.\n# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less\n# common and fewer objects in the cluster watch \"all Leases\".\n- apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs:\n - create\n - get\n - update\n---\n# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cilium\n labels:\n app.kubernetes.io/part-of: cilium\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cilium\nsubjects:\n- kind: ServiceAccount\n name: \"cilium\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cilium-operator\n labels:\n app.kubernetes.io/part-of: cilium\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cilium-operator\nsubjects:\n- kind: ServiceAccount\n name: \"cilium-operator\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-agent/role.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: cilium-config-agent\n namespace: kube-system\n labels:\n app.kubernetes.io/part-of: cilium\nrules:\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n verbs:\n - get\n - list\n - watch\n---\n# Source: cilium/templates/cilium-agent/role.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: cilium-tlsinterception-secrets\n namespace: \"cilium-secrets\"\n labels:\n app.kubernetes.io/part-of: cilium\nrules:\n- apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - get\n - list\n - watch\n---\n# Source: cilium/templates/cilium-operator/role.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: cilium-operator-tlsinterception-secrets\n namespace: \"cilium-secrets\"\n labels:\n app.kubernetes.io/part-of: cilium\nrules:\n- apiGroups:\n - \"\"\n resources:\n - secrets\n verbs:\n - create\n - delete\n - update\n - patch\n---\n# Source: cilium/templates/cilium-agent/rolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: cilium-config-agent\n namespace: kube-system\n labels:\n app.kubernetes.io/part-of: cilium\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: cilium-config-agent\nsubjects:\n - kind: ServiceAccount\n name: \"cilium\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-agent/rolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: cilium-tlsinterception-secrets\n namespace: \"cilium-secrets\"\n labels:\n app.kubernetes.io/part-of: cilium\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: cilium-tlsinterception-secrets\nsubjects:\n- kind: ServiceAccount\n name: \"cilium\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-operator/rolebinding.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: cilium-operator-tlsinterception-secrets\n namespace: \"cilium-secrets\"\n labels:\n app.kubernetes.io/part-of: cilium\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: cilium-operator-tlsinterception-secrets\nsubjects:\n- kind: ServiceAccount\n name: \"cilium-operator\"\n namespace: kube-system\n---\n# Source: cilium/templates/cilium-envoy/service.yaml\napiVersion: v1\nkind: Service\nmetadata:\n name: cilium-envoy\n namespace: kube-system\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"9964\"\n labels:\n k8s-app: cilium-envoy\n app.kubernetes.io/name: cilium-envoy\n app.kubernetes.io/part-of: cilium\n io.cilium/app: proxy\nspec:\n clusterIP: None\n type: ClusterIP\n selector:\n k8s-app: cilium-envoy\n ports:\n - name: envoy-metrics\n port: 9964\n protocol: TCP\n targetPort: envoy-metrics\n---\n# Source: cilium/templates/cilium-agent/daemonset.yaml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: cilium\n namespace: kube-system\n labels:\n k8s-app: cilium\n app.kubernetes.io/part-of: cilium\n app.kubernetes.io/name: cilium-agent\nspec:\n selector:\n matchLabels:\n k8s-app: cilium\n updateStrategy:\n rollingUpdate:\n maxUnavailable: 2\n type: RollingUpdate\n template:\n metadata:\n annotations:\n kubectl.kubernetes.io/default-container: cilium-agent\n labels:\n k8s-app: cilium\n app.kubernetes.io/name: cilium-agent\n app.kubernetes.io/part-of: cilium\n spec:\n securityContext:\n appArmorProfile:\n type: Unconfined\n seccompProfile:\n type: Unconfined\n containers:\n - name: cilium-agent\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n command:\n - cilium-agent\n args:\n - --config-dir=/tmp/cilium/config-map\n startupProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9879\n scheme: HTTP\n httpHeaders:\n - name: \"brief\"\n value: \"true\"\n failureThreshold: 300\n periodSeconds: 2\n successThreshold: 1\n initialDelaySeconds: 5\n livenessProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9879\n scheme: HTTP\n httpHeaders:\n - name: \"brief\"\n value: \"true\"\n - name: \"require-k8s-connectivity\"\n value: \"false\"\n periodSeconds: 30\n successThreshold: 1\n failureThreshold: 10\n timeoutSeconds: 5\n readinessProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9879\n scheme: HTTP\n httpHeaders:\n - name: \"brief\"\n value: \"true\"\n periodSeconds: 30\n successThreshold: 1\n failureThreshold: 3\n timeoutSeconds: 5\n env:\n - name: K8S_NODE_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: spec.nodeName\n - name: CILIUM_K8S_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: CILIUM_CLUSTERMESH_CONFIG\n value: /var/lib/cilium/clustermesh/\n - name: GOMEMLIMIT\n valueFrom:\n resourceFieldRef:\n resource: limits.memory\n divisor: '1'\n - name: KUBE_CLIENT_BACKOFF_BASE\n value: \"1\"\n - name: KUBE_CLIENT_BACKOFF_DURATION\n value: \"120\"\n lifecycle:\n postStart:\n exec:\n command:\n - \"bash\"\n - \"-c\"\n - |\n set -o errexit\n set -o pipefail\n set -o nounset\n\n # When running in AWS ENI mode, it's likely that 'aws-node' has\n # had a chance to install SNAT iptables rules. These can result\n # in dropped traffic, so we should attempt to remove them.\n # We do it using a 'postStart' hook since this may need to run\n # for nodes which might have already been init'ed but may still\n # have dangling rules. This is safe because there are no\n # dependencies on anything that is part of the startup script\n # itself, and can be safely run multiple times per node (e.g. in\n # case of a restart).\n if [[ \"$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')\" != \"0\" ]];\n then\n echo 'Deleting iptables rules created by the AWS CNI VPC plugin'\n iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore\n fi\n echo 'Done!'\n\n preStop:\n exec:\n command:\n - /cni-uninstall.sh\n securityContext:\n seLinuxOptions:\n level: s0\n type: spc_t\n capabilities:\n add:\n - CHOWN\n - KILL\n - NET_ADMIN\n - NET_RAW\n - IPC_LOCK\n - SYS_MODULE\n - SYS_ADMIN\n - SYS_RESOURCE\n - DAC_OVERRIDE\n - FOWNER\n - SETGID\n - SETUID\n drop:\n - ALL\n terminationMessagePolicy: FallbackToLogsOnError\n volumeMounts:\n - name: envoy-sockets\n mountPath: /var/run/cilium/envoy/sockets\n readOnly: false\n # Unprivileged containers need to mount /proc/sys/net from the host\n # to have write access\n - mountPath: /host/proc/sys/net\n name: host-proc-sys-net\n # Unprivileged containers need to mount /proc/sys/kernel from the host\n # to have write access\n - mountPath: /host/proc/sys/kernel\n name: host-proc-sys-kernel\n - name: bpf-maps\n mountPath: /sys/fs/bpf\n # Unprivileged containers can't set mount propagation to bidirectional\n # in this case we will mount the bpf fs from an init container that\n # is privileged and set the mount propagation from host to container\n # in Cilium.\n mountPropagation: HostToContainer\n - name: cilium-run\n mountPath: /var/run/cilium\n - name: cilium-netns\n mountPath: /var/run/cilium/netns\n mountPropagation: HostToContainer\n - name: etc-cni-netd\n mountPath: /host/etc/cni/net.d\n - name: clustermesh-secrets\n mountPath: /var/lib/cilium/clustermesh\n readOnly: true\n # Needed to be able to load kernel modules\n - name: lib-modules\n mountPath: /lib/modules\n readOnly: true\n - name: xtables-lock\n mountPath: /run/xtables.lock\n - name: tmp\n mountPath: /tmp\n\n initContainers:\n - name: config\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n command:\n - cilium-dbg\n - build-config\n env:\n - name: K8S_NODE_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: spec.nodeName\n - name: CILIUM_K8S_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n volumeMounts:\n - name: tmp\n mountPath: /tmp\n terminationMessagePolicy: FallbackToLogsOnError\n # Required to mount cgroup2 filesystem on the underlying Kubernetes node.\n # We use nsenter command with host's cgroup and mount namespaces enabled.\n - name: mount-cgroup\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n env:\n - name: CGROUP_ROOT\n value: /run/cilium/cgroupv2\n - name: BIN_PATH\n value: /opt/cni/bin\n command:\n - sh\n - -ec\n # The statically linked Go program binary is invoked to avoid any\n # dependency on utilities like sh and mount that can be missing on certain\n # distros installed on the underlying host. Copy the binary to the\n # same directory where we install cilium cni plugin so that exec permissions\n # are available.\n - |\n cp /usr/bin/cilium-mount /hostbin/cilium-mount;\n nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt \"${BIN_PATH}/cilium-mount\" $CGROUP_ROOT;\n rm /hostbin/cilium-mount\n volumeMounts:\n - name: hostproc\n mountPath: /hostproc\n - name: cni-path\n mountPath: /hostbin\n terminationMessagePolicy: FallbackToLogsOnError\n securityContext:\n seLinuxOptions:\n level: s0\n type: spc_t\n capabilities:\n add:\n - SYS_ADMIN\n - SYS_CHROOT\n - SYS_PTRACE\n drop:\n - ALL\n - name: apply-sysctl-overwrites\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n env:\n - name: BIN_PATH\n value: /opt/cni/bin\n command:\n - sh\n - -ec\n # The statically linked Go program binary is invoked to avoid any\n # dependency on utilities like sh that can be missing on certain\n # distros installed on the underlying host. Copy the binary to the\n # same directory where we install cilium cni plugin so that exec permissions\n # are available.\n - |\n cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;\n nsenter --mount=/hostproc/1/ns/mnt \"${BIN_PATH}/cilium-sysctlfix\";\n rm /hostbin/cilium-sysctlfix\n volumeMounts:\n - name: hostproc\n mountPath: /hostproc\n - name: cni-path\n mountPath: /hostbin\n terminationMessagePolicy: FallbackToLogsOnError\n securityContext:\n seLinuxOptions:\n level: s0\n type: spc_t\n capabilities:\n add:\n - SYS_ADMIN\n - SYS_CHROOT\n - SYS_PTRACE\n drop:\n - ALL\n # Mount the bpf fs if it is not mounted. We will perform this task\n # from a privileged container because the mount propagation bidirectional\n # only works from privileged containers.\n - name: mount-bpf-fs\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n args:\n - 'mount | grep \"/sys/fs/bpf type bpf\" || mount -t bpf bpf /sys/fs/bpf'\n command:\n - /bin/bash\n - -c\n - --\n terminationMessagePolicy: FallbackToLogsOnError\n securityContext:\n privileged: true\n volumeMounts:\n - name: bpf-maps\n mountPath: /sys/fs/bpf\n mountPropagation: Bidirectional\n - name: clean-cilium-state\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n command:\n - /init-container.sh\n env:\n - name: CILIUM_ALL_STATE\n valueFrom:\n configMapKeyRef:\n name: cilium-config\n key: clean-cilium-state\n optional: true\n - name: CILIUM_BPF_STATE\n valueFrom:\n configMapKeyRef:\n name: cilium-config\n key: clean-cilium-bpf-state\n optional: true\n - name: WRITE_CNI_CONF_WHEN_READY\n valueFrom:\n configMapKeyRef:\n name: cilium-config\n key: write-cni-conf-when-ready\n optional: true\n terminationMessagePolicy: FallbackToLogsOnError\n securityContext:\n seLinuxOptions:\n level: s0\n type: spc_t\n capabilities:\n add:\n - NET_ADMIN\n - SYS_MODULE\n - SYS_ADMIN\n - SYS_RESOURCE\n drop:\n - ALL\n volumeMounts:\n - name: bpf-maps\n mountPath: /sys/fs/bpf\n # Required to mount cgroup filesystem from the host to cilium agent pod\n - name: cilium-cgroup\n mountPath: /run/cilium/cgroupv2\n mountPropagation: HostToContainer\n - name: cilium-run\n mountPath: /var/run/cilium # wait-for-kube-proxy\n # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent\n - name: install-cni-binaries\n image: \"quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4\"\n imagePullPolicy: IfNotPresent\n command:\n - \"/install-plugin.sh\"\n resources:\n requests:\n cpu: 100m\n memory: 10Mi\n securityContext:\n seLinuxOptions:\n level: s0\n type: spc_t\n capabilities:\n drop:\n - ALL\n terminationMessagePolicy: FallbackToLogsOnError\n volumeMounts:\n - name: cni-path\n mountPath: /host/opt/cni/bin # .Values.cni.install\n restartPolicy: Always\n priorityClassName: system-node-critical\n serviceAccountName: \"cilium\"\n automountServiceAccountToken: true\n terminationGracePeriodSeconds: 1\n hostNetwork: true\n\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n k8s-app: cilium\n topologyKey: kubernetes.io/hostname\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n - operator: Exists\n volumes:\n # For sharing configuration between the \"config\" initContainer and the agent\n - name: tmp\n emptyDir: {}\n # To keep state between restarts / upgrades\n - name: cilium-run\n hostPath:\n path: /var/run/cilium\n type: DirectoryOrCreate\n # To exec into pod network namespaces\n - name: cilium-netns\n hostPath:\n path: /var/run/netns\n type: DirectoryOrCreate\n # To keep state between restarts / upgrades for bpf maps\n - name: bpf-maps\n hostPath:\n path: /sys/fs/bpf\n type: DirectoryOrCreate\n # To mount cgroup2 filesystem on the host or apply sysctlfix\n - name: hostproc\n hostPath:\n path: /proc\n type: Directory\n # To keep state between restarts / upgrades for cgroup2 filesystem\n - name: cilium-cgroup\n hostPath:\n path: /run/cilium/cgroupv2\n type: DirectoryOrCreate\n # To install cilium cni plugin in the host\n - name: cni-path\n hostPath:\n path: /opt/cni/bin\n type: DirectoryOrCreate\n # To install cilium cni configuration in the host\n - name: etc-cni-netd\n hostPath:\n path: /etc/cni/net.d\n type: DirectoryOrCreate\n # To be able to load kernel modules\n - name: lib-modules\n hostPath:\n path: /lib/modules\n # To access iptables concurrently with other processes (e.g. kube-proxy)\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n # Sharing socket with Cilium Envoy on the same node by using a host path\n - name: envoy-sockets\n hostPath:\n path: \"/var/run/cilium/envoy/sockets\"\n type: DirectoryOrCreate\n # To read the clustermesh configuration\n - name: clustermesh-secrets\n projected:\n # note: the leading zero means this number is in octal representation: do not remove it\n defaultMode: 0400\n sources:\n - secret:\n name: cilium-clustermesh\n optional: true\n # note: items are not explicitly listed here, since the entries of this secret\n # depend on the peers configured, and that would cause a restart of all agents\n # at every addition/removal. Leaving the field empty makes each secret entry\n # to be automatically projected into the volume as a file whose name is the key.\n - secret:\n name: clustermesh-apiserver-remote-cert\n optional: true\n items:\n - key: tls.key\n path: common-etcd-client.key\n - key: tls.crt\n path: common-etcd-client.crt\n - key: ca.crt\n path: common-etcd-client-ca.crt\n # note: we configure the volume for the kvstoremesh-specific certificate\n # regardless of whether KVStoreMesh is enabled or not, so that it can be\n # automatically mounted in case KVStoreMesh gets subsequently enabled,\n # without requiring an agent restart.\n - secret:\n name: clustermesh-apiserver-local-cert\n optional: true\n items:\n - key: tls.key\n path: local-etcd-client.key\n - key: tls.crt\n path: local-etcd-client.crt\n - key: ca.crt\n path: local-etcd-client-ca.crt\n - name: host-proc-sys-net\n hostPath:\n path: /proc/sys/net\n type: Directory\n - name: host-proc-sys-kernel\n hostPath:\n path: /proc/sys/kernel\n type: Directory\n---\n# Source: cilium/templates/cilium-envoy/daemonset.yaml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: cilium-envoy\n namespace: kube-system\n labels:\n k8s-app: cilium-envoy\n app.kubernetes.io/part-of: cilium\n app.kubernetes.io/name: cilium-envoy\n name: cilium-envoy\nspec:\n selector:\n matchLabels:\n k8s-app: cilium-envoy\n updateStrategy:\n rollingUpdate:\n maxUnavailable: 2\n type: RollingUpdate\n template:\n metadata:\n annotations:\n labels:\n k8s-app: cilium-envoy\n name: cilium-envoy\n app.kubernetes.io/name: cilium-envoy\n app.kubernetes.io/part-of: cilium\n spec:\n securityContext:\n appArmorProfile:\n type: Unconfined\n containers:\n - name: cilium-envoy\n image: \"quay.io/cilium/cilium-envoy:v1.35.9-1767794330-db497dd19e346b39d81d7b5c0dedf6c812bcc5c9@sha256:81398e449f2d3d0a6a70527e4f641aaa685d3156bea0bb30712fae3fd8822b86\"\n imagePullPolicy: IfNotPresent\n command:\n - /usr/bin/cilium-envoy-starter\n args:\n - '--'\n - '-c /var/run/cilium/envoy/bootstrap-config.json'\n - '--base-id 0'\n - '--log-level info'\n startupProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9878\n scheme: HTTP\n failureThreshold: 105\n periodSeconds: 2\n successThreshold: 1\n initialDelaySeconds: 5\n livenessProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9878\n scheme: HTTP\n periodSeconds: 30\n successThreshold: 1\n failureThreshold: 10\n timeoutSeconds: 5\n readinessProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9878\n scheme: HTTP\n periodSeconds: 30\n successThreshold: 1\n failureThreshold: 3\n timeoutSeconds: 5\n env:\n - name: K8S_NODE_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: spec.nodeName\n - name: CILIUM_K8S_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n ports:\n - name: envoy-metrics\n containerPort: 9964\n hostPort: 9964\n protocol: TCP\n securityContext:\n seLinuxOptions:\n level: s0\n type: spc_t\n capabilities:\n add:\n - NET_ADMIN\n - SYS_ADMIN\n drop:\n - ALL\n terminationMessagePolicy: FallbackToLogsOnError\n volumeMounts:\n - name: envoy-sockets\n mountPath: /var/run/cilium/envoy/sockets\n readOnly: false\n - name: envoy-artifacts\n mountPath: /var/run/cilium/envoy/artifacts\n readOnly: true\n - name: envoy-config\n mountPath: /var/run/cilium/envoy/\n readOnly: true\n - name: bpf-maps\n mountPath: /sys/fs/bpf\n mountPropagation: HostToContainer\n restartPolicy: Always\n priorityClassName: system-node-critical\n serviceAccountName: \"cilium-envoy\"\n automountServiceAccountToken: true\n terminationGracePeriodSeconds: 1\n hostNetwork: true\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: cilium.io/no-schedule\n operator: NotIn\n values:\n - \"true\"\n podAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n k8s-app: cilium\n topologyKey: kubernetes.io/hostname\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n k8s-app: cilium-envoy\n topologyKey: kubernetes.io/hostname\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n - operator: Exists\n volumes:\n - name: envoy-sockets\n hostPath:\n path: \"/var/run/cilium/envoy/sockets\"\n type: DirectoryOrCreate\n - name: envoy-artifacts\n hostPath:\n path: \"/var/run/cilium/envoy/artifacts\"\n type: DirectoryOrCreate\n - name: envoy-config\n configMap:\n name: \"cilium-envoy-config\"\n # note: the leading zero means this number is in octal representation: do not remove it\n defaultMode: 0400\n items:\n - key: bootstrap-config.json\n path: bootstrap-config.json\n # To keep state between restarts / upgrades\n # To keep state between restarts / upgrades for bpf maps\n - name: bpf-maps\n hostPath:\n path: /sys/fs/bpf\n type: DirectoryOrCreate\n---\n# Source: cilium/templates/cilium-operator/deployment.yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: cilium-operator\n namespace: kube-system\n labels:\n io.cilium/app: operator\n name: cilium-operator\n app.kubernetes.io/part-of: cilium\n app.kubernetes.io/name: cilium-operator\nspec:\n # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go\n # for more details.\n replicas: 2\n selector:\n matchLabels:\n io.cilium/app: operator\n name: cilium-operator\n # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case\n # of one replica and no user configured Recreate strategy.\n # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the\n # podAntiAffinity which prevents deployments of multiple operator replicas on the same node.\n strategy:\n rollingUpdate:\n maxSurge: 25%\n maxUnavailable: 50%\n type: RollingUpdate\n template:\n metadata:\n annotations:\n prometheus.io/port: \"9963\"\n prometheus.io/scrape: \"true\"\n labels:\n io.cilium/app: operator\n name: cilium-operator\n app.kubernetes.io/part-of: cilium\n app.kubernetes.io/name: cilium-operator\n spec:\n securityContext:\n seccompProfile:\n type: RuntimeDefault\n containers:\n - name: cilium-operator\n image: \"quay.io/cilium/operator-generic:v1.18.6@sha256:34a827ce9ed021c8adf8f0feca131f53b3c54a3ef529053d871d0347ec4d69af\"\n imagePullPolicy: IfNotPresent\n command:\n - cilium-operator-generic\n args:\n - --config-dir=/tmp/cilium/config-map\n - --debug=$(CILIUM_DEBUG)\n env:\n - name: K8S_NODE_NAME\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: spec.nodeName\n - name: CILIUM_K8S_NAMESPACE\n valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: metadata.namespace\n - name: CILIUM_DEBUG\n valueFrom:\n configMapKeyRef:\n key: debug\n name: cilium-config\n optional: true\n ports:\n - name: prometheus\n containerPort: 9963\n hostPort: 9963\n protocol: TCP\n livenessProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9234\n scheme: HTTP\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 3\n readinessProbe:\n httpGet:\n host: \"127.0.0.1\"\n path: /healthz\n port: 9234\n scheme: HTTP\n initialDelaySeconds: 0\n periodSeconds: 5\n timeoutSeconds: 3\n failureThreshold: 5\n volumeMounts:\n - name: cilium-config-path\n mountPath: /tmp/cilium/config-map\n readOnly: true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n terminationMessagePolicy: FallbackToLogsOnError\n hostNetwork: true\n restartPolicy: Always\n priorityClassName: system-cluster-critical\n serviceAccountName: \"cilium-operator\"\n automountServiceAccountToken: true\n # In HA mode, cilium-operator pods must not be scheduled on the same\n # node as they will clash with each other.\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n io.cilium/app: operator\n topologyKey: kubernetes.io/hostname\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n - key: node-role.kubernetes.io/control-plane\n operator: Exists\n - key: node-role.kubernetes.io/master\n operator: Exists\n - key: node.kubernetes.io/not-ready\n operator: Exists\n - key: node.cloudprovider.kubernetes.io/uninitialized\n operator: Exists\n - key: node.cilium.io/agent-not-ready\n operator: Exists\n\n volumes:\n # To read the configuration from the config map\n - name: cilium-config-path\n configMap:\n name: cilium-config\n" }, { "path": "tests/files/custom_cni/values.yaml", "content": "---\n\n# We disable hubble so that helm doesn't try to generate any certificate.\n# This is not needed to test network_plugin/custom_cni anyway.\nhubble:\n enabled: false\n\nipam:\n operator:\n # Set the appropriate pods subnet\n clusterPoolIPv4PodCIDRList: [\"{{ kube_pods_subnet }}\"]\n" }, { "path": "tests/files/debian11-calico-collection.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Use static containerd binary for older distributions like Debian 11.\ncontainerd_static_binary: true\n" }, { "path": "tests/files/debian11-calico-upgrade", "content": "UPGRADE_TEST=graceful\n" }, { "path": "tests/files/debian11-calico-upgrade-once", "content": "UPGRADE_TEST=graceful\n" }, { "path": "tests/files/debian11-calico-upgrade-once.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Kubespray settings\ndownload_run_once: true\n\n# Pin disabling ipip mode to ensure proper upgrade\nipip: false\ncalico_pool_blocksize: 26\ncalico_vxlan_mode: Always\ncalico_network_backend: bird\n\n# Needed to bypass deprecation check\nignore_assert_errors: true\n\n# Use static containerd binary for older distributions like Debian 11.\ncontainerd_static_binary: true\n" }, { "path": "tests/files/debian11-calico-upgrade.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Pin disabling ipip mode to ensure proper upgrade\nipip: false\ncalico_pool_blocksize: 26\ncalico_vxlan_mode: Always\ncalico_network_backend: bird\n\n# Needed to bypass deprecation check\nignore_assert_errors: true\n\n# Remove anonymous access to cluster\nremove_anonymous_access: true\n\n# Use static containerd binary for older distributions like Debian 11.\ncontainerd_static_binary: true\n" }, { "path": "tests/files/debian11-custom-cni.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Kubespray settings\nkube_owner: root\nkube_network_plugin: custom_cni\ncustom_cni_manifests:\n - \"{{ playbook_dir }}/../tests/files/custom_cni/cilium.yaml\"\n\n# Use static containerd binary for older distributions like Debian 11.\ncontainerd_static_binary: true\n" }, { "path": "tests/files/debian11-docker.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Use docker\ncontainer_manager: docker\netcd_deployment_type: docker\nresolvconf_mode: docker_dns\n" }, { "path": "tests/files/debian11-kubelet-csr-approver.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Kubespray settings\nkubelet_rotate_server_certificates: true\nkubelet_csr_approver_enabled: true\nkubelet_csr_approver_values:\n # Do not check DNS resolution in testing (not recommended in production)\n bypassDnsResolution: true\n\n# Use static containerd binary for older distributions like Debian 11.\ncontainerd_static_binary: true\n" }, { "path": "tests/files/debian11-macvlan.yml", "content": "---\n# Instance settings\ncloud_image: debian-11\n\n# Kubespray settings\nkube_network_plugin: macvlan\nenable_nodelocaldns: false\nkube_proxy_masquerade_all: true\nmacvlan_interface: \"eth0\"\nauto_renew_certificates: true\n\n# Use static containerd binary for older distributions like Debian 11.\ncontainerd_static_binary: true\n" }, { "path": "tests/files/debian12-calico.yml", "content": "---\n# Instance settings\ncloud_image: debian-12\n\n# Kubespray settings\n\ndns_mode: coredns_dual\nkube_asymmetric_encryption_algorithm: \"RSA-3072\"\n" }, { "path": "tests/files/debian12-cilium-svc-proxy.yml", "content": "---\n# Instance settings\ncloud_image: debian-12\nmode: ha\n\n# Kubespray settings\nkube_network_plugin: cilium\nenable_network_policy: true\n\ncilium_kube_proxy_replacement: true\n\nkube_owner: root\n" }, { "path": "tests/files/debian12-cilium.yml", "content": "---\n# Instance settings\ncloud_image: debian-12\n\n# Kubespray settings\nkube_network_plugin: cilium\n\n# ntp settings\nntp_enabled: true\nntp_package: ntp\n\nkube_owner: root\n" }, { "path": "tests/files/debian12-custom-cni-helm.yml", "content": "---\n# Instance settings\ncloud_image: debian-12\n\n# Kubespray settings\nkube_owner: root\nkube_network_plugin: custom_cni\ncustom_cni_chart_namespace: kube-system\ncustom_cni_chart_release_name: cilium\ncustom_cni_chart_repository_name: cilium\ncustom_cni_chart_repository_url: https://helm.cilium.io\ncustom_cni_chart_ref: cilium/cilium\ncustom_cni_chart_version: 1.18.6\ncustom_cni_chart_values:\n cluster:\n name: kubespray\n hubble:\n enabled: false\n ipam:\n operator:\n clusterPoolIPv4PodCIDRList:\n - \"{{ kube_pods_subnet }}\"\n" }, { "path": "tests/files/debian12-docker.yml", "content": "---\n# Instance settings\ncloud_image: debian-12\n\n# Use docker\ncontainer_manager: docker\netcd_deployment_type: docker\nresolvconf_mode: docker_dns\ndocker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg\n" }, { "path": "tests/files/debian13-calico.yml", "content": "---\n# Instance settings\ncloud_image: debian-13\n\n# Kubespray settings\ngateway_api_enabled: true\n\ndns_mode: coredns_dual\nkube_asymmetric_encryption_algorithm: \"RSA-3072\"\n" }, { "path": "tests/files/debian13-cilium.yml", "content": "---\n# Instance settings\ncloud_image: debian-13\n\n# Kubespray settings\nkube_network_plugin: cilium\n\nkube_owner: root\n\nprometheus_operator_crds_enabled: true\n" }, { "path": "tests/files/fedora39-calico-selinux.yml", "content": "---\n# Instance settings\ncloud_image: fedora-39\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Test with SELinux in enforcing mode\npreinstall_selinux_state: enforcing\n" }, { "path": "tests/files/fedora39-calico-swap-selinux.yml", "content": "---\n# Instance settings\ncloud_image: fedora-39\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Test with SELinux in enforcing mode\npreinstall_selinux_state: enforcing\n\n# Test Alpha swap feature by leveraging zswap default config in Fedora 35\nkubelet_fail_swap_on: false\nkube_feature_gates:\n - \"NodeSwap=True\"\n" }, { "path": "tests/files/fedora39-crio.yml", "content": "---\n# Instance settings\ncloud_image: fedora-39\n\n# Kubespray settings\ncontainer_manager: crio\nauto_renew_certificates: true\n\n# Test with SELinux in enforcing mode\npreinstall_selinux_state: enforcing\n" }, { "path": "tests/files/fedora39-kube-router.yml", "content": "---\ncloud_image: fedora-39\ncluster_layout:\n - node_groups: ['kube_control_plane', 'etcd', 'kube_node']\n - node_groups: ['kube_node']\n\nkube_network_plugin: \"kube-router\"\n" }, { "path": "tests/files/fedora40-docker-calico.yml", "content": "---\n# Instance settings\ncloud_image: fedora-40\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Docker specific settings:\ncontainer_manager: docker\netcd_deployment_type: docker\n" }, { "path": "tests/files/fedora40-docker.calico", "content": "RESET_CHECK=true\n" }, { "path": "tests/files/fedora40-flannel-crio-collection-scale.yml", "content": "---\ncloud_image: fedora-40\nnetwork_plugin: flannel\ncontainer_manager: crio\n\ncluster_layout:\n - node_groups: ['kube_control_plane', 'etcd']\n - node_groups: ['kube_node']\n - node_groups: ['kube_node', 'for_scale']\n" }, { "path": "tests/files/fedora41-calico-selinux.yml", "content": "---\n# Instance settings\ncloud_image: fedora-41\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Test with SELinux in enforcing mode\npreinstall_selinux_state: enforcing\n" }, { "path": "tests/files/fedora41-calico-swap-selinux.yml", "content": "---\n# Instance settings\ncloud_image: fedora-41\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Test with SELinux in enforcing mode\npreinstall_selinux_state: enforcing\n\n# Test Alpha swap feature by leveraging zswap default config in Fedora 35\nkubelet_fail_swap_on: false\nkube_feature_gates:\n - \"NodeSwap=True\"\n" }, { "path": "tests/files/fedora41-crio.yml", "content": "---\n# Instance settings\ncloud_image: fedora-41\n\n# Kubespray settings\ncontainer_manager: crio\nauto_renew_certificates: true\n" }, { "path": "tests/files/fedora41-kube-router.yml", "content": "---\ncloud_image: fedora-41\ncluster_layout:\n - node_groups: ['kube_control_plane', 'etcd', 'kube_node']\n - node_groups: ['kube_node']\n\nkube_network_plugin: \"kube-router\"\n" }, { "path": "tests/files/fedora42-calico.yml", "content": "---\n# Instance settings\ncloud_image: fedora-42\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Test with SELinux in enforcing mode\npreinstall_selinux_state: enforcing\n" }, { "path": "tests/files/flatcar4081-calico.yml", "content": "---\n# Instance settings\ncloud_image: flatcar-4081\nmode: default\nvm_memory: 3072\n\n# Kubespray settings\nmetrics_server_enabled: true\nloadbalancer_apiserver_type: haproxy\n" }, { "path": "tests/files/openeuler24-calico.yml", "content": "---\n# Instance settings\ncloud_image: openeuler-2403\nvm_memory: 3072\n\n# Use metalink for faster package downloads (auto-selects closest mirror)\nopeneuler_metalink_enabled: true\n\n# CI package installation takes ~7min; default 5min is too tight, use 15min for margin\npkg_install_timeout: \"{{ 15 * 60 }}\"\n\n# Work around so the Kubernetes 1.35 tests can pass. We will discuss the openeuler support later.\nkubeadm_ignore_preflight_errors:\n - SystemVerification\n\nkubelet_fail_cgroup_v1: false\n" }, { "path": "tests/files/rockylinux10-calico.yml", "content": "---\n# Instance settings\ncloud_image: rockylinux-10-extra\nvm_memory: 3072\n\n# Kubespray settings\nmetrics_server_enabled: true\nloadbalancer_apiserver_type: haproxy\n" }, { "path": "tests/files/rockylinux10-cilium", "content": "RESET_CHECK=true\n" }, { "path": "tests/files/rockylinux10-cilium.yml", "content": "---\n# Instance settings\ncloud_image: rockylinux-10-extra\nvm_memory: 3072\n\n# Kubespray settings\nkube_network_plugin: cilium\n\ncilium_kube_proxy_replacement: true\n\nkube_owner: root\n\n# Node Feature Discovery\nnode_feature_discovery_enabled: true\nkube_asymmetric_encryption_algorithm: \"ECDSA-P256\"\n\n# Testing no_proxy setup\n# The proxy is not intended to be accessed at all, we're only testing\n# the no_proxy construction\nhttps_proxy: \"http://some-proxy.invalid\"\nhttp_proxy: \"http://some-proxy.invalid\"\nadditional_no_proxy_list:\n - github.com\n - githubusercontent.com\n - k8s.io\n - rockylinux.org\n - docker.io\n - googleapis.com\n - quay.io\n - pkg.dev\n - amazonaws.com\n - cilium.io\nskip_http_proxy_on_os_packages: true\n" }, { "path": "tests/files/rockylinux9-calico.yml", "content": "---\n# Instance settings\ncloud_image: rockylinux-9\nvm_memory: 3072\n\n# Kubespray settings\nmetrics_server_enabled: true\nloadbalancer_apiserver_type: haproxy\n" }, { "path": "tests/files/rockylinux9-cilium", "content": "RESET_CHECK=true\n" }, { "path": "tests/files/rockylinux9-cilium.yml", "content": "---\n# Instance settings\ncloud_image: rockylinux-9\nvm_memory: 3072\n\n# Kubespray settings\nkube_network_plugin: cilium\n\ncilium_kube_proxy_replacement: true\n\nkube_owner: root\n\n# Node Feature Discovery\nnode_feature_discovery_enabled: true\nkube_asymmetric_encryption_algorithm: \"ECDSA-P256\"\n\nloadbalancer_apiserver_localhost: false\n" }, { "path": "tests/files/tf-elastx_ubuntu24-calico.yml", "content": "---\nsonobuoy_enabled: true\n\n# Ignore ping errors\nignore_assert_errors: true\n" }, { "path": "tests/files/ubuntu22-all-in-one-docker.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2204\nmode: all-in-one\nvm_memory: 3072\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# Use docker\ncontainer_manager: docker\netcd_deployment_type: docker\nresolvconf_mode: docker_dns\ndocker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg\n" }, { "path": "tests/files/ubuntu22-calico-all-in-one-upgrade", "content": "UPGRADE_TEST=graceful\n" }, { "path": "tests/files/ubuntu22-calico-all-in-one-upgrade.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2204\nmode: all-in-one\nvm_memory: 3072\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# Single node don't need the DNS autoscaler\nenable_dns_autoscaler: false\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - prefix: 172.19.16.11:5000\n mirrors:\n - host: http://172.19.16.11:5000\n capabilities: [\"pull\", \"resolve\", \"push\"]\n skip_verify: true\n" }, { "path": "tests/files/ubuntu22-calico-all-in-one.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2204\nmode: all-in-one\nvm_memory: 3072\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - prefix: 172.19.16.11:5000\n mirrors:\n - host: http://172.19.16.11:5000\n capabilities: [\"pull\", \"resolve\", \"push\"]\n skip_verify: true\n" }, { "path": "tests/files/ubuntu22-crio.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2204\n\n# Kubespray settings\ncontainer_manager: crio\n\ndownload_localhost: false\ndownload_run_once: true\n" }, { "path": "tests/files/ubuntu24-all-in-one-docker.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: all-in-one\nvm_memory: 3072\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# Use docker\ncontainer_manager: docker\netcd_deployment_type: docker\nresolvconf_mode: docker_dns\ndocker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg\n" }, { "path": "tests/files/ubuntu24-calico-all-in-one", "content": "RESET_CHECK=true\n" }, { "path": "tests/files/ubuntu24-calico-all-in-one-hardening.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: all-in-one\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# The followings are for hardening\n## kube-apiserver\nauthorization_modes: [\"Node\", \"RBAC\"]\nkube_apiserver_request_timeout: 120s\nkube_apiserver_service_account_lookup: true\n\n# enable kubernetes audit\nkubernetes_audit: true\naudit_log_path: \"/var/log/kube-apiserver-log.json\"\naudit_log_maxage: 30\naudit_log_maxbackups: 10\naudit_log_maxsize: 100\n\ntls_min_version: VersionTLS12\ntls_cipher_suites:\n - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n\n# enable encryption at rest\nkube_encrypt_secret_data: true\nkube_encryption_resources: [secrets]\nkube_encryption_algorithm: \"secretbox\"\n\nkube_apiserver_enable_admission_plugins:\n - EventRateLimit\n - AlwaysPullImages\n - ServiceAccount\n - NamespaceLifecycle\n - NodeRestriction\n - LimitRanger\n - ResourceQuota\n - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - PodNodeSelector\n - PodSecurity\nkube_apiserver_admission_control_config_file: true\n# EventRateLimit plugin configuration\nkube_apiserver_admission_event_rate_limits:\n limit_1:\n type: Namespace\n qps: 50\n burst: 100\n cache_size: 2000\n limit_2:\n type: User\n qps: 50\n burst: 100\nkube_profiling: false\n\n## kube-controller-manager\nkube_controller_manager_bind_address: 127.0.0.1\nkube_controller_terminated_pod_gc_threshold: 50\nkube_controller_feature_gates: [\"RotateKubeletServerCertificate=true\"]\n\n## kube-scheduler\nkube_scheduler_bind_address: 127.0.0.1\n\n## etcd\netcd_deployment_type: kubeadm\n\n## kubelet\nkubelet_authentication_token_webhook: true\nkube_read_only_port: 0\nkubelet_rotate_server_certificates: true\nkubelet_csr_approver_enabled: true # For hydrophone\nkubelet_csr_approver_values:\n # Do not check DNS resolution in testing (not recommended in production)\n bypassDnsResolution: true\nkubelet_protect_kernel_defaults: true\nkubelet_event_record_qps: 1\nkubelet_rotate_certificates: true\nkubelet_streaming_connection_idle_timeout: \"5m\"\nkubelet_make_iptables_util_chains: true\nkubelet_feature_gates: [\"RotateKubeletServerCertificate=true\"]\nkubelet_seccomp_default: true\nkubelet_systemd_hardening: true\n# In case you have multiple interfaces in your\n# control plane nodes and you want to specify the right\n# IP addresses, kubelet_secure_addresses allows you\n# to specify the IP from which the kubelet\n# will receive the packets.\n# kubelet_secure_addresses: \"192.168.10.110 192.168.10.111 192.168.10.112\"\n\n# additional configurations\nkube_owner: root\nkube_cert_group: root\n\n# create a default Pod Security Configuration and deny running of insecure pods\n# kube-system namespace is exempted by default\nkube_pod_security_use_default: true\nkube_pod_security_default_enforce: restricted\n\n# Remove anonymous access to cluster\nremove_anonymous_access: true\n" }, { "path": "tests/files/ubuntu24-calico-all-in-one.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: all-in-one\nvm_memory: 3072\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - prefix: 172.19.16.11:5000\n mirrors:\n - host: http://172.19.16.11:5000\n capabilities: [\"pull\", \"resolve\", \"push\"]\n skip_verify: true\n\nkube_reserved: true\nsystem_reserved: true\n" }, { "path": "tests/files/ubuntu24-calico-dual-stack.rb", "content": "$os = \"ubuntu2404\"\n\n$vm_cpus = 2\n$libvirt_volume_cache = \"unsafe\"\n\n# Checking for box update can trigger API rate limiting\n# https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html\n$box_check_update = false\n$network_plugin = \"calico\"\n" }, { "path": "tests/files/ubuntu24-calico-dual-stack.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\n\n# Kubespray settings\nipv4_stack: true\nipv6_stack: true\n" }, { "path": "tests/files/ubuntu24-calico-etcd-datastore.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: node-etcd-client\nvm_memory: 3072\n\n# Kubespray settings\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: nftables\nenable_nodelocaldns: false\n\ncontainerd_registries_mirrors:\n - prefix: docker.io\n mirrors:\n - host: https://mirror.gcr.io\n capabilities: [\"pull\", \"resolve\"]\n skip_verify: false\n - prefix: 172.19.16.11:5000\n mirrors:\n - host: http://172.19.16.11:5000\n capabilities: [\"pull\", \"resolve\", \"push\"]\n skip_verify: true\n\ncalico_datastore: \"etcd\"\n\n# Test kubeadm patches\nkubeadm_patches:\n - target: kube-apiserver\n patch:\n metadata:\n annotations:\n example.com/test: \"true\"\n labels:\n example.com/prod_level: \"prep\"\n - target: kube-controller-manager\n patch:\n metadata:\n annotations:\n example.com/test: \"false\"\n labels:\n example.com/prod_level: \"prep\"\n\n# ntp settings\nntp_enabled: true\nntp_package: ntpsec\n" }, { "path": "tests/files/ubuntu24-calico-etcd-kubeadm-upgrade-ha", "content": "UPGRADE_TEST=basic\n" }, { "path": "tests/files/ubuntu24-calico-etcd-kubeadm-upgrade-ha.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: ha\n\n# use the kubeadm etcd setting to test the upgrade\netcd_deployment_type: kubeadm\n\nupgrade_cluster_setup: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# Pin disabling ipip mode to ensure proper upgrade\nipip: false\ncalico_vxlan_mode: Always\ncalico_network_backend: bird\n\n# Needed to bypass deprecation check\nignore_assert_errors: true\n### FIXME FLORYUT Needed for upgrade job, will be removed when releasing kubespray 2.20\ncalico_pool_blocksize: 24\n### /FIXME\n" }, { "path": "tests/files/ubuntu24-calico-etcd-kubeadm.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\n\n# use the kubeadm etcd setting to test the upgrade\netcd_deployment_type: kubeadm\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# Remove anonymous access to cluster\nremove_anonymous_access: true\n" }, { "path": "tests/files/ubuntu24-calico-ha-recover", "content": "RECOVER_CONTROL_PLANE_TEST=true\nRECOVER_CONTROL_PLANE_TEST_GROUPS=\"etcd[2:]:kube_control_plane[1:]\"\n" }, { "path": "tests/files/ubuntu24-calico-ha-recover-noquorum", "content": "RECOVER_CONTROL_PLANE_TEST=true\nRECOVER_CONTROL_PLANE_TEST_GROUPS=\"etcd[1:]:kube_control_plane[1:]\"\n" }, { "path": "tests/files/ubuntu24-calico-ha-recover-noquorum.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: ha-recover-noquorum\n" }, { "path": "tests/files/ubuntu24-calico-ha-recover.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: ha-recover\n" }, { "path": "tests/files/ubuntu24-calico-ha-wireguard.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: ha\n\n# Kubespray settings\ncalico_wireguard_enabled: true\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\n# KVM kernel used by packet instances is missing the dummy.ko kernel module so it cannot enable nodelocaldns\nenable_nodelocaldns: false\n" }, { "path": "tests/files/ubuntu24-calico-ipv6only-stack.rb", "content": "$os = \"ubuntu2404\"\n\n$vm_cpus = 2\n$libvirt_volume_cache = \"unsafe\"\n\n# Checking for box update can trigger API rate limiting\n# https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html\n$box_check_update = false\n$network_plugin = \"calico\"\n" }, { "path": "tests/files/ubuntu24-calico-ipv6only-stack.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\n\n# Kubespray settings\nipv4_stack: false\nipv6_stack: true\nkube_network_plugin: calico\netcd_deployment_type: kubeadm\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n" }, { "path": "tests/files/ubuntu24-cilium-sep.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: separate\n\n# Kubespray settings\nkube_network_plugin: cilium\nenable_network_policy: true\nauto_renew_certificates: true\n\nkube_owner: root\n" }, { "path": "tests/files/ubuntu24-crio-scale.yml", "content": "---\ncloud_image: ubuntu-2404\ncontainer_manager: crio\n\ncluster_layout:\n - node_groups: [\"kube_control_plane\", \"etcd\"]\n - node_groups: [\"kube_node\"]\n - node_groups: [\"kube_node\", \"for_scale\"]\n" }, { "path": "tests/files/ubuntu24-crio-upgrade", "content": "UPGRADE_TEST=graceful\n" }, { "path": "tests/files/ubuntu24-crio-upgrade.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: all-in-one\nvm_memory: 3072\n\n# Kubespray settings\ncontainer_manager: crio\nauto_renew_certificates: true\n\n# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko\nkube_proxy_mode: iptables\nenable_nodelocaldns: false\n\n# Single node don't need the DNS autoscaler\nenable_dns_autoscaler: false\n" }, { "path": "tests/files/ubuntu24-flannel-collection.yml", "content": "---\ncloud_image: ubuntu-2404\ncluster_layout:\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"etcd\", \"kube_node\"]\n\nkube_network_plugin: flannel\n" }, { "path": "tests/files/ubuntu24-flannel-ha-once.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: ha\n\n# Kubespray settings\nkubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085\nkube_proxy_mode: iptables\nkube_network_plugin: flannel\nhelm_enabled: true\nkubernetes_audit: true\netcd_events_cluster_enabled: true\nlocal_volume_provisioner_enabled: true\nkube_encrypt_secret_data: true\ningress_nginx_enabled: true\ncert_manager_enabled: true\n# Disable as health checks are still unstable and slow to respond.\nmetrics_server_enabled: false\nmetrics_server_kubelet_insecure_tls: true\nkube_token_auth: true\nenable_nodelocaldns: false\n" }, { "path": "tests/files/ubuntu24-flannel-ha.yml", "content": "---\n# Instance settings\ncloud_image: ubuntu-2404\nmode: ha\n\n# Kubespray settings\nkube_network_plugin: flannel\netcd_deployment_type: kubeadm\nkubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085\nskip_non_kubeadm_warning: true\nkube_asymmetric_encryption_algorithm: \"RSA-4096\"\n\n# This test the variable usage, it is not a prerequisite of the test itself\nkubeadm_ignore_preflight_errors:\n - all\n" }, { "path": "tests/files/ubuntu24-flannel.yml", "content": "---\ncloud_image: ubuntu-2404\ncluster_layout:\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"etcd\", \"kube_node\"]\n\nkube_network_plugin: flannel\n" }, { "path": "tests/files/ubuntu24-ha-separate-etcd", "content": "REMOVE_NODE_CHECK=true\nREMOVE_NODE_NAME=etcd[2]\n" }, { "path": "tests/files/ubuntu24-ha-separate-etcd.yml", "content": "---\ncloud_image: ubuntu-2404\ncluster_layout:\n - node_groups: ['kube_control_plane']\n - node_groups: ['kube_control_plane']\n - node_groups: ['kube_control_plane']\n - node_groups: ['kube_node']\n - node_groups: ['etcd']\n - node_groups: ['etcd']\n - node_groups: ['etcd']\n\nkube_network_plugin: calico\ncalico_datastore: etcd\n" }, { "path": "tests/files/ubuntu24-kube-router-sep.yml", "content": "---\ncloud_image: ubuntu-2404\ncluster_layout:\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"kube_node\"]\n\nkube_network_plugin: \"kube-router\"\n" }, { "path": "tests/files/ubuntu24-kube-router-svc-proxy.yml", "content": "---\ncloud_image: ubuntu-2404\ncluster_layout:\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"kube_control_plane\", \"etcd\", \"kube_node\"]\n - node_groups: [\"etcd\", \"kube_node\"]\n\nkube_network_plugin: \"kube-router\"\n\nkube_router_run_service_proxy: true\n" }, { "path": "tests/requirements.txt", "content": "-r ../requirements.txt\ndistlib==0.4.0 # required for building collections\nmolecule==26.3.0\npytest-testinfra==10.2.2\n" }, { "path": "tests/scripts/check-templates.py", "content": "#!/usr/bin/env python\n\nimport sys\nimport traceback\nfrom jinja2 import Environment\nfrom jinja2.exceptions import TemplateSyntaxError\n\n\nenv = Environment()\nerrors = False\nfor template in sys.argv[1:]:\n try:\n with open(template) as t:\n env.parse(t.read())\n except TemplateSyntaxError as e:\n print (template)\n traceback.print_exc()\n errors = True\nif errors:\n exit (1)\n" }, { "path": "tests/scripts/collection-build-install.sh", "content": "#!/bin/sh -e\nexport ANSIBLE_COLLECTIONS_PATH=\"./ansible_collections\"\nansible-galaxy collection build --force\nansible-galaxy collection install kubernetes_sigs-kubespray-$(grep \"^version:\" galaxy.yml | awk '{print $2}').tar.gz\nansible-galaxy collection list $(egrep -i '(name:\\s+|namespace:\\s+)' galaxy.yml | awk '{print $2}' | tr '\\n' '.' | sed 's|\\.$||g') | grep \"^kubernetes_sigs.kubespray\"\ntest -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml\ntest -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml\n" }, { "path": "tests/scripts/md-table/main.py", "content": "#!/usr/bin/env python\nimport argparse\nimport sys\nimport glob\nfrom pathlib import Path\nimport yaml\nimport re\nimport jinja2\nimport sys\n\nfrom pprint import pprint\n\n\nparser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')\nparser.add_argument('--dir', default='tests/files/', help='folder with test yml files')\nparser.add_argument('--output', default='docs/developers/ci.md', help='output file')\n\n\nargs = parser.parse_args()\np = Path(args.dir)\n\nenv = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))\n\n# Data represents CI coverage data matrix\nclass Data:\n def __init__(self):\n self.container_managers = set()\n self.network_plugins = set()\n self.os = set()\n self.combination = set()\n\n\n def set(self, container_manager, network_plugin, os):\n self.container_managers.add(container_manager)\n self.network_plugins.add(network_plugin)\n self.os.add(os)\n self.combination.add(container_manager+network_plugin+os)\n\n def exists(self, container_manager, network_plugin, os):\n return (container_manager+network_plugin+os) in self.combination\n\n def jinja(self):\n template = env.get_template('table.md.j2')\n container_engines = sorted(self.container_managers)\n network_plugins = sorted(self.network_plugins)\n operating_systems = sorted(self.os)\n\n return template.render(\n container_engines=container_engines,\n network_plugins=network_plugins,\n operating_systems=operating_systems,\n exists=self.exists\n )\n\n def markdown(self):\n out = ''\n for container_manager in self.db.get_unique_ids('container_manager'):\n # Prepare the headers\n out += \"# \" + container_manager + \"\\n\"\n headers = '|OS / CNI| '\n underline = '|----|'\n for network_plugin in self.db.get_unique_ids(\"network_plugin\"):\n headers += network_plugin + ' | '\n underline += '----|'\n out += headers + \"\\n\" + underline + \"\\n\"\n for operating_system in self.db.get_unique_ids(\"operating_system\"):\n out += '| ' + operating_system + ' | '\n for network_plugin in self.db.get_unique_ids(\"network_plugin\"):\n if self.exists(container_manager, network_plugin, operating_system):\n emoji = ':white_check_mark:'\n else:\n emoji = ':x:'\n out += emoji + ' | '\n out += \"\\n\"\n\n pprint(self.db.get_unique_ids('operating_system'))\n pprint(self.db.get_unique_ids('network_plugin'))\n return out\n\n\n\nif not p.is_dir():\n print(\"Path is not a directory\")\n sys.exit(2)\n\ndata = Data()\nfiles = p.glob('*.yml')\nfor f in files:\n y = yaml.load(f.open(), Loader=yaml.FullLoader)\n\n container_manager = y.get('container_manager', 'containerd')\n network_plugin = y.get('kube_network_plugin', 'calico')\n x = re.match(r\"^([a-z-]+_)?([a-z0-9]+).*\", f.name)\n operating_system = x.group(2)\n data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system)\nprint(data.jinja(), file=open(args.output, 'w'))\n" }, { "path": "tests/scripts/md-table/table.md.j2", "content": "# CI test coverage\n\nTo generate this Matrix run `./tests/scripts/md-table/main.py`\n\n{%- for container_engine in container_engines %}\n\n## {{ container_engine }}\n\n| OS / CNI |{% for cni in network_plugins %} {{ cni }} |{% endfor %}\n|---|{% for cni in network_plugins %} --- |{% endfor %}\n{%- for os in operating_systems %}\n{{ os }} | {% for cni in network_plugins %} {{ ':white_check_mark:' if exists(container_engine, cni, os) else ':x:' }} |{% endfor %}\n{%- endfor %}\n\n{%- endfor %}\n" }, { "path": "tests/scripts/molecule_run.sh", "content": "#!/bin/bash\nset -euxo pipefail -o noglob\n\nexport LC_ALL=C.UTF-8\nexport LANG=C.UTF-8\n\n_PATH='roles'\n_EXCLUDE=\"\"\n\nwhile [[ $# -gt 0 ]] ; do\n case $1 in\n -e|--exclude)\n _EXCLUDE=\"${_EXCLUDE} -not -path ${_PATH}/$2/*\"\n shift\n shift\n ;;\n -i|--include)\n _PATH=\"${_PATH}/$2\"\n shift\n shift\n ;;\n -h|--help)\n echo \"Usage: molecule_run.sh [-h|--help] [-e|--exclude] [-i|--include]\"\n exit 0\n ;;\n esac\ndone\n\nfor d in $(find ${_PATH} ${_EXCLUDE} -name molecule -type d)\ndo\n pushd $(dirname $d)\n molecule test --all\n popd\ndone\n" }, { "path": "tests/scripts/opentofu_install.sh", "content": "#!/bin/bash\nset -euxo pipefail\n\ncurl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh\nchmod +x install-opentofu.sh\n./install-opentofu.sh --install-method standalone\nrm -f install-opentofu.sh\ntofu --version\n" }, { "path": "tests/scripts/rebase.sh", "content": "#!/bin/sh\nset -ex\n\nif [ \"${GITHUB_BASE_REF}\" ]; then\n git pull --rebase origin $GITHUB_BASE_REF\nfi\n" }, { "path": "tests/scripts/testcases_run.sh", "content": "#!/bin/bash\nset -euxo pipefail\n\nif [[ -v TESTCASE ]]; then\n TESTCASE_FILE=files/${TESTCASE}.yml\nelse\n TESTCASE_FILE=common_vars.yml\n TESTCASE=default\nfi\n\necho \"TESTCASE is $TESTCASE\"\n\nsource tests/files/$TESTCASE || true\n\n# Check out latest tag if testing upgrade\nif [ \"${UPGRADE_TEST}\" != \"false\" ]; then\n git fetch --all && git checkout $(git describe --tags --abbrev=0)\n # Checkout the current tests/ directory ; even when testing old version,\n # we want the up-to-date test setup/provisionning\n git checkout \"${CI_COMMIT_SHA}\" -- tests/\n pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt\nfi\n\nexport ANSIBLE_BECOME=true\nexport ANSIBLE_BECOME_USER=root\n\nrun_playbook () {\nif [[ \"${TESTCASE}\" =~ \"collection\" ]]; then\n playbook=kubernetes_sigs.kubespray.$1\n # Handle upgrade case properly\n rm -f kubernetes_sigs-kubespray-*.tar.gz\n ansible-galaxy collection build\n ansible-galaxy collection install kubernetes_sigs-kubespray-*.tar.gz\nelse\n playbook=$1.yml\nfi\nshift\n\nansible-playbook \\\n -e @tests/common_vars.yml \\\n -e @tests/${TESTCASE_FILE} \\\n \"$@\" \\\n ${playbook}\n}\n\n\n## START KUBESPRAY\n\n# Create cluster\nif [[ \"${TESTCASE}\" =~ \"scale\" ]]; then\n run_playbook cluster --limit '!for_scale'\n run_playbook scale --limit 'for_scale'\nelse\n run_playbook cluster\nfi\n\n# Repeat deployment if testing upgrade\nif [ \"${UPGRADE_TEST}\" != \"false\" ]; then\n git checkout \"${CI_COMMIT_SHA}\"\n\n pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt\n\n case \"${UPGRADE_TEST}\" in\n \"basic\")\n run_playbook cluster\n ;;\n \"graceful\")\n run_playbook upgrade_cluster\n ;;\n *)\n ;;\n esac\nfi\n\n# Test control plane recovery\nif [ \"${RECOVER_CONTROL_PLANE_TEST}\" != \"false\" ]; then\n run_playbook reset --limit \"${RECOVER_CONTROL_PLANE_TEST_GROUPS}\" -e reset_confirmation=yes\n run_playbook recover-control-plane -e etcd_retries=10 --limit \"etcd:kube_control_plane\"\nfi\n\n# Run tests\nansible-playbook \\\n -e @tests/common_vars.yml \\\n -e @tests/${TESTCASE_FILE} \\\n -e local_release_dir=${PWD}/downloads \\\n tests/testcases/tests.yml\n\n# Test node removal procedure\nif [ \"${REMOVE_NODE_CHECK}\" = \"true\" ]; then\n run_playbook remove-node -e skip_confirmation=yes -e node=\"${REMOVE_NODE_NAME}\"\nfi\n\n# Clean up at the end, this is to allow stage1 tests to include cleanup test\nif [ \"${RESET_CHECK}\" = \"true\" ]; then\n run_playbook reset -e reset_confirmation=yes\nfi\n" }, { "path": "tests/scripts/vagrant-install.sh", "content": "#!/bin/bash\n\n# install_vagrant() {\n# sudo apt install vagrant-libvirt vagrant -y\n# sudo vagrant plugin install vagrant-libvirt\n# }\n\n# prep(){\n# \tsudo apt-get update -y\n# \tsudo apt-get install ca-certificates curl libvirt-daemon-system\\\n# \t\tlibvirt-clients qemu-utils qemu-kvm htop atop -y\n\n# \tsudo install -m 0755 -d /etc/apt/keyrings\n# }\n# install_docker() {\n# \tVERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble\n# \tsudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc\n# \tsudo chmod a+r /etc/apt/keyrings/docker.asc\n\n# \t# Add the repository to Apt sources:\n# \techo \\\n# \t\t\"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \\\n# \t\t$(. /etc/os-release && echo \"$VERSION_CODENAME\") stable\" | \\\n# \t\t\tsudo tee /etc/apt/sources.list.d/docker.list > /dev/null\n# \tsudo apt-get update -y\n\n# \tsudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y\n# }\n# install_docker_auto () {\n# \t curl -fsSL https://get.docker.com -o get-docker.sh\n# \t sudo sh ./get-docker.sh --dry-run\n# }\n\n\n\nVAGRANT_VERSION=2.4.1\nVAGRANT_DEFAULT_PROVIDER=libvirt\nVAGRANT_ANSIBLE_TAGS=facts\nLANG=C.UTF-8\nDEBIAN_FRONTEND=noninteractive\nPYTHONDONTWRITEBYTECODE=1\nKUBE_VERSION=1.29.5\npipeline_install() {\n cp /etc/apt/sources.list /etc/apt/sources.list.\"$(date +\"%F\")\"\n sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list\n sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources\n\n apt update\n # libssl-dev \\\n # python3-dev \\\n # # jq \\\n # moreutils \\\n # libvirt-dev \\\n # # rsync \\\n # git \\\n # # htop \\\n # gpg \\\n # atop\n\n # gnupg2 \\\n# software-properties-common\n#\n apt install --no-install-recommends -y \\\n git \\\n make \\\n python3-pip \\\n sshpass \\\n apt-transport-https \\\n openssh-client \\\n ca-certificates \\\n curl \\\n libfuse2 \\\n unzip \\\n qemu-utils \\\n libvirt-daemon-system \\\n libvirt-clients \\\n qemu-kvm \\\n ebtables libguestfs-tools \\\n ruby-fog-libvirt \\\n libvirt-dev \\\n gcc \\\n build-essential \\\n ruby-libvirt \\\n libxslt-dev libxml2-dev zlib1g-dev \\\n python3-venv python3-full \\\n dnsmasq\n\n apt-get build-dep -y ruby-libvirt ruby-dev\n ### VAGRANT ###\n # apt-get install -y unzip\n curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip\n unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip\n mv vagrant /usr/local/bin/vagrant\n chmod a+x /usr/local/bin/vagrant\n # ls -la /usr/local/bin/vagrant\n /usr/local/bin/vagrant plugin install vagrant-libvirt\n usermod -aG kvm kubespray\n usermod -aG libvirt kubespray\n\n ### DOCKER ###\n curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -\n add-apt-repository -y \"deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\"\n apt update\n apt install --no-install-recommends -y docker-ce\n apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*\n\n ### KUBECTL ###\n curl -LO \"https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl\"\n mv kubectl /usr/local/bin/kubectl\n chmod a+x /usr/local/bin/kubectl\n systemctl restart libvirtd\n # Install Vagrant\n # apt update -y\n # echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt\n # apt --fix-broken install -y\n # dpkg --configure -a -y\n\n\n}\n# wrapped up in a function so that we have some protection against only getting\n# half the file during \"curl | sh\"\npipeline_install\n" }, { "path": "tests/scripts/vagrant-validate.sh", "content": "#!/bin/bash\nset -euxo pipefail\n\ncurl -sL \"https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_amd64.deb\" -o \"/tmp/vagrant_${VAGRANT_VERSION}-1_amd64.deb\"\ndpkg -i \"/tmp/vagrant_${VAGRANT_VERSION}-1_amd64.deb\"\nvagrant validate --ignore-provider\n" }, { "path": "tests/scripts/vagrant_clean.sh", "content": "#!/bin/bash\nset -euxo pipefail\n\n# Cleanup vagrant VMs to avoid name conflicts\n\nfor i in $(virsh list --name)\ndo\n virsh destroy \"$i\"\n virsh undefine \"$i\"\ndone\n\n\n# Cleanup domain volumes\nfor i in $(virsh vol-list default|grep \\.img |grep -v VAGRANTSLASH | cut -f 2 -d ' ')\ndo\n virsh vol-delete \"$i\" --pool default\ndone\n" }, { "path": "tests/testcases/000_install-hydrophone.yml", "content": "---\n- name: Download hydrophone\n get_url:\n url: \"https://github.com/kubernetes-sigs/hydrophone/releases/download/v{{ hydrophone_version }}/hydrophone_Linux_{{ hydrophone_arch }}.tar.gz\"\n dest: /tmp/hydrophone.tar.gz\n checksum: \"{{ hydrophone_checksum }}\"\n mode: \"0644\"\n\n- name: Extract hydrophone\n unarchive:\n src: /tmp/hydrophone.tar.gz\n dest: \"{{ bin_dir }}\"\n copy: false\n" }, { "path": "tests/testcases/010_check-apiserver.yml", "content": "---\n- name: Check the API servers are responding\n uri:\n url: \"https://{{ (access_ip if (ipv4_stack | default(true)) else access_ip6) | default(ansible_default_ipv4.address if (ipv4_stack | default(true)) else ansible_default_ipv6.address) | ansible.utils.ipwrap }}:{{ kube_apiserver_port | default(6443) }}/version\"\n validate_certs: false\n status_code: 200\n register: apiserver_response\n retries: 12\n delay: 5\n until: apiserver_response is success\n\n- name: Check API servers version\n assert:\n that:\n - apiserver_response.json.gitVersion == ('v' + kube_version)\n fail_msg: \"apiserver is {{ apiserver_response.json.gitVersion }}, expected {{ kube_version }}\"\n" }, { "path": "tests/testcases/015_check-nodes-ready.yml", "content": "---\n- import_role: # noqa name[missing]\n name: cluster-dump\n\n- name: Check kubectl output\n command: \"{{ bin_dir }}/kubectl get nodes\"\n changed_when: false\n register: get_nodes\n\n- name: Check that all nodes are running and ready\n command: \"{{ bin_dir }}/kubectl get nodes --no-headers -o yaml\"\n changed_when: false\n register: get_nodes_yaml\n until:\n # Check that all nodes are Status=Ready\n - '(get_nodes_yaml.stdout | from_yaml)[\"items\"] | map(attribute = \"status.conditions\") | map(\"items2dict\", key_name=\"type\", value_name=\"status\") | map(attribute=\"Ready\") | list | min'\n retries: 30\n delay: 10\n" }, { "path": "tests/testcases/020_check-pods-running.yml", "content": "---\n- import_role: # noqa name[missing]\n name: cluster-dump\n\n- name: Check kubectl output\n command: \"{{ bin_dir }}/kubectl get pods --all-namespaces -owide\"\n changed_when: false\n\n- name: Check pods\n vars:\n query_pods_not_running: \"items[?status.phase != 'Running']\"\n query_pods_not_ready: \"items[?(status.conditions[?type == 'Ready'])[0].status != 'True']\"\n pods_not_running: \"{{ run_pods_log.stdout | from_json | json_query(query_pods_not_running + '.metadata') }}\"\n pods_not_ready: \"{{ run_pods_log.stdout | from_json | json_query(query_pods_not_ready + '.metadata') }}\"\n block:\n - name: Check that all pods are running\n command: \"{{ bin_dir }}/kubectl get pods --all-namespaces -o json\"\n register: run_pods_log\n changed_when: false\n until:\n # Check that all pods are running\n - run_pods_log.stdout | from_json | json_query(query_pods_not_running) == []\n # Check that all pods are ready\n - run_pods_log.stdout | from_json | json_query(query_pods_not_ready) == []\n retries: 30\n delay: 10\n rescue:\n - name: Describe broken pods\n command: \"{{ bin_dir }}/kubectl describe pod -n {{ item.namespace }} {{ item.name }}\"\n loop: \"{{ pods_not_running + pods_not_ready }}\"\n loop_control:\n label: \"{{ item.namespace }}/{{ item.name }}\"\n - name: Get logs from broken pods\n command: \"{{ bin_dir }}/kubectl logs -n {{ item.namespace }} {{ item.name }}\"\n loop: \"{{ pods_not_running + pods_not_ready }}\"\n loop_control:\n label: \"{{ item.namespace }}/{{ item.name }}\"\n - name: Fail CI\n fail: {}\n" }, { "path": "tests/testcases/025_check-csr-request.yml", "content": "---\n- name: Check kubelet serving certificates approved with kubelet_csr_approver\n when:\n - kubelet_rotate_server_certificates | default(false)\n - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))\n vars:\n csrs: \"{{ csr_json.stdout | from_json }}\"\n block:\n\n - name: Get certificate signing requests\n command: \"{{ bin_dir }}/kubectl get csr -o jsonpath-as-json={.items[*]}\"\n register: csr_json\n changed_when: false\n\n - name: Check there are csrs\n assert:\n that: csrs | length > 0\n fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found\n\n - name: Check there are Denied/Pending csrs\n assert:\n that:\n - csrs | rejectattr('status') | length == 0 # Pending == no status\n - csrs | map(attribute='status.conditions') | flatten | selectattr('type', 'equalto', 'Denied') | length == 0 # Denied\n\n fail_msg: kubelet_csr_approver is enabled but CSRs are not approved\n\n- name: Approve kubelet serving certificates\n when:\n - kubelet_rotate_server_certificates | default(false)\n - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)))\n block:\n\n - name: Get certificate signing requests\n command: \"{{ bin_dir }}/kubectl get csr -o name\"\n register: get_csr\n changed_when: false\n\n - name: Check there are csrs\n assert:\n that: get_csr.stdout_lines | length > 0\n fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found\n\n - name: Approve certificates\n command: \"{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}\"\n register: certificate_approve\n when: get_csr.stdout_lines | length > 0\n changed_when: certificate_approve.stdout\n" }, { "path": "tests/testcases/030_check-network.yml", "content": "---\n- name: Run the hydrophone checks\n vars:\n networking_check: \"\\\\[sig-network\\\\] Networking Granular Checks.+\\\\[Conformance\\\\]\"\n block:\n - name: Run the networking granular checks\n command: \"{{ hydrophone_path }} --focus=\\\"{{ networking_check }}\\\" --parallel {{ hydrophone_parallel }}\"\n rescue:\n - name: List pods cluster-wide\n command: \"{{ bin_dir }}/kubectl get pods --all-namespaces -owide\"\n changed_when: false\n\n - import_role: # noqa name[missing]\n name: cluster-dump\n - fail: # noqa name[missing]\n" }, { "path": "tests/testcases/040_check-network-adv.yml", "content": "---\n- name: Test tunl0 routes\n command: \"/sbin/ip route\"\n register: routes\n failed_when: routes.stdout_lines\n | select('contains', '/' ~ calico_pool_blocksize|d(26))\n | select('contains', 'tunl0') | length == 0\n when:\n - ('kube_node' in group_names)\n - (calico_ipip_mode is defined and calico_ipip_mode != 'Never')\n - kube_network_plugin | default('calico') == 'calico'\n\n- import_role: # noqa name[missing]\n name: cluster-dump\n\n- name: Create macvlan network conf\n command:\n cmd: \"{{ bin_dir }}/kubectl create -f -\"\n stdin: |\n apiVersion: \"k8s.cni.cncf.io/v1\"\n kind: NetworkAttachmentDefinition\n metadata:\n name: macvlan-conf\n spec:\n config: '{\n \"cniVersion\": \"0.4.0\",\n \"type\": \"macvlan\",\n \"master\": \"eth0\",\n \"mode\": \"bridge\",\n \"ipam\": {\n \"type\": \"host-local\",\n \"subnet\": \"192.168.1.0/24\",\n \"rangeStart\": \"192.168.1.200\",\n \"rangeEnd\": \"192.168.1.216\",\n \"routes\": [\n { \"dst\": \"0.0.0.0/0\" }\n ],\n \"gateway\": \"192.168.1.1\"\n }\n }'\n ---\n apiVersion: v1\n kind: Pod\n metadata:\n name: samplepod\n annotations:\n k8s.v1.cni.cncf.io/networks: macvlan-conf\n spec:\n containers:\n - name: samplepod\n command: [\"/bin/bash\", \"-c\", \"sleep 2000000000000\"]\n image: dougbtv/centos-network\n delegate_to: groups['kube_control_plane'][0]\n run_once: true\n when:\n - kube_network_plugin_multus | default(false) | bool\n\n- name: Check secondary macvlan interface\n command: \"{{ bin_dir }}/kubectl exec samplepod -- ip addr show dev net1\"\n register: output\n until: output.rc == 0\n retries: 90\n changed_when: false\n delegate_to: groups['kube_control_plane'][0]\n run_once: true\n when:\n - kube_network_plugin_multus | default(false) | bool\n" }, { "path": "tests/testcases/100_check-k8s-conformance.yml", "content": "---\n- name: Download sonobuoy\n get_url:\n url: \"https://github.com/vmware-tanzu/sonobuoy/releases/download/v{{ sonobuoy_version }}/sonobuoy_{{ sonobuoy_version }}_linux_{{ sonobuoy_arch }}.tar.gz\"\n dest: /tmp/sonobuoy.tar.gz\n mode: \"0644\"\n\n- name: Extract sonobuoy\n unarchive:\n src: /tmp/sonobuoy.tar.gz\n dest: /usr/local/bin/\n copy: false\n\n- name: Run sonobuoy\n command: \"{{ sonobuoy_path }} run --mode {{ sonobuoy_mode }} --e2e-parallel {{ sonobuoy_parallel }} --wait\"\n\n- name: Run sonobuoy retrieve\n command: \"{{ sonobuoy_path }} retrieve\"\n register: sonobuoy_retrieve\n\n- name: Run inspect results\n command: \"{{ sonobuoy_path }} results {{ sonobuoy_retrieve.stdout }} --plugin e2e --mode report\"\n" }, { "path": "tests/testcases/roles/cluster-dump/tasks/main.yml", "content": "---\n- name: Generate dump folder\n command: \"{{ bin_dir }}/kubectl cluster-info dump --all-namespaces --output-directory /tmp/cluster-dump\"\n when: inventory_hostname in groups['kube_control_plane']\n\n- name: Compress directory cluster-dump\n community.general.archive:\n path: /tmp/cluster-dump\n dest: /tmp/cluster-dump.tgz\n mode: \"0644\"\n when: inventory_hostname in groups['kube_control_plane']\n\n- name: Fetch dump file\n fetch:\n src: /tmp/cluster-dump.tgz\n dest: \"{{ lookup('env', 'CI_PROJECT_DIR') }}/cluster-dump/{{ inventory_hostname }}.tgz\"\n flat: true\n when: inventory_hostname in groups['kube_control_plane']\n" }, { "path": "tests/testcases/tests.yml", "content": "---\n- name: Define dynamic groups\n import_playbook: ../../playbooks/boilerplate.yml\n\n- name: Kubespray CI tests\n hosts: k8s_cluster\n gather_facts: false\n vars:\n testcase: \"{{ lookup('env', 'TESTCASE') }}\"\n tasks:\n - name: Import Kubespray variables\n import_role:\n name: ../../roles/kubespray_defaults\n - name: Install the Hydrophone for tests\n import_tasks: 000_install-hydrophone.yml\n - name: Testcases for apiserver\n import_tasks: 010_check-apiserver.yml\n when:\n - ('kube_control_plane') in group_names\n - name: Test using API\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n block:\n - name: Testcases checking nodes\n import_tasks: 015_check-nodes-ready.yml\n - name: Testcases checking pods\n import_tasks: 020_check-pods-running.yml\n when: ('macvlan' not in testcase)\n - name: Checking CSR approver\n import_tasks: 025_check-csr-request.yml\n - name: Testcases for network\n import_tasks: 030_check-network.yml\n when: ('macvlan' not in testcase)\n - name: Testcases for calico / advanced network\n import_tasks: 040_check-network-adv.yml\n when:\n - ('macvlan' not in testcase)\n - ('hardening' not in testcase)\n - name: Testcases for kubernetes conformance\n import_tasks: 100_check-k8s-conformance.yml\n delegate_to: \"{{ groups['kube_control_plane'][0] }}\"\n run_once: true\n when:\n - sonobuoy_enabled is defined\n - sonobuoy_enabled\n vars:\n sonobuoy_version: 0.57.3\n sonobuoy_arch: amd64\n sonobuoy_parallel: 30\n sonobuoy_path: /usr/local/bin/sonobuoy\n sonobuoy_mode: Quick\n" }, { "path": "upgrade-cluster.yml", "content": "---\n- name: Upgrade cluster\n ansible.builtin.import_playbook: playbooks/upgrade_cluster.yml\n" }, { "path": "upgrade_cluster.yml", "content": "---\n- name: Upgrade cluster\n ansible.builtin.import_playbook: playbooks/upgrade_cluster.yml\n" } ]