Repository: kubernetes-sigs/kubespray Branch: master Commit: 341da0e8cb8c Files: 1193 Total size: 3.1 MB Directory structure: gitextract_cnwkyuho/ ├── .ansible-lint ├── .ansible-lint-ignore ├── .editorconfig ├── .gitattributes ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.yaml │ │ ├── config.yml │ │ ├── enhancement.yaml │ │ └── failing-test.yaml │ ├── PULL_REQUEST_TEMPLATE.md │ ├── dependabot.yml │ └── workflows/ │ ├── auto-label-os.yml │ ├── upgrade-patch-versions-schedule.yml │ └── upgrade-patch-versions.yml ├── .gitignore ├── .gitlab-ci/ │ ├── build.yml │ ├── kubevirt.yml │ ├── lint.yml │ ├── molecule.yml │ ├── terraform.yml │ └── vagrant.yml ├── .gitlab-ci.yml ├── .gitmodules ├── .md_style.rb ├── .mdlrc ├── .nojekyll ├── .pre-commit-config.yaml ├── .yamllint ├── CHANGELOG.md ├── CNAME ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── OWNERS ├── OWNERS_ALIASES ├── README.md ├── RELEASE.md ├── SECURITY_CONTACTS ├── Vagrantfile ├── _config.yml ├── ansible.cfg ├── cluster.yml ├── code-of-conduct.md ├── contrib/ │ ├── aws_iam/ │ │ ├── kubernetes-master-policy.json │ │ ├── kubernetes-master-role.json │ │ ├── kubernetes-minion-policy.json │ │ └── kubernetes-minion-role.json │ ├── aws_inventory/ │ │ ├── kubespray-aws-inventory.py │ │ └── requirements.txt │ ├── azurerm/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── apply-rg.sh │ │ ├── clear-rg.sh │ │ ├── generate-inventory.sh │ │ ├── generate-inventory.yml │ │ ├── generate-inventory_2.yml │ │ ├── generate-templates.yml │ │ ├── group_vars/ │ │ │ └── all │ │ └── roles/ │ │ ├── generate-inventory/ │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── inventory.j2 │ │ ├── generate-inventory_2/ │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── inventory.j2 │ │ │ └── loadbalancer_vars.j2 │ │ └── generate-templates/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── templates/ │ │ ├── availability-sets.json │ │ ├── bastion.json │ │ ├── clear-rg.json │ │ ├── masters.json │ │ ├── minions.json │ │ ├── network.json │ │ └── storage.json │ ├── collection.sh │ ├── offline/ │ │ ├── README.md │ │ ├── docker-daemon.json │ │ ├── generate_list.sh │ │ ├── generate_list.yml │ │ ├── manage-offline-container-images.sh │ │ ├── manage-offline-files.sh │ │ ├── nginx.conf │ │ ├── registries.conf │ │ └── upload2artifactory.py │ ├── os-services/ │ │ ├── os-services.yml │ │ └── roles/ │ │ └── prepare/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ └── tasks/ │ │ └── main.yml │ └── terraform/ │ ├── aws/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── create-infrastructure.tf │ │ ├── credentials.tfvars.example │ │ ├── modules/ │ │ │ ├── iam/ │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ └── variables.tf │ │ │ ├── nlb/ │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ └── variables.tf │ │ │ └── vpc/ │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ ├── output.tf │ │ ├── sample-inventory/ │ │ │ └── cluster.tfvars │ │ ├── templates/ │ │ │ └── inventory.tpl │ │ ├── terraform.tfvars │ │ ├── terraform.tfvars.example │ │ └── variables.tf │ ├── exoscale/ │ │ ├── README.md │ │ ├── default.tfvars │ │ ├── main.tf │ │ ├── modules/ │ │ │ └── kubernetes-cluster/ │ │ │ ├── main.tf │ │ │ ├── output.tf │ │ │ ├── templates/ │ │ │ │ └── cloud-init.tmpl │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ ├── output.tf │ │ ├── sample-inventory/ │ │ │ └── cluster.tfvars │ │ ├── templates/ │ │ │ └── inventory.tpl │ │ ├── variables.tf │ │ └── versions.tf │ ├── gcp/ │ │ ├── README.md │ │ ├── generate-inventory.sh │ │ ├── main.tf │ │ ├── modules/ │ │ │ └── kubernetes-cluster/ │ │ │ ├── main.tf │ │ │ ├── output.tf │ │ │ └── variables.tf │ │ ├── output.tf │ │ ├── tfvars.json │ │ └── variables.tf │ ├── hetzner/ │ │ ├── README.md │ │ ├── default.tfvars │ │ ├── main.tf │ │ ├── modules/ │ │ │ ├── kubernetes-cluster/ │ │ │ │ ├── main.tf │ │ │ │ ├── output.tf │ │ │ │ ├── templates/ │ │ │ │ │ └── cloud-init.tmpl │ │ │ │ ├── variables.tf │ │ │ │ └── versions.tf │ │ │ └── kubernetes-cluster-flatcar/ │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ ├── templates/ │ │ │ │ └── machine.yaml.tmpl │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ ├── output.tf │ │ ├── sample-inventory/ │ │ │ └── cluster.tfvars │ │ ├── templates/ │ │ │ └── inventory.tpl │ │ ├── variables.tf │ │ └── versions.tf │ ├── openstack/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── kubespray.tf │ │ ├── modules/ │ │ │ ├── compute/ │ │ │ │ ├── ansible_bastion_template.txt │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ ├── templates/ │ │ │ │ │ └── cloudinit.yaml.tmpl │ │ │ │ ├── variables.tf │ │ │ │ └── versions.tf │ │ │ ├── ips/ │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ ├── variables.tf │ │ │ │ └── versions.tf │ │ │ ├── loadbalancer/ │ │ │ │ ├── main.tf │ │ │ │ ├── variables.tf │ │ │ │ └── versions.tf │ │ │ └── network/ │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── terraform.py │ ├── upcloud/ │ │ ├── README.md │ │ ├── cluster-settings.tfvars │ │ ├── main.tf │ │ ├── modules/ │ │ │ └── kubernetes-cluster/ │ │ │ ├── main.tf │ │ │ ├── output.tf │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ ├── output.tf │ │ ├── sample-inventory/ │ │ │ └── cluster.tfvars │ │ ├── templates/ │ │ │ └── inventory.tpl │ │ ├── variables.tf │ │ └── versions.tf │ └── vsphere/ │ ├── README.md │ ├── default.tfvars │ ├── main.tf │ ├── modules/ │ │ └── kubernetes-cluster/ │ │ ├── main.tf │ │ ├── output.tf │ │ ├── templates/ │ │ │ ├── cloud-init.tpl │ │ │ ├── metadata.tpl │ │ │ └── vapp-cloud-init.tpl │ │ ├── variables.tf │ │ └── versions.tf │ ├── output.tf │ ├── sample-inventory/ │ │ └── cluster.tfvars │ ├── templates/ │ │ └── inventory.tpl │ ├── variables.tf │ └── versions.tf ├── docs/ │ ├── CNI/ │ │ ├── calico.md │ │ ├── cilium.md │ │ ├── cni.md │ │ ├── flannel.md │ │ ├── kube-ovn.md │ │ ├── kube-router.md │ │ ├── macvlan.md │ │ └── multus.md │ ├── CRI/ │ │ ├── containerd.md │ │ ├── cri-o.md │ │ ├── docker.md │ │ ├── gvisor.md │ │ └── kata-containers.md │ ├── CSI/ │ │ ├── aws-ebs-csi.md │ │ ├── azure-csi.md │ │ ├── cinder-csi.md │ │ ├── gcp-pd-csi.md │ │ └── vsphere-csi.md │ ├── _sidebar.md │ ├── advanced/ │ │ ├── arch.md │ │ ├── cert_manager.md │ │ ├── dns-stack.md │ │ ├── downloads.md │ │ ├── gcp-lb.md │ │ ├── kubernetes-reliability.md │ │ ├── netcheck.md │ │ ├── ntp.md │ │ ├── proxy.md │ │ └── registry.md │ ├── ansible/ │ │ ├── ansible.md │ │ ├── ansible_collection.md │ │ ├── inventory.md │ │ └── vars.md │ ├── calico_peer_example/ │ │ ├── new-york.yml │ │ └── paris.yml │ ├── cloud_controllers/ │ │ ├── openstack.md │ │ └── vsphere.md │ ├── cloud_providers/ │ │ ├── aws.md │ │ ├── azure.md │ │ └── cloud.md │ ├── developers/ │ │ ├── ci-setup.md │ │ ├── ci.md │ │ ├── test_cases.md │ │ └── vagrant.md │ ├── external_storage_provisioners/ │ │ ├── local_volume_provisioner.md │ │ └── scheduler_plugins.md │ ├── getting_started/ │ │ ├── comparisons.md │ │ ├── getting-started.md │ │ └── setting-up-your-first-cluster.md │ ├── ingress/ │ │ ├── alb_ingress_controller.md │ │ ├── kube-vip.md │ │ └── metallb.md │ ├── operating_systems/ │ │ ├── amazonlinux.md │ │ ├── bootstrap-os.md │ │ ├── fcos.md │ │ ├── flatcar.md │ │ ├── kylinlinux.md │ │ ├── openeuler.md │ │ ├── opensuse.md │ │ ├── rhel.md │ │ └── uoslinux.md │ ├── operations/ │ │ ├── cgroups.md │ │ ├── encrypting-secret-data-at-rest.md │ │ ├── etcd.md │ │ ├── ha-mode.md │ │ ├── hardening.md │ │ ├── integration.md │ │ ├── kernel-requirements.md │ │ ├── large-deployments.md │ │ ├── mirror.md │ │ ├── nodes.md │ │ ├── offline-environment.md │ │ ├── port-requirements.md │ │ ├── recover-control-plane.md │ │ └── upgrades.md │ ├── roadmap/ │ │ └── roadmap.md │ └── upgrades/ │ └── migrate_docker2containerd.md ├── extra_playbooks/ │ ├── files/ │ │ └── get_cinder_pvs.sh │ ├── migrate_openstack_provider.yml │ ├── upgrade-only-k8s.yml │ └── wait-for-cloud-init.yml ├── galaxy.yml ├── index.html ├── inventory/ │ ├── local/ │ │ └── hosts.ini │ └── sample/ │ ├── group_vars/ │ │ ├── all/ │ │ │ ├── all.yml │ │ │ ├── aws.yml │ │ │ ├── azure.yml │ │ │ ├── containerd.yml │ │ │ ├── coreos.yml │ │ │ ├── cri-o.yml │ │ │ ├── docker.yml │ │ │ ├── etcd.yml │ │ │ ├── gcp.yml │ │ │ ├── hcloud.yml │ │ │ ├── huaweicloud.yml │ │ │ ├── oci.yml │ │ │ ├── offline.yml │ │ │ ├── openstack.yml │ │ │ ├── upcloud.yml │ │ │ └── vsphere.yml │ │ └── k8s_cluster/ │ │ ├── addons.yml │ │ ├── k8s-cluster.yml │ │ ├── k8s-net-calico.yml │ │ ├── k8s-net-cilium.yml │ │ ├── k8s-net-custom-cni.yml │ │ ├── k8s-net-flannel.yml │ │ ├── k8s-net-kube-ovn.yml │ │ ├── k8s-net-kube-router.yml │ │ ├── k8s-net-macvlan.yml │ │ └── kube_control_plane.yml │ └── inventory.ini ├── logo/ │ ├── LICENSE │ └── usage_guidelines.md ├── meta/ │ └── runtime.yml ├── pipeline.Dockerfile ├── playbooks/ │ ├── ansible_version.yml │ ├── boilerplate.yml │ ├── cluster.yml │ ├── facts.yml │ ├── install_etcd.yml │ ├── internal_facts.yml │ ├── recover_control_plane.yml │ ├── remove_node.yml │ ├── reset.yml │ ├── scale.yml │ └── upgrade_cluster.yml ├── plugins/ │ └── modules/ │ └── kube.py ├── recover-control-plane.yml ├── remove-node.yml ├── remove_node.yml ├── requirements.txt ├── reset.yml ├── roles/ │ ├── adduser/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── molecule/ │ │ │ └── default/ │ │ │ ├── converge.yml │ │ │ └── molecule.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── vars/ │ │ ├── coreos.yml │ │ ├── debian.yml │ │ └── redhat.yml │ ├── bastion-ssh-config/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── molecule/ │ │ │ └── default/ │ │ │ ├── converge.yml │ │ │ └── molecule.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── templates/ │ │ └── ssh-bastion.conf.j2 │ ├── bootstrap-os/ │ │ └── tasks/ │ │ └── main.yml │ ├── bootstrap_os/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── files/ │ │ │ └── bootstrap.sh │ │ ├── handlers/ │ │ │ └── main.yml │ │ ├── meta/ │ │ │ └── main.yml │ │ ├── molecule/ │ │ │ └── default/ │ │ │ ├── converge.yml │ │ │ ├── molecule.yml │ │ │ └── tests/ │ │ │ └── test_default.py │ │ ├── tasks/ │ │ │ ├── almalinux.yml │ │ │ ├── amzn.yml │ │ │ ├── centos.yml │ │ │ ├── debian.yml │ │ │ ├── fedora-coreos.yml │ │ │ ├── fedora.yml │ │ │ ├── flatcar.yml │ │ │ ├── main.yml │ │ │ ├── openEuler.yml │ │ │ ├── opensuse-leap.yml │ │ │ ├── opensuse-tumbleweed.yml │ │ │ ├── opensuse.yml │ │ │ ├── rhel.yml │ │ │ ├── rocky.yml │ │ │ └── ubuntu.yml │ │ └── vars/ │ │ ├── fedora-coreos.yml │ │ └── flatcar.yml │ ├── container-engine/ │ │ ├── containerd/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ ├── main.yml │ │ │ │ └── reset.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── molecule/ │ │ │ │ └── default/ │ │ │ │ ├── converge.yml │ │ │ │ ├── molecule.yml │ │ │ │ └── verify.yml │ │ │ ├── tasks/ │ │ │ │ ├── main.yml │ │ │ │ └── reset.yml │ │ │ └── templates/ │ │ │ ├── config-v1.toml.j2 │ │ │ ├── config.toml.j2 │ │ │ ├── containerd.service.j2 │ │ │ ├── hosts.toml.j2 │ │ │ └── http-proxy.conf.j2 │ │ ├── containerd-common/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── vars/ │ │ │ ├── amazon.yml │ │ │ └── suse.yml │ │ ├── cri-dockerd/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── molecule/ │ │ │ │ └── default/ │ │ │ │ ├── converge.yml │ │ │ │ ├── files/ │ │ │ │ │ ├── 10-mynet.conf │ │ │ │ │ ├── container.json │ │ │ │ │ └── sandbox.json │ │ │ │ ├── molecule.yml │ │ │ │ └── verify.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── cri-dockerd.service.j2 │ │ │ └── cri-dockerd.socket.j2 │ │ ├── cri-o/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── molecule/ │ │ │ │ └── default/ │ │ │ │ ├── converge.yml │ │ │ │ ├── molecule.yml │ │ │ │ └── verify.yml │ │ │ ├── tasks/ │ │ │ │ ├── load_vars.yml │ │ │ │ ├── main.yaml │ │ │ │ ├── reset.yml │ │ │ │ └── setup-amazon.yaml │ │ │ ├── templates/ │ │ │ │ ├── config.json.j2 │ │ │ │ ├── crio.conf.j2 │ │ │ │ ├── http-proxy.conf.j2 │ │ │ │ ├── mounts.conf.j2 │ │ │ │ ├── registry.conf.j2 │ │ │ │ └── unqualified.conf.j2 │ │ │ └── vars/ │ │ │ ├── v1.29.yml │ │ │ └── v1.31.yml │ │ ├── crictl/ │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── crictl.yaml.j2 │ │ ├── crun/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── docker/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── files/ │ │ │ │ └── cleanup-docker-orphans.sh │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── docker_plugin.yml │ │ │ │ ├── main.yml │ │ │ │ ├── pre-upgrade.yml │ │ │ │ ├── reset.yml │ │ │ │ ├── set_facts_dns.yml │ │ │ │ └── systemd.yml │ │ │ ├── templates/ │ │ │ │ ├── docker-dns.conf.j2 │ │ │ │ ├── docker-options.conf.j2 │ │ │ │ ├── docker-orphan-cleanup.conf.j2 │ │ │ │ ├── docker.service.j2 │ │ │ │ ├── fedora_docker.repo.j2 │ │ │ │ ├── http-proxy.conf.j2 │ │ │ │ └── rh_docker.repo.j2 │ │ │ └── vars/ │ │ │ ├── amazon.yml │ │ │ ├── clearlinux.yml │ │ │ ├── debian.yml │ │ │ ├── fedora.yml │ │ │ ├── kylin.yml │ │ │ ├── redhat.yml │ │ │ ├── suse.yml │ │ │ ├── ubuntu.yml │ │ │ └── uniontech.yml │ │ ├── gvisor/ │ │ │ ├── molecule/ │ │ │ │ └── default/ │ │ │ │ ├── converge.yml │ │ │ │ ├── files/ │ │ │ │ │ ├── 10-mynet.conf │ │ │ │ │ ├── container.json │ │ │ │ │ └── sandbox.json │ │ │ │ ├── molecule.yml │ │ │ │ └── verify.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── kata-containers/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── molecule/ │ │ │ │ └── default/ │ │ │ │ ├── converge.yml │ │ │ │ ├── files/ │ │ │ │ │ ├── 10-mynet.conf │ │ │ │ │ ├── container.json │ │ │ │ │ └── sandbox.json │ │ │ │ ├── molecule.yml │ │ │ │ └── verify.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── configuration-qemu.toml.j2 │ │ │ └── containerd-shim-kata-v2.j2 │ │ ├── molecule/ │ │ │ ├── files/ │ │ │ │ └── 10-mynet.conf │ │ │ ├── prepare.yml │ │ │ ├── templates/ │ │ │ │ ├── container.json.j2 │ │ │ │ └── sandbox.json.j2 │ │ │ ├── test_cri.yml │ │ │ └── test_runtime.yml │ │ ├── nerdctl/ │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── nerdctl.toml.j2 │ │ ├── runc/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── skopeo/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ ├── validate-container-engine/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ └── youki/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── molecule/ │ │ │ └── default/ │ │ │ ├── converge.yml │ │ │ ├── files/ │ │ │ │ ├── 10-mynet.conf │ │ │ │ ├── container.json │ │ │ │ └── sandbox.json │ │ │ ├── molecule.yml │ │ │ └── verify.yml │ │ └── tasks/ │ │ └── main.yml │ ├── download/ │ │ ├── meta/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ ├── check_pull_required.yml │ │ │ ├── download_container.yml │ │ │ ├── download_file.yml │ │ │ ├── extract_file.yml │ │ │ ├── main.yml │ │ │ ├── prep_download.yml │ │ │ ├── prep_kubeadm_images.yml │ │ │ └── set_container_facts.yml │ │ └── templates/ │ │ └── kubeadm-images.yaml.j2 │ ├── dynamic_groups/ │ │ └── tasks/ │ │ └── main.yml │ ├── etcd/ │ │ ├── handlers/ │ │ │ ├── backup.yml │ │ │ ├── backup_cleanup.yml │ │ │ └── main.yml │ │ ├── meta/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ ├── check_certs.yml │ │ │ ├── clean_v2_store.yml │ │ │ ├── configure.yml │ │ │ ├── gen_certs_script.yml │ │ │ ├── gen_nodes_certs_script.yml │ │ │ ├── install_docker.yml │ │ │ ├── install_host.yml │ │ │ ├── join_etcd-events_member.yml │ │ │ ├── join_etcd_member.yml │ │ │ ├── main.yml │ │ │ ├── refresh_config.yml │ │ │ └── upd_ca_trust.yml │ │ └── templates/ │ │ ├── etcd-docker.service.j2 │ │ ├── etcd-events-docker.service.j2 │ │ ├── etcd-events-host.service.j2 │ │ ├── etcd-events.env.j2 │ │ ├── etcd-events.j2 │ │ ├── etcd-host.service.j2 │ │ ├── etcd.env.j2 │ │ ├── etcd.j2 │ │ ├── make-ssl-etcd.sh.j2 │ │ └── openssl.conf.j2 │ ├── etcd_defaults/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ └── vars/ │ │ └── main.yml │ ├── etcdctl_etcdutl/ │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── templates/ │ │ └── etcdctl.sh.j2 │ ├── helm-apps/ │ │ ├── README.md │ │ ├── meta/ │ │ │ ├── argument_specs.yml │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── vars/ │ │ └── main.yml │ ├── kubernetes/ │ │ ├── client/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── control-plane/ │ │ │ ├── defaults/ │ │ │ │ └── main/ │ │ │ │ ├── etcd.yml │ │ │ │ ├── kube-proxy.yml │ │ │ │ ├── kube-scheduler.yml │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── check-api.yml │ │ │ │ ├── encrypt-at-rest.yml │ │ │ │ ├── kubeadm-backup.yml │ │ │ │ ├── kubeadm-etcd.yml │ │ │ │ ├── kubeadm-secondary.yml │ │ │ │ ├── kubeadm-setup.yml │ │ │ │ ├── kubeadm-upgrade.yml │ │ │ │ ├── main.yml │ │ │ │ └── pre-upgrade.yml │ │ │ ├── templates/ │ │ │ │ ├── admission-controls.yaml.j2 │ │ │ │ ├── apiserver-audit-policy.yaml.j2 │ │ │ │ ├── apiserver-audit-webhook-config.yaml.j2 │ │ │ │ ├── apiserver-tracing.yaml.j2 │ │ │ │ ├── eventratelimit.yaml.j2 │ │ │ │ ├── k8s-certs-renew.service.j2 │ │ │ │ ├── k8s-certs-renew.sh.j2 │ │ │ │ ├── k8s-certs-renew.timer.j2 │ │ │ │ ├── kubeadm-config.v1beta4.yaml.j2 │ │ │ │ ├── kubeadm-controlplane.yaml.j2 │ │ │ │ ├── kubescheduler-config.yaml.j2 │ │ │ │ ├── podnodeselector.yaml.j2 │ │ │ │ ├── podsecurity.yaml.j2 │ │ │ │ ├── resourcequota.yaml.j2 │ │ │ │ ├── secrets_encryption.yaml.j2 │ │ │ │ ├── webhook-authorization-config.yaml.j2 │ │ │ │ └── webhook-token-auth-config.yaml.j2 │ │ │ └── vars/ │ │ │ └── main.yaml │ │ ├── kubeadm/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── kubeadm_etcd_node.yml │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── kubeadm-client.conf.j2 │ │ ├── kubeadm_common/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── node/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── facts.yml │ │ │ │ ├── install.yml │ │ │ │ ├── kubelet.yml │ │ │ │ ├── loadbalancer/ │ │ │ │ │ ├── haproxy.yml │ │ │ │ │ ├── kube-vip.yml │ │ │ │ │ └── nginx-proxy.yml │ │ │ │ └── main.yml │ │ │ ├── templates/ │ │ │ │ ├── http-proxy.conf.j2 │ │ │ │ ├── kubelet-config.v1beta1.yaml.j2 │ │ │ │ ├── kubelet.env.v1beta1.j2 │ │ │ │ ├── kubelet.service.j2 │ │ │ │ ├── loadbalancer/ │ │ │ │ │ ├── haproxy.cfg.j2 │ │ │ │ │ └── nginx.conf.j2 │ │ │ │ └── manifests/ │ │ │ │ ├── haproxy.manifest.j2 │ │ │ │ ├── kube-vip.manifest.j2 │ │ │ │ └── nginx-proxy.manifest.j2 │ │ │ └── vars/ │ │ │ ├── fedora.yml │ │ │ ├── ubuntu-18.yml │ │ │ ├── ubuntu-20.yml │ │ │ ├── ubuntu-22.yml │ │ │ └── ubuntu-24.yml │ │ ├── node-label/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── node-taint/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ └── preinstall/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── files/ │ │ │ └── dhclient_nodnsupdate │ │ ├── handlers/ │ │ │ └── main.yml │ │ ├── meta/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ ├── 0010-swapoff.yml │ │ │ ├── 0020-set_facts.yml │ │ │ ├── 0040-verify-settings.yml │ │ │ ├── 0050-create_directories.yml │ │ │ ├── 0060-resolvconf.yml │ │ │ ├── 0061-systemd-resolved.yml │ │ │ ├── 0062-networkmanager-unmanaged-devices.yml │ │ │ ├── 0063-networkmanager-dns.yml │ │ │ ├── 0080-system-configurations.yml │ │ │ ├── 0081-ntp-configurations.yml │ │ │ ├── 0100-dhclient-hooks.yml │ │ │ ├── 0110-dhclient-hooks-undo.yml │ │ │ └── main.yml │ │ ├── templates/ │ │ │ ├── ansible_git.j2 │ │ │ ├── chrony.conf.j2 │ │ │ ├── dhclient_dnsupdate.sh.j2 │ │ │ ├── dhclient_dnsupdate_rh.sh.j2 │ │ │ ├── ntp.conf.j2 │ │ │ ├── resolvconf.j2 │ │ │ └── resolved.conf.j2 │ │ └── vars/ │ │ └── main.yml │ ├── kubernetes-apps/ │ │ ├── ansible/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ ├── templates/ │ │ │ │ ├── coredns-clusterrole.yml.j2 │ │ │ │ ├── coredns-clusterrolebinding.yml.j2 │ │ │ │ ├── coredns-config.yml.j2 │ │ │ │ ├── coredns-deployment.yml.j2 │ │ │ │ ├── coredns-poddisruptionbudget.yml.j2 │ │ │ │ ├── coredns-sa.yml.j2 │ │ │ │ ├── coredns-svc.yml.j2 │ │ │ │ ├── dns-autoscaler-clusterrole.yml.j2 │ │ │ │ ├── dns-autoscaler-clusterrolebinding.yml.j2 │ │ │ │ ├── dns-autoscaler-sa.yml.j2 │ │ │ │ ├── dns-autoscaler.yml.j2 │ │ │ │ ├── etcd_metrics-endpoints.yml.j2 │ │ │ │ ├── etcd_metrics-service.yml.j2 │ │ │ │ ├── nodelocaldns-config.yml.j2 │ │ │ │ ├── nodelocaldns-daemonset.yml.j2 │ │ │ │ ├── nodelocaldns-sa.yml.j2 │ │ │ │ └── nodelocaldns-second-daemonset.yml.j2 │ │ │ └── vars/ │ │ │ └── main.yml │ │ ├── argocd/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── argocd-namespace.yml.j2 │ │ ├── cluster_roles/ │ │ │ ├── files/ │ │ │ │ └── k8s-cluster-critical-pc.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── namespace.j2 │ │ │ ├── node-crb.yml.j2 │ │ │ └── vsphere-rbac.yml.j2 │ │ ├── common_crds/ │ │ │ ├── gateway_api/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ └── tasks/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ └── prometheus_operator_crds/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── container_engine_accelerator/ │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ └── nvidia_gpu/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ ├── templates/ │ │ │ │ ├── k8s-device-plugin-nvidia-daemonset.yml.j2 │ │ │ │ └── nvidia-driver-install-daemonset.yml.j2 │ │ │ └── vars/ │ │ │ ├── ubuntu-16.yml │ │ │ └── ubuntu-18.yml │ │ ├── container_runtimes/ │ │ │ ├── crun/ │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yaml │ │ │ │ └── templates/ │ │ │ │ └── runtimeclass-crun.yml │ │ │ ├── gvisor/ │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yaml │ │ │ │ └── templates/ │ │ │ │ └── runtimeclass-gvisor.yml.j2 │ │ │ ├── kata_containers/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yaml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yaml │ │ │ │ └── templates/ │ │ │ │ └── runtimeclass-kata-qemu.yml.j2 │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ └── youki/ │ │ │ ├── tasks/ │ │ │ │ └── main.yaml │ │ │ └── templates/ │ │ │ └── runtimeclass-youki.yml │ │ ├── csi_driver/ │ │ │ ├── aws_ebs/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── aws-ebs-csi-controllerservice-rbac.yml.j2 │ │ │ │ ├── aws-ebs-csi-controllerservice.yml.j2 │ │ │ │ ├── aws-ebs-csi-driver.yml.j2 │ │ │ │ └── aws-ebs-csi-nodeservice.yml.j2 │ │ │ ├── azuredisk/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ ├── azure-credential-check.yml │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── azure-csi-azuredisk-controller-rbac.yml.j2 │ │ │ │ ├── azure-csi-azuredisk-controller.yml.j2 │ │ │ │ ├── azure-csi-azuredisk-driver.yml.j2 │ │ │ │ ├── azure-csi-azuredisk-node-rbac.yml.j2 │ │ │ │ ├── azure-csi-azuredisk-node.yml.j2 │ │ │ │ ├── azure-csi-cloud-config-secret.yml.j2 │ │ │ │ └── azure-csi-cloud-config.j2 │ │ │ ├── cinder/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ ├── cinder-credential-check.yml │ │ │ │ │ ├── cinder-write-cacert.yml │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── cinder-csi-cloud-config-secret.yml.j2 │ │ │ │ ├── cinder-csi-cloud-config.j2 │ │ │ │ ├── cinder-csi-controllerplugin-rbac.yml.j2 │ │ │ │ ├── cinder-csi-controllerplugin.yml.j2 │ │ │ │ ├── cinder-csi-driver.yml.j2 │ │ │ │ ├── cinder-csi-nodeplugin-rbac.yml.j2 │ │ │ │ ├── cinder-csi-nodeplugin.yml.j2 │ │ │ │ └── cinder-csi-poddisruptionbudget.yml.j2 │ │ │ ├── csi_crd/ │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── volumegroupsnapshotclasses.yml.j2 │ │ │ │ ├── volumegroupsnapshotcontents.yml.j2 │ │ │ │ ├── volumegroupsnapshots.yml.j2 │ │ │ │ ├── volumesnapshotclasses.yml.j2 │ │ │ │ ├── volumesnapshotcontents.yml.j2 │ │ │ │ └── volumesnapshots.yml.j2 │ │ │ ├── gcp_pd/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── gcp-pd-csi-controller.yml.j2 │ │ │ │ ├── gcp-pd-csi-cred-secret.yml.j2 │ │ │ │ ├── gcp-pd-csi-node.yml.j2 │ │ │ │ ├── gcp-pd-csi-sc-regional.yml.j2 │ │ │ │ ├── gcp-pd-csi-sc-zonal.yml.j2 │ │ │ │ └── gcp-pd-csi-setup.yml.j2 │ │ │ ├── upcloud/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── upcloud-csi-controller.yml.j2 │ │ │ │ ├── upcloud-csi-cred-secret.yml.j2 │ │ │ │ ├── upcloud-csi-driver.yml.j2 │ │ │ │ ├── upcloud-csi-node.yml.j2 │ │ │ │ └── upcloud-csi-setup.yml.j2 │ │ │ └── vsphere/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── main.yml │ │ │ │ └── vsphere-credentials-check.yml │ │ │ └── templates/ │ │ │ ├── vsphere-csi-cloud-config.j2 │ │ │ ├── vsphere-csi-controller-config.yml.j2 │ │ │ ├── vsphere-csi-controller-deployment.yml.j2 │ │ │ ├── vsphere-csi-controller-rbac.yml.j2 │ │ │ ├── vsphere-csi-controller-service.yml.j2 │ │ │ ├── vsphere-csi-driver.yml.j2 │ │ │ ├── vsphere-csi-namespace.yml.j2 │ │ │ ├── vsphere-csi-node-rbac.yml.j2 │ │ │ └── vsphere-csi-node.yml.j2 │ │ ├── external_cloud_controller/ │ │ │ ├── hcloud/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 │ │ │ │ ├── external-hcloud-cloud-controller-manager-ds.yml.j2 │ │ │ │ ├── external-hcloud-cloud-role-bindings.yml.j2 │ │ │ │ ├── external-hcloud-cloud-secret.yml.j2 │ │ │ │ └── external-hcloud-cloud-service-account.yml.j2 │ │ │ ├── huaweicloud/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ ├── huaweicloud-credential-check.yml │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── external-huawei-cloud-config-secret.yml.j2 │ │ │ │ ├── external-huawei-cloud-config.j2 │ │ │ │ ├── external-huawei-cloud-controller-manager-ds.yml.j2 │ │ │ │ ├── external-huawei-cloud-controller-manager-role-bindings.yml.j2 │ │ │ │ └── external-huawei-cloud-controller-manager-roles.yml.j2 │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── oci/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── external-oci-cloud-config-secret.yml.j2 │ │ │ │ ├── external-oci-cloud-config.yml.j2 │ │ │ │ ├── external-oci-cloud-controller-manager-rbac.yml.j2 │ │ │ │ └── external-oci-cloud-controller-manager.yml.j2 │ │ │ ├── openstack/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ ├── main.yml │ │ │ │ │ └── openstack-credential-check.yml │ │ │ │ └── templates/ │ │ │ │ ├── external-openstack-cloud-config-secret.yml.j2 │ │ │ │ ├── external-openstack-cloud-config.j2 │ │ │ │ ├── external-openstack-cloud-controller-manager-ds.yml.j2 │ │ │ │ ├── external-openstack-cloud-controller-manager-role-bindings.yml.j2 │ │ │ │ └── external-openstack-cloud-controller-manager-roles.yml.j2 │ │ │ └── vsphere/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── main.yml │ │ │ │ └── vsphere-credentials-check.yml │ │ │ └── templates/ │ │ │ ├── external-vsphere-cloud-controller-manager-ds.yml.j2 │ │ │ ├── external-vsphere-cloud-controller-manager-role-bindings.yml.j2 │ │ │ ├── external-vsphere-cloud-controller-manager-roles.yml.j2 │ │ │ ├── external-vsphere-cpi-cloud-config-secret.yml.j2 │ │ │ └── external-vsphere-cpi-cloud-config.j2 │ │ ├── external_provisioner/ │ │ │ ├── local_path_provisioner/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── local-path-storage-clusterrolebinding.yml.j2 │ │ │ │ ├── local-path-storage-cm.yml.j2 │ │ │ │ ├── local-path-storage-cr.yml.j2 │ │ │ │ ├── local-path-storage-deployment.yml.j2 │ │ │ │ ├── local-path-storage-ns.yml.j2 │ │ │ │ ├── local-path-storage-sa.yml.j2 │ │ │ │ └── local-path-storage-sc.yml.j2 │ │ │ ├── local_volume_provisioner/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ ├── basedirs.yml │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── local-volume-provisioner-clusterrole.yml.j2 │ │ │ │ ├── local-volume-provisioner-clusterrolebinding.yml.j2 │ │ │ │ ├── local-volume-provisioner-cm.yml.j2 │ │ │ │ ├── local-volume-provisioner-ds.yml.j2 │ │ │ │ ├── local-volume-provisioner-ns.yml.j2 │ │ │ │ ├── local-volume-provisioner-sa.yml.j2 │ │ │ │ └── local-volume-provisioner-sc.yml.j2 │ │ │ └── meta/ │ │ │ └── main.yml │ │ ├── helm/ │ │ │ ├── .gitkeep │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── main.yml │ │ │ │ └── pyyaml-flatcar.yml │ │ │ └── vars/ │ │ │ ├── amazon.yml │ │ │ ├── centos-7.yml │ │ │ ├── centos.yml │ │ │ ├── debian.yml │ │ │ ├── fedora.yml │ │ │ ├── redhat-7.yml │ │ │ ├── redhat.yml │ │ │ ├── suse.yml │ │ │ └── ubuntu.yml │ │ ├── ingress_controller/ │ │ │ ├── alb_ingress_controller/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── alb-ingress-clusterrole.yml.j2 │ │ │ │ ├── alb-ingress-clusterrolebinding.yml.j2 │ │ │ │ ├── alb-ingress-deploy.yml.j2 │ │ │ │ ├── alb-ingress-ns.yml.j2 │ │ │ │ └── alb-ingress-sa.yml.j2 │ │ │ ├── cert_manager/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── cert-manager.crds.yml.j2 │ │ │ │ └── cert-manager.yml.j2 │ │ │ └── meta/ │ │ │ └── main.yml │ │ ├── kubelet-csr-approver/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── meta/ │ │ │ └── main.yml │ │ ├── meta/ │ │ │ └── main.yml │ │ ├── metallb/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── layer2.yaml.j2 │ │ │ ├── layer3.yaml.j2 │ │ │ ├── metallb.yaml.j2 │ │ │ └── pools.yaml.j2 │ │ ├── metrics_server/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── auth-delegator.yaml.j2 │ │ │ ├── auth-reader.yaml.j2 │ │ │ ├── metrics-apiservice.yaml.j2 │ │ │ ├── metrics-server-deployment.yaml.j2 │ │ │ ├── metrics-server-sa.yaml.j2 │ │ │ ├── metrics-server-service.yaml.j2 │ │ │ ├── resource-reader-clusterrolebinding.yaml.j2 │ │ │ └── resource-reader.yaml.j2 │ │ ├── node_feature_discovery/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── nfd-api-crds.yaml.j2 │ │ │ ├── nfd-clusterrole.yaml.j2 │ │ │ ├── nfd-clusterrolebinding.yaml.j2 │ │ │ ├── nfd-gc.yaml.j2 │ │ │ ├── nfd-master-conf.yaml.j2 │ │ │ ├── nfd-master.yaml.j2 │ │ │ ├── nfd-ns.yaml.j2 │ │ │ ├── nfd-role.yaml.j2 │ │ │ ├── nfd-rolebinding.yaml.j2 │ │ │ ├── nfd-service.yaml.j2 │ │ │ ├── nfd-serviceaccount.yaml.j2 │ │ │ ├── nfd-topologyupdater-conf.yaml.j2 │ │ │ ├── nfd-worker-conf.yaml.j2 │ │ │ └── nfd-worker.yaml.j2 │ │ ├── persistent_volumes/ │ │ │ ├── aws-ebs-csi/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ └── aws-ebs-csi-storage-class.yml.j2 │ │ │ ├── azuredisk-csi/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ └── azure-csi-storage-class.yml.j2 │ │ │ ├── cinder-csi/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ └── cinder-csi-storage-class.yml.j2 │ │ │ ├── gcp-pd-csi/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ └── gcp-pd-csi-storage-class.yml.j2 │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ └── upcloud-csi/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── upcloud-csi-storage-class.yml.j2 │ │ ├── policy_controller/ │ │ │ ├── calico/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ ├── calico-kube-controllers.yml.j2 │ │ │ │ ├── calico-kube-cr.yml.j2 │ │ │ │ ├── calico-kube-crb.yml.j2 │ │ │ │ └── calico-kube-sa.yml.j2 │ │ │ └── meta/ │ │ │ └── main.yml │ │ ├── registry/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── registry-cm.yml.j2 │ │ │ ├── registry-ing.yml.j2 │ │ │ ├── registry-ns.yml.j2 │ │ │ ├── registry-pvc.yml.j2 │ │ │ ├── registry-rs.yml.j2 │ │ │ ├── registry-sa.yml.j2 │ │ │ ├── registry-secrets.yml.j2 │ │ │ └── registry-svc.yml.j2 │ │ ├── scheduler_plugins/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 │ │ │ ├── cm-scheduler-plugins.yaml.j2 │ │ │ ├── deploy-scheduler-plugins.yaml.j2 │ │ │ ├── namespace.yaml.j2 │ │ │ ├── networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 │ │ │ ├── rbac-scheduler-plugins.yaml.j2 │ │ │ ├── sa-scheduler-plugins.yaml.j2 │ │ │ ├── scheduling.x-k8s.io_elasticquotas.yaml.j2 │ │ │ ├── scheduling.x-k8s.io_podgroups.yaml.j2 │ │ │ └── topology.node.k8s.io_noderesourcetopologies.yaml.j2 │ │ ├── snapshots/ │ │ │ ├── cinder-csi/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ ├── tasks/ │ │ │ │ │ └── main.yml │ │ │ │ └── templates/ │ │ │ │ └── cinder-csi-snapshot-class.yml.j2 │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ └── snapshot-controller/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── rbac-snapshot-controller.yml.j2 │ │ │ ├── snapshot-controller.yml.j2 │ │ │ └── snapshot-ns.yml.j2 │ │ └── utils/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ └── vars/ │ │ └── main.yml │ ├── kubespray-defaults/ │ │ └── tasks/ │ │ └── main.yml │ ├── kubespray_defaults/ │ │ ├── defaults/ │ │ │ └── main/ │ │ │ ├── download.yml │ │ │ └── main.yml │ │ └── vars/ │ │ └── main/ │ │ ├── checksums.yml │ │ └── main.yml │ ├── network_facts/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── meta/ │ │ │ └── main.yml │ │ └── tasks/ │ │ └── main.yaml │ ├── network_plugin/ │ │ ├── calico/ │ │ │ ├── files/ │ │ │ │ └── openssl.conf │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── rr/ │ │ │ │ ├── defaults/ │ │ │ │ │ └── main.yml │ │ │ │ └── tasks/ │ │ │ │ ├── main.yml │ │ │ │ ├── pre.yml │ │ │ │ └── update-node.yml │ │ │ ├── tasks/ │ │ │ │ ├── calico_apiserver_certs.yml │ │ │ │ ├── check.yml │ │ │ │ ├── install.yml │ │ │ │ ├── main.yml │ │ │ │ ├── peer_with_calico_rr.yml │ │ │ │ ├── peer_with_router.yml │ │ │ │ ├── pre.yml │ │ │ │ ├── repos.yml │ │ │ │ ├── reset.yml │ │ │ │ └── typha_certs.yml │ │ │ ├── templates/ │ │ │ │ ├── calico-apiserver-ns.yml.j2 │ │ │ │ ├── calico-apiserver.yml.j2 │ │ │ │ ├── calico-config.yml.j2 │ │ │ │ ├── calico-cr.yml.j2 │ │ │ │ ├── calico-crb.yml.j2 │ │ │ │ ├── calico-ipamconfig.yml.j2 │ │ │ │ ├── calico-node-sa.yml.j2 │ │ │ │ ├── calico-node.yml.j2 │ │ │ │ ├── calico-typha.yml.j2 │ │ │ │ ├── calicoctl.etcd.sh.j2 │ │ │ │ ├── calicoctl.kdd.sh.j2 │ │ │ │ ├── kubernetes-services-endpoint.yml.j2 │ │ │ │ └── make-ssl-calico.sh.j2 │ │ │ └── vars/ │ │ │ ├── amazon.yml │ │ │ ├── centos-9.yml │ │ │ ├── debian.yml │ │ │ ├── fedora.yml │ │ │ ├── opensuse.yml │ │ │ ├── redhat-9.yml │ │ │ ├── redhat.yml │ │ │ └── rocky-9.yml │ │ ├── calico_defaults/ │ │ │ └── defaults/ │ │ │ └── main.yml │ │ ├── cilium/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── apply.yml │ │ │ │ ├── check.yml │ │ │ │ ├── install.yml │ │ │ │ ├── main.yml │ │ │ │ ├── reset.yml │ │ │ │ └── reset_iface.yml │ │ │ └── templates/ │ │ │ ├── cilium/ │ │ │ │ ├── cilium-bgp-advertisement.yml.j2 │ │ │ │ ├── cilium-bgp-cluster-config.yml.j2 │ │ │ │ ├── cilium-bgp-node-config-override.yml.j2 │ │ │ │ ├── cilium-bgp-peer-config.yml.j2 │ │ │ │ ├── cilium-bgp-peering-policy.yml.j2 │ │ │ │ └── cilium-loadbalancer-ip-pool.yml.j2 │ │ │ └── values.yaml.j2 │ │ ├── cni/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── custom_cni/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── flannel/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── main.yml │ │ │ │ └── reset.yml │ │ │ └── templates/ │ │ │ ├── cni-flannel-rbac.yml.j2 │ │ │ └── cni-flannel.yml.j2 │ │ ├── kube-ovn/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── cni-kube-ovn-crd.yml.j2 │ │ │ ├── cni-kube-ovn.yml.j2 │ │ │ └── cni-ovn.yml.j2 │ │ ├── kube-router/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── annotate.yml │ │ │ │ ├── main.yml │ │ │ │ └── reset.yml │ │ │ └── templates/ │ │ │ ├── cni-conf.json.j2 │ │ │ ├── kube-router.yml.j2 │ │ │ └── kubeconfig.yml.j2 │ │ ├── macvlan/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── files/ │ │ │ │ ├── ifdown-local │ │ │ │ ├── ifdown-macvlan │ │ │ │ ├── ifup-local │ │ │ │ └── ifup-macvlan │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ ├── 10-macvlan.conf.j2 │ │ │ ├── 99-loopback.conf.j2 │ │ │ ├── centos-network-macvlan.cfg.j2 │ │ │ ├── centos-postdown-macvlan.cfg.j2 │ │ │ ├── centos-postup-macvlan.cfg.j2 │ │ │ ├── centos-routes-macvlan.cfg.j2 │ │ │ ├── coreos-device-macvlan.cfg.j2 │ │ │ ├── coreos-interface-macvlan.cfg.j2 │ │ │ ├── coreos-network-macvlan.cfg.j2 │ │ │ ├── coreos-service-nat_ouside.j2 │ │ │ └── debian-network-macvlan.cfg.j2 │ │ ├── multus/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ ├── files/ │ │ │ │ ├── multus-clusterrole.yml │ │ │ │ ├── multus-clusterrolebinding.yml │ │ │ │ ├── multus-crd.yml │ │ │ │ └── multus-serviceaccount.yml │ │ │ ├── meta/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ └── main.yml │ │ │ └── templates/ │ │ │ └── multus-daemonset.yml.j2 │ │ ├── ovn4nfv/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ └── tasks/ │ │ └── main.yml │ ├── recover_control_plane/ │ │ ├── control-plane/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── etcd/ │ │ │ └── tasks/ │ │ │ ├── main.yml │ │ │ └── recover_lost_quorum.yml │ │ └── post-recover/ │ │ └── tasks/ │ │ └── main.yml │ ├── remove-node/ │ │ ├── post-remove/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── pre-remove/ │ │ │ └── tasks/ │ │ │ └── main.yml │ │ └── remove-etcd-node/ │ │ └── tasks/ │ │ └── main.yml │ ├── remove_node/ │ │ └── pre_remove/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ └── tasks/ │ │ └── main.yml │ ├── reset/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ └── tasks/ │ │ └── main.yml │ ├── system_packages/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── vars/ │ │ └── main.yml │ ├── upgrade/ │ │ ├── post-upgrade/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ ├── pre-upgrade/ │ │ │ ├── defaults/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ └── main.yml │ │ └── system-upgrade/ │ │ └── tasks/ │ │ ├── apt.yml │ │ ├── main.yml │ │ └── yum.yml │ ├── validate_inventory/ │ │ ├── meta/ │ │ │ └── main.yml │ │ └── tasks/ │ │ └── main.yml │ └── win_nodes/ │ └── kubernetes_patch/ │ ├── defaults/ │ │ └── main.yml │ └── tasks/ │ └── main.yml ├── scale.yml ├── scripts/ │ ├── Dockerfile.j2 │ ├── assert-sorted-checksums.yml │ ├── collect-info.yaml │ ├── component_hash_update/ │ │ ├── pyproject.toml │ │ └── src/ │ │ └── component_hash_update/ │ │ ├── __init__.py │ │ ├── components.py │ │ ├── download.py │ │ └── list_releases.graphql │ ├── galaxy_version.py │ ├── gen_docs_sidebar.sh │ ├── get_node_ids.sh │ ├── gitlab-runner.sh │ ├── openstack-cleanup/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── main.py │ │ └── requirements.txt │ ├── pipeline.Dockerfile.j2 │ ├── propagate_ansible_variables.yml │ └── readme_versions.md.j2 ├── test-infra/ │ ├── image-builder/ │ │ ├── Makefile │ │ ├── OWNERS │ │ ├── README.md │ │ ├── cluster.yml │ │ ├── hosts.ini │ │ └── roles/ │ │ └── kubevirt-images/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ └── templates/ │ │ └── Dockerfile │ └── vagrant-docker/ │ ├── Dockerfile │ ├── README.md │ └── build.sh ├── tests/ │ ├── Makefile │ ├── ansible.cfg │ ├── cloud_playbooks/ │ │ ├── create-kubevirt.yml │ │ └── roles/ │ │ └── packet-ci/ │ │ ├── defaults/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ └── main.yml │ │ ├── templates/ │ │ │ └── vm.yml.j2 │ │ └── vars/ │ │ └── main.yml │ ├── common_vars.yml │ ├── files/ │ │ ├── almalinux9-calico-ha-ebpf.yml │ │ ├── almalinux9-calico-nodelocaldns-secondary.yml │ │ ├── almalinux9-calico-remove-node │ │ ├── almalinux9-calico-remove-node.yml │ │ ├── almalinux9-calico.yml │ │ ├── almalinux9-crio.yml │ │ ├── almalinux9-docker.yml │ │ ├── almalinux9-kube-ovn.yml │ │ ├── amazon-linux-2-all-in-one.yml │ │ ├── custom_cni/ │ │ │ ├── README.md │ │ │ ├── cilium.yaml │ │ │ └── values.yaml │ │ ├── debian11-calico-collection.yml │ │ ├── debian11-calico-upgrade │ │ ├── debian11-calico-upgrade-once │ │ ├── debian11-calico-upgrade-once.yml │ │ ├── debian11-calico-upgrade.yml │ │ ├── debian11-custom-cni.yml │ │ ├── debian11-docker.yml │ │ ├── debian11-kubelet-csr-approver.yml │ │ ├── debian11-macvlan.yml │ │ ├── debian12-calico.yml │ │ ├── debian12-cilium-svc-proxy.yml │ │ ├── debian12-cilium.yml │ │ ├── debian12-custom-cni-helm.yml │ │ ├── debian12-docker.yml │ │ ├── debian13-calico.yml │ │ ├── debian13-cilium.yml │ │ ├── fedora39-calico-selinux.yml │ │ ├── fedora39-calico-swap-selinux.yml │ │ ├── fedora39-crio.yml │ │ ├── fedora39-kube-router.yml │ │ ├── fedora40-docker-calico.yml │ │ ├── fedora40-docker.calico │ │ ├── fedora40-flannel-crio-collection-scale.yml │ │ ├── fedora41-calico-selinux.yml │ │ ├── fedora41-calico-swap-selinux.yml │ │ ├── fedora41-crio.yml │ │ ├── fedora41-kube-router.yml │ │ ├── fedora42-calico.yml │ │ ├── flatcar4081-calico.yml │ │ ├── openeuler24-calico.yml │ │ ├── rockylinux10-calico.yml │ │ ├── rockylinux10-cilium │ │ ├── rockylinux10-cilium.yml │ │ ├── rockylinux9-calico.yml │ │ ├── rockylinux9-cilium │ │ ├── rockylinux9-cilium.yml │ │ ├── tf-elastx_ubuntu24-calico.yml │ │ ├── ubuntu22-all-in-one-docker.yml │ │ ├── ubuntu22-calico-all-in-one-upgrade │ │ ├── ubuntu22-calico-all-in-one-upgrade.yml │ │ ├── ubuntu22-calico-all-in-one.yml │ │ ├── ubuntu22-crio.yml │ │ ├── ubuntu24-all-in-one-docker.yml │ │ ├── ubuntu24-calico-all-in-one │ │ ├── ubuntu24-calico-all-in-one-hardening.yml │ │ ├── ubuntu24-calico-all-in-one.yml │ │ ├── ubuntu24-calico-dual-stack.rb │ │ ├── ubuntu24-calico-dual-stack.yml │ │ ├── ubuntu24-calico-etcd-datastore.yml │ │ ├── ubuntu24-calico-etcd-kubeadm-upgrade-ha │ │ ├── ubuntu24-calico-etcd-kubeadm-upgrade-ha.yml │ │ ├── ubuntu24-calico-etcd-kubeadm.yml │ │ ├── ubuntu24-calico-ha-recover │ │ ├── ubuntu24-calico-ha-recover-noquorum │ │ ├── ubuntu24-calico-ha-recover-noquorum.yml │ │ ├── ubuntu24-calico-ha-recover.yml │ │ ├── ubuntu24-calico-ha-wireguard.yml │ │ ├── ubuntu24-calico-ipv6only-stack.rb │ │ ├── ubuntu24-calico-ipv6only-stack.yml │ │ ├── ubuntu24-cilium-sep.yml │ │ ├── ubuntu24-crio-scale.yml │ │ ├── ubuntu24-crio-upgrade │ │ ├── ubuntu24-crio-upgrade.yml │ │ ├── ubuntu24-flannel-collection.yml │ │ ├── ubuntu24-flannel-ha-once.yml │ │ ├── ubuntu24-flannel-ha.yml │ │ ├── ubuntu24-flannel.yml │ │ ├── ubuntu24-ha-separate-etcd │ │ ├── ubuntu24-ha-separate-etcd.yml │ │ ├── ubuntu24-kube-router-sep.yml │ │ └── ubuntu24-kube-router-svc-proxy.yml │ ├── requirements.txt │ ├── scripts/ │ │ ├── check-templates.py │ │ ├── collection-build-install.sh │ │ ├── md-table/ │ │ │ ├── main.py │ │ │ └── table.md.j2 │ │ ├── molecule_run.sh │ │ ├── opentofu_install.sh │ │ ├── rebase.sh │ │ ├── testcases_run.sh │ │ ├── vagrant-install.sh │ │ ├── vagrant-validate.sh │ │ └── vagrant_clean.sh │ └── testcases/ │ ├── 000_install-hydrophone.yml │ ├── 010_check-apiserver.yml │ ├── 015_check-nodes-ready.yml │ ├── 020_check-pods-running.yml │ ├── 025_check-csr-request.yml │ ├── 030_check-network.yml │ ├── 040_check-network-adv.yml │ ├── 100_check-k8s-conformance.yml │ ├── roles/ │ │ └── cluster-dump/ │ │ └── tasks/ │ │ └── main.yml │ └── tests.yml ├── upgrade-cluster.yml └── upgrade_cluster.yml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .ansible-lint ================================================ --- skip_list: # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules # DO NOT add any other rules to this skip_list, instead use local `# noqa` with a comment explaining WHY it is necessary # These rules are intentionally skipped: # # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern # Meta roles in Kubespray don't need proper names # (Disabled in June 2021) - 'role-name' # [var-naming] # In Kubespray we use variables that use camelCase to match their k8s counterparts # (Disabled in June 2021) - 'var-naming[pattern]' # Variables names from within roles in kubespray don't need role name as a prefix - 'var-naming[no-role-prefix]' # [fqcn-builtins] # Roles in kubespray don't need fully qualified collection names # (Disabled in Feb 2023) - 'fqcn-builtins' # We use template in names - 'name[template]' # No changed-when on commands # (Disabled in June 2023 after ansible upgrade; FIXME) - 'no-changed-when' # Disable run-once check with free strategy # (Disabled in June 2023 after ansible upgrade; FIXME) - 'run-once[task]' - 'jinja[spacing]' exclude_paths: # Generated files - tests/files/custom_cni/cilium.yaml - venv - .github - .ansible - .cache - .gitlab-ci.yml - .gitlab-ci mock_modules: - gluster.gluster.gluster_volume ================================================ FILE: .ansible-lint-ignore ================================================ # This file contains ignores rule violations for ansible-lint inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml jinja[spacing] roles/kubernetes/control-plane/defaults/main/kube-proxy.yml jinja[spacing] roles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing] roles/kubernetes/kubeadm/defaults/main.yml jinja[spacing] roles/kubernetes/node/defaults/main.yml jinja[spacing] roles/kubernetes/preinstall/defaults/main.yml jinja[spacing] roles/kubespray-defaults/defaults/main/main.yml jinja[spacing] ================================================ FILE: .editorconfig ================================================ root = true [*.{yaml,yml,yml.j2,yaml.j2}] indent_style = space indent_size = 2 trim_trailing_whitespace = true insert_final_newline = true charset = utf-8 [{Dockerfile}] indent_style = space indent_size = 2 trim_trailing_whitespace = true insert_final_newline = true charset = utf-8 ================================================ FILE: .gitattributes ================================================ docs/_sidebar.md linguist-generated=true ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.yaml ================================================ --- name: Bug Report description: Report a bug encountered while using Kubespray labels: kind/bug body: - type: markdown attributes: value: | Please, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. - type: textarea id: problem attributes: label: What happened? description: | Please provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. validations: required: true - type: textarea id: expected attributes: label: What did you expect to happen? validations: required: true - type: textarea id: repro attributes: label: How can we reproduce it (as minimally and precisely as possible)? validations: required: true - type: markdown attributes: value: '### Environment' - type: dropdown id: os attributes: label: OS options: - 'RHEL 9' - 'RHEL 8' - 'Fedora 40' - 'Ubuntu 24' - 'Ubuntu 22' - 'Ubuntu 20' - 'Debian 12' - 'Debian 11' - 'Flatcar Container Linux' - 'openSUSE Leap' - 'openSUSE Tumbleweed' - 'Oracle Linux 9' - 'Oracle Linux 8' - 'AlmaLinux 9' - 'AlmaLinux 8' - 'Rocky Linux 9' - 'Rocky Linux 8' - 'Amazon Linux 2' - 'Kylin Linux Advanced Server V10' - 'UOS Linux 20' - 'openEuler 24' - 'openEuler 22' - 'openEuler 20' - 'Other|Unsupported' validations: required: true - type: textarea id: ansible_version attributes: label: Version of Ansible placeholder: 'ansible --version' validations: required: true - type: input id: python_version attributes: label: Version of Python placeholder: 'python --version' validations: required: true - type: input id: kubespray_version attributes: label: Version of Kubespray (commit) placeholder: 'git rev-parse --short HEAD' validations: required: true - type: dropdown id: network_plugin attributes: label: Network plugin used options: - calico - cilium - cni - custom_cni - flannel - kube-ovn - kube-router - macvlan - meta - multus - ovn4nfv validations: required: true - type: textarea id: inventory attributes: label: Full inventory with variables placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"' description: We recommend using snippets services like https://gist.github.com/ etc. validations: required: true - type: input id: ansible_command attributes: label: Command used to invoke ansible validations: required: true - type: textarea id: ansible_output attributes: label: Output of ansible run description: We recommend using snippets services like https://gist.github.com/ etc. validations: required: true - type: textarea id: anything_else attributes: label: Anything else we need to know description: | By running scripts/collect-info.yaml you can get a lot of useful informations. Script can be started by: ansible-playbook -i -u -e ansible_ssh_user= -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml (If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python'). After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ --- blank_issues_enabled: false contact_links: - name: Support Request url: https://kubernetes.slack.com/channels/kubespray about: Support request or question relating to Kubernetes ================================================ FILE: .github/ISSUE_TEMPLATE/enhancement.yaml ================================================ --- name: Enhancement Request description: Suggest an enhancement to the Kubespray project labels: kind/feature body: - type: markdown attributes: value: Please only use this template for submitting enhancement requests - type: textarea id: what attributes: label: What would you like to be added validations: required: true - type: textarea id: why attributes: label: Why is this needed validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/failing-test.yaml ================================================ --- name: Failing Test description: Report test failures in Kubespray CI jobs labels: kind/failing-test body: - type: markdown attributes: value: Please only use this template for submitting reports about failing tests in Kubespray CI jobs - type: textarea id: failing_jobs attributes: label: Which jobs are failing ? validations: required: true - type: textarea id: failing_tests attributes: label: Which tests are failing ? validations: required: true - type: input id: since_when attributes: label: Since when has it been failing ? validations: required: true - type: textarea id: failure_reason attributes: label: Reason for failure description: If you don't know and have no guess, just put "Unknown" validations: required: true - type: textarea id: anything_else attributes: label: Anything else we need to know ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ **What type of PR is this?** > Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line: > > /kind api-change > /kind bug > /kind cleanup > /kind design > /kind documentation > /kind failing-test > /kind feature > /kind flake **What this PR does / why we need it**: **Which issue(s) this PR fixes**: Fixes # **Special notes for your reviewer**: **Does this PR introduce a user-facing change?**: ```release-note ``` ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "pip" directory: "/" schedule: interval: "weekly" labels: - dependencies - release-note-none groups: molecule: patterns: - molecule - molecule-plugins* - package-ecosystem: "github-actions" directory: "/" labels: - release-note-none - ci-short schedule: interval: "weekly" ================================================ FILE: .github/workflows/auto-label-os.yml ================================================ name: Issue labeler on: issues: types: [opened] permissions: contents: read jobs: label-component: runs-on: ubuntu-latest permissions: issues: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - name: Parse issue form uses: stefanbuck/github-issue-parser@10dcc54158ba4c137713d9d69d70a2da63b6bda3 id: issue-parser with: template-path: .github/ISSUE_TEMPLATE/bug-report.yaml - name: Set labels based on OS field uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e with: issue-form: ${{ steps.issue-parser.outputs.jsonString }} section: os block-list: | None Other token: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/upgrade-patch-versions-schedule.yml ================================================ name: Upgrade Kubespray components with new patches versions - all branches on: schedule: - cron: '22 2 * * *' # every day, 02:22 UTC workflow_dispatch: permissions: {} jobs: get-releases-branches: if: github.repository == 'kubernetes-sigs/kubespray' runs-on: ubuntu-latest outputs: branches: ${{ steps.get-branches.outputs.data }} steps: - uses: octokit/graphql-action@ddde8ebb2493e79f390e6449c725c21663a67505 id: get-branches with: query: | query get_release_branches($owner:String!, $name:String!) { repository(owner:$owner, name:$name) { refs(refPrefix: "refs/heads/", first: 3, query: "release-", orderBy: { field: ALPHABETICAL, direction: DESC }) { nodes { name } } } } variables: | owner: ${{ github.repository_owner }} name: ${{ github.event.repository.name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} update-versions: needs: get-releases-branches strategy: fail-fast: false matrix: branch: - name: ${{ github.event.repository.default_branch }} - ${{ fromJSON(needs.get-releases-branches.outputs.branches).repository.refs.nodes }} uses: ./.github/workflows/upgrade-patch-versions.yml permissions: contents: write pull-requests: write name: Update patch updates on ${{ matrix.branch.name }} with: branch: ${{ matrix.branch.name }} ================================================ FILE: .github/workflows/upgrade-patch-versions.yml ================================================ on: workflow_call: inputs: branch: description: Which branch to update with new patch versions default: master required: true type: string jobs: update-patch-versions: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: ref: ${{ inputs.branch }} - uses: actions/setup-python@v6 with: python-version: '3.13' cache: 'pip' - run: pip install scripts/component_hash_update pre-commit - run: update-hashes env: API_KEY: ${{ secrets.GITHUB_TOKEN }} - uses: actions/cache@v5 with: key: pre-commit-hook-propagate path: | ~/.cache/pre-commit - run: pre-commit run --all-files propagate-ansible-variables continue-on-error: true - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 with: commit-message: Patch versions updates title: Patch versions updates - ${{ inputs.branch }} labels: bot branch: component_hash_update/${{ inputs.branch }} sign-commits: true body: | /kind feature ```release-note NONE ``` ================================================ FILE: .gitignore ================================================ .vagrant *.retry **/vagrant_ansible_inventory *.iml temp contrib/offline/container-images contrib/offline/container-images.tar.gz contrib/offline/offline-files contrib/offline/offline-files.tar.gz .idea .vscode .tox .cache *.bak *.tfstate *.tfstate*backup *.lock.hcl .terraform/ contrib/terraform/aws/credentials.tfvars .terraform.lock.hcl /ssh-bastion.conf **/*.sw[pon] *~ vagrant/ plugins/mitogen # Ansible inventory inventory/* !inventory/local !inventory/sample inventory/*/artifacts/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # Distribution / packaging .Python env/ build/ credentials/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # IPython Notebook .ipynb_checkpoints # pyenv .python-version # dotenv .env # virtualenv venv/ ENV/ # molecule roles/**/molecule/**/__pycache__/ # macOS .DS_Store # Temp location used by our scripts scripts/tmp/ tmp.md # Ansible collection files kubernetes_sigs-kubespray*tar.gz ansible_collections ================================================ FILE: .gitlab-ci/build.yml ================================================ --- pipeline-image: cache: key: $CI_COMMIT_REF_SLUG paths: - image-cache tags: - ffci stage: build image: moby/buildkit:rootless variables: BUILDKITD_FLAGS: --oci-worker-no-process-sandbox CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache # TODO: remove the override # currently rebase.sh depends on bash (not available in the kaniko image) # once we have a simpler rebase (which should be easy if the target branch ref is available as variable # we'll be able to rebase here as well hopefully before_script: - mkdir -p ~/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json script: - | buildctl-daemonless.sh build \ --frontend dockerfile.v0 \ --local context=$CI_PROJECT_DIR \ --local dockerfile=$CI_PROJECT_DIR \ --opt filename=pipeline.Dockerfile \ --export-cache type=registry,ref=$CACHE_IMAGE \ --import-cache type=registry,ref=$CACHE_IMAGE \ --output type=image,name=$PIPELINE_IMAGE,push=true ================================================ FILE: .gitlab-ci/kubevirt.yml ================================================ --- .kubevirt: extends: .job-moderated interruptible: true script: - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml -e @"tests/files/${TESTCASE}.yml" - ./tests/scripts/testcases_run.sh variables: ANSIBLE_TIMEOUT: "120" tags: - ffci needs: - pipeline-image # TODO: generate testcases matrixes from the files in tests/files/ # this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch # (currently, a removed job in the target branch breaks the tests, because the # pipeline definition is parsed by gitlab before the rebase.sh script) # CI template for PRs pr: stage: deploy-part1 rules: - if: $PR_LABELS =~ /.*ci-short.*/ when: manual allow_failure: true - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/ when: on_success - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success - when: manual allow_failure: true extends: .kubevirt parallel: matrix: - TESTCASE: - almalinux9-crio - almalinux9-kube-ovn - debian11-calico-collection - debian11-macvlan - debian12-cilium - debian13-cilium - fedora39-kube-router - fedora41-kube-router - fedora42-calico - rockylinux9-cilium - rockylinux10-cilium - ubuntu22-calico-all-in-one - ubuntu22-calico-all-in-one-upgrade - ubuntu24-calico-etcd-datastore - ubuntu24-calico-all-in-one-hardening - ubuntu24-cilium-sep - ubuntu24-crio-scale - ubuntu24-crio-upgrade - ubuntu24-flannel-collection - ubuntu24-kube-router-sep - ubuntu24-kube-router-svc-proxy - ubuntu24-ha-separate-etcd - fedora40-flannel-crio-collection-scale - openeuler24-calico # This is for flakey test so they don't disrupt the PR worklflow too much. # Jobs here MUST have a open issue so we don't lose sight of them pr-flakey: extends: pr retry: 1 parallel: matrix: - TESTCASE: - flatcar4081-calico # https://github.com/kubernetes-sigs/kubespray/issues/12309 # The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken ubuntu24-calico-all-in-one: stage: deploy-part1 extends: .kubevirt variables: TESTCASE: ubuntu24-calico-all-in-one rules: - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/ when: on_success - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success - when: manual allow_failure: true pr_full: extends: .kubevirt stage: deploy-extended rules: - if: $PR_LABELS =~ /.*ci-full.*/ when: on_success - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success # Else run as manual - when: manual allow_failure: true parallel: matrix: - TESTCASE: - almalinux9-calico-ha-ebpf - almalinux9-calico-nodelocaldns-secondary - debian11-custom-cni - debian11-kubelet-csr-approver - debian12-custom-cni-helm - fedora39-calico-swap-selinux - fedora39-crio - fedora41-calico-swap-selinux - fedora41-crio - ubuntu24-calico-ha-wireguard - ubuntu24-flannel-ha - ubuntu24-flannel-ha-once # Need an update of the container image to use schema v2 # update: quay.io/kubespray/vm-amazon-linux-2:latest manual: extends: pr_full parallel: matrix: - TESTCASE: - amazon-linux-2-all-in-one rules: - when: manual allow_failure: true pr_extended: extends: .kubevirt stage: deploy-extended rules: - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/ when: on_success - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success - when: manual allow_failure: true parallel: matrix: - TESTCASE: - almalinux9-calico - almalinux9-calico-remove-node - almalinux9-docker - debian11-docker - debian12-calico - debian12-docker - debian13-calico - rockylinux9-calico - rockylinux10-calico - ubuntu22-all-in-one-docker - ubuntu24-all-in-one-docker - ubuntu24-calico-all-in-one - ubuntu24-calico-etcd-kubeadm - ubuntu24-flannel # TODO: migrate to pr-full, fix the broken ones periodic: allow_failure: true extends: .kubevirt rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success parallel: matrix: - TESTCASE: - debian11-calico-upgrade - debian11-calico-upgrade-once - debian12-cilium-svc-proxy - fedora39-calico-selinux - fedora40-docker-calico - fedora41-calico-selinux - ubuntu24-calico-etcd-kubeadm-upgrade-ha - ubuntu24-calico-ha-recover - ubuntu24-calico-ha-recover-noquorum ================================================ FILE: .gitlab-ci/lint.yml ================================================ --- pre-commit: stage: test tags: - ffci image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9' variables: PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit ANSIBLE_STDOUT_CALLBACK: default script: - pre-commit run --all-files --show-diff-on-failure cache: key: pre-commit-2 paths: - ${PRE_COMMIT_HOME} when: 'always' needs: [] vagrant-validate: extends: .job stage: test tags: [ffci] variables: VAGRANT_VERSION: 2.3.7 script: - ./tests/scripts/vagrant-validate.sh ================================================ FILE: .gitlab-ci/molecule.yml ================================================ --- .molecule: tags: [ffci] rules: # run on ci-short as well - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/ when: on_success - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success - when: manual allow_failure: true stage: deploy-part1 image: $PIPELINE_IMAGE needs: - pipeline-image script: - ./tests/scripts/molecule_run.sh after_script: - rm -fr molecule_logs - mkdir -p molecule_logs - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar - gzip molecule_logs/molecule.tar artifacts: when: always paths: - molecule_logs/ molecule: extends: .molecule script: - ./tests/scripts/molecule_run.sh -i $ROLE parallel: matrix: - ROLE: - container-engine/cri-dockerd - container-engine/containerd - container-engine/cri-o - container-engine/gvisor - container-engine/youki - adduser - bastion-ssh-config - bootstrap_os molecule_full: allow_failure: true rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success - when: manual allow_failure: true extends: molecule parallel: matrix: - ROLE: # FIXME : tests below are perma-failing - container-engine/kata-containers ================================================ FILE: .gitlab-ci/terraform.yml ================================================ --- # Tests for contrib/terraform/ .terraform_install: extends: .job needs: - pipeline-image variables: TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub" TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE CLUSTER: $CI_COMMIT_REF_NAME TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR stage: deploy-part1 before_script: - ./tests/scripts/rebase.sh - mkdir -p cluster-dump $ANSIBLE_INVENTORY - ./tests/scripts/opentofu_install.sh - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars . - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts - tofu -chdir="contrib/terraform/$PROVIDER" init terraform_validate: extends: .terraform_install tags: [ffci] only: ['master', /^pr-.*$/] script: - tofu -chdir="contrib/terraform/$PROVIDER" validate - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff stage: test needs: - pipeline-image parallel: matrix: - PROVIDER: - openstack - aws - exoscale - hetzner - vsphere - upcloud .terraform_apply: extends: .terraform_install tags: [ffci] stage: deploy-extended when: manual only: [/^pr-.*$/] variables: ANSIBLE_INVENTORY_UNPARSED_FAILED: "true" ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly ANSIBLE_SSH_RETRIES: 15 TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER TF_VAR_cluster_name: $CI_JOB_ID script: # Set Ansible config - cp ansible.cfg ~/.ansible.cfg - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa - mkdir -p contrib/terraform/$PROVIDER/group_vars # Random subnet to avoid routing conflicts - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24" - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1 - tests/scripts/testcases_run.sh after_script: # Cleanup regardless of exit code - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve # Elastx is generously donating resources for Kubespray on Openstack CI # Contacts: @gix @bl0m1 .elastx_variables: &elastx_variables OS_AUTH_URL: https://ops.elastx.cloud:5000 OS_PROJECT_ID: 564c6b461c6b44b1bb19cdb9c2d928e4 OS_PROJECT_NAME: kubespray_ci OS_USER_DOMAIN_NAME: Default OS_PROJECT_DOMAIN_ID: default OS_USERNAME: kubespray@root314.com OS_REGION_NAME: se-sto OS_INTERFACE: public OS_IDENTITY_API_VERSION: "3" TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df" tf-elastx_cleanup: tags: [ffci] image: python variables: <<: *elastx_variables before_script: - pip install -r scripts/openstack-cleanup/requirements.txt script: - ./scripts/openstack-cleanup/main.py allow_failure: true tf-elastx_ubuntu24-calico: extends: .terraform_apply stage: deploy-part1 when: on_success variables: <<: *elastx_variables PROVIDER: openstack ANSIBLE_TIMEOUT: "60" TF_VAR_number_of_k8s_masters: "1" TF_VAR_number_of_k8s_masters_no_floating_ip: "0" TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0" TF_VAR_number_of_etcd: "0" TF_VAR_number_of_k8s_nodes: "1" TF_VAR_number_of_k8s_nodes_no_floating_ip: "0" TF_VAR_number_of_gfs_nodes_no_floating_ip: "0" TF_VAR_number_of_bastions: "0" TF_VAR_number_of_k8s_masters_no_etcd: "0" TF_VAR_floatingip_pool: "elx-public1" TF_VAR_dns_nameservers: '["1.1.1.1", "8.8.8.8", "8.8.4.4"]' TF_VAR_use_access_ip: "0" TF_VAR_external_net: "600b8501-78cb-4155-9c9f-23dfcba88828" TF_VAR_network_name: "ci-$CI_JOB_ID" TF_VAR_az_list: '["sto1"]' TF_VAR_az_list_node: '["sto1"]' TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b # v1-standard-2 TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b # v1-standard-2 TF_VAR_image: ubuntu-24.04-server-latest TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]' TESTCASE: $CI_JOB_NAME ================================================ FILE: .gitlab-ci/vagrant.yml ================================================ --- vagrant: extends: .job-moderated variables: CI_PLATFORM: "vagrant" SSH_USER: "vagrant" VAGRANT_DEFAULT_PROVIDER: "libvirt" KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb DOCKER_NAME: vagrant VAGRANT_ANSIBLE_TAGS: facts VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d" PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" tags: [ffci-vm-large] image: quay.io/kubespray/vm-kubespray-ci:v13 services: [] before_script: - echo $USER - python3 -m venv citest - source citest/bin/activate - vagrant plugin expunge --reinstall --force --no-tty - vagrant plugin install vagrant-libvirt - pip install --no-compile --no-cache-dir pip -U - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt - ./tests/scripts/vagrant_clean.sh script: - vagrant up - ./tests/scripts/testcases_run.sh after_script: - vagrant destroy -f cache: key: $CI_JOB_NAME_SLUG paths: - .vagrant.d/boxes - .cache/pip policy: pull-push # TODO: change to "pull" when not on main stage: deploy-extended rules: - if: $PR_LABELS =~ /.*ci-full.*/ when: on_success - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci" when: on_success - when: manual allow_failure: true parallel: matrix: - TESTCASE: - ubuntu24-calico-dual-stack - ubuntu24-calico-ipv6only-stack ================================================ FILE: .gitlab-ci.yml ================================================ --- stages: - build # build docker image used in most other jobs - test # unit tests - deploy-part1 # kubespray runs - common setup - deploy-extended # kubespray runs - rarer or costlier (to test) setups variables: FAILFASTCI_NAMESPACE: 'kargo-ci' GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' GIT_CONFIG_COUNT: 2 GIT_CONFIG_KEY_0: user.email GIT_CONFIG_VALUE_0: "ci@kubespray.io" GIT_CONFIG_KEY_1: user.name GIT_CONFIG_VALUE_1: "Kubespray CI" ANSIBLE_FORCE_COLOR: "true" MAGIC: "ci check this" GS_ACCESS_KEY_ID: $GS_KEY GS_SECRET_ACCESS_KEY: $GS_SECRET CONTAINER_ENGINE: docker GCE_PREEMPTIBLE: "false" ANSIBLE_KEEP_REMOTE_FILES: "1" ANSIBLE_CONFIG: ./tests/ansible.cfg ANSIBLE_REMOTE_USER: kubespray ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa ANSIBLE_INVENTORY: /tmp/inventory ANSIBLE_STDOUT_CALLBACK: "default" RESET_CHECK: "false" REMOVE_NODE_CHECK: "false" UPGRADE_TEST: "false" MITOGEN_ENABLE: "false" ANSIBLE_VERBOSITY: 2 RECOVER_CONTROL_PLANE_TEST: "false" RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]" OPENTOFU_VERSION: v1.9.1 PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" before_script: - ./tests/scripts/rebase.sh - mkdir -p cluster-dump $ANSIBLE_INVENTORY .job: &job tags: - ffci image: $PIPELINE_IMAGE artifacts: when: always paths: - cluster-dump/ needs: - pipeline-image .job-moderated: extends: .job needs: - pipeline-image - pre-commit # lint - vagrant-validate # lint include: - .gitlab-ci/build.yml - .gitlab-ci/lint.yml - .gitlab-ci/terraform.yml - .gitlab-ci/kubevirt.yml - .gitlab-ci/vagrant.yml - .gitlab-ci/molecule.yml ================================================ FILE: .gitmodules ================================================ ================================================ FILE: .md_style.rb ================================================ all exclude_rule 'MD013' exclude_rule 'MD029' rule 'MD007', :indent => 2 ================================================ FILE: .mdlrc ================================================ style "#{File.dirname(__FILE__)}/.md_style.rb" ================================================ FILE: .nojekyll ================================================ ================================================ FILE: .pre-commit-config.yaml ================================================ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-executables-have-shebangs - id: check-xml - id: check-merge-conflict - id: detect-private-key - id: end-of-file-fixer - id: forbid-new-submodules - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/adrienverge/yamllint.git rev: v1.37.1 hooks: - id: yamllint args: [--strict] - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.11.0.1 hooks: - id: shellcheck args: ["--severity=error"] exclude: "^.git" files: "\\.sh$" - repo: https://github.com/ansible/ansible-lint rev: v25.11.0 hooks: - id: ansible-lint additional_dependencies: - jmespath==1.0.1 - netaddr==1.3.0 - distlib - repo: https://github.com/golangci/misspell rev: v0.7.0 hooks: - id: misspell exclude: "OWNERS_ALIASES$" - repo: local hooks: - id: collection-build-install name: Build and install kubernetes-sigs.kubespray Ansible collection language: python additional_dependencies: - ansible-core>=2.16.4 - distlib entry: tests/scripts/collection-build-install.sh pass_filenames: false - id: generate-docs-sidebar name: generate-docs-sidebar entry: scripts/gen_docs_sidebar.sh language: script pass_filenames: false - id: ci-matrix name: ci-matrix entry: tests/scripts/md-table/main.py language: python pass_filenames: false additional_dependencies: - jinja2 - pathlib - pyaml - id: check-galaxy-version name: Verify correct version for galaxy.yml entry: scripts/galaxy_version.py language: python pass_filenames: false additional_dependencies: - ruamel.yaml - id: jinja-syntax-check name: jinja-syntax-check entry: tests/scripts/check-templates.py language: python types: - jinja additional_dependencies: - jinja2 - id: propagate-ansible-variables name: Update static files referencing default kubespray values language: python additional_dependencies: - ansible-core>=2.16.4 entry: scripts/propagate_ansible_variables.yml pass_filenames: false - id: check-checksums-sorted name: Check that our checksums are correctly sorted by version entry: scripts/assert-sorted-checksums.yml language: python pass_filenames: false additional_dependencies: - ansible - repo: https://github.com/markdownlint/markdownlint rev: v0.12.0 hooks: - id: markdownlint exclude: "^.github|(^docs/_sidebar\\.md$)" ================================================ FILE: .yamllint ================================================ --- extends: default ignore: | .git/ .github/ # Generated file tests/files/custom_cni/cilium.yaml # https://ansible.readthedocs.io/projects/lint/rules/yaml/ rules: braces: min-spaces-inside: 0 max-spaces-inside: 1 brackets: min-spaces-inside: 0 max-spaces-inside: 1 comments: min-spaces-from-content: 1 # https://github.com/adrienverge/yamllint/issues/384 comments-indentation: false indentation: spaces: 2 indent-sequences: consistent line-length: disable new-line-at-end-of-file: disable octal-values: forbid-implicit-octal: true # yamllint defaults to false forbid-explicit-octal: true # yamllint defaults to false ================================================ FILE: CHANGELOG.md ================================================ # See our release notes on [GitHub](https://github.com/kubernetes-sigs/kubespray/releases) ================================================ FILE: CNAME ================================================ kubespray.io ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing guidelines ## How to become a contributor and submit your own code ### Environment setup It is recommended to use filter to manage the GitHub email notification, see [examples for setting filters to Kubernetes Github notifications](https://github.com/kubernetes/community/blob/master/communication/best-practices.md#examples-for-setting-filters-to-kubernetes-github-notifications) To install development dependencies you can set up a python virtual env with the necessary dependencies: ```ShellSession virtualenv venv source venv/bin/activate pip install -r tests/requirements.txt ``` #### Linting Kubespray uses [pre-commit](https://pre-commit.com) hook configuration to run several linters, please install this tool and use it to run validation tests before submitting a PR. ```ShellSession pre-commit install pre-commit run -a # To run pre-commit hook on all files in the repository, even if they were not modified ``` #### Molecule [molecule](https://github.com/ansible-community/molecule) is designed to help the development and testing of Ansible roles. In Kubespray you can run it all for all roles with `./tests/scripts/molecule_run.sh` or for a specific role (that you are working with) with `molecule test` from the role directory (`cd roles/my-role`). When developing or debugging a role it can be useful to run `molecule create` and `molecule converge` separately. Then you can use `molecule login` to SSH into the test environment. #### Vagrant Vagrant with VirtualBox or libvirt driver helps you to quickly spin test clusters to test things end to end. See [README.md#vagrant](README.md) ### Contributing A Patch 1. Submit an issue describing your proposed change to the repo in question. 2. The [repo owners](OWNERS) will respond to your issue promptly. 3. Fork the desired repo, develop and test your code changes. 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo. 5. Address any pre-commit validation failures. 6. Sign the CNCF CLA () 7. Submit a pull request. 8. Work with the reviewers on their suggestions. 9. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits () before final merger of your contribution. ================================================ FILE: Dockerfile ================================================ # syntax=docker/dockerfile:1 # Use immutable image tags rather than mutable tags (like ubuntu:24.04) FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b # Some tools like yamllint need this # Pip needs this as well at the moment to install ansible # (and potentially other packages) # See: https://github.com/pypa/pip/issues/10219 ENV LANG=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 WORKDIR /kubespray # hadolint ignore=DL3008 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update -q \ && apt-get install -yq --no-install-recommends \ curl \ python3 \ python3-pip \ sshpass \ vim \ rsync \ openssh-client \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/log/* RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \ --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \ pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt \ && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \; SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \ && curl -L "https://dl.k8s.io/release/v1.35.1/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \ && echo "$(curl -L "https://dl.k8s.io/release/v1.35.1/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl COPY *.yml ./ COPY *.cfg ./ COPY roles ./roles COPY contrib ./contrib COPY inventory ./inventory COPY library ./library COPY extra_playbooks ./extra_playbooks COPY playbooks ./playbooks COPY plugins ./plugins ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016 Kubespray Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners approvers: - kubespray-approvers reviewers: - kubespray-reviewers emeritus_approvers: - kubespray-emeritus_approvers ================================================ FILE: OWNERS_ALIASES ================================================ aliases: kubespray-approvers: - ant31 - mzaian - tico88612 - vannten - yankay kubespray-reviewers: - cyclinder - erikjiang - mzaian - tico88612 - vannten - yankay kubespray-emeritus_approvers: - atoms - chadswen - cristicalin - floryut - liupeng0518 - luckysb - mattymo - miouge1 - oomichi - riverzhang - woopstar ================================================ FILE: README.md ================================================ # Deploy a Production Ready Kubernetes Cluster ![Kubernetes Logo](https://raw.githubusercontent.com/kubernetes-sigs/kubespray/master/docs/img/kubernetes-logo.png) If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**. You can get your invite [here](http://slack.k8s.io/) - Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_controllers/openstack.md), [vSphere](docs/cloud_controllers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal** - **Highly available** cluster - **Composable** (Choice of the network plugin for instance) - Supports most popular **Linux distributions** - **Continuous integration tests** ## Quick Start Below are several ways to use Kubespray to deploy a Kubernetes cluster. ### Docker Ensure you have installed Docker then ```ShellSession docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ quay.io/kubespray/kubespray:v2.30.0 bash # Inside the container you may now run the kubespray playbooks: ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` ### Ansible #### Usage See [Getting started](/docs/getting_started/getting-started.md) #### Collection See [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection ### Vagrant For Vagrant we need to install Python dependencies for provisioning tasks. Check that ``Python`` and ``pip`` are installed: ```ShellSession python -V && pip -V ``` If this returns the version of the software, you're good to go. If not, download and install Python from here Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible) then run the following step: ```ShellSession vagrant up ``` ## Documents - [Requirements](#requirements) - [Kubespray vs ...](docs/getting_started/comparisons.md) - [Getting started](docs/getting_started/getting-started.md) - [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md) - [Ansible inventory and tags](docs/ansible/ansible.md) - [Integration with existing ansible repo](docs/operations/integration.md) - [Deployment data variables](docs/ansible/vars.md) - [DNS stack](docs/advanced/dns-stack.md) - [HA mode](docs/operations/ha-mode.md) - [Network plugins](#network-plugins) - [Vagrant install](docs/developers/vagrant.md) - [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md) - [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md) - [openSUSE setup](docs/operating_systems/opensuse.md) - [Downloaded artifacts](docs/advanced/downloads.md) - [Equinix Metal](docs/cloud_providers/equinix-metal.md) - [OpenStack](docs/cloud_controllers/openstack.md) - [vSphere](docs/cloud_controllers/vsphere.md) - [Large deployments](docs/operations/large-deployments.md) - [Adding/replacing a node](docs/operations/nodes.md) - [Upgrades basics](docs/operations/upgrades.md) - [Air-Gap installation](docs/operations/offline-environment.md) - [NTP](docs/advanced/ntp.md) - [Hardening](docs/operations/hardening.md) - [Mirror](docs/operations/mirror.md) - [Roadmap](docs/roadmap/roadmap.md) ## Supported Linux Distributions - **Flatcar Container Linux by Kinvolk** - **Debian** Bookworm, Bullseye, Trixie - **Ubuntu** 22.04, 24.04 - **CentOS Stream / RHEL** 9, 10 - **Fedora** 39, 40, 41, 42 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md)) - **openSUSE** Leap 15.x/Tumbleweed - **Oracle Linux** 9, 10 - **Alma Linux** 9, 10 - **Rocky Linux** 9, 10 (experimental in 10: see [Rocky Linux 10 notes](docs/operating_systems/rhel.md#rocky-linux-10)) - **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md)) - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md)) - **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md)) - **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md)) Note: - Upstart/SysV init based OS types are not supported. - [Kernel requirements](docs/operations/kernel-requirements.md) (please read if the OS kernel version is < 4.19). ## Supported Components - Core - [kubernetes](https://github.com/kubernetes/kubernetes) 1.35.1 - [etcd](https://github.com/etcd-io/etcd) 3.6.8 - [docker](https://www.docker.com/) 28.3 - [containerd](https://containerd.io/) 2.2.1 - [cri-o](http://cri-o.io/) 1.35.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0 - [calico](https://github.com/projectcalico/calico) 3.30.6 - [cilium](https://github.com/cilium/cilium) 1.19.1 - [flannel](https://github.com/flannel-io/flannel) 0.27.3 - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21 - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1 - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2 - [kube-vip](https://github.com/kube-vip/kube-vip) 1.0.3 - Application - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3 - [coredns](https://github.com/coredns/coredns) 1.12.4 - [argocd](https://argoproj.github.io/) 2.14.5 - [helm](https://helm.sh/) 3.18.4 - [metallb](https://metallb.universe.tf/) 0.13.9 - [registry](https://github.com/distribution/distribution) 2.8.1 - Storage Plugin - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0 - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0 - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0 - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2 - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32 - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0 - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4 ## Container Runtime Notes - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20) ## Requirements - **Minimum required version of Kubernetes is v1.30** - **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands** - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md)) - The target servers are configured to allow **IPv4 forwarding**. - If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**. - The **firewalls are not managed**, you'll need to implement your own rules the way you used to. in order to avoid any issue during deployment you should disable your firewall. - If kubespray is run from non-root user account, correct privilege escalation method should be configured in the target servers. Then the `ansible_become` flag or command parameters `--become or -b` should be specified. Hardware: These limits are safeguarded by Kubespray. Actual requirements for your workload can differ. For a sizing guide go to the [Building Large Clusters](https://kubernetes.io/docs/setup/cluster-large/#size-of-master-and-master-components) guide. - Control Plane - Memory: 2 GB - Worker Node - Memory: 1 GB ## Network Plugins You can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`) - [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking. - [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options designed to give you the most efficient networking across a range of situations, including non-overlay and overlay networks, with or without BGP. Calico uses the same engine to enforce network policy for hosts, pods, and (if using Istio and Envoy) applications at the service mesh layer. - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic. - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises. - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy), iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers). It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs. - [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network. - [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. - [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray. See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart. The network plugin to use is defined by the variable `kube_network_plugin`. There is also an option to leverage built-in cloud provider networking instead. See also [Network checker](docs/advanced/netcheck.md). ## Ingress Plugins - [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider. ## Community docs and resources - [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/) - [kubespray, monitoring and logging](https://github.com/gregbkr/kubernetes-kargo-logging-monitoring) by @gregbkr - [Deploy Kubernetes w/ Ansible & Terraform](https://rsmitty.github.io/Terraform-Ansible-Kubernetes/) by @rsmitty - [Deploy a Kubernetes Cluster with Kubespray (video)](https://www.youtube.com/watch?v=CJ5G4GpqDy0) ## Tools and projects on top of Kubespray - [Digital Rebar Provision](https://github.com/digitalrebar/provision/blob/v4/doc/integrations/ansible.rst) - [Terraform Contrib](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/terraform) - [Kubean](https://github.com/kubean-io/kubean) ## CI Tests [![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/pipelines) CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/). See the [test matrix](docs/developers/test_cases.md) for details. ================================================ FILE: RELEASE.md ================================================ # Release Process The Kubespray Project is released on an as-needed basis. The process is as follows: 1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325) 1. At least one of the [approvers](OWNERS_ALIASES) must approve this release 1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1` 1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables. 1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details. 1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes 1. (Only for major releases) An approver creates a release branch in the form `release-X.Y` 1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request. 1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request. 1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details. 1. The release issue is closed 1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released` 1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...` 1. Create/Update Issue for upgrading kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance) ## Major/minor releases and milestones * For major releases (vX.Y) Kubespray maintains one branch (`release-X.Y`). Minor releases (vX.Y.Z) are available only as tags. * Security patches and bugs might be backported. * Fixes for major releases (vX.Y) and minor releases (vX.Y.Z) are delivered via maintenance releases (vX.Y.Z) and assigned to the corresponding open [GitHub milestone](https://github.com/kubernetes-sigs/kubespray/milestones). That milestone remains open for the major/minor releases support lifetime, which ends once the milestone is closed. Then only a next major or minor release can be done. * Kubespray major and minor releases are bound to the given `kube_version` major/minor version numbers and other components' arbitrary versions, like etcd or network plugins. Older or newer component versions are not supported and not tested for the given release (even if included in the checksum variables, like `kubeadm_checksums`). * There is no unstable releases and no APIs, thus Kubespray doesn't follow [semver](https://semver.org/). Every version describes only a stable release. Breaking changes, if any introduced by changed defaults or non-contrib ansible roles' playbooks, shall be described in the release notes. Other breaking changes, if any in the contributed addons or bound versions of Kubernetes and other components, are considered out of Kubespray scope and are up to the components' teams to deal with and document. * Minor releases can change components' versions, but not the major `kube_version`. Greater `kube_version` requires a new major or minor release. For example, if Kubespray v2.0.0 is bound to `kube_version: 1.4.x`, `calico_version: 0.22.0`, `etcd_version: 3.0.6`, then Kubespray v2.1.0 may be bound to only minor changes to `kube_version`, like v1.5.1 and *any* changes to other components, like etcd v4, or calico 1.2.3. And Kubespray v3.x.x shall be bound to `kube_version: 2.x.x` respectively. ## Release note creation You can create a release note with: ```shell export GITHUB_TOKEN= export ORG=kubernetes-sigs export REPO=kubespray release-notes --start-sha --end-sha --dependencies=false --output=/tmp/kubespray-release-note --required-author="" ``` If the release note file(/tmp/kubespray-release-note) contains "### Uncategorized" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.). It is necessary to put a valid label on each pull request and run the above release-notes command again to get a better release note ## Container image creation The container image `quay.io/kubespray/kubespray:vX.Y.Z` can be created from Dockerfile of the kubespray root directory: ```shell cd kubespray/ nerdctl build -t quay.io/kubespray/kubespray:vX.Y.Z . nerdctl push quay.io/kubespray/kubespray:vX.Y.Z ``` The container image `quay.io/kubespray/vagrant:vX.Y.Z` can be created from build.sh of test-infra/vagrant-docker/: ```shell cd kubespray/test-infra/vagrant-docker/ ./build vX.Y.Z ``` Please note that the above operation requires the permission to push container images into quay.io/kubespray/. If you don't have the permission, please ask it on the #kubespray-dev channel. ================================================ FILE: SECURITY_CONTACTS ================================================ # Defined below are the security contacts for this repo. # # They are the contact point for the Product Security Committee to reach out # to for triaging and handling of incoming issues. # # The below names agree to abide by the # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) # and will be removed and replaced if they violate that agreement. # # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE # INSTRUCTIONS AT https://kubernetes.io/security/ floryut ant31 VannTen yankay ================================================ FILE: Vagrantfile ================================================ # -*- mode: ruby -*- # # vi: set ft=ruby : # For help on using kubespray with vagrant, check out docs/developers/vagrant.md require 'fileutils' require 'ipaddr' require 'socket' Vagrant.require_version ">= 2.0.0" CONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb') FLATCAR_URL_TEMPLATE = "https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json" # Uniq disk UUID for libvirt DISK_UUID = Time.now.utc.to_i SUPPORTED_OS = { "flatcar-stable" => {box: "flatcar-stable", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["stable"]}, "flatcar-beta" => {box: "flatcar-beta", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["beta"]}, "flatcar-alpha" => {box: "flatcar-alpha", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["alpha"]}, "flatcar-edge" => {box: "flatcar-edge", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]}, "ubuntu2004" => {box: "generic/ubuntu2004", user: "vagrant"}, "ubuntu2204" => {box: "generic/ubuntu2204", user: "vagrant"}, "ubuntu2404" => {box: "bento/ubuntu-24.04", user: "vagrant"}, "centos8" => {box: "centos/8", user: "vagrant"}, "centos8-bento" => {box: "bento/centos-8", user: "vagrant"}, "almalinux8" => {box: "almalinux/8", user: "vagrant"}, "almalinux8-bento" => {box: "bento/almalinux-8", user: "vagrant"}, "almalinux9" => {box: "almalinux/9", user: "vagrant"}, "rockylinux8" => {box: "rockylinux/8", user: "vagrant"}, "rockylinux9" => {box: "rockylinux/9", user: "vagrant"}, "fedora39" => {box: "fedora/39-cloud-base", user: "vagrant"}, "fedora40" => {box: "fedora/40-cloud-base", user: "vagrant"}, "fedora39-arm64" => {box: "bento/fedora-39-arm64", user: "vagrant"}, "fedora40-arm64" => {box: "bento/fedora-40", user: "vagrant"}, "fedora41" => {box: "fedora/41-cloud-base", user: "vagrant"}, "fedora42" => {box: "fedora/42-cloud-base", user: "vagrant"}, "fedora41-bento" => {box: "bento/fedora-41", user: "vagrant"}, "opensuse" => {box: "opensuse/Leap-15.6.x86_64", user: "vagrant"}, "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"}, "oraclelinux" => {box: "generic/oracle7", user: "vagrant"}, "oraclelinux8" => {box: "generic/oracle8", user: "vagrant"}, "rhel8" => {box: "generic/rhel8", user: "vagrant"}, "debian11" => {box: "debian/bullseye64", user: "vagrant"}, "debian12" => {box: "debian/bookworm64", user: "vagrant"}, } if File.exist?(CONFIG) require CONFIG end # Defaults for config options defined in CONFIG $num_instances ||= 3 $instance_name_prefix ||= "k8s" $vm_gui ||= false $vm_memory ||= 2048 $vm_cpus ||= 2 $shared_folders ||= {} $forwarded_ports ||= {} $subnet ||= "172.18.8" $subnet_ipv6 ||= "fd3c:b398:0698:0756" $os ||= "ubuntu2004" $network_plugin ||= "flannel" $inventories ||= [] # Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni $multi_networking ||= "False" $download_run_once ||= "True" $download_force_cache ||= "False" # Modify those to have separate groups (for instance, to test separate etcd:) # first_control_plane = 1 # first_etcd = 4 # control_plane_instances = 3 # etcd_instances = 3 $first_node ||= 1 $first_control_plane ||= 1 $first_etcd ||= 1 # The first three nodes are etcd servers $etcd_instances ||= [$num_instances, 3].min # The first two nodes are kube masters $control_plane_instances ||= [$num_instances, 2].min # All nodes are kube nodes $kube_node_instances ||= $num_instances - $first_node + 1 # The following only works when using the libvirt provider $kube_node_instances_with_disks ||= false $kube_node_instances_with_disks_size ||= "20G" $kube_node_instances_with_disks_number ||= 2 $override_disk_size ||= false $disk_size ||= "20GB" $local_path_provisioner_enabled ||= "False" $local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/" $libvirt_nested ||= false # boolean or string (e.g. "-vvv") $ansible_verbosity ||= false $ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || "" $vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant") $playbook ||= "cluster.yml" $extra_vars ||= {} host_vars = {} def collect_networks(subnet, subnet_ipv6) Socket.getifaddrs.filter_map do |iface| next unless iface&.netmask&.ip_address && iface.addr is_ipv6 = iface.addr.ipv6? ip = IPAddr.new(iface.addr.ip_address.split('%').first) ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0") prefix = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1') network = ip.mask(prefix) [IPAddr.new("#{network}/#{prefix}"), ip_test] end end def subnet_in_use?(network_ips) network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net } end network_ips = collect_networks($subnet, $subnet_ipv6) if subnet_in_use?(network_ips) puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0" puts "Subnets in use: #{network_ips.inspect}" exit 1 end # throw error if os is not supported if ! SUPPORTED_OS.key?($os) puts "Unsupported OS: #{$os}" puts "Supported OS are: #{SUPPORTED_OS.keys.join(', ')}" exit 1 end $box = SUPPORTED_OS[$os][:box] if Vagrant.has_plugin?("vagrant-proxyconf") $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" (1..$num_instances).each do |i| $no_proxy += ",#{$subnet}.#{i+100}" end end Vagrant.configure("2") do |config| config.vm.box = $box if SUPPORTED_OS[$os].has_key? :box_url config.vm.box_url = SUPPORTED_OS[$os][:box_url] end config.ssh.username = SUPPORTED_OS[$os][:user] # plugin conflict if Vagrant.has_plugin?("vagrant-vbguest") then config.vbguest.auto_update = false end # always use Vagrants insecure key config.ssh.insert_key = false if ($override_disk_size) unless Vagrant.has_plugin?("vagrant-disksize") system "vagrant plugin install vagrant-disksize" end config.disksize.size = $disk_size end (1..$num_instances).each do |i| config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node| node.vm.hostname = vm_name if Vagrant.has_plugin?("vagrant-proxyconf") node.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || "" node.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || "" node.proxy.no_proxy = $no_proxy end ["vmware_fusion", "vmware_workstation"].each do |vmware| node.vm.provider vmware do |v| v.vmx['memsize'] = $vm_memory v.vmx['numvcpus'] = $vm_cpus end end node.vm.provider :virtualbox do |vb| vb.memory = $vm_memory vb.cpus = $vm_cpus vb.gui = $vm_gui vb.linked_clone = true vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM vb.customize ["modifyvm", :id, "--audio", "none"] end node.vm.provider :libvirt do |lv| lv.nested = $libvirt_nested lv.cpu_mode = "host-model" lv.memory = $vm_memory lv.cpus = $vm_cpus lv.default_prefix = 'kubespray' # Fix kernel panic on fedora 28 if $os == "fedora" lv.cpu_mode = "host-passthrough" end end if $kube_node_instances_with_disks # Libvirt driverletters = ('a'..'z').to_a node.vm.provider :libvirt do |lv| # always make /dev/sd{a/b/c} so that CI can ensure that # virtualbox and libvirt will have the same devices to use for OSDs (1..$kube_node_instances_with_disks_number).each do |d| lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi" end end node.vm.provider :virtualbox do |vb| # always make /dev/sd{a/b/c} so that CI can ensure that # virtualbox and libvirt will have the same devices to use for OSDs (1..$kube_node_instances_with_disks_number).each do |d| vb.customize ['createhd', '--filename', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--size', $kube_node_instances_with_disks_size] # 10GB disk vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', d, '--device', 0, '--type', 'hdd', '--medium', "disk-#{i}-#{driverletters[d]}-#{DISK_UUID}.disk", '--nonrotational', 'on', '--mtype', 'normal'] end end end if $expose_docker_tcp node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true end $forwarded_ports.each do |guest, host| node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true end if ["rhel8"].include? $os # Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot # be installed until the host is registered with a valid Red Hat support subscription node.vm.synced_folder ".", "/vagrant", disabled: false $shared_folders.each do |src, dst| node.vm.synced_folder src, dst end else node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv'] $shared_folders.each do |src, dst| node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] end end ip = "#{$subnet}.#{i+100}" ip6 = "#{$subnet_ipv6}::#{i+100}" node.vm.network :private_network, :ip => ip, :libvirt__guest_ipv6 => 'yes', :libvirt__ipv6_address => ip6, :libvirt__ipv6_prefix => "64", :libvirt__forward_mode => "none", :libvirt__dhcp_enabled => false # libvirt__ipv6_address does not work as intended, the address is obtained with the desired prefix, but auto-generated(like fd3c:b398:698:756:5054:ff:fe48:c61e/64) # add default route for detect ansible_default_ipv6 # TODO: fix libvirt__ipv6 or use $subnet in shell config.vm.provision "shell", inline: "ip -6 r a fd3c:b398:698:756::/64 dev eth1;ip -6 r add default via fd3c:b398:0698:0756::1 dev eth1 || true" # Disable swap for each vm node.vm.provision "shell", inline: "swapoff -a" # ubuntu2004 and ubuntu2204 have IPv6 explicitly disabled. This undoes that. if ["ubuntu2004", "ubuntu2204"].include? $os node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf" node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf" end # Hack for fedora39/40 to get the IP address of the second interface if ["fedora39", "fedora40", "fedora39-arm64", "fedora40-arm64"].include? $os config.vm.provision "shell", inline: <<-SHELL nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2)/24 nmcli conn modify 'Wired connection 2' ipv4.method manual service NetworkManager restart SHELL end # Rockylinux boxes needs UEFI if ["rockylinux8", "rockylinux9"].include? $os config.vm.provider "libvirt" do |domain| domain.loader = "/usr/share/OVMF/x64/OVMF_CODE.fd" end end # Disable firewalld on oraclelinux/redhat vms if ["oraclelinux","oraclelinux8", "rhel8","rockylinux8"].include? $os node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld" end host_vars[vm_name] = { "ip": ip, "flannel_interface": "eth1", "kube_network_plugin": $network_plugin, "kube_network_plugin_multus": $multi_networking, "download_run_once": $download_run_once, "download_localhost": "False", "download_cache_dir": ENV['HOME'] + "/kubespray_cache", # Make kubespray cache even when download_run_once is false "download_force_cache": $download_force_cache, # Keeping the cache on the nodes can improve provisioning speed while debugging kubespray "download_keep_remote_cache": "False", "docker_rpm_keepcache": "1", # These two settings will put kubectl and admin.config in $inventory/artifacts "kubeconfig_localhost": "True", "kubectl_localhost": "True", "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}", "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}", "ansible_ssh_user": SUPPORTED_OS[$os][:user], "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"), "unsafe_show_logs": "True" } # Only execute the Ansible provisioner once, when all the machines are up and ready. # And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh if i == $num_instances node.vm.provision "ansible" do |ansible| ansible.playbook = $playbook ansible.compatibility_mode = "2.0" ansible.verbose = $ansible_verbosity ansible.become = true ansible.limit = "all,localhost" ansible.host_key_checking = false ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"] + $inventories.map {|inv| ["-i", inv]}.flatten ansible.host_vars = host_vars ansible.extra_vars = $extra_vars if $ansible_tags != "" ansible.tags = [$ansible_tags] end ansible.groups = { "etcd" => ["#{$instance_name_prefix}-[#{$first_etcd}:#{$etcd_instances + $first_etcd - 1}]"], "kube_control_plane" => ["#{$instance_name_prefix}-[#{$first_control_plane}:#{$control_plane_instances + $first_control_plane - 1}]"], "kube_node" => ["#{$instance_name_prefix}-[#{$first_node}:#{$kube_node_instances + $first_node - 1}]"], "k8s_cluster:children" => ["kube_control_plane", "kube_node"], } end end end end end ================================================ FILE: _config.yml ================================================ --- theme: jekyll-theme-slate ================================================ FILE: ansible.cfg ================================================ [ssh_connection] pipelining=True ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null #control_path = ~/.ssh/ansible-%%r@%%h:%%p [defaults] # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .) force_valid_group_names = ignore host_key_checking=False gathering = smart fact_caching = jsonfile fact_caching_connection = /tmp fact_caching_timeout = 86400 timeout = 300 stdout_callback = default display_skipped_hosts = no library = ./library callbacks_enabled = profile_tasks roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles deprecation_warnings=False inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg [inventory] ignore_patterns = artifacts, credentials ================================================ FILE: cluster.yml ================================================ --- - name: Install Kubernetes ansible.builtin.import_playbook: playbooks/cluster.yml ================================================ FILE: code-of-conduct.md ================================================ # Kubernetes Community Code of Conduct Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) ================================================ FILE: contrib/aws_iam/kubernetes-master-policy.json ================================================ { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": ["ec2:*"], "Resource": ["*"] }, { "Effect": "Allow", "Action": ["elasticloadbalancing:*"], "Resource": ["*"] }, { "Effect": "Allow", "Action": ["route53:*"], "Resource": ["*"] }, { "Effect": "Allow", "Action": "s3:*", "Resource": [ "arn:aws:s3:::kubernetes-*" ] } ] } ================================================ FILE: contrib/aws_iam/kubernetes-master-role.json ================================================ { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole" } ] } ================================================ FILE: contrib/aws_iam/kubernetes-minion-policy.json ================================================ { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "s3:*", "Resource": [ "arn:aws:s3:::kubernetes-*" ] }, { "Effect": "Allow", "Action": "ec2:Describe*", "Resource": "*" }, { "Effect": "Allow", "Action": "ec2:AttachVolume", "Resource": "*" }, { "Effect": "Allow", "Action": "ec2:DetachVolume", "Resource": "*" }, { "Effect": "Allow", "Action": ["route53:*"], "Resource": ["*"] }, { "Effect": "Allow", "Action": [ "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:GetRepositoryPolicy", "ecr:DescribeRepositories", "ecr:ListImages", "ecr:BatchGetImage" ], "Resource": "*" } ] } ================================================ FILE: contrib/aws_iam/kubernetes-minion-role.json ================================================ { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole" } ] } ================================================ FILE: contrib/aws_inventory/kubespray-aws-inventory.py ================================================ #!/usr/bin/env python from __future__ import print_function import boto3 import os import argparse import json class SearchEC2Tags(object): def __init__(self): self.parse_args() if self.args.list: self.search_tags() if self.args.host: data = {} print(json.dumps(data, indent=2)) def parse_args(self): ##Check if VPC_VISIBILITY is set, if not default to private if "VPC_VISIBILITY" in os.environ: self.vpc_visibility = os.environ['VPC_VISIBILITY'] else: self.vpc_visibility = "private" ##Support --list and --host flags. We largely ignore the host one. parser = argparse.ArgumentParser() parser.add_argument('--list', action='store_true', default=False, help='List instances') parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance') self.args = parser.parse_args() def search_tags(self): hosts = {} hosts['_meta'] = { 'hostvars': {} } ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value. for group in ["kube_control_plane", "kube_node", "etcd"]: hosts[group] = [] tag_key = "kubespray-role" tag_value = ["*"+group+"*"] region = os.environ['AWS_REGION'] ec2 = boto3.resource('ec2', region) filters = [{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}] cluster_name = os.getenv('CLUSTER_NAME') if cluster_name: filters.append({'Name': 'tag-key', 'Values': ['kubernetes.io/cluster/'+cluster_name]}) instances = ec2.instances.filter(Filters=filters) for instance in instances: ##Suppose default vpc_visibility is private dns_name = instance.private_dns_name ansible_host = { 'ansible_ssh_host': instance.private_ip_address } ##Override when vpc_visibility actually is public if self.vpc_visibility == "public": dns_name = instance.public_dns_name ansible_host = { 'ansible_ssh_host': instance.public_ip_address } ##Set when instance actually has node_labels node_labels_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-labels', instance.tags)) if node_labels_tag: ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ]) ##Set when instance actually has node_taints node_taints_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-taints', instance.tags)) if node_taints_tag: ansible_host['node_taints'] = list([ taint.strip() for taint in node_taints_tag[0]['Value'].split(',') ]) hosts[group].append(dns_name) hosts['_meta']['hostvars'][dns_name] = ansible_host hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']} print(json.dumps(hosts, sort_keys=True, indent=2)) SearchEC2Tags() ================================================ FILE: contrib/aws_inventory/requirements.txt ================================================ boto3 # Apache-2.0 ================================================ FILE: contrib/azurerm/.gitignore ================================================ .generated /inventory ================================================ FILE: contrib/azurerm/README.md ================================================ # Kubernetes on Azure with Azure Resource Group Templates Provision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates) ## Status This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course). ## Requirements - [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) - [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest) - Dedicated Resource Group created in the Azure Portal or through azure-cli ## Configuration through group_vars/all You have to modify at least two variables in group_vars/all. The one is the **cluster_name** variable, it must be globally unique due to some restrictions in Azure. The other one is the **ssh_public_keys** variable, it must be your ssh public key to access your azure virtual machines. Most other variables should be self explanatory if you have some basic Kubernetes experience. ## Bastion host You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option also removes all public IPs from all other VMs. ## Generating and applying To generate and apply the templates, call: ```shell ./apply-rg.sh ``` If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will take care about creating/modifying whatever is needed. ## Clearing a resource group If you need to delete all resources from a resource group, simply call: ```shell ./clear-rg.sh ``` **WARNING** this really deletes everything from your resource group, including everything that was later created by you! ## Installing Ansible and the dependencies Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible) ## Generating an inventory for kubespray After you have applied the templates, you can generate an inventory with this call: ```shell ./generate-inventory.sh ``` It will create the file ./inventory which can then be used with kubespray, e.g.: ```shell cd kubespray-root-dir ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all/all.yml" cluster.yml ``` ================================================ FILE: contrib/azurerm/apply-rg.sh ================================================ #!/usr/bin/env bash set -e AZURE_RESOURCE_GROUP="$1" if [ "$AZURE_RESOURCE_GROUP" == "" ]; then echo "AZURE_RESOURCE_GROUP is missing" exit 1 fi ansible-playbook generate-templates.yml az deployment group create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP az deployment group create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP az deployment group create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP az deployment group create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP az deployment group create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP az deployment group create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP ================================================ FILE: contrib/azurerm/clear-rg.sh ================================================ #!/usr/bin/env bash set -e AZURE_RESOURCE_GROUP="$1" if [ "$AZURE_RESOURCE_GROUP" == "" ]; then echo "AZURE_RESOURCE_GROUP is missing" exit 1 fi ansible-playbook generate-templates.yml az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete ================================================ FILE: contrib/azurerm/generate-inventory.sh ================================================ #!/usr/bin/env bash set -e AZURE_RESOURCE_GROUP="$1" if [ "$AZURE_RESOURCE_GROUP" == "" ]; then echo "AZURE_RESOURCE_GROUP is missing" exit 1 fi # check if azure cli 2.0 exists else use azure cli 1.0 if az &>/dev/null; then ansible-playbook generate-inventory_2.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP" elif azure &>/dev/null; then ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP" else echo "Azure cli not found" fi ================================================ FILE: contrib/azurerm/generate-inventory.yml ================================================ --- - name: Generate Azure inventory hosts: localhost gather_facts: false roles: - generate-inventory ================================================ FILE: contrib/azurerm/generate-inventory_2.yml ================================================ --- - name: Generate Azure inventory hosts: localhost gather_facts: false roles: - generate-inventory_2 ================================================ FILE: contrib/azurerm/generate-templates.yml ================================================ --- - name: Generate Azure templates hosts: localhost gather_facts: false roles: - generate-templates ================================================ FILE: contrib/azurerm/group_vars/all ================================================ # Due to some Azure limitations (ex:- Storage Account's name must be unique), # this name must be globally unique - it will be used as a prefix for azure components cluster_name: example # Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion # node that can be used to access the masters and minions use_bastion: false # Set this to a preferred name that will be used as the first part of the dns name for your bastotion host. For example: k8s-bastion..cloudapp.azure.com. # This is convenient when exceptions have to be configured on a firewall to allow ssh to the given bastion host. # bastion_domain_prefix: k8s-bastion number_of_k8s_masters: 3 number_of_k8s_nodes: 3 masters_vm_size: Standard_A2 masters_os_disk_size: 1000 minions_vm_size: Standard_A2 minions_os_disk_size: 1000 admin_username: devops admin_password: changeme # MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines ssh_public_keys: - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy" # Disable using ssh using password. Change it to false to allow to connect to ssh by password disablePasswordAuthentication: true # Azure CIDRs azure_vnet_cidr: 10.0.0.0/8 azure_admin_cidr: 10.241.2.0/24 azure_masters_cidr: 10.0.4.0/24 azure_minions_cidr: 10.240.0.0/16 # Azure loadbalancer port to use to access your cluster kube_apiserver_port: 6443 # Azure Netwoking and storage naming to use with inventory/all.yml #azure_virtual_network_name: KubeVNET #azure_subnet_admin_name: ad-subnet #azure_subnet_masters_name: master-subnet #azure_subnet_minions_name: minion-subnet #azure_route_table_name: routetable #azure_security_group_name: secgroup # Storage types available are: "Standard_LRS","Premium_LRS" #azure_storage_account_type: Standard_LRS ================================================ FILE: contrib/azurerm/roles/generate-inventory/tasks/main.yml ================================================ --- - name: Query Azure VMs command: azure vm list-ip-address --json {{ azure_resource_group }} register: vm_list_cmd - name: Set vm_list set_fact: vm_list: "{{ vm_list_cmd.stdout }}" - name: Generate inventory template: src: inventory.j2 dest: "{{ playbook_dir }}/inventory" mode: "0644" ================================================ FILE: contrib/azurerm/roles/generate-inventory/templates/inventory.j2 ================================================ {% for vm in vm_list %} {% if not use_bastion or vm.name == 'bastion' %} {{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }} {% else %} {{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }} {% endif %} {% endfor %} [kube_control_plane] {% for vm in vm_list %} {% if 'kube_control_plane' in vm.tags.roles %} {{ vm.name }} {% endif %} {% endfor %} [etcd] {% for vm in vm_list %} {% if 'etcd' in vm.tags.roles %} {{ vm.name }} {% endif %} {% endfor %} [kube_node] {% for vm in vm_list %} {% if 'kube_node' in vm.tags.roles %} {{ vm.name }} {% endif %} {% endfor %} [k8s_cluster:children] kube_node kube_control_plane ================================================ FILE: contrib/azurerm/roles/generate-inventory_2/tasks/main.yml ================================================ --- - name: Query Azure VMs IPs command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }} register: vm_ip_list_cmd - name: Query Azure VMs Roles command: az vm list -o json --resource-group {{ azure_resource_group }} register: vm_list_cmd - name: Query Azure Load Balancer Public IP command: az network public-ip show -o json -g {{ azure_resource_group }} -n kubernetes-api-pubip register: lb_pubip_cmd - name: Set VM IP, roles lists and load balancer public IP set_fact: vm_ip_list: "{{ vm_ip_list_cmd.stdout }}" vm_roles_list: "{{ vm_list_cmd.stdout }}" lb_pubip: "{{ lb_pubip_cmd.stdout }}" - name: Generate inventory template: src: inventory.j2 dest: "{{ playbook_dir }}/inventory" mode: "0644" - name: Generate Load Balancer variables template: src: loadbalancer_vars.j2 dest: "{{ playbook_dir }}/loadbalancer_vars.yml" mode: "0644" ================================================ FILE: contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2 ================================================ {% for vm in vm_ip_list %} {% if not use_bastion or vm.virtualMachine.name == 'bastion' %} {{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }} {% else %} {{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.privateIpAddresses[0] }} {% endif %} {% endfor %} [kube_control_plane] {% for vm in vm_roles_list %} {% if 'kube_control_plane' in vm.tags.roles %} {{ vm.name }} {% endif %} {% endfor %} [etcd] {% for vm in vm_roles_list %} {% if 'etcd' in vm.tags.roles %} {{ vm.name }} {% endif %} {% endfor %} [kube_node] {% for vm in vm_roles_list %} {% if 'kube_node' in vm.tags.roles %} {{ vm.name }} {% endif %} {% endfor %} [k8s_cluster:children] kube_node kube_control_plane ================================================ FILE: contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2 ================================================ ## External LB example config apiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }} loadbalancer_apiserver: address: {{ lb_pubip.ipAddress }} port: 6443 ## Internal loadbalancers for apiservers loadbalancer_apiserver_localhost: false ================================================ FILE: contrib/azurerm/roles/generate-templates/defaults/main.yml ================================================ --- apiVersion: "2015-06-15" virtualNetworkName: "{{ azure_virtual_network_name | default('KubeVNET') }}" subnetAdminName: "{{ azure_subnet_admin_name | default('ad-subnet') }}" subnetMastersName: "{{ azure_subnet_masters_name | default('master-subnet') }}" subnetMinionsName: "{{ azure_subnet_minions_name | default('minion-subnet') }}" routeTableName: "{{ azure_route_table_name | default('routetable') }}" securityGroupName: "{{ azure_security_group_name | default('secgroup') }}" nameSuffix: "{{ cluster_name }}" availabilitySetMasters: "master-avs" availabilitySetMinions: "minion-avs" faultDomainCount: 3 updateDomainCount: 10 bastionVmSize: Standard_A0 bastionVMName: bastion bastionIPAddressName: bastion-pubip disablePasswordAuthentication: true sshKeyPath: "/home/{{ admin_username }}/.ssh/authorized_keys" imageReference: publisher: "OpenLogic" offer: "CentOS" sku: "7.5" version: "latest" imageReferenceJson: "{{ imageReference | to_json }}" storageAccountName: "sa{{ nameSuffix | replace('-', '') }}" storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}" ================================================ FILE: contrib/azurerm/roles/generate-templates/tasks/main.yml ================================================ --- - name: Set base_dir set_fact: base_dir: "{{ playbook_dir }}/.generated/" - name: Create base_dir file: path: "{{ base_dir }}" state: directory recurse: true mode: "0755" - name: Store json files in base_dir template: src: "{{ item }}" dest: "{{ base_dir }}/{{ item }}" mode: "0644" with_items: - network.json - storage.json - availability-sets.json - bastion.json - masters.json - minions.json - clear-rg.json ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/availability-sets.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { }, "variables": { }, "resources": [ { "type": "Microsoft.Compute/availabilitySets", "name": "{{availabilitySetMasters}}", "apiVersion": "{{apiVersion}}", "location": "[resourceGroup().location]", "properties": { "PlatformFaultDomainCount": "{{faultDomainCount}}", "PlatformUpdateDomainCount": "{{updateDomainCount}}" } }, { "type": "Microsoft.Compute/availabilitySets", "name": "{{availabilitySetMinions}}", "apiVersion": "{{apiVersion}}", "location": "[resourceGroup().location]", "properties": { "PlatformFaultDomainCount": "{{faultDomainCount}}", "PlatformUpdateDomainCount": "{{updateDomainCount}}" } } ] } ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/bastion.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { }, "variables": { "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]", "subnetAdminRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]" }, "resources": [ {% if use_bastion %} { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/publicIPAddresses", "name": "{{bastionIPAddressName}}", "location": "[resourceGroup().location]", "properties": { "publicIPAllocationMethod": "Static", "dnsSettings": { {% if bastion_domain_prefix %} "domainNameLabel": "{{ bastion_domain_prefix }}" {% endif %} } } }, { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/networkInterfaces", "name": "{{bastionVMName}}-nic", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]" ], "properties": { "ipConfigurations": [ { "name": "BastionIpConfig", "properties": { "privateIPAllocationMethod": "Dynamic", "publicIPAddress": { "id": "[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]" }, "subnet": { "id": "[variables('subnetAdminRef')]" } } } ] } }, { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Compute/virtualMachines", "name": "{{bastionVMName}}", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]" ], "tags": { "roles": "bastion" }, "properties": { "hardwareProfile": { "vmSize": "{{bastionVmSize}}" }, "osProfile": { "computerName": "{{bastionVMName}}", "adminUsername": "{{admin_username}}", "adminPassword": "{{admin_password}}", "linuxConfiguration": { "disablePasswordAuthentication": "true", "ssh": { "publicKeys": [ {% for key in ssh_public_keys %} { "path": "{{sshKeyPath}}", "keyData": "{{key}}" }{% if loop.index < ssh_public_keys | length %},{% endif %} {% endfor %} ] } } }, "storageProfile": { "imageReference": {{imageReferenceJson}}, "osDisk": { "name": "osdisk", "vhd": { "uri": "[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]" }, "caching": "ReadWrite", "createOption": "FromImage" } }, "networkProfile": { "networkInterfaces": [ { "id": "[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]" } ] } } } {% endif %} ] } ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/clear-rg.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": {}, "variables": {}, "resources": [], "outputs": {} } ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/masters.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { }, "variables": { "lbDomainName": "{{nameSuffix}}-api", "lbPublicIPAddressName": "kubernetes-api-pubip", "lbPublicIPAddressType": "Static", "lbPublicIPAddressID": "[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]", "lbName": "kubernetes-api", "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]", "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]", "kubeMastersSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]" }, "resources": [ { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/publicIPAddresses", "name": "[variables('lbPublicIPAddressName')]", "location": "[resourceGroup().location]", "properties": { "publicIPAllocationMethod": "[variables('lbPublicIPAddressType')]", "dnsSettings": { "domainNameLabel": "[variables('lbDomainName')]" } } }, { "apiVersion": "{{apiVersion}}", "name": "[variables('lbName')]", "type": "Microsoft.Network/loadBalancers", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]" ], "properties": { "frontendIPConfigurations": [ { "name": "kube-api-frontend", "properties": { "publicIPAddress": { "id": "[variables('lbPublicIPAddressID')]" } } } ], "backendAddressPools": [ { "name": "kube-api-backend" } ], "loadBalancingRules": [ { "name": "kube-api", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]" }, "backendAddressPool": { "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]" }, "protocol": "tcp", "frontendPort": "{{kube_apiserver_port}}", "backendPort": "{{kube_apiserver_port}}", "enableFloatingIP": false, "idleTimeoutInMinutes": 5, "probe": { "id": "[concat(variables('lbID'), '/probes/kube-api')]" } } } ], "probes": [ { "name": "kube-api", "properties": { "protocol": "tcp", "port": "{{kube_apiserver_port}}", "intervalInSeconds": 5, "numberOfProbes": 2 } } ] } }, {% for i in range(number_of_k8s_masters) %} {% if not use_bastion %} { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/publicIPAddresses", "name": "master-{{i}}-pubip", "location": "[resourceGroup().location]", "properties": { "publicIPAllocationMethod": "Static" } }, {% endif %} { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/networkInterfaces", "name": "master-{{i}}-nic", "location": "[resourceGroup().location]", "dependsOn": [ {% if not use_bastion %} "[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]", {% endif %} "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]" ], "properties": { "ipConfigurations": [ { "name": "MastersIpConfig", "properties": { "privateIPAllocationMethod": "Dynamic", {% if not use_bastion %} "publicIPAddress": { "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]" }, {% endif %} "subnet": { "id": "[variables('kubeMastersSubnetRef')]" }, "loadBalancerBackendAddressPools": [ { "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]" } ] } } ], "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]" }, "enableIPForwarding": true } }, { "type": "Microsoft.Compute/virtualMachines", "name": "master-{{i}}", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]" ], "tags": { "roles": "kube_control_plane,etcd" }, "apiVersion": "{{apiVersion}}", "properties": { "availabilitySet": { "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]" }, "hardwareProfile": { "vmSize": "{{masters_vm_size}}" }, "osProfile": { "computerName": "master-{{i}}", "adminUsername": "{{admin_username}}", "adminPassword": "{{admin_password}}", "linuxConfiguration": { "disablePasswordAuthentication": "{{disablePasswordAuthentication}}", "ssh": { "publicKeys": [ {% for key in ssh_public_keys %} { "path": "{{sshKeyPath}}", "keyData": "{{key}}" }{% if loop.index < ssh_public_keys | length %},{% endif %} {% endfor %} ] } } }, "storageProfile": { "imageReference": {{imageReferenceJson}}, "osDisk": { "name": "ma{{nameSuffix}}{{i}}", "vhd": { "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]" }, "caching": "ReadWrite", "createOption": "FromImage", "diskSizeGB": "{{masters_os_disk_size}}" } }, "networkProfile": { "networkInterfaces": [ { "id": "[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]" } ] } } } {% if not loop.last %},{% endif %} {% endfor %} ] } ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/minions.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { }, "variables": { "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]", "kubeMinionsSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]" }, "resources": [ {% for i in range(number_of_k8s_nodes) %} {% if not use_bastion %} { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/publicIPAddresses", "name": "minion-{{i}}-pubip", "location": "[resourceGroup().location]", "properties": { "publicIPAllocationMethod": "Static" } }, {% endif %} { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/networkInterfaces", "name": "minion-{{i}}-nic", "location": "[resourceGroup().location]", "dependsOn": [ {% if not use_bastion %} "[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]" {% endif %} ], "properties": { "ipConfigurations": [ { "name": "MinionsIpConfig", "properties": { "privateIPAllocationMethod": "Dynamic", {% if not use_bastion %} "publicIPAddress": { "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]" }, {% endif %} "subnet": { "id": "[variables('kubeMinionsSubnetRef')]" } } } ], "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]" }, "enableIPForwarding": true } }, { "type": "Microsoft.Compute/virtualMachines", "name": "minion-{{i}}", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]" ], "tags": { "roles": "kube_node" }, "apiVersion": "{{apiVersion}}", "properties": { "availabilitySet": { "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]" }, "hardwareProfile": { "vmSize": "{{minions_vm_size}}" }, "osProfile": { "computerName": "minion-{{i}}", "adminUsername": "{{admin_username}}", "adminPassword": "{{admin_password}}", "linuxConfiguration": { "disablePasswordAuthentication": "{{disablePasswordAuthentication}}", "ssh": { "publicKeys": [ {% for key in ssh_public_keys %} { "path": "{{sshKeyPath}}", "keyData": "{{key}}" }{% if loop.index < ssh_public_keys | length %},{% endif %} {% endfor %} ] } } }, "storageProfile": { "imageReference": {{imageReferenceJson}}, "osDisk": { "name": "mi{{nameSuffix}}{{i}}", "vhd": { "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]" }, "caching": "ReadWrite", "createOption": "FromImage", "diskSizeGB": "{{minions_os_disk_size}}" } }, "networkProfile": { "networkInterfaces": [ { "id": "[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]" } ] } } } {% if not loop.last %},{% endif %} {% endfor %} ] } ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/network.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { }, "variables": { }, "resources": [ { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/routeTables", "name": "{{routeTableName}}", "location": "[resourceGroup().location]", "properties": { "routes": [ ] } }, { "type": "Microsoft.Network/virtualNetworks", "name": "{{virtualNetworkName}}", "location": "[resourceGroup().location]", "apiVersion": "{{apiVersion}}", "dependsOn": [ "[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]" ], "properties": { "addressSpace": { "addressPrefixes": [ "{{azure_vnet_cidr}}" ] }, "subnets": [ { "name": "{{subnetMastersName}}", "properties": { "addressPrefix": "{{azure_masters_cidr}}", "routeTable": { "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]" } } }, { "name": "{{subnetMinionsName}}", "properties": { "addressPrefix": "{{azure_minions_cidr}}", "routeTable": { "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]" } } } {% if use_bastion %} ,{ "name": "{{subnetAdminName}}", "properties": { "addressPrefix": "{{azure_admin_cidr}}", "routeTable": { "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]" } } } {% endif %} ] } }, { "apiVersion": "{{apiVersion}}", "type": "Microsoft.Network/networkSecurityGroups", "name": "{{securityGroupName}}", "location": "[resourceGroup().location]", "properties": { "securityRules": [ {% if not use_bastion %} { "name": "ssh", "properties": { "description": "Allow SSH", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "22", "sourceAddressPrefix": "Internet", "destinationAddressPrefix": "*", "access": "Allow", "priority": 100, "direction": "Inbound" } }, {% endif %} { "name": "kube-api", "properties": { "description": "Allow secure kube-api", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "{{kube_apiserver_port}}", "sourceAddressPrefix": "Internet", "destinationAddressPrefix": "*", "access": "Allow", "priority": 101, "direction": "Inbound" } } ] }, "resources": [], "dependsOn": [] } ] } ================================================ FILE: contrib/azurerm/roles/generate-templates/templates/storage.json ================================================ { "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { }, "variables": { }, "resources": [ { "type": "Microsoft.Storage/storageAccounts", "name": "{{storageAccountName}}", "location": "[resourceGroup().location]", "apiVersion": "{{apiVersion}}", "properties": { "accountType": "{{storageAccountType}}" } } ] } ================================================ FILE: contrib/collection.sh ================================================ #!/bin/bash -eux # Install collection from source assuming dependencies are present. # Run in SemaphoreUI this bash script can install Kubespray from the repo NAMESPACE=kubernetes_sigs COLLECTION=kubespray MY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //') ansible-galaxy collection build --force --output-path . ansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz ================================================ FILE: contrib/offline/README.md ================================================ # Offline deployment ## manage-offline-container-images.sh Container image collecting script for offline deployment This script has two features: (1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE environment variable to get images from a file (e.g. temp/images.list after running the ./generate_list.sh script). (2) Deploy local container registry and register the container images to the registry. Step(1) should be done online site as a preparation, then we bring the gotten images to the target offline environment. if images are from a private registry, you need to set `PRIVATE_REGISTRY` environment variable. Then we will run step(2) for registering the images to local registry, or to an existing registry set by the `DESTINATION_REGISTRY` environment variable. By default, the local registry will run on port 5000. This can be changed with the `REGISTRY_PORT` environment variable Step(1) can be operated with: ```shell manage-offline-container-images.sh create ``` Step(2) can be operated with: ```shell manage-offline-container-images.sh register ``` ## generate_list.sh This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file. Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files, all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template. ```shell ./generate_list.sh tree temp temp ├── files.list ├── files.list.template ├── images.list └── images.list.template 0 directories, 5 files ``` In some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars, then run `./generate_list.sh -i [inventory_file]` to update file.list and images.list. ## manage-offline-files.sh This script will download all files according to `temp/files.list` and run nginx container to provide offline file download. Step(1) generate `files.list` ```shell ./generate_list.sh ``` Step(2) download files and run nginx container ```shell ./manage-offline-files.sh ``` when nginx container is running, it can be accessed through . ## upload2artifactory.py After the steps above, this script can recursively upload each file under a directory to a generic repository in Artifactory. Environment Variables: - USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'. - TOKEN -- Generate this with 'Set Me Up' in your user. - BASE_URL -- The URL including the repository name. Step(3) (optional) upload files to Artifactory ```shell cd kubespray/contrib/offline/offline-files export USERNAME=admin export TOKEN=... export BASE_URL=https://artifactory.example.com/artifactory/a-generic-repo/ ./upload2artifactory.py ``` ================================================ FILE: contrib/offline/docker-daemon.json ================================================ { "insecure-registries":["HOSTNAME:5000"] } ================================================ FILE: contrib/offline/generate_list.sh ================================================ #!/bin/bash set -eo pipefail CURRENT_DIR=$(cd $(dirname $0); pwd) TEMP_DIR="${CURRENT_DIR}/temp" REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}" : ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"} mkdir -p ${TEMP_DIR} # generate all download files url template grep 'download_url:' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \ | sed 's/^.*_url: //g;s/\"//g' > ${TEMP_DIR}/files.list.template # generate all images list template sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \ | sed -n "s/repo: //p;s/tag: //p" | tr -d ' ' \ | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template # add kube-* images to images list template # Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml # doesn't contain those images. That is reason why here needs to put those images into the # list separately. KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy" for i in $KUBE_IMAGES; do echo "{{ kube_image_repo }}/$i:v{{ kube_version }}" >> ${TEMP_DIR}/images.list.template done # run ansible to expand templates /bin/cp ${CURRENT_DIR}/generate_list.yml ${REPO_ROOT_DIR} (cd ${REPO_ROOT_DIR} && ansible-playbook $* generate_list.yml && /bin/rm generate_list.yml) || exit 1 ================================================ FILE: contrib/offline/generate_list.yml ================================================ --- - name: Collect container images for offline deployment hosts: localhost become: false roles: # Just load default variables from roles. - role: kubespray_defaults when: false - role: download when: false tasks: # Generate files.list and images.list files from templates. - name: Collect container images for offline deployment template: src: ./contrib/offline/temp/{{ item }}.list.template dest: ./contrib/offline/temp/{{ item }}.list mode: "0644" with_items: - files - images ================================================ FILE: contrib/offline/manage-offline-container-images.sh ================================================ #!/usr/bin/env bash OPTION=$1 CURRENT_DIR=$(cd $(dirname $0); pwd) TEMP_DIR="${CURRENT_DIR}/temp" IMAGE_TAR_FILE="${CURRENT_DIR}/container-images.tar.gz" IMAGE_DIR="${CURRENT_DIR}/container-images" IMAGE_LIST="${IMAGE_DIR}/container-images.txt" RETRY_COUNT=5 function create_container_image_tar() { set -e if [ -z "${IMAGES_FROM_FILE}" ]; then echo "Getting images from current \"$(kubectl config current-context)\"" IMAGES=$(mktemp --suffix=-images) trap 'rm -f "${IMAGES}"' EXIT kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}" # NOTE: etcd and pause cannot be seen as pods. kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}" else echo "Getting images from file \"${IMAGES_FROM_FILE}\"" if [ ! -f "${IMAGES_FROM_FILE}" ]; then echo "${IMAGES_FROM_FILE} is not a file" exit 1 fi IMAGES=$(realpath $IMAGES_FROM_FILE) fi rm -f ${IMAGE_TAR_FILE} rm -rf ${IMAGE_DIR} mkdir ${IMAGE_DIR} cd ${IMAGE_DIR} sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest sudo ${runtime} save -o registry-latest.tar registry:latest while read -r image do FILE_NAME="$(echo ${image} | sed s@"/"@"-"@g | sed s/":"/"-"/g | sed -E 's/\@.*//g')".tar set +e for step in $(seq 1 ${RETRY_COUNT}) do sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image} if [ $? -eq 0 ]; then break fi echo "Failed to pull ${image} at step ${step}" if [ ${step} -eq ${RETRY_COUNT} ]; then exit 1 fi done set -e sudo ${runtime} save -o ${FILE_NAME} ${image} # NOTE: Here removes the following repo parts from each image # so that these parts will be replaced with Kubespray. # - kube_image_repo: "registry.k8s.io" # - gcr_image_repo: "gcr.io" # - ghcr_image_repo: "ghcr.io" # - docker_image_repo: "docker.io" # - quay_image_repo: "quay.io" FIRST_PART=$(echo ${image} | awk -F"/" '{print $1}') if [ "${FIRST_PART}" = "registry.k8s.io" ] || [ "${FIRST_PART}" = "gcr.io" ] || [ "${FIRST_PART}" = "ghcr.io" ] || [ "${FIRST_PART}" = "docker.io" ] || [ "${FIRST_PART}" = "quay.io" ] || [ "${FIRST_PART}" = "${PRIVATE_REGISTRY}" ]; then image=$(echo ${image} | sed s@"${FIRST_PART}/"@@ | sed -E 's/\@.*/\n/g') fi echo "${FILE_NAME} ${image}" >> ${IMAGE_LIST} done < "${IMAGES}" cd .. sudo chown ${USER} ${IMAGE_DIR}/* tar -zcvf ${IMAGE_TAR_FILE} ./container-images rm -rf ${IMAGE_DIR} echo "" echo "${IMAGE_TAR_FILE} is created to contain your container images." echo "Please keep this file and bring it to your offline environment." } function register_container_images() { create_registry=false REGISTRY_PORT=${REGISTRY_PORT:-"5000"} if [ -z "${DESTINATION_REGISTRY}" ]; then echo "DESTINATION_REGISTRY not set, will create local registry" create_registry=true DESTINATION_REGISTRY="$(hostname):${REGISTRY_PORT}" fi echo "Images will be pushed to ${DESTINATION_REGISTRY}" if [ ! -f ${IMAGE_TAR_FILE} ]; then echo "${IMAGE_TAR_FILE} should exist." exit 1 fi if [ ! -d ${TEMP_DIR} ]; then mkdir ${TEMP_DIR} fi # To avoid "http: server gave http response to https client" error. if [ -d /etc/docker/ ]; then set -e # Ubuntu18.04, RHEL7/CentOS7 cp ${CURRENT_DIR}/docker-daemon.json ${TEMP_DIR}/docker-daemon.json sed -i s@"HOSTNAME"@"$(hostname)"@ ${TEMP_DIR}/docker-daemon.json sudo cp ${TEMP_DIR}/docker-daemon.json /etc/docker/daemon.json elif [ -d /etc/containers/ ]; then set -e # RHEL8/CentOS8 cp ${CURRENT_DIR}/registries.conf ${TEMP_DIR}/registries.conf sed -i s@"HOSTNAME"@"$(hostname)"@ ${TEMP_DIR}/registries.conf sudo cp ${TEMP_DIR}/registries.conf /etc/containers/registries.conf elif [ "$(uname)" == "Darwin" ]; then echo "This is a Mac, no configuration changes are required" else echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed" exit 1 fi tar -zxvf ${IMAGE_TAR_FILE} if ${create_registry}; then sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar set +e sudo ${runtime} container inspect registry >/dev/null 2>&1 if [ $? -ne 0 ]; then sudo ${runtime} run --restart=always -d -p "${REGISTRY_PORT}":"${REGISTRY_PORT}" --name registry registry:latest fi set -e fi while read -r line; do file_name=$(echo ${line} | awk '{print $1}') raw_image=$(echo ${line} | awk '{print $2}') new_image="${DESTINATION_REGISTRY}/${raw_image}" load_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1) org_image=$(echo "${load_image}" | awk '{print $3}') # special case for tags containing the digest when using docker or podman as the container runtime if [ "${org_image}" == "ID:" ]; then org_image=$(echo "${load_image}" | awk '{print $4}') fi image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}") if [ -z "${file_name}" ]; then echo "Failed to get file_name for line ${line}" exit 1 fi if [ -z "${raw_image}" ]; then echo "Failed to get raw_image for line ${line}" exit 1 fi if [ -z "${org_image}" ]; then echo "Failed to get org_image for line ${line}" exit 1 fi if [ -z "${image_id}" ]; then echo "Failed to get image_id for file ${file_name}" exit 1 fi sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} sudo ${runtime} tag ${image_id} ${new_image} sudo ${runtime} push ${new_image} done <<< "$(cat ${IMAGE_LIST})" echo "Succeeded to register container images to local registry." echo "Please specify \"${DESTINATION_REGISTRY}\" for the following options in your inventry:" echo "- kube_image_repo" echo "- gcr_image_repo" echo "- docker_image_repo" echo "- quay_image_repo" } # get runtime command if command -v nerdctl 1>/dev/null 2>&1; then runtime="nerdctl" elif command -v podman 1>/dev/null 2>&1; then runtime="podman" elif command -v docker 1>/dev/null 2>&1; then runtime="docker" else echo "No supported container runtime found" exit 1 fi if [ "${OPTION}" == "create" ]; then create_container_image_tar elif [ "${OPTION}" == "register" ]; then register_container_images else echo "This script has two features:" echo "(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE" echo " environment variable to get images from a file (e.g. temp/images.list after running the" echo " ./generate_list.sh script)." echo "(2) Deploy local container registry and register the container images to the registry." echo "" echo "Step(1) should be done online site as a preparation, then we bring" echo "the gotten images to the target offline environment. if images are from" echo "a private registry, you need to set PRIVATE_REGISTRY environment variable." echo "Then we will run step(2) for registering the images to local registry, or to an existing" echo "registry set by the DESTINATION_REGISTRY environment variable. By default, the local registry" echo "will run on port 5000. This can be changed with the REGISTRY_PORT environment variable" echo "" echo "${IMAGE_TAR_FILE} is created to contain your container images." echo "Please keep this file and bring it to your offline environment." echo "" echo "Step(1) can be operated with:" echo " $ ./manage-offline-container-images.sh create" echo "" echo "Step(2) can be operated with:" echo " $ ./manage-offline-container-images.sh register" echo "" echo "Please specify 'create' or 'register'." echo "" exit 1 fi ================================================ FILE: contrib/offline/manage-offline-files.sh ================================================ #!/bin/bash CURRENT_DIR=$( dirname "$(readlink -f "$0")" ) OFFLINE_FILES_DIR_NAME="offline-files" OFFLINE_FILES_DIR="${CURRENT_DIR}/${OFFLINE_FILES_DIR_NAME}" OFFLINE_FILES_ARCHIVE="${CURRENT_DIR}/offline-files.tar.gz" FILES_LIST=${FILES_LIST:-"${CURRENT_DIR}/temp/files.list"} NGINX_PORT=8080 # download files if [ ! -f "${FILES_LIST}" ]; then echo "${FILES_LIST} should exist, run ./generate_list.sh first." exit 1 fi rm -rf "${OFFLINE_FILES_DIR}" rm "${OFFLINE_FILES_ARCHIVE}" mkdir "${OFFLINE_FILES_DIR}" while read -r url; do if ! wget -x -P "${OFFLINE_FILES_DIR}" "${url}"; then exit 1 fi done < "${FILES_LIST}" tar -czvf "${OFFLINE_FILES_ARCHIVE}" "${OFFLINE_FILES_DIR_NAME}" [ -n "$NO_HTTP_SERVER" ] && echo "skip to run nginx" && exit 0 # run nginx container server if command -v nerdctl 1>/dev/null 2>&1; then runtime="nerdctl" elif command -v podman 1>/dev/null 2>&1; then runtime="podman" elif command -v docker 1>/dev/null 2>&1; then runtime="docker" else echo "No supported container runtime found" exit 1 fi sudo "${runtime}" container inspect nginx >/dev/null 2>&1 if [ $? -ne 0 ]; then sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \ --restart=always -d -p ${NGINX_PORT}:80 \ --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \ --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \ --name nginx nginx:alpine fi ================================================ FILE: contrib/offline/nginx.conf ================================================ user nginx; worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; include /usr/share/nginx/modules/*.conf; events { worker_connections 1024; } http { log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; server { listen 80 default_server; listen [::]:80 default_server; server_name _; include /etc/nginx/default.d/*.conf; location / { root /usr/share/nginx/html/download; autoindex on; autoindex_exact_size off; autoindex_localtime on; } error_page 404 /404.html; location = /40x.html { } error_page 500 502 503 504 /50x.html; location = /50x.html { } } } ================================================ FILE: contrib/offline/registries.conf ================================================ [registries.search] registries = ['registry.access.redhat.com', 'registry.redhat.io', 'docker.io'] [registries.insecure] registries = ['HOSTNAME:5000'] [registries.block] registries = [] ================================================ FILE: contrib/offline/upload2artifactory.py ================================================ #!/usr/bin/env python3 """This is a helper script to manage-offline-files.sh. After running manage-offline-files.sh, you can run upload2artifactory.py to recursively upload each file to a generic repository in Artifactory. This script recurses the current working directory and is intended to be started from 'kubespray/contrib/offline/offline-files' Environment Variables: USERNAME -- At least permissions'Deploy/Cache' and 'Delete/Overwrite'. TOKEN -- Generate this with 'Set Me Up' in your user. BASE_URL -- The URL including the repository name. """ import os import urllib.request import base64 def upload_file(file_path, destination_url, username, token): """Helper function to upload a single file""" try: with open(file_path, 'rb') as f: file_data = f.read() request = urllib.request.Request(destination_url, data=file_data, method='PUT') # NOQA auth_header = base64.b64encode(f"{username}:{token}".encode()).decode() request.add_header("Authorization", f"Basic {auth_header}") with urllib.request.urlopen(request) as response: if response.status in [200, 201]: print(f"Success: Uploaded {file_path}") else: print(f"Failed: {response.status} {response.read().decode('utf-8')}") # NOQA except urllib.error.HTTPError as e: print(f"HTTPError: {e.code} {e.reason} for {file_path}") except urllib.error.URLError as e: print(f"URLError: {e.reason} for {file_path}") except OSError as e: print(f"OSError: {e.strerror} for {file_path}") def upload_files(base_url, username, token): """ Recurse current dir and upload each file using urllib.request """ for root, _, files in os.walk(os.getcwd()): for file in files: file_path = os.path.join(root, file) relative_path = os.path.relpath(file_path, os.getcwd()) destination_url = f"{base_url}/{relative_path}" print(f"Uploading {file_path} to {destination_url}") upload_file(file_path, destination_url, username, token) if __name__ == "__main__": a_user = os.getenv("USERNAME") a_token = os.getenv("TOKEN") a_url = os.getenv("BASE_URL") if not a_user or not a_token or not a_url: print( "Error: Environment variables USERNAME, TOKEN, and BASE_URL must be set." # NOQA ) exit() upload_files(a_url, a_user, a_token) ================================================ FILE: contrib/os-services/os-services.yml ================================================ --- - name: Disable firewalld/ufw hosts: all roles: - { role: prepare } ================================================ FILE: contrib/os-services/roles/prepare/defaults/main.yml ================================================ --- disable_service_firewall: false ================================================ FILE: contrib/os-services/roles/prepare/tasks/main.yml ================================================ --- - name: Disable firewalld and ufw when: - disable_service_firewall is defined and disable_service_firewall block: - name: List services service_facts: - name: Disable service firewalld systemd_service: name: firewalld state: stopped enabled: false when: "'firewalld.service' in services and services['firewalld.service'].status != 'not-found'" - name: Disable service ufw systemd_service: name: ufw state: stopped enabled: false when: "'ufw.service' in services and services['ufw.service'].status != 'not-found'" ================================================ FILE: contrib/terraform/aws/.gitignore ================================================ *.tfstate* .terraform.lock.hcl .terraform ================================================ FILE: contrib/terraform/aws/README.md ================================================ # Kubernetes on AWS with Terraform ## Overview This project will create: - VPC with Public and Private Subnets in # Availability Zones - Bastion Hosts and NAT Gateways in the Public Subnet - A dynamic number of masters, etcd, and worker nodes in the Private Subnet - even distributed over the # of Availability Zones - AWS ELB in the Public Subnet for accessing the Kubernetes API from the internet ## Requirements - Terraform 0.12.0 or newer ## How to Use - Export the variables for your AWS credentials or edit `credentials.tfvars`: ```commandline export TF_VAR_AWS_ACCESS_KEY_ID="www" export TF_VAR_AWS_SECRET_ACCESS_KEY ="xxx" export TF_VAR_AWS_SSH_KEY_NAME="yyy" export TF_VAR_AWS_DEFAULT_REGION="zzz" ``` - Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use Ubuntu 18.04 LTS (Bionic) as base image. If you want to change this behaviour, see note "Using other distrib than Ubuntu" below. - Create an AWS EC2 SSH Key - Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials Example: ```commandline terraform apply -var-file=credentials.tfvars ``` - Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory` - Ansible will automatically generate an ssh config file for your bastion hosts. To connect to hosts with ssh using bastion host use generated `ssh-bastion.conf`. Ansible automatically detects bastion and changes `ssh_args` ```commandline ssh -F ./ssh-bastion.conf user@$ip ``` - Once the infrastructure is created, you can run the kubespray playbooks and supply inventory/hosts with the `-i` flag. Example (this one assumes you are using Ubuntu) ```commandline ansible-playbook -i ./inventory/hosts ./cluster.yml -e ansible_user=ubuntu -b --become-user=root --flush-cache ``` ## Using other distrib than Ubuntu*** To leverage a Linux distribution other than Ubuntu 18.04 (Bionic) LTS for your Terraform configurations, you can adjust the AMI search filters within the 'data "aws_ami" "distro"' block by utilizing variables in your `terraform.tfvars` file. This approach ensures a flexible configuration that adapts to various Linux distributions without directly modifying the core Terraform files. ### Example Usages - **Debian Jessie**: To configure the usage of Debian Jessie, insert the subsequent lines into your `terraform.tfvars`: ```hcl ami_name_pattern = "debian-jessie-amd64-hvm-*" ami_owners = ["379101102735"] ``` - **Ubuntu 16.04**: To utilize Ubuntu 16.04 instead, apply the following configuration in your `terraform.tfvars`: ```hcl ami_name_pattern = "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-*" ami_owners = ["099720109477"] ``` - **Centos 7**: For employing Centos 7, incorporate these lines into your `terraform.tfvars`: ```hcl ami_name_pattern = "dcos-centos7-*" ami_owners = ["688023202711"] ``` ## Connecting to Kubernetes You can use the following set of commands to get the kubeconfig file from your newly created cluster. Before running the commands, make sure you are in the project's root folder. ```commandline # Get the controller's IP address. CONTROLLER_HOST_NAME=$(cat ./inventory/hosts | grep "\[kube_control_plane\]" -A 1 | tail -n 1) CONTROLLER_IP=$(cat ./inventory/hosts | grep $CONTROLLER_HOST_NAME | grep ansible_host | cut -d'=' -f2) # Get the hostname of the load balancer. LB_HOST=$(cat inventory/hosts | grep apiserver_loadbalancer_domain_name | cut -d'"' -f2) # Get the controller's SSH fingerprint. ssh-keygen -R $CONTROLLER_IP > /dev/null 2>&1 ssh-keyscan -H $CONTROLLER_IP >> ~/.ssh/known_hosts 2>/dev/null # Get the kubeconfig from the controller. mkdir -p ~/.kube ssh -F ssh-bastion.conf centos@$CONTROLLER_IP "sudo chmod 644 /etc/kubernetes/admin.conf" scp -F ssh-bastion.conf centos@$CONTROLLER_IP:/etc/kubernetes/admin.conf ~/.kube/config sed -i "s^server:.*^server: https://$LB_HOST:6443^" ~/.kube/config kubectl get nodes ``` ## Troubleshooting ### Remaining AWS IAM Instance Profile If the cluster was destroyed without using Terraform it is possible that the AWS IAM Instance Profiles still remain. To delete them you can use the `AWS CLI` with the following command: ```commandline aws iam delete-instance-profile --region --instance-profile-name ``` ### Ansible Inventory doesn't get created It could happen that Terraform doesn't create an Ansible Inventory file automatically. If this is the case copy the output after `inventory=` and create a file named `hosts`in the directory `inventory` and paste the inventory into the file. ## Architecture Pictured is an AWS Infrastructure created with this Terraform project distributed over two Availability Zones. ![AWS Infrastructure with Terraform ](docs/aws_kubespray.png) ================================================ FILE: contrib/terraform/aws/create-infrastructure.tf ================================================ terraform { required_version = ">= 0.12.0" required_providers { aws = { source = "hashicorp/aws" version = "~> 5.0" } } } provider "aws" { access_key = var.AWS_ACCESS_KEY_ID secret_key = var.AWS_SECRET_ACCESS_KEY region = var.AWS_DEFAULT_REGION } data "aws_availability_zones" "available" {} /* * Calling modules who create the initial AWS VPC / AWS ELB * and AWS IAM Roles for Kubernetes Deployment */ module "aws-vpc" { source = "./modules/vpc" aws_cluster_name = var.aws_cluster_name aws_vpc_cidr_block = var.aws_vpc_cidr_block aws_avail_zones = data.aws_availability_zones.available.names aws_cidr_subnets_private = var.aws_cidr_subnets_private aws_cidr_subnets_public = var.aws_cidr_subnets_public default_tags = var.default_tags } module "aws-nlb" { source = "./modules/nlb" aws_cluster_name = var.aws_cluster_name aws_vpc_id = module.aws-vpc.aws_vpc_id aws_avail_zones = data.aws_availability_zones.available.names aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public aws_nlb_api_port = var.aws_nlb_api_port k8s_secure_api_port = var.k8s_secure_api_port default_tags = var.default_tags } module "aws-iam" { source = "./modules/iam" aws_cluster_name = var.aws_cluster_name } /* * Create Bastion Instances in AWS * */ resource "aws_instance" "bastion-server" { ami = data.aws_ami.distro.id instance_type = var.aws_bastion_size count = var.aws_bastion_num associate_public_ip_address = true subnet_id = element(module.aws-vpc.aws_subnet_ids_public, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group key_name = var.AWS_SSH_KEY_NAME tags = merge(var.default_tags, tomap({ Name = "kubernetes-${var.aws_cluster_name}-bastion-${count.index}" Cluster = var.aws_cluster_name Role = "bastion-${var.aws_cluster_name}-${count.index}" })) } /* * Create K8s Master and worker nodes and etcd instances * */ resource "aws_instance" "k8s-master" { ami = data.aws_ami.distro.id instance_type = var.aws_kube_master_size count = var.aws_kube_master_num subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group root_block_device { volume_size = var.aws_kube_master_disk_size } iam_instance_profile = module.aws-iam.kube_control_plane-profile key_name = var.AWS_SSH_KEY_NAME tags = merge(var.default_tags, tomap({ Name = "kubernetes-${var.aws_cluster_name}-master${count.index}" "kubernetes.io/cluster/${var.aws_cluster_name}" = "member" Role = "master" })) } resource "aws_lb_target_group_attachment" "tg-attach_master_nodes" { count = var.aws_kube_master_num target_group_arn = module.aws-nlb.aws_nlb_api_tg_arn target_id = element(aws_instance.k8s-master.*.private_ip, count.index) } resource "aws_instance" "k8s-etcd" { ami = data.aws_ami.distro.id instance_type = var.aws_etcd_size count = var.aws_etcd_num subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group root_block_device { volume_size = var.aws_etcd_disk_size } key_name = var.AWS_SSH_KEY_NAME tags = merge(var.default_tags, tomap({ Name = "kubernetes-${var.aws_cluster_name}-etcd${count.index}" "kubernetes.io/cluster/${var.aws_cluster_name}" = "member" Role = "etcd" })) } resource "aws_instance" "k8s-worker" { ami = data.aws_ami.distro.id instance_type = var.aws_kube_worker_size count = var.aws_kube_worker_num subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group root_block_device { volume_size = var.aws_kube_worker_disk_size } iam_instance_profile = module.aws-iam.kube-worker-profile key_name = var.AWS_SSH_KEY_NAME tags = merge(var.default_tags, tomap({ Name = "kubernetes-${var.aws_cluster_name}-worker${count.index}" "kubernetes.io/cluster/${var.aws_cluster_name}" = "member" Role = "worker" })) } /* * Create Kubespray Inventory File * */ data "template_file" "inventory" { template = file("${path.module}/templates/inventory.tpl") vars = { public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip)) connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip)) connection_strings_node = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip)) list_master = join("\n", aws_instance.k8s-master.*.private_dns) list_node = join("\n", aws_instance.k8s-worker.*.private_dns) connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip)) list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns))) nlb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-nlb.aws_nlb_api_fqdn}\"" } } resource "null_resource" "inventories" { provisioner "local-exec" { command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}" } triggers = { template = data.template_file.inventory.rendered } } ================================================ FILE: contrib/terraform/aws/credentials.tfvars.example ================================================ #AWS Access Key AWS_ACCESS_KEY_ID = "" #AWS Secret Key AWS_SECRET_ACCESS_KEY = "" #EC2 SSH Key Name AWS_SSH_KEY_NAME = "" #AWS Region AWS_DEFAULT_REGION = "eu-central-1" ================================================ FILE: contrib/terraform/aws/modules/iam/main.tf ================================================ #Add AWS Roles for Kubernetes resource "aws_iam_role" "kube_control_plane" { name = "kubernetes-${var.aws_cluster_name}-master" assume_role_policy = < 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip))) } output "aws_nlb_api_fqdn" { value = "${module.aws-nlb.aws_nlb_api_fqdn}:${var.aws_nlb_api_port}" } output "inventory" { value = data.template_file.inventory.rendered } output "default_tags" { value = var.default_tags } ================================================ FILE: contrib/terraform/aws/sample-inventory/cluster.tfvars ================================================ #Global Vars aws_cluster_name = "devtest" #VPC Vars aws_vpc_cidr_block = "10.250.192.0/18" aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"] aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"] #Bastion Host aws_bastion_num = 1 aws_bastion_size = "t2.medium" #Kubernetes Cluster aws_kube_master_num = 3 aws_kube_master_size = "t2.medium" aws_kube_master_disk_size = 50 aws_etcd_num = 3 aws_etcd_size = "t2.medium" aws_etcd_disk_size = 50 aws_kube_worker_num = 4 aws_kube_worker_size = "t2.medium" aws_kube_worker_disk_size = 50 #Settings AWS NLB aws_nlb_api_port = 6443 k8s_secure_api_port = 6443 default_tags = { # Env = "devtest" # Product = "kubernetes" } inventory_file = "../../../inventory/hosts" ## Credentials #AWS Access Key AWS_ACCESS_KEY_ID = "" #AWS Secret Key AWS_SECRET_ACCESS_KEY = "" #EC2 SSH Key Name AWS_SSH_KEY_NAME = "" #AWS Region AWS_DEFAULT_REGION = "eu-central-1" ================================================ FILE: contrib/terraform/aws/templates/inventory.tpl ================================================ [all] ${connection_strings_master} ${connection_strings_node} ${connection_strings_etcd} ${public_ip_address_bastion} [bastion] ${public_ip_address_bastion} [kube_control_plane] ${list_master} [kube_node] ${list_node} [etcd] ${list_etcd} [calico_rr] [k8s_cluster:children] kube_node kube_control_plane calico_rr [k8s_cluster:vars] ${nlb_api_fqdn} ================================================ FILE: contrib/terraform/aws/terraform.tfvars ================================================ #Global Vars aws_cluster_name = "devtest" #VPC Vars aws_vpc_cidr_block = "10.250.192.0/18" aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"] aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"] # single AZ deployment #aws_cidr_subnets_private = ["10.250.192.0/20"] #aws_cidr_subnets_public = ["10.250.224.0/20"] # 3+ AZ deployment #aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"] #aws_cidr_subnets_public = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"] #Bastion Host aws_bastion_num = 1 aws_bastion_size = "t3.small" #Kubernetes Cluster aws_kube_master_num = 3 aws_kube_master_size = "t3.medium" aws_kube_master_disk_size = 50 aws_etcd_num = 0 aws_etcd_size = "t3.medium" aws_etcd_disk_size = 50 aws_kube_worker_num = 4 aws_kube_worker_size = "t3.medium" aws_kube_worker_disk_size = 50 #Settings AWS ELB aws_nlb_api_port = 6443 k8s_secure_api_port = 6443 default_tags = { # Env = "devtest" # Product = "kubernetes" } inventory_file = "../../../inventory/hosts" ================================================ FILE: contrib/terraform/aws/terraform.tfvars.example ================================================ #Global Vars aws_cluster_name = "devtest" #VPC Vars aws_vpc_cidr_block = "10.250.192.0/18" aws_cidr_subnets_private = ["10.250.192.0/20","10.250.208.0/20"] aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"] aws_avail_zones = ["eu-central-1a","eu-central-1b"] #Bastion Host aws_bastion_num = 1 aws_bastion_size = "t3.small" #Kubernetes Cluster aws_kube_master_num = 3 aws_kube_master_size = "t3.medium" aws_kube_master_disk_size = 50 aws_etcd_num = 3 aws_etcd_size = "t3.medium" aws_etcd_disk_size = 50 aws_kube_worker_num = 4 aws_kube_worker_size = "t3.medium" aws_kube_worker_disk_size = 50 #Settings AWS ELB aws_nlb_api_port = 6443 k8s_secure_api_port = 6443 default_tags = { } inventory_file = "../../../inventory/hosts" ================================================ FILE: contrib/terraform/aws/variables.tf ================================================ variable "AWS_ACCESS_KEY_ID" { description = "AWS Access Key" } variable "AWS_SECRET_ACCESS_KEY" { description = "AWS Secret Key" } variable "AWS_SSH_KEY_NAME" { description = "Name of the SSH keypair to use in AWS." } variable "AWS_DEFAULT_REGION" { description = "AWS Region" } //General Cluster Settings variable "aws_cluster_name" { description = "Name of AWS Cluster" } variable "ami_name_pattern" { description = "The name pattern to use for AMI lookup" type = string default = "debian-10-amd64-*" } variable "ami_virtualization_type" { description = "The virtualization type to use for AMI lookup" type = string default = "hvm" } variable "ami_owners" { description = "The owners to use for AMI lookup" type = list(string) default = ["136693071363"] } data "aws_ami" "distro" { most_recent = true filter { name = "name" values = [var.ami_name_pattern] } filter { name = "virtualization-type" values = [var.ami_virtualization_type] } owners = var.ami_owners } //AWS VPC Variables variable "aws_vpc_cidr_block" { description = "CIDR Block for VPC" } variable "aws_cidr_subnets_private" { description = "CIDR Blocks for private subnets in Availability Zones" type = list(string) } variable "aws_cidr_subnets_public" { description = "CIDR Blocks for public subnets in Availability Zones" type = list(string) } //AWS EC2 Settings variable "aws_bastion_size" { description = "EC2 Instance Size of Bastion Host" } /* * AWS EC2 Settings * The number should be divisable by the number of used * AWS Availability Zones without an remainder. */ variable "aws_bastion_num" { description = "Number of Bastion Nodes" } variable "aws_kube_master_num" { description = "Number of Kubernetes Master Nodes" } variable "aws_kube_master_disk_size" { description = "Disk size for Kubernetes Master Nodes (in GiB)" } variable "aws_kube_master_size" { description = "Instance size of Kube Master Nodes" } variable "aws_etcd_num" { description = "Number of etcd Nodes" } variable "aws_etcd_disk_size" { description = "Disk size for etcd Nodes (in GiB)" } variable "aws_etcd_size" { description = "Instance size of etcd Nodes" } variable "aws_kube_worker_num" { description = "Number of Kubernetes Worker Nodes" } variable "aws_kube_worker_disk_size" { description = "Disk size for Kubernetes Worker Nodes (in GiB)" } variable "aws_kube_worker_size" { description = "Instance size of Kubernetes Worker Nodes" } /* * AWS NLB Settings * */ variable "aws_nlb_api_port" { description = "Port for AWS NLB" } variable "k8s_secure_api_port" { description = "Secure Port of K8S API Server" } variable "default_tags" { description = "Default tags for all resources" type = map(string) } variable "inventory_file" { description = "Where to store the generated inventory file" } ================================================ FILE: contrib/terraform/exoscale/README.md ================================================ # Kubernetes on Exoscale with Terraform Provision a Kubernetes cluster on [Exoscale](https://www.exoscale.com/) using Terraform and Kubespray ## Overview The setup looks like following ```text Kubernetes cluster +-----------------------+ +---------------+ | +--------------+ | | | | | +--------------+ | | API server LB +---------> | | | | | | | | | Master/etcd | | +---------------+ | | | node(s) | | | +-+ | | | +--------------+ | | ^ | | | | | v | +---------------+ | +--------------+ | | | | | +--------------+ | | Ingress LB +---------> | | | | | | | | | Worker | | +---------------+ | | | node(s) | | | +-+ | | | +--------------+ | +-----------------------+ ``` ## Requirements * Terraform 0.13.0 or newer (0.12 also works if you modify the provider block to include version and remove all `versions.tf` files) ## Quickstart NOTE: *Assumes you are at the root of the kubespray repo* Copy the sample inventory for your cluster and copy the default terraform variables. ```bash CLUSTER=my-exoscale-cluster cp -r inventory/sample inventory/$CLUSTER cp contrib/terraform/exoscale/default.tfvars inventory/$CLUSTER/ cd inventory/$CLUSTER ``` Edit `default.tfvars` to match your setup. You MUST, at the very least, change `ssh_public_keys`. ```bash # Ensure $EDITOR points to your favorite editor, e.g., vim, emacs, VS Code, etc. $EDITOR default.tfvars ``` For authentication you can use the credentials file `~/.cloudstack.ini` or `./cloudstack.ini`. The file should look like something like this: ```ini [cloudstack] key = secret = ``` Follow the [Exoscale IAM Quick-start](https://community.exoscale.com/documentation/iam/quick-start/) to learn how to generate API keys. ### Encrypted credentials To have the credentials encrypted at rest, you can use [sops](https://github.com/mozilla/sops) and only decrypt the credentials at runtime. ```bash cat << EOF > cloudstack.ini [cloudstack] key = secret = EOF sops --encrypt --in-place --pgp cloudstack.ini sops cloudstack.ini ``` Run terraform to create the infrastructure ```bash terraform init ../../contrib/terraform/exoscale terraform apply -var-file default.tfvars ../../contrib/terraform/exoscale ``` If your cloudstack credentials file is encrypted using sops, run the following: ```bash terraform init ../../contrib/terraform/exoscale sops exec-file -no-fifo cloudstack.ini 'CLOUDSTACK_CONFIG={} terraform apply -var-file default.tfvars ../../contrib/terraform/exoscale' ``` You should now have a inventory file named `inventory.ini` that you can use with kubespray. You can now copy your inventory file and use it with kubespray to set up a cluster. You can type `terraform output` to find out the IP addresses of the nodes, as well as control-plane and data-plane load-balancer. It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by: ```bash ansible -i inventory.ini -m ping all ``` Example to use this with the default sample inventory: ```bash ansible-playbook -i inventory.ini ../../cluster.yml -b -v ``` ## Teardown The Kubernetes cluster cannot create any load-balancers or disks, hence, teardown is as simple as Terraform destroy: ```bash terraform destroy -var-file default.tfvars ../../contrib/terraform/exoscale ``` ## Variables ### Required * `ssh_public_keys`: List of public SSH keys to install on all machines * `zone`: The zone where to run the cluster * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* * `size`: The size to use * `boot_disk`: The boot disk to use * `image_name`: Name of the image * `root_partition_size`: Size *(in GB)* for the root partition * `ceph_partition_size`: Size *(in GB)* for the partition for rook to use as ceph storage. *(Set to 0 to disable)* * `node_local_partition_size`: Size *(in GB)* for the partition for node-local-storage. *(Set to 0 to disable)* * `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes * `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server * `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports) ### Optional * `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)* An example variables file can be found `default.tfvars` ## Known limitations ### Only single disk Since Exoscale doesn't support additional disks to be mounted onto an instance, this script has the ability to create partitions for [Rook](https://rook.io/) and [node-local-storage](https://kubernetes.io/docs/concepts/storage/volumes/#local). ### No Kubernetes API The current solution doesn't use the [Exoscale Kubernetes cloud controller](https://github.com/exoscale/exoscale-cloud-controller-manager). This means that we need to set up a HTTP(S) loadbalancer in front of all workers and set the Ingress controller to DaemonSet mode. ================================================ FILE: contrib/terraform/exoscale/default.tfvars ================================================ prefix = "default" zone = "ch-gva-2" inventory_file = "inventory.ini" ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] machines = { "master-0" : { "node_type" : "master", "size" : "standard.medium", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } }, "worker-0" : { "node_type" : "worker", "size" : "standard.large", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } }, "worker-1" : { "node_type" : "worker", "size" : "standard.large", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } }, "worker-2" : { "node_type" : "worker", "size" : "standard.large", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } } } nodeport_whitelist = [ "0.0.0.0/0" ] ssh_whitelist = [ "0.0.0.0/0" ] api_server_whitelist = [ "0.0.0.0/0" ] ================================================ FILE: contrib/terraform/exoscale/main.tf ================================================ provider "exoscale" {} module "kubernetes" { source = "./modules/kubernetes-cluster" prefix = var.prefix zone = var.zone machines = var.machines ssh_public_keys = var.ssh_public_keys ssh_whitelist = var.ssh_whitelist api_server_whitelist = var.api_server_whitelist nodeport_whitelist = var.nodeport_whitelist } # # Generate ansible inventory # data "template_file" "inventory" { template = file("${path.module}/templates/inventory.tpl") vars = { connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d", keys(module.kubernetes.master_ip_addresses), values(module.kubernetes.master_ip_addresses).*.public_ip, values(module.kubernetes.master_ip_addresses).*.private_ip, range(1, length(module.kubernetes.master_ip_addresses) + 1))) connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s", keys(module.kubernetes.worker_ip_addresses), values(module.kubernetes.worker_ip_addresses).*.public_ip, values(module.kubernetes.worker_ip_addresses).*.private_ip)) list_master = join("\n", keys(module.kubernetes.master_ip_addresses)) list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses)) api_lb_ip_address = module.kubernetes.control_plane_lb_ip_address } } resource "null_resource" "inventories" { provisioner "local-exec" { command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}" } triggers = { template = data.template_file.inventory.rendered } } ================================================ FILE: contrib/terraform/exoscale/modules/kubernetes-cluster/main.tf ================================================ data "exoscale_template" "os_image" { for_each = var.machines zone = var.zone name = each.value.boot_disk.image_name } data "exoscale_compute_instance" "master_nodes" { for_each = exoscale_compute_instance.master id = each.value.id zone = var.zone } data "exoscale_compute_instance" "worker_nodes" { for_each = exoscale_compute_instance.worker id = each.value.id zone = var.zone } resource "exoscale_private_network" "private_network" { zone = var.zone name = "${var.prefix}-network" start_ip = cidrhost(var.private_network_cidr, 1) # cidr -1 = Broadcast address # cidr -2 = DHCP server address (exoscale specific) end_ip = cidrhost(var.private_network_cidr, -3) netmask = cidrnetmask(var.private_network_cidr) } resource "exoscale_compute_instance" "master" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "master" } name = "${var.prefix}-${each.key}" template_id = data.exoscale_template.os_image[each.key].id type = each.value.size disk_size = each.value.boot_disk.root_partition_size + each.value.boot_disk.node_local_partition_size + each.value.boot_disk.ceph_partition_size state = "Running" zone = var.zone security_group_ids = [exoscale_security_group.master_sg.id] network_interface { network_id = exoscale_private_network.private_network.id } elastic_ip_ids = [exoscale_elastic_ip.control_plane_lb.id] user_data = templatefile( "${path.module}/templates/cloud-init.tmpl", { eip_ip_address = exoscale_elastic_ip.ingress_controller_lb.ip_address node_local_partition_size = each.value.boot_disk.node_local_partition_size ceph_partition_size = each.value.boot_disk.ceph_partition_size root_partition_size = each.value.boot_disk.root_partition_size node_type = "master" ssh_public_keys = var.ssh_public_keys } ) } resource "exoscale_compute_instance" "worker" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "worker" } name = "${var.prefix}-${each.key}" template_id = data.exoscale_template.os_image[each.key].id type = each.value.size disk_size = each.value.boot_disk.root_partition_size + each.value.boot_disk.node_local_partition_size + each.value.boot_disk.ceph_partition_size state = "Running" zone = var.zone security_group_ids = [exoscale_security_group.worker_sg.id] network_interface { network_id = exoscale_private_network.private_network.id } elastic_ip_ids = [exoscale_elastic_ip.ingress_controller_lb.id] user_data = templatefile( "${path.module}/templates/cloud-init.tmpl", { eip_ip_address = exoscale_elastic_ip.ingress_controller_lb.ip_address node_local_partition_size = each.value.boot_disk.node_local_partition_size ceph_partition_size = each.value.boot_disk.ceph_partition_size root_partition_size = each.value.boot_disk.root_partition_size node_type = "worker" ssh_public_keys = var.ssh_public_keys } ) } resource "exoscale_security_group" "master_sg" { name = "${var.prefix}-master-sg" description = "Security group for Kubernetes masters" } resource "exoscale_security_group_rule" "master_sg_rule_ssh" { security_group_id = exoscale_security_group.master_sg.id for_each = toset(var.ssh_whitelist) # SSH type = "INGRESS" start_port = 22 end_port = 22 protocol = "TCP" cidr = each.value } resource "exoscale_security_group_rule" "master_sg_rule_k8s_api" { security_group_id = exoscale_security_group.master_sg.id for_each = toset(var.api_server_whitelist) # Kubernetes API type = "INGRESS" start_port = 6443 end_port = 6443 protocol = "TCP" cidr = each.value } resource "exoscale_security_group" "worker_sg" { name = "${var.prefix}-worker-sg" description = "security group for kubernetes worker nodes" } resource "exoscale_security_group_rule" "worker_sg_rule_ssh" { security_group_id = exoscale_security_group.worker_sg.id # SSH for_each = toset(var.ssh_whitelist) type = "INGRESS" start_port = 22 end_port = 22 protocol = "TCP" cidr = each.value } resource "exoscale_security_group_rule" "worker_sg_rule_http" { security_group_id = exoscale_security_group.worker_sg.id # HTTP(S) for_each = toset(["80", "443"]) type = "INGRESS" start_port = each.value end_port = each.value protocol = "TCP" cidr = "0.0.0.0/0" } resource "exoscale_security_group_rule" "worker_sg_rule_nodeport" { security_group_id = exoscale_security_group.worker_sg.id # HTTP(S) for_each = toset(var.nodeport_whitelist) type = "INGRESS" start_port = 30000 end_port = 32767 protocol = "TCP" cidr = each.value } resource "exoscale_elastic_ip" "ingress_controller_lb" { zone = var.zone healthcheck { mode = "http" port = 80 uri = "/healthz" interval = 10 timeout = 2 strikes_ok = 2 strikes_fail = 3 } } resource "exoscale_elastic_ip" "control_plane_lb" { zone = var.zone healthcheck { mode = "tcp" port = 6443 interval = 10 timeout = 2 strikes_ok = 2 strikes_fail = 3 } } ================================================ FILE: contrib/terraform/exoscale/modules/kubernetes-cluster/output.tf ================================================ output "master_ip_addresses" { value = { for key, instance in exoscale_compute_instance.master : instance.name => { "private_ip" = contains(keys(data.exoscale_compute_instance.master_nodes), key) ? data.exoscale_compute_instance.master_nodes[key].private_network_ip_addresses[0] : "" "public_ip" = exoscale_compute_instance.master[key].ip_address } } } output "worker_ip_addresses" { value = { for key, instance in exoscale_compute_instance.worker : instance.name => { "private_ip" = contains(keys(data.exoscale_compute_instance.worker_nodes), key) ? data.exoscale_compute_instance.worker_nodes[key].private_network_ip_addresses[0] : "" "public_ip" = exoscale_compute_instance.worker[key].ip_address } } } output "cluster_private_network_cidr" { value = var.private_network_cidr } output "ingress_controller_lb_ip_address" { value = exoscale_elastic_ip.ingress_controller_lb.ip_address } output "control_plane_lb_ip_address" { value = exoscale_elastic_ip.control_plane_lb.ip_address } ================================================ FILE: contrib/terraform/exoscale/modules/kubernetes-cluster/templates/cloud-init.tmpl ================================================ #cloud-config %{ if ceph_partition_size > 0 || node_local_partition_size > 0} bootcmd: - [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, /dev/vda ] %{ if node_local_partition_size > 0 } # Create partition for node local storage - [ cloud-init-per, once, create-node-local-part, parted, --script, /dev/vda, 'mkpart extended ext4 ${root_partition_size}GB %{ if ceph_partition_size == 0 }-1%{ else }${root_partition_size + node_local_partition_size}GB%{ endif }' ] - [ cloud-init-per, once, create-fs-node-local-part, mkfs.ext4, /dev/vda2 ] %{ endif } %{ if ceph_partition_size > 0 } # Create partition for rook to use for ceph - [ cloud-init-per, once, create-ceph-part, parted, --script, /dev/vda, 'mkpart extended ${root_partition_size + node_local_partition_size}GB -1' ] %{ endif } %{ endif } ssh_authorized_keys: %{ for ssh_public_key in ssh_public_keys ~} - ${ssh_public_key} %{ endfor ~} write_files: - path: /etc/netplan/eth1.yaml content: | network: version: 2 ethernets: eth1: dhcp4: true %{ if node_type == "worker" } # TODO: When a VM is seen as healthy and is added to the EIP loadbalancer # pool it no longer can send traffic back to itself via the EIP IP # address. # Remove this if it ever gets solved. - path: /etc/netplan/20-eip-fix.yaml content: | network: version: 2 ethernets: "lo:0": match: name: lo dhcp4: false addresses: - ${eip_ip_address}/32 %{ endif } runcmd: - netplan apply %{ if node_local_partition_size > 0 } - mkdir -p /mnt/disks/node-local-storage - chown nobody:nogroup /mnt/disks/node-local-storage - mount /dev/vda2 /mnt/disks/node-local-storage %{ endif } ================================================ FILE: contrib/terraform/exoscale/modules/kubernetes-cluster/variables.tf ================================================ variable "zone" { type = string # This is currently the only zone that is supposed to be supporting # so called "managed private networks". # See: https://www.exoscale.com/syslog/introducing-managed-private-networks default = "ch-gva-2" } variable "prefix" {} variable "machines" { type = map(object({ node_type = string size = string boot_disk = object({ image_name = string root_partition_size = number ceph_partition_size = number node_local_partition_size = number }) })) } variable "ssh_public_keys" { type = list(string) } variable "ssh_whitelist" { type = list(string) } variable "api_server_whitelist" { type = list(string) } variable "nodeport_whitelist" { type = list(string) } variable "private_network_cidr" { default = "172.0.10.0/24" } ================================================ FILE: contrib/terraform/exoscale/modules/kubernetes-cluster/versions.tf ================================================ terraform { required_providers { exoscale = { source = "exoscale/exoscale" version = ">= 0.21" } } required_version = ">= 0.13" } ================================================ FILE: contrib/terraform/exoscale/output.tf ================================================ output "master_ips" { value = module.kubernetes.master_ip_addresses } output "worker_ips" { value = module.kubernetes.worker_ip_addresses } output "ingress_controller_lb_ip_address" { value = module.kubernetes.ingress_controller_lb_ip_address } output "control_plane_lb_ip_address" { value = module.kubernetes.control_plane_lb_ip_address } ================================================ FILE: contrib/terraform/exoscale/sample-inventory/cluster.tfvars ================================================ prefix = "default" zone = "ch-gva-2" inventory_file = "inventory.ini" ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] machines = { "master-0" : { "node_type" : "master", "size" : "Small", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } }, "worker-0" : { "node_type" : "worker", "size" : "Large", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } }, "worker-1" : { "node_type" : "worker", "size" : "Large", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } }, "worker-2" : { "node_type" : "worker", "size" : "Large", "boot_disk" : { "image_name" : "Linux Ubuntu 20.04 LTS 64-bit", "root_partition_size" : 50, "node_local_partition_size" : 0, "ceph_partition_size" : 0 } } } nodeport_whitelist = [ "0.0.0.0/0" ] ssh_whitelist = [ "0.0.0.0/0" ] api_server_whitelist = [ "0.0.0.0/0" ] ================================================ FILE: contrib/terraform/exoscale/templates/inventory.tpl ================================================ [all] ${connection_strings_master} ${connection_strings_worker} [kube_control_plane] ${list_master} [kube_control_plane:vars] supplementary_addresses_in_ssl_keys = [ "${api_lb_ip_address}" ] [etcd] ${list_master} [kube_node] ${list_worker} [k8s_cluster:children] kube_control_plane kube_node ================================================ FILE: contrib/terraform/exoscale/variables.tf ================================================ variable "zone" { description = "The zone where to run the cluster" } variable "prefix" { description = "Prefix for resource names" default = "default" } variable "machines" { description = "Cluster machines" type = map(object({ node_type = string size = string boot_disk = object({ image_name = string root_partition_size = number ceph_partition_size = number node_local_partition_size = number }) })) } variable "ssh_public_keys" { description = "List of public SSH keys which are injected into the VMs." type = list(string) } variable "ssh_whitelist" { description = "List of IP ranges (CIDR) to whitelist for ssh" type = list(string) } variable "api_server_whitelist" { description = "List of IP ranges (CIDR) to whitelist for kubernetes api server" type = list(string) } variable "nodeport_whitelist" { description = "List of IP ranges (CIDR) to whitelist for kubernetes nodeports" type = list(string) } variable "inventory_file" { description = "Where to store the generated inventory file" } ================================================ FILE: contrib/terraform/exoscale/versions.tf ================================================ terraform { required_providers { exoscale = { source = "exoscale/exoscale" version = ">= 0.21" } null = { source = "hashicorp/null" } template = { source = "hashicorp/template" } } required_version = ">= 0.13" } ================================================ FILE: contrib/terraform/gcp/README.md ================================================ # Kubernetes on GCP with Terraform Provision a Kubernetes cluster on GCP using Terraform and Kubespray ## Overview The setup looks like following ```text Kubernetes cluster +-----------------------+ +---------------+ | +--------------+ | | | | | +--------------+ | | API server LB +---------> | | | | | | | | | Master/etcd | | +---------------+ | | | node(s) | | | +-+ | | | +--------------+ | | ^ | | | | | v | +---------------+ | +--------------+ | | | | | +--------------+ | | Ingress LB +---------> | | | | | | | | | Worker | | +---------------+ | | | node(s) | | | +-+ | | | +--------------+ | +-----------------------+ ``` ## Requirements * Terraform 0.12.0 or newer ## Quickstart To get a cluster up and running you'll need a JSON keyfile. Set the path to the file in the `tfvars.json` file and run the following: ```bash terraform apply -var-file tfvars.json -state dev-cluster.tfstate -var gcp_project_id= -var keyfile_location= ``` To generate kubespray inventory based on the terraform state file you can run the following: ```bash ./generate-inventory.sh dev-cluster.tfstate > inventory.ini ``` You should now have a inventory file named `inventory.ini` that you can use with kubespray, e.g. ```bash ansible-playbook -i contrib/terraform/gcp/inventory.ini cluster.yml -b -v ``` ## Variables ### Required * `keyfile_location`: Location to the keyfile to use as credentials for the google terraform provider * `gcp_project_id`: ID of the GCP project to deploy the cluster in * `ssh_pub_key`: Path to public ssh key to use for all machines * `region`: The region where to run the cluster * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* * `size`: The size to use * `zone`: The zone the machine should run in * `additional_disks`: Extra disks to add to the machine. Key of this object will be used as the disk name * `size`: Size of the disk (in GB) * `boot_disk`: The boot disk to use * `image_name`: Name of the image * `size`: Size of the boot disk (in GB) * `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes * `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server * `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports) * `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to ingress on ports 80 and 443 * `extra_ingress_firewalls`: Additional ingress firewall rules. Key will be used as the name of the rule * `source_ranges`: List of IP ranges (CIDR). Example: `["8.8.8.8"]` * `protocol`: Protocol. Example `"tcp"` * `ports`: List of ports, as string. Example `["53"]` * `target_tags`: List of target tag (either the machine name or `control-plane` or `worker`). Example: `["control-plane", "worker-0"]` ### Optional * `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project *(Defaults to `default`)* * `master_sa_email`: Service account email to use for the control plane nodes *(Defaults to `""`, auto generate one)* * `master_sa_scopes`: Service account email to use for the control plane nodes *(Defaults to `["https://www.googleapis.com/auth/cloud-platform"]`)* * `master_preemptible`: Enable [preemptible](https://cloud.google.com/compute/docs/instances/preemptible) for the control plane nodes *(Defaults to `false`)* * `master_additional_disk_type`: [Disk type](https://cloud.google.com/compute/docs/disks/#disk-types) for extra disks added on the control plane nodes *(Defaults to `"pd-ssd"`)* * `worker_sa_email`: Service account email to use for the worker nodes *(Defaults to `""`, auto generate one)* * `worker_sa_scopes`: Service account email to use for the worker nodes *(Defaults to `["https://www.googleapis.com/auth/cloud-platform"]`)* * `worker_preemptible`: Enable [preemptible](https://cloud.google.com/compute/docs/instances/preemptible) for the worker nodes *(Defaults to `false`)* * `worker_additional_disk_type`: [Disk type](https://cloud.google.com/compute/docs/disks/#disk-types) for extra disks added on the worker nodes *(Defaults to `"pd-ssd"`)* An example variables file can be found `tfvars.json` ## Known limitations This solution does not provide a solution to use a bastion host. Thus all the nodes must expose a public IP for kubespray to work. ================================================ FILE: contrib/terraform/gcp/generate-inventory.sh ================================================ #!/bin/bash # # Generates a inventory file based on the terraform output. # After provisioning a cluster, simply run this command and supply the terraform state file # Default state file is terraform.tfstate # set -e usage () { echo "Usage: $0 " >&2 exit 1 } if [[ $# -ne 1 ]]; then usage fi TF_STATE_FILE=${1} if [[ ! -f "${TF_STATE_FILE}" ]]; then echo "ERROR: state file ${TF_STATE_FILE} doesn't exist" >&2 usage fi TF_OUT=$(terraform output -state "${TF_STATE_FILE}" -json) MASTERS=$(jq -r '.master_ips.value | to_entries[]' <(echo "${TF_OUT}")) WORKERS=$(jq -r '.worker_ips.value | to_entries[]' <(echo "${TF_OUT}")) mapfile -t MASTER_NAMES < <(jq -r '.key' <(echo "${MASTERS}")) mapfile -t WORKER_NAMES < <(jq -r '.key' <(echo "${WORKERS}")) API_LB=$(jq -r '.control_plane_lb_ip_address.value' <(echo "${TF_OUT}")) # Generate master hosts i=1 for name in "${MASTER_NAMES[@]}"; do private_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.private_ip' <(echo "${MASTERS}")) public_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.public_ip' <(echo "${MASTERS}")) echo "${name} ansible_user=ubuntu ansible_host=${public_ip} ip=${private_ip} etcd_member_name=etcd${i}" i=$(( i + 1 )) done # Generate worker hosts for name in "${WORKER_NAMES[@]}"; do private_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.private_ip' <(echo "${WORKERS}")) public_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.public_ip' <(echo "${WORKERS}")) echo "${name} ansible_user=ubuntu ansible_host=${public_ip} ip=${private_ip}" done echo "" echo "[kube_control_plane]" for name in "${MASTER_NAMES[@]}"; do echo "${name}" done echo "" echo "[kube_control_plane:vars]" echo "supplementary_addresses_in_ssl_keys = [ '${API_LB}' ]" # Add LB address to API server certificate echo "" echo "[etcd]" for name in "${MASTER_NAMES[@]}"; do echo "${name}" done echo "" echo "[kube_node]" for name in "${WORKER_NAMES[@]}"; do echo "${name}" done echo "" echo "[k8s_cluster:children]" echo "kube_control_plane" echo "kube_node" ================================================ FILE: contrib/terraform/gcp/main.tf ================================================ terraform { required_providers { google = { source = "hashicorp/google" version = "~> 4.0" } } } provider "google" { credentials = file(var.keyfile_location) region = var.region project = var.gcp_project_id } module "kubernetes" { source = "./modules/kubernetes-cluster" region = var.region prefix = var.prefix machines = var.machines ssh_pub_key = var.ssh_pub_key master_sa_email = var.master_sa_email master_sa_scopes = var.master_sa_scopes master_preemptible = var.master_preemptible master_additional_disk_type = var.master_additional_disk_type worker_sa_email = var.worker_sa_email worker_sa_scopes = var.worker_sa_scopes worker_preemptible = var.worker_preemptible worker_additional_disk_type = var.worker_additional_disk_type ssh_whitelist = var.ssh_whitelist api_server_whitelist = var.api_server_whitelist nodeport_whitelist = var.nodeport_whitelist ingress_whitelist = var.ingress_whitelist extra_ingress_firewalls = var.extra_ingress_firewalls } ================================================ FILE: contrib/terraform/gcp/modules/kubernetes-cluster/main.tf ================================================ ################################################# ## ## General ## resource "google_compute_network" "main" { name = "${var.prefix}-network" auto_create_subnetworks = false } resource "google_compute_subnetwork" "main" { name = "${var.prefix}-subnet" network = google_compute_network.main.name ip_cidr_range = var.private_network_cidr region = var.region } resource "google_compute_firewall" "deny_all" { name = "${var.prefix}-default-firewall" network = google_compute_network.main.name priority = 1000 source_ranges = ["0.0.0.0/0"] deny { protocol = "all" } } resource "google_compute_firewall" "allow_internal" { name = "${var.prefix}-internal-firewall" network = google_compute_network.main.name priority = 500 source_ranges = [var.private_network_cidr] allow { protocol = "all" } } resource "google_compute_firewall" "ssh" { count = length(var.ssh_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-ssh-firewall" network = google_compute_network.main.name priority = 100 source_ranges = var.ssh_whitelist allow { protocol = "tcp" ports = ["22"] } } resource "google_compute_firewall" "api_server" { count = length(var.api_server_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-api-server-firewall" network = google_compute_network.main.name priority = 100 source_ranges = var.api_server_whitelist allow { protocol = "tcp" ports = ["6443"] } } resource "google_compute_firewall" "nodeport" { count = length(var.nodeport_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-nodeport-firewall" network = google_compute_network.main.name priority = 100 source_ranges = var.nodeport_whitelist allow { protocol = "tcp" ports = ["30000-32767"] } } resource "google_compute_firewall" "ingress_http" { count = length(var.ingress_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-http-ingress-firewall" network = google_compute_network.main.name priority = 100 source_ranges = var.ingress_whitelist allow { protocol = "tcp" ports = ["80"] } } resource "google_compute_firewall" "ingress_https" { count = length(var.ingress_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-https-ingress-firewall" network = google_compute_network.main.name priority = 100 source_ranges = var.ingress_whitelist allow { protocol = "tcp" ports = ["443"] } } ################################################# ## ## Local variables ## locals { master_target_list = [ for name, machine in google_compute_instance.master : "${machine.zone}/${machine.name}" ] worker_target_list = [ for name, machine in google_compute_instance.worker : "${machine.zone}/${machine.name}" ] master_disks = flatten([ for machine_name, machine in var.machines : [ for disk_name, disk in machine.additional_disks : { "${machine_name}-${disk_name}" = { "machine_name": machine_name, "machine": machine, "disk_size": disk.size, "disk_name": disk_name } } ] if machine.node_type == "master" ]) worker_disks = flatten([ for machine_name, machine in var.machines : [ for disk_name, disk in machine.additional_disks : { "${machine_name}-${disk_name}" = { "machine_name": machine_name, "machine": machine, "disk_size": disk.size, "disk_name": disk_name } } ] if machine.node_type == "worker" ]) } ################################################# ## ## Master ## resource "google_compute_address" "master" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "master" } name = "${var.prefix}-${each.key}-pip" address_type = "EXTERNAL" region = var.region } resource "google_compute_disk" "master" { for_each = { for item in local.master_disks : keys(item)[0] => values(item)[0] } name = "${var.prefix}-${each.key}" type = var.master_additional_disk_type zone = each.value.machine.zone size = each.value.disk_size physical_block_size_bytes = 4096 } resource "google_compute_attached_disk" "master" { for_each = { for item in local.master_disks : keys(item)[0] => values(item)[0] } disk = google_compute_disk.master[each.key].id instance = google_compute_instance.master[each.value.machine_name].id } resource "google_compute_instance" "master" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "master" } name = "${var.prefix}-${each.key}" machine_type = each.value.size zone = each.value.zone tags = ["control-plane", "master", each.key] boot_disk { initialize_params { image = each.value.boot_disk.image_name size = each.value.boot_disk.size } } network_interface { subnetwork = google_compute_subnetwork.main.name access_config { nat_ip = google_compute_address.master[each.key].address } } metadata = { ssh-keys = "ubuntu:${trimspace(file(pathexpand(var.ssh_pub_key)))}" } service_account { email = var.master_sa_email scopes = var.master_sa_scopes } # Since we use google_compute_attached_disk we need to ignore this lifecycle { ignore_changes = [attached_disk] } scheduling { preemptible = var.master_preemptible automatic_restart = !var.master_preemptible } } resource "google_compute_forwarding_rule" "master_lb" { count = length(var.api_server_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-master-lb-forward-rule" port_range = "6443" target = google_compute_target_pool.master_lb[count.index].id } resource "google_compute_target_pool" "master_lb" { count = length(var.api_server_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-master-lb-pool" instances = local.master_target_list } ################################################# ## ## Worker ## resource "google_compute_disk" "worker" { for_each = { for item in local.worker_disks : keys(item)[0] => values(item)[0] } name = "${var.prefix}-${each.key}" type = var.worker_additional_disk_type zone = each.value.machine.zone size = each.value.disk_size physical_block_size_bytes = 4096 } resource "google_compute_attached_disk" "worker" { for_each = { for item in local.worker_disks : keys(item)[0] => values(item)[0] } disk = google_compute_disk.worker[each.key].id instance = google_compute_instance.worker[each.value.machine_name].id } resource "google_compute_address" "worker" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "worker" } name = "${var.prefix}-${each.key}-pip" address_type = "EXTERNAL" region = var.region } resource "google_compute_instance" "worker" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "worker" } name = "${var.prefix}-${each.key}" machine_type = each.value.size zone = each.value.zone tags = ["worker", each.key] boot_disk { initialize_params { image = each.value.boot_disk.image_name size = each.value.boot_disk.size } } network_interface { subnetwork = google_compute_subnetwork.main.name access_config { nat_ip = google_compute_address.worker[each.key].address } } metadata = { ssh-keys = "ubuntu:${trimspace(file(pathexpand(var.ssh_pub_key)))}" } service_account { email = var.worker_sa_email scopes = var.worker_sa_scopes } # Since we use google_compute_attached_disk we need to ignore this lifecycle { ignore_changes = [attached_disk] } scheduling { preemptible = var.worker_preemptible automatic_restart = !var.worker_preemptible } } resource "google_compute_address" "worker_lb" { count = length(var.ingress_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-worker-lb-address" address_type = "EXTERNAL" region = var.region } resource "google_compute_forwarding_rule" "worker_http_lb" { count = length(var.ingress_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-worker-http-lb-forward-rule" ip_address = google_compute_address.worker_lb[count.index].address port_range = "80" target = google_compute_target_pool.worker_lb[count.index].id } resource "google_compute_forwarding_rule" "worker_https_lb" { count = length(var.ingress_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-worker-https-lb-forward-rule" ip_address = google_compute_address.worker_lb[count.index].address port_range = "443" target = google_compute_target_pool.worker_lb[count.index].id } resource "google_compute_target_pool" "worker_lb" { count = length(var.ingress_whitelist) > 0 ? 1 : 0 name = "${var.prefix}-worker-lb-pool" instances = local.worker_target_list } resource "google_compute_firewall" "extra_ingress_firewall" { for_each = { for name, firewall in var.extra_ingress_firewalls : name => firewall } name = "${var.prefix}-${each.key}-ingress" network = google_compute_network.main.name priority = 100 source_ranges = each.value.source_ranges target_tags = each.value.target_tags allow { protocol = each.value.protocol ports = each.value.ports } } ================================================ FILE: contrib/terraform/gcp/modules/kubernetes-cluster/output.tf ================================================ output "master_ip_addresses" { value = { for key, instance in google_compute_instance.master : instance.name => { "private_ip" = instance.network_interface.0.network_ip "public_ip" = instance.network_interface.0.access_config.0.nat_ip } } } output "worker_ip_addresses" { value = { for key, instance in google_compute_instance.worker : instance.name => { "private_ip" = instance.network_interface.0.network_ip "public_ip" = instance.network_interface.0.access_config.0.nat_ip } } } output "ingress_controller_lb_ip_address" { value = length(var.ingress_whitelist) > 0 ? google_compute_address.worker_lb.0.address : "" } output "control_plane_lb_ip_address" { value = length(var.api_server_whitelist) > 0 ? google_compute_forwarding_rule.master_lb.0.ip_address : "" } ================================================ FILE: contrib/terraform/gcp/modules/kubernetes-cluster/variables.tf ================================================ variable "region" { type = string } variable "prefix" {} variable "machines" { type = map(object({ node_type = string size = string zone = string additional_disks = map(object({ size = number })) boot_disk = object({ image_name = string size = number }) })) } variable "master_sa_email" { type = string } variable "master_sa_scopes" { type = list(string) } variable "master_preemptible" { type = bool } variable "master_additional_disk_type" { type = string } variable "worker_sa_email" { type = string } variable "worker_sa_scopes" { type = list(string) } variable "worker_preemptible" { type = bool } variable "worker_additional_disk_type" { type = string } variable "ssh_pub_key" {} variable "ssh_whitelist" { type = list(string) } variable "api_server_whitelist" { type = list(string) } variable "nodeport_whitelist" { type = list(string) } variable "ingress_whitelist" { type = list(string) default = ["0.0.0.0/0"] } variable "private_network_cidr" { default = "10.0.10.0/24" } variable "extra_ingress_firewalls" { type = map(object({ source_ranges = set(string) protocol = string ports = list(string) target_tags = set(string) })) default = {} } ================================================ FILE: contrib/terraform/gcp/output.tf ================================================ output "master_ips" { value = module.kubernetes.master_ip_addresses } output "worker_ips" { value = module.kubernetes.worker_ip_addresses } output "ingress_controller_lb_ip_address" { value = module.kubernetes.ingress_controller_lb_ip_address } output "control_plane_lb_ip_address" { value = module.kubernetes.control_plane_lb_ip_address } ================================================ FILE: contrib/terraform/gcp/tfvars.json ================================================ { "gcp_project_id": "GCP_PROJECT_ID", "region": "us-central1", "ssh_pub_key": "~/.ssh/id_rsa.pub", "keyfile_location": "service-account.json", "prefix": "development", "ssh_whitelist": [ "1.2.3.4/32" ], "api_server_whitelist": [ "1.2.3.4/32" ], "nodeport_whitelist": [ "1.2.3.4/32" ], "ingress_whitelist": [ "0.0.0.0/0" ], "machines": { "master-0": { "node_type": "master", "size": "n1-standard-2", "zone": "us-central1-a", "additional_disks": {}, "boot_disk": { "image_name": "ubuntu-os-cloud/ubuntu-2004-focal-v20220118", "size": 50 } }, "worker-0": { "node_type": "worker", "size": "n1-standard-8", "zone": "us-central1-a", "additional_disks": { "extra-disk-1": { "size": 100 } }, "boot_disk": { "image_name": "ubuntu-os-cloud/ubuntu-2004-focal-v20220118", "size": 50 } }, "worker-1": { "node_type": "worker", "size": "n1-standard-8", "zone": "us-central1-a", "additional_disks": { "extra-disk-1": { "size": 100 } }, "boot_disk": { "image_name": "ubuntu-os-cloud/ubuntu-2004-focal-v20220118", "size": 50 } } } } ================================================ FILE: contrib/terraform/gcp/variables.tf ================================================ variable keyfile_location { description = "Location of the json keyfile to use with the google provider" type = string } variable region { description = "Region of all resources" type = string } variable gcp_project_id { description = "ID of the project" type = string } variable prefix { description = "Prefix for resource names" default = "default" } variable machines { description = "Cluster machines" type = map(object({ node_type = string size = string zone = string additional_disks = map(object({ size = number })) boot_disk = object({ image_name = string size = number }) })) } variable "master_sa_email" { type = string default = "" } variable "master_sa_scopes" { type = list(string) default = ["https://www.googleapis.com/auth/cloud-platform"] } variable "master_preemptible" { type = bool default = false } variable "master_additional_disk_type" { type = string default = "pd-ssd" } variable "worker_sa_email" { type = string default = "" } variable "worker_sa_scopes" { type = list(string) default = ["https://www.googleapis.com/auth/cloud-platform"] } variable "worker_preemptible" { type = bool default = false } variable "worker_additional_disk_type" { type = string default = "pd-ssd" } variable ssh_pub_key { description = "Path to public SSH key file which is injected into the VMs." type = string } variable ssh_whitelist { type = list(string) } variable api_server_whitelist { type = list(string) } variable nodeport_whitelist { type = list(string) } variable "ingress_whitelist" { type = list(string) default = ["0.0.0.0/0"] } variable "extra_ingress_firewalls" { type = map(object({ source_ranges = set(string) protocol = string ports = list(string) target_tags = set(string) })) default = {} } ================================================ FILE: contrib/terraform/hetzner/README.md ================================================ # Kubernetes on Hetzner with Terraform Provision a Kubernetes cluster on [Hetzner](https://www.hetzner.com/cloud) using Terraform and Kubespray ## Overview The setup looks like following ```text Kubernetes cluster +--------------------------+ | +--------------+ | | | +--------------+ | | --> | | | | | | | Master/etcd | | | | | node(s) | | | +-+ | | | +--------------+ | | ^ | | | | | v | | +--------------+ | | | +--------------+ | | --> | | | | | | | Worker | | | | | node(s) | | | +-+ | | | +--------------+ | +--------------------------+ ``` The nodes uses a private network for node to node communication and a public interface for all external communication. ## Requirements * Terraform 0.14.0 or newer ## Quickstart NOTE: Assumes you are at the root of the kubespray repo. For authentication in your cluster you can use the environment variables. ```bash export HCLOUD_TOKEN=api-token ``` Copy the cluster configuration file. ```bash CLUSTER=my-hetzner-cluster cp -r inventory/sample inventory/$CLUSTER cp contrib/terraform/hetzner/default.tfvars inventory/$CLUSTER/ cd inventory/$CLUSTER ``` Edit `default.tfvars` to match your requirement. Flatcar Container Linux instead of the basic Hetzner Images. ```bash cd ../../contrib/terraform/hetzner ``` Edit `main.tf` and reactivate the module `source = "./modules/kubernetes-cluster-flatcar"`and comment out the `#source = "./modules/kubernetes-cluster"`. activate `ssh_private_key_path = var.ssh_private_key_path`. The VM boots into Rescue-Mode with the selected image of the `var.machines` but installs Flatcar instead. Run Terraform to create the infrastructure. ```bash cd ./kubespray terraform -chdir=./contrib/terraform/hetzner/ init terraform -chdir=./contrib/terraform/hetzner/ apply --var-file=../../../inventory/$CLUSTER/default.tfvars ``` You should now have a inventory file named `inventory.ini` that you can use with kubespray. You can use the inventory file with kubespray to set up a cluster. It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by: ```bash ansible -i inventory.ini -m ping all ``` You can setup Kubernetes with kubespray using the generated inventory: ```bash ansible-playbook -i inventory.ini ../../cluster.yml -b -v ``` ## Cloud controller For better support with the cloud you can install the [hcloud cloud controller](https://github.com/hetznercloud/hcloud-cloud-controller-manager) and [CSI driver](https://github.com/hetznercloud/csi-driver). Please read the instructions in both repos on how to install it. ## Teardown You can teardown your infrastructure using the following Terraform command: ```bash cd ./kubespray terraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars ``` ## Variables * `prefix`: Prefix to add to all resources, if set to "" don't set any prefix * `ssh_public_keys`: List of public SSH keys to install on all machines * `zone`: The zone where to run the cluster * `network_zone`: the network zone where the cluster is running * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* * `size`: Size of the VM * `image`: The image to use for the VM * `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes * `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server * `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports) * `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to kubernetes workers on port 80 and 443 ================================================ FILE: contrib/terraform/hetzner/default.tfvars ================================================ prefix = "default" zone = "hel1" network_zone = "eu-central" inventory_file = "inventory.ini" ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] ssh_private_key_path = "~/.ssh/id_rsa" machines = { "master-0" : { "node_type" : "master", "size" : "cx21", "image" : "ubuntu-22.04", }, "worker-0" : { "node_type" : "worker", "size" : "cx21", "image" : "ubuntu-22.04", }, "worker-1" : { "node_type" : "worker", "size" : "cx21", "image" : "ubuntu-22.04", } } nodeport_whitelist = [ "0.0.0.0/0" ] ingress_whitelist = [ "0.0.0.0/0" ] ssh_whitelist = [ "0.0.0.0/0" ] api_server_whitelist = [ "0.0.0.0/0" ] ================================================ FILE: contrib/terraform/hetzner/main.tf ================================================ provider "hcloud" {} module "kubernetes" { source = "./modules/kubernetes-cluster" # source = "./modules/kubernetes-cluster-flatcar" prefix = var.prefix zone = var.zone machines = var.machines #only for flatcar #ssh_private_key_path = var.ssh_private_key_path ssh_public_keys = var.ssh_public_keys network_zone = var.network_zone ssh_whitelist = var.ssh_whitelist api_server_whitelist = var.api_server_whitelist nodeport_whitelist = var.nodeport_whitelist ingress_whitelist = var.ingress_whitelist } # # Generate ansible inventory # locals { inventory = templatefile( "${path.module}/templates/inventory.tpl", { connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d", keys(module.kubernetes.master_ip_addresses), values(module.kubernetes.master_ip_addresses).*.public_ip, values(module.kubernetes.master_ip_addresses).*.private_ip, range(1, length(module.kubernetes.master_ip_addresses) + 1))) connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s", keys(module.kubernetes.worker_ip_addresses), values(module.kubernetes.worker_ip_addresses).*.public_ip, values(module.kubernetes.worker_ip_addresses).*.private_ip)) list_master = join("\n", keys(module.kubernetes.master_ip_addresses)) list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses)) network_id = module.kubernetes.network_id } ) } resource "null_resource" "inventories" { provisioner "local-exec" { command = "echo '${local.inventory}' > ${var.inventory_file}" } triggers = { template = local.inventory } } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf ================================================ resource "hcloud_network" "kubernetes" { name = "${var.prefix}-network" ip_range = var.private_network_cidr } resource "hcloud_network_subnet" "kubernetes" { type = "cloud" network_id = hcloud_network.kubernetes.id network_zone = var.network_zone ip_range = var.private_subnet_cidr } resource "hcloud_server" "master" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "master" } name = "${var.prefix}-${each.key}" image = each.value.image server_type = each.value.size location = var.zone user_data = templatefile( "${path.module}/templates/cloud-init.tmpl", { ssh_public_keys = var.ssh_public_keys } ) firewall_ids = [hcloud_firewall.master.id] } resource "hcloud_server_network" "master" { for_each = hcloud_server.master server_id = each.value.id subnet_id = hcloud_network_subnet.kubernetes.id } resource "hcloud_server" "worker" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "worker" } name = "${var.prefix}-${each.key}" image = each.value.image server_type = each.value.size location = var.zone user_data = templatefile( "${path.module}/templates/cloud-init.tmpl", { ssh_public_keys = var.ssh_public_keys } ) firewall_ids = [hcloud_firewall.worker.id] } resource "hcloud_server_network" "worker" { for_each = hcloud_server.worker server_id = each.value.id subnet_id = hcloud_network_subnet.kubernetes.id } resource "hcloud_firewall" "master" { name = "${var.prefix}-master-firewall" rule { direction = "in" protocol = "tcp" port = "22" source_ips = var.ssh_whitelist } rule { direction = "in" protocol = "tcp" port = "6443" source_ips = var.api_server_whitelist } } resource "hcloud_firewall" "worker" { name = "${var.prefix}-worker-firewall" rule { direction = "in" protocol = "tcp" port = "22" source_ips = var.ssh_whitelist } rule { direction = "in" protocol = "tcp" port = "80" source_ips = var.ingress_whitelist } rule { direction = "in" protocol = "tcp" port = "443" source_ips = var.ingress_whitelist } rule { direction = "in" protocol = "tcp" port = "30000-32767" source_ips = var.nodeport_whitelist } } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf ================================================ output "master_ip_addresses" { value = { for key, instance in hcloud_server.master : instance.name => { "private_ip" = hcloud_server_network.master[key].ip "public_ip" = hcloud_server.master[key].ipv4_address } } } output "worker_ip_addresses" { value = { for key, instance in hcloud_server.worker : instance.name => { "private_ip" = hcloud_server_network.worker[key].ip "public_ip" = hcloud_server.worker[key].ipv4_address } } } output "cluster_private_network_cidr" { value = var.private_subnet_cidr } output "network_id" { value = hcloud_network.kubernetes.id } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl ================================================ #cloud-config users: - default - name: ubuntu shell: /bin/bash sudo: "ALL=(ALL) NOPASSWD:ALL" ssh_authorized_keys: %{ for ssh_public_key in ssh_public_keys ~} - ${ssh_public_key} %{ endfor ~} ssh_authorized_keys: %{ for ssh_public_key in ssh_public_keys ~} - ${ssh_public_key} %{ endfor ~} ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf ================================================ variable "zone" { type = string } variable "prefix" {} variable "machines" { type = map(object({ node_type = string size = string image = string })) } variable "ssh_public_keys" { type = list(string) } variable "ssh_whitelist" { type = list(string) } variable "api_server_whitelist" { type = list(string) } variable "nodeport_whitelist" { type = list(string) } variable "ingress_whitelist" { type = list(string) } variable "private_network_cidr" { default = "10.0.0.0/16" } variable "private_subnet_cidr" { default = "10.0.10.0/24" } variable "network_zone" { default = "eu-central" } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf ================================================ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" version = "1.38.2" } } required_version = ">= 0.14" } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf ================================================ resource "hcloud_network" "kubernetes" { name = "${var.prefix}-network" ip_range = var.private_network_cidr } resource "hcloud_network_subnet" "kubernetes" { type = "cloud" network_id = hcloud_network.kubernetes.id network_zone = var.network_zone ip_range = var.private_subnet_cidr } resource "hcloud_ssh_key" "first" { name = var.prefix public_key = var.ssh_public_keys.0 } resource "hcloud_server" "machine" { for_each = { for name, machine in var.machines : name => machine } name = "${var.prefix}-${each.key}" ssh_keys = [hcloud_ssh_key.first.id] # boot into rescue OS rescue = "linux64" # dummy value for the OS because Flatcar is not available image = each.value.image server_type = each.value.size location = var.zone connection { host = self.ipv4_address timeout = "5m" private_key = file(var.ssh_private_key_path) } firewall_ids = each.value.node_type == "master" ? [hcloud_firewall.master.id] : [hcloud_firewall.worker.id] provisioner "file" { content = data.ct_config.machine-ignitions[each.key].rendered destination = "/root/ignition.json" } provisioner "remote-exec" { inline = [ "set -ex", "apt update", "apt install -y gawk", "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install", "chmod +x flatcar-install", "./flatcar-install -s -i /root/ignition.json -C stable", "shutdown -r +1", ] } # optional: provisioner "remote-exec" { connection { host = self.ipv4_address private_key = file(var.ssh_private_key_path) timeout = "3m" user = var.user_flatcar } inline = [ "sudo hostnamectl set-hostname ${self.name}", ] } } resource "hcloud_server_network" "machine" { for_each = { for name, machine in var.machines : name => hcloud_server.machine[name] } server_id = each.value.id subnet_id = hcloud_network_subnet.kubernetes.id } data "ct_config" "machine-ignitions" { for_each = { for name, machine in var.machines : name => machine } strict = false content = templatefile( "${path.module}/templates/machine.yaml.tmpl", { ssh_keys = jsonencode(var.ssh_public_keys) user_flatcar = var.user_flatcar name = each.key } ) } resource "hcloud_firewall" "master" { name = "${var.prefix}-master-firewall" rule { direction = "in" protocol = "tcp" port = "22" source_ips = var.ssh_whitelist } rule { direction = "in" protocol = "tcp" port = "6443" source_ips = var.api_server_whitelist } } resource "hcloud_firewall" "worker" { name = "${var.prefix}-worker-firewall" rule { direction = "in" protocol = "tcp" port = "22" source_ips = var.ssh_whitelist } rule { direction = "in" protocol = "tcp" port = "80" source_ips = var.ingress_whitelist } rule { direction = "in" protocol = "tcp" port = "443" source_ips = var.ingress_whitelist } rule { direction = "in" protocol = "tcp" port = "30000-32767" source_ips = var.nodeport_whitelist } } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf ================================================ output "master_ip_addresses" { value = { for name, machine in var.machines : name => { "private_ip" = hcloud_server_network.machine[name].ip "public_ip" = hcloud_server.machine[name].ipv4_address } if machine.node_type == "master" } } output "worker_ip_addresses" { value = { for name, machine in var.machines : name => { "private_ip" = hcloud_server_network.machine[name].ip "public_ip" = hcloud_server.machine[name].ipv4_address } if machine.node_type == "worker" } } output "cluster_private_network_cidr" { value = var.private_subnet_cidr } output "network_id" { value = hcloud_network.kubernetes.id } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl ================================================ variant: flatcar version: 1.0.0 passwd: users: - name: ${user_flatcar} ssh_authorized_keys: ${ssh_keys} storage: files: - path: /home/core/works filesystem: root mode: 0755 contents: inline: | #!/bin/bash set -euo pipefail hostname="$(hostname)" echo My name is ${name} and the hostname is $${hostname} ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf ================================================ variable "zone" { type = string default = "fsn1" } variable "prefix" { default = "k8s" } variable "user_flatcar" { type = string default = "core" } variable "machines" { type = map(object({ node_type = string size = string image = string })) } variable "ssh_public_keys" { type = list(string) } variable "ssh_private_key_path" { type = string default = "~/.ssh/id_rsa" } variable "ssh_whitelist" { type = list(string) } variable "api_server_whitelist" { type = list(string) } variable "nodeport_whitelist" { type = list(string) } variable "ingress_whitelist" { type = list(string) } variable "private_network_cidr" { default = "10.0.0.0/16" } variable "private_subnet_cidr" { default = "10.0.10.0/24" } variable "network_zone" { default = "eu-central" } ================================================ FILE: contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf ================================================ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" } ct = { source = "poseidon/ct" version = "0.11.0" } null = { source = "hashicorp/null" } } } ================================================ FILE: contrib/terraform/hetzner/output.tf ================================================ output "master_ips" { value = module.kubernetes.master_ip_addresses } output "worker_ips" { value = module.kubernetes.worker_ip_addresses } ================================================ FILE: contrib/terraform/hetzner/sample-inventory/cluster.tfvars ================================================ prefix = "default" zone = "hel1" network_zone = "eu-central" inventory_file = "inventory.ini" ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] ssh_private_key_path = "~/.ssh/id_rsa" machines = { "master-0" : { "node_type" : "master", "size" : "cx21", "image" : "ubuntu-22.04", }, "worker-0" : { "node_type" : "worker", "size" : "cx21", "image" : "ubuntu-22.04", }, "worker-1" : { "node_type" : "worker", "size" : "cx21", "image" : "ubuntu-22.04", } } nodeport_whitelist = [ "0.0.0.0/0" ] ingress_whitelist = [ "0.0.0.0/0" ] ssh_whitelist = [ "0.0.0.0/0" ] api_server_whitelist = [ "0.0.0.0/0" ] ================================================ FILE: contrib/terraform/hetzner/templates/inventory.tpl ================================================ [all] ${connection_strings_master} ${connection_strings_worker} [kube_control_plane] ${list_master} [etcd] ${list_master} [kube_node] ${list_worker} [k8s_cluster:children] kube_control_plane kube_node [k8s_cluster:vars] network_id=${network_id} ================================================ FILE: contrib/terraform/hetzner/variables.tf ================================================ variable "zone" { description = "The zone where to run the cluster" } variable "network_zone" { description = "The network zone where the cluster is running" default = "eu-central" } variable "prefix" { description = "Prefix for resource names" default = "default" } variable "machines" { description = "Cluster machines" type = map(object({ node_type = string size = string image = string })) } variable "ssh_public_keys" { description = "Public SSH key which are injected into the VMs." type = list(string) } variable "ssh_private_key_path" { description = "Private SSH key which connect to the VMs." type = string default = "~/.ssh/id_rsa" } variable "ssh_whitelist" { description = "List of IP ranges (CIDR) to whitelist for ssh" type = list(string) } variable "api_server_whitelist" { description = "List of IP ranges (CIDR) to whitelist for kubernetes api server" type = list(string) } variable "nodeport_whitelist" { description = "List of IP ranges (CIDR) to whitelist for kubernetes nodeports" type = list(string) } variable "ingress_whitelist" { description = "List of IP ranges (CIDR) to whitelist for HTTP" type = list(string) } variable "inventory_file" { description = "Where to store the generated inventory file" } ================================================ FILE: contrib/terraform/hetzner/versions.tf ================================================ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" version = "1.38.2" } null = { source = "hashicorp/null" } } required_version = ">= 0.14" } ================================================ FILE: contrib/terraform/openstack/.gitignore ================================================ .terraform *.tfvars !sample-inventory/cluster.tfvars *.tfstate *.tfstate.backup ================================================ FILE: contrib/terraform/openstack/README.md ================================================ # Kubernetes on OpenStack with Terraform Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on OpenStack. ## Status This will install a Kubernetes cluster on an OpenStack Cloud. It should work on most modern installs of OpenStack that support the basic services. ### Known compatible public clouds - [Auro](https://auro.io/) - [Betacloud](https://www.betacloud.io/) - [CityCloud](https://www.citycloud.com/) - [DreamHost](https://www.dreamhost.com/cloud/computing/) - [ELASTX](https://elastx.se/) - [EnterCloudSuite](https://www.entercloudsuite.com/) - [FugaCloud](https://fuga.cloud/) - [Open Telekom Cloud](https://cloud.telekom.de/) - [OVH](https://www.ovh.com/) - [Rackspace](https://www.rackspace.com/) - [Safespring](https://www.safespring.com) - [Ultimum](https://ultimum.io/) - [VexxHost](https://vexxhost.com/) - [Zetta](https://www.zetta.io/) - [Cloudify](https://www.cloudify.ro/en) ## Approach The terraform configuration inspects variables found in [variables.tf](variables.tf) to create resources in your OpenStack cluster. There is a [python script](../terraform.py) that reads the generated`.tfstate` file to generate a dynamic inventory that is consumed by the main ansible script to actually install kubernetes and stand up the cluster. ### Networking The configuration includes creating a private subnet with a router to the external net. It will allocate floating IPs from a pool and assign them to the hosts where that makes sense. You have the option of creating bastion hosts inside the private subnet to access the nodes there. Alternatively, a node with a floating IP can be used as a jump host to nodes without. #### Using an existing router It is possible to use an existing router instead of creating one. To use an existing router set the router\_id variable to the uuid of the router you wish to use. For example: ```ShellSession router_id = "00c542e7-6f46-4535-ae95-984c7f0391a3" ``` ### Kubernetes Nodes You can create many different kubernetes topologies by setting the number of different classes of hosts. For each class there are options for allocating floating IP addresses or not. - Control plane nodes with etcd - Control plane nodes without etcd - Standalone etcd hosts - Kubernetes worker nodes Note that the Ansible script will report an invalid configuration if you wind up with an even number of etcd instances since that is not a valid configuration. This restriction includes standalone etcd nodes that are deployed in a cluster along with control plane nodes with etcd replicas. As an example, if you have three control plane nodes with etcd replicas and three standalone etcd nodes, the script will fail since there are now six total etcd replicas. ### GlusterFS shared file system The Terraform configuration supports provisioning of an optional GlusterFS shared file system based on a separate set of VMs. To enable this, you need to specify: - the number of Gluster hosts (minimum 2) - Size of the non-ephemeral volumes to be attached to store the GlusterFS bricks - Other properties related to provisioning the hosts Even if you are using Flatcar Container Linux by Kinvolk for your cluster, you will still need the GlusterFS VMs to be based on either Debian or RedHat based images. Flatcar Container Linux by Kinvolk cannot serve GlusterFS, but can connect to it through binaries available on hyperkube v1.4.3_coreos.0 or higher. ## Requirements - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.14 or later - [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) - you already have a suitable OS image in Glance - you already have a floating IP pool created - you have security groups enabled - you have a pair of keys generated that can be used to secure the new hosts ## Module Architecture The configuration is divided into four modules: - Network - Loadbalancer - IPs - Compute The main reason for splitting the configuration up in this way is to easily accommodate situations where floating IPs are limited by a quota or if you have any external references to the floating IP (e.g. DNS) that would otherwise have to be updated. You can force your existing IPs by modifying the compute variables in `kubespray.tf` as follows: ```ini k8s_master_fips = ["151.101.129.67"] k8s_node_fips = ["151.101.129.68"] ``` ## Terraform Terraform will be used to provision all of the OpenStack resources with base software as appropriate. ### Configuration #### Inventory files Create an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state): ```ShellSession cp -LRp contrib/terraform/openstack/sample-inventory inventory/$CLUSTER cd inventory/$CLUSTER ln -s ../../contrib/terraform/openstack/hosts ln -s ../../contrib ``` This will be the base for subsequent Terraform commands. #### OpenStack access and credentials No provider variables are hardcoded inside `variables.tf` because Terraform supports various authentication methods for OpenStack: the older script and environment method (using `openrc`) as well as a newer declarative method, and different OpenStack environments may support Identity API version 2 or 3. These are examples and may vary depending on your OpenStack cloud provider, for an exhaustive list on how to authenticate on OpenStack with Terraform please read the [OpenStack provider documentation](https://www.terraform.io/docs/providers/openstack/). ##### Declarative method (recommended) The recommended authentication method is to describe credentials in a YAML file `clouds.yaml` that can be stored in: - the current directory - `~/.config/openstack` - `/etc/openstack` `clouds.yaml`: ```yaml clouds: mycloud: auth: auth_url: https://openstack:5000/v3 username: "username" project_name: "projectname" project_id: projectid user_domain_name: "Default" password: "password" region_name: "RegionOne" interface: "public" identity_api_version: 3 ``` If you have multiple clouds defined in your `clouds.yaml` file you can choose the one you want to use with the environment variable `OS_CLOUD`: ```ShellSession export OS_CLOUD=mycloud ``` ##### Openrc method When using classic environment variables, Terraform uses default `OS_*` environment variables. A script suitable for your environment may be available from Horizon under *Project* -> *Compute* -> *Access & Security* -> *API Access*. With identity v2: ```ShellSession source openrc env | grep OS OS_AUTH_URL=https://openstack:5000/v2.0 OS_PROJECT_ID=projectid OS_PROJECT_NAME=projectname OS_USERNAME=username OS_PASSWORD=password OS_REGION_NAME=RegionOne OS_INTERFACE=public OS_IDENTITY_API_VERSION=2 ``` With identity v3: ```ShellSession source openrc env | grep OS OS_AUTH_URL=https://openstack:5000/v3 OS_PROJECT_ID=projectid OS_PROJECT_NAME=username OS_PROJECT_DOMAIN_ID=default OS_USERNAME=username OS_PASSWORD=password OS_REGION_NAME=RegionOne OS_INTERFACE=public OS_IDENTITY_API_VERSION=3 OS_USER_DOMAIN_NAME=Default ``` Terraform does not support a mix of DomainName and DomainID, choose one or the other: - provider.openstack: You must provide exactly one of DomainID or DomainName to authenticate by Username ```ShellSession unset OS_USER_DOMAIN_NAME export OS_USER_DOMAIN_ID=default ``` or ```ShellSession unset OS_PROJECT_DOMAIN_ID set OS_PROJECT_DOMAIN_NAME=Default ``` #### Cluster variables The construction of the cluster is driven by values found in [variables.tf](variables.tf). For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |Variable | Description | |---------|-------------| |`cluster_name` | All OpenStack resources will use the Terraform variable`cluster_name` (default`example`) in their name to make it easier to track. For example the first compute resource will be named`example-kubernetes-1`. | |`az_list` | List of Availability Zones available in your OpenStack cluster. | |`network_name` | The name to be given to the internal network that will be generated | |`use_existing_network`| Use an existing network with the name of `network_name`. `false` by default | |`network_dns_domain` | (Optional) The dns_domain for the internal network that will be generated | |`dns_nameservers`| An array of DNS name server names to be used by hosts in the internal subnet. | |`floatingip_pool` | Name of the pool from which floating IPs will be allocated | |`k8s_master_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to master nodes instead of creating new random floating IPs. | |`bastion_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to bastion node instead of creating new random floating IPs. | |`external_net` | UUID of the external network that will be routed to | |`flavor_k8s_master`,`flavor_k8s_node`,`flavor_etcd`, `flavor_bastion`,`flavor_gfs_node` | Flavor depends on your openstack installation, you can get available flavor IDs through `openstack flavor list` | |`image`,`image_gfs`, `image_master` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. | |`image_uuid`,`image_gfs_uuid`, `image_master_uuid` | UUID of the image to use in provisioning the compute resources. Should already be loaded into glance. | |`ssh_user`,`ssh_user_gfs` | The username to ssh into the image with. This usually depends on the image you have selected | |`public_key_path` | Path on your local workstation to the public key file you wish to use in creating the key pairs | |`number_of_k8s_masters`, `number_of_k8s_masters_no_floating_ip` | Number of nodes that serve as both master and etcd. These can be provisioned with or without floating IP addresses| |`number_of_k8s_masters_no_etcd`, `number_of_k8s_masters_no_floating_ip_no_etcd` | Number of nodes that serve as just master with no etcd. These can be provisioned with or without floating IP addresses | |`number_of_etcd` | Number of pure etcd nodes | |`number_of_k8s_nodes`, `number_of_k8s_nodes_no_floating_ip` | Kubernetes worker nodes. These can be provisioned with or without floating ip addresses. | |`number_of_bastions` | Number of bastion hosts to create. Scripts assume this is really just zero or one | |`number_of_gfs_nodes_no_floating_ip` | Number of gluster servers to provision. | | `gfs_volume_size_in_gb` | Size of the non-ephemeral volumes to be attached to store the GlusterFS bricks | |`supplementary_master_groups` | To add ansible groups to the masters, such as `kube_node` for tainting them as nodes, empty by default. | |`supplementary_node_groups` | To add ansible groups to the nodes, such as `kube_ingress` for running ingress controller pods, empty by default. | |`bastion_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, `["0.0.0.0/0"]` by default | |`bastion_allowed_remote_ipv6_ips` | List of IPv6 CIDR allowed to initiate a SSH connection, `["::/0"]` by default | |`master_allowed_remote_ips` | List of CIDR blocks allowed to initiate an API connection, `["0.0.0.0/0"]` by default | |`master_allowed_remote_ipv6_ips` | List of IPv6 CIDR blocks allowed to initiate an API connection, `["::/0"]` by default | |`bastion_allowed_ports` | List of ports to open on bastion node, `[]` by default | |`bastion_allowed_ports_ipv6` | List of ports to open on bastion node for IPv6 CIDR blocks, `[]` by default | |`k8s_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, empty by default | |`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default | |`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `["::/0"]` by default | |`worker_allowed_ports` | List of ports to open on worker nodes, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "0.0.0.0/0"}]` by default | |`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}, { "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default | |`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "0.0.0.0/0"}]`, empty by default | |`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, `[{ "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default | |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage | |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage | |`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default | |`node_volume_type` | Volume type of the root volume for nodes, 'Default' by default | |`gfs_root_volume_size_in_gb` | Size of the root volume for gluster, 0 to use ephemeral storage | |`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage | |`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage | |`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: "" (disabled) | |`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: "" (disabled) | |`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: "" (disabled) | |`additional_server_groups` | Extra server groups to create. Set "policy" to the policy for the group, expected format is `{"new-server-group" = {"policy" = "anti-affinity"}}`, default: {} (to not create any extra groups) | |`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. | |`port_security_enabled` | Allow to disable port security by setting this to `false`. `true` by default | |`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default | |`k8s_nodes` | Map containing worker node definition, see explanation below | |`k8s_masters` | Map containing master node definition, see explanation for k8s_nodes and `sample-inventory/cluster.tfvars` | |`k8s_master_loadbalancer_enabled` | Enable and use an Octavia load balancer for the K8s master nodes | |`k8s_master_loadbalancer_listener_port` | Define via which port the K8s Api should be exposed. `6443` by default | |`k8s_master_loadbalancer_server_port` | Define via which port the K8S api is available on the master nodes. `6443` by default | |`k8s_master_loadbalancer_public_ip` | Specify if an existing floating IP should be used for the load balancer. A new floating IP is assigned by default | ##### k8s_nodes Allows a custom definition of worker nodes giving the operator full control over individual node flavor and availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and `number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration using the `k8s_nodes` variable. The `az`, `flavor` and `floating_ip` parameters are mandatory. The optional parameter `extra_groups` (a comma-delimited string) can be used to define extra inventory group memberships for specific nodes. ```yaml k8s_nodes: node-name: az: string # Name of the AZ flavor: string # Flavor ID to use floating_ip: bool # If floating IPs should be used or not reserved_floating_ip: string # If floating_ip is true use existing floating IP, if reserved_floating_ip is an empty string and floating_ip is true, a new floating IP will be created extra_groups: string # (optional) Additional groups to add for kubespray, defaults to no groups image_id: string # (optional) Image ID to use, defaults to var.image_id or var.image root_volume_size_in_gb: number # (optional) Size of the block storage to use as root disk, defaults to var.node_root_volume_size_in_gb or to use volume from flavor otherwise volume_type: string # (optional) Volume type to use, defaults to var.node_volume_type network_id: string # (optional) Use this network_id for the node, defaults to either var.network_id or ID of var.network_name server_group: string # (optional) Server group to add this node to. If set, this has to be one specified in additional_server_groups, defaults to use the server group specified in node_server_group_policy cloudinit: # (optional) Options for cloud-init extra_partitions: # List of extra partitions (other than the root partition) to setup during creation volume_path: string # Path to the volume to create partition for (e.g. /dev/vda ) partition_path: string # Path to the partition (e.g. /dev/vda2 ) mount_path: string # Path to where the partition should be mounted partition_start: string # Where the partition should start (e.g. 10GB ). Note, if you set the partition_start to 0 there will be no space left for the root partition partition_end: string # Where the partition should end (e.g. 10GB or -1 for end of volume) netplan_critical_dhcp_interface: string # Name of interface to set the dhcp flag critical = true, to circumvent [this issue](https://bugs.launchpad.net/ubuntu/+source/systemd/+bug/1776013). ``` For example: ```ini k8s_nodes = { "1" = { "az" = "sto1" "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" "floating_ip" = true }, "2" = { "az" = "sto2" "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" "floating_ip" = true }, "3" = { "az" = "sto3" "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" "floating_ip" = true "extra_groups" = "calico_rr" } } ``` Would result in the same configuration as: ```ini number_of_k8s_nodes = 3 flavor_k8s_node = "83d8b44a-26a0-4f02-a981-079446926445" az_list = ["sto1", "sto2", "sto3"] ``` And: ```ini k8s_nodes = { "ing-1" = { "az" = "sto1" "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" "floating_ip" = true }, "ing-2" = { "az" = "sto2" "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" "floating_ip" = true }, "ing-3" = { "az" = "sto3" "flavor" = "83d8b44a-26a0-4f02-a981-079446926445" "floating_ip" = true }, "big-1" = { "az" = "sto1" "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b" "floating_ip" = false }, "big-2" = { "az" = "sto2" "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b" "floating_ip" = false }, "big-3" = { "az" = "sto3" "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b" "floating_ip" = false }, "small-1" = { "az" = "sto1" "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e" "floating_ip" = false }, "small-2" = { "az" = "sto2" "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e" "floating_ip" = false }, "small-3" = { "az" = "sto3" "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e" "floating_ip" = false } } ``` Would result in three nodes in each availability zone each with their own separate naming, flavor and floating ip configuration. The "schema": ```ini k8s_nodes = { "key | node name suffix, must be unique" = { "az" = string "flavor" = string "floating_ip" = bool }, } ``` All values are required. #### Terraform state files In the cluster's inventory folder, the following files might be created (either by Terraform or manually), to prevent you from pushing them accidentally they are in a `.gitignore` file in the `terraform/openstack` directory : - `.terraform` - `.tfvars` - `.tfstate` - `.tfstate.backup` You can still add them manually if you want to. ### Initialization Before Terraform can operate on your cluster you need to install the required plugins. This is accomplished as follows: ```ShellSession cd inventory/$CLUSTER terraform -chdir="../../contrib/terraform/openstack" init ``` This should finish fairly quickly telling you Terraform has successfully initialized and loaded necessary modules. ### Customizing with cloud-init You can apply cloud-init based customization for the openstack instances before provisioning your cluster. One common template is used for all instances. Adjust the file shown below: `contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml.tmpl` For example, to enable openstack novnc access and ansible_user=root SSH access: ```ShellSession #cloud-config ## in some cases novnc console access is required ## it requires ssh password to be set ssh_pwauth: yes chpasswd: list: | root:secret expire: False ## in some cases direct root ssh access via ssh key is required disable_root: false ``` ### Provisioning cluster You can apply the Terraform configuration to your cluster with the following command issued from your cluster's inventory directory (`inventory/$CLUSTER`): ```ShellSession terraform -chdir="../../contrib/terraform/openstack" apply -var-file=cluster.tfvars ``` if you chose to create a bastion host, this script will create `contrib/terraform/openstack/k8s_cluster.yml` with an ssh command for Ansible to be able to access your machines tunneling through the bastion's IP address. If you want to manually handle the ssh tunneling to these machines, please delete or move that file. If you want to use this, just leave it there, as ansible will pick it up automatically. ### Destroying cluster You can destroy your new cluster with the following command issued from the cluster's inventory directory: ```ShellSession terraform -chdir="../../contrib/terraform/openstack" destroy -var-file=cluster.tfvars ``` If you've started the Ansible run, it may also be a good idea to do some manual cleanup: - remove SSH keys from the destroyed cluster from your `~/.ssh/known_hosts` file - clean up any temporary cache files: `rm /tmp/$CLUSTER-*` ### Debugging You can enable debugging output from Terraform by setting `OS_DEBUG` to 1 and`TF_LOG` to`DEBUG` before running the Terraform command. ### Terraform output Terraform can output values that are useful for configure Neutron/Octavia LBaaS or Cinder persistent volume provisioning as part of your Kubernetes deployment: - `private_subnet_id`: the subnet where your instances are running is used for `openstack_lbaas_subnet_id` - `floating_network_id`: the network_id where the floating IP are provisioned is used for `openstack_lbaas_floating_network_id` ## Ansible ### Node access #### SSH Ensure your local ssh-agent is running and your ssh key has been added. This step is required by the terraform provisioner: ```ShellSession eval $(ssh-agent -s) ssh-add ~/.ssh/id_rsa ``` If you have deployed and destroyed a previous iteration of your cluster, you will need to clear out any stale keys from your SSH "known hosts" file ( `~/.ssh/known_hosts`). #### Metadata variables The [python script](../terraform.py) that reads the generated`.tfstate` file to generate a dynamic inventory recognizes some variables within a "metadata" block, defined in a "resource" block (example): ```ini resource "openstack_compute_instance_v2" "example" { ... metadata { ssh_user = "ubuntu" prefer_ipv6 = true python_bin = "/usr/bin/python3" } ... } ``` As the example shows, these let you define the SSH username for Ansible, a Python binary which is needed by Ansible if `/usr/bin/python` doesn't exist, and whether the IPv6 address of the instance should be preferred over IPv4. #### Bastion host Bastion access will be determined by: - Your choice on the amount of bastion hosts (set by `number_of_bastions` terraform variable). - The existence of nodes/masters with floating IPs (set by `number_of_k8s_masters`, `number_of_k8s_nodes`, `number_of_k8s_masters_no_etcd` terraform variables). If you have a bastion host, your ssh traffic will be directly routed through it. This is regardless of whether you have masters/nodes with a floating IP assigned. If you don't have a bastion host, but at least one of your masters/nodes have a floating IP, then ssh traffic will be tunneled by one of these machines. So, either a bastion host, or at least master/node with a floating IP are required. #### Test access Make sure you can connect to the hosts. Note that Flatcar Container Linux by Kinvolk will have a state `FAILED` due to Python not being present. This is okay, because Python will be installed during bootstrapping, so long as the hosts are not `UNREACHABLE`. ```ShellSession $ ansible -i inventory/$CLUSTER/hosts -m ping all example-k8s_node-1 | SUCCESS => { "changed": false, "ping": "pong" } example-etcd-1 | SUCCESS => { "changed": false, "ping": "pong" } example-k8s-master-1 | SUCCESS => { "changed": false, "ping": "pong" } ``` If it fails try to connect manually via SSH. It could be something as simple as a stale host key. ### Configure cluster variables Edit `inventory/$CLUSTER/group_vars/all/all.yml`: - **bin_dir**: ```yml # Directory where the binaries will be installed # Default: # bin_dir: /usr/local/bin # For Flatcar Container Linux by Kinvolk: bin_dir: /opt/bin ``` - and **cloud_provider**: ```yml cloud_provider: openstack ``` Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`: - Set variable **kube_network_plugin** to your desired networking plugin. - **flannel** works out-of-the-box - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets ```yml # Choose network plugin (calico or flannel) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: flannel ``` - Set variable **resolvconf_mode** ```yml # Can be docker_dns, host_resolvconf or none # Default: # resolvconf_mode: docker_dns # For Flatcar Container Linux by Kinvolk: resolvconf_mode: host_resolvconf ``` - Set max amount of attached cinder volume per host (default 256) ```yml node_volume_attach_limit: 26 ``` ### Deploy Kubernetes ```ShellSession ansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml ``` This will take some time as there are many tasks to run. ## Kubernetes ### Set up kubectl 1. [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your workstation 2. Add a route to the internal IP of a master node (if needed): ```ShellSession sudo route add [master-internal-ip] gw [router-ip] ``` or ```ShellSession sudo route add -net [internal-subnet]/24 gw [router-ip] ``` 1. List Kubernetes certificates & keys: ```ShellSession ssh [os-user]@[master-ip] sudo ls /etc/kubernetes/ssl/ ``` 1. Get `admin`'s certificates and keys: ```ShellSession ssh [os-user]@[master-ip] sudo cat /etc/kubernetes/ssl/admin-kube-master-1-key.pem > admin-key.pem ssh [os-user]@[master-ip] sudo cat /etc/kubernetes/ssl/admin-kube-master-1.pem > admin.pem ssh [os-user]@[master-ip] sudo cat /etc/kubernetes/ssl/ca.pem > ca.pem ``` 1. Configure kubectl: ```ShellSession $ kubectl config set-cluster default-cluster --server=https://[master-internal-ip]:6443 \ --certificate-authority=ca.pem $ kubectl config set-credentials default-admin \ --certificate-authority=ca.pem \ --client-key=admin-key.pem \ --client-certificate=admin.pem $ kubectl config set-context default-system --cluster=default-cluster --user=default-admin $ kubectl config use-context default-system ``` 1. Check it: ```ShellSession kubectl version ``` ## GlusterFS GlusterFS is not deployed by the standard `cluster.yml` playbook, see the [GlusterFS playbook documentation](../../network-storage/glusterfs/README.md) for instructions. Basically you will install Gluster as ```ShellSession ansible-playbook --become -i inventory/$CLUSTER/hosts ./contrib/network-storage/glusterfs/glusterfs.yml ``` ## What's next Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/). ## Appendix ### Migration from `number_of_k8s_nodes*` to `k8s_nodes` If you currently have a cluster defined using the `number_of_k8s_nodes*` variables and wish to migrate to the `k8s_nodes` style you can do it like so: ```ShellSession $ terraform state list module.compute.data.openstack_images_image_v2.gfs_image module.compute.data.openstack_images_image_v2.vm_image module.compute.openstack_compute_floatingip_associate_v2.k8s_master[0] module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0] module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1] module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2] module.compute.openstack_compute_instance_v2.k8s_master[0] module.compute.openstack_compute_instance_v2.k8s_node[0] module.compute.openstack_compute_instance_v2.k8s_node[1] module.compute.openstack_compute_instance_v2.k8s_node[2] module.compute.openstack_compute_keypair_v2.k8s module.compute.openstack_compute_servergroup_v2.k8s_etcd[0] module.compute.openstack_compute_servergroup_v2.k8s_master[0] module.compute.openstack_compute_servergroup_v2.k8s_node[0] module.compute.openstack_networking_secgroup_rule_v2.bastion[0] module.compute.openstack_networking_secgroup_rule_v2.egress[0] module.compute.openstack_networking_secgroup_rule_v2.k8s module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[0] module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[1] module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[2] module.compute.openstack_networking_secgroup_rule_v2.k8s_master[0] module.compute.openstack_networking_secgroup_rule_v2.worker[0] module.compute.openstack_networking_secgroup_rule_v2.worker[1] module.compute.openstack_networking_secgroup_rule_v2.worker[2] module.compute.openstack_networking_secgroup_rule_v2.worker[3] module.compute.openstack_networking_secgroup_rule_v2.worker[4] module.compute.openstack_networking_secgroup_v2.bastion[0] module.compute.openstack_networking_secgroup_v2.k8s module.compute.openstack_networking_secgroup_v2.k8s_master module.compute.openstack_networking_secgroup_v2.worker module.ips.null_resource.dummy_dependency module.ips.openstack_networking_floatingip_v2.k8s_master[0] module.ips.openstack_networking_floatingip_v2.k8s_node[0] module.ips.openstack_networking_floatingip_v2.k8s_node[1] module.ips.openstack_networking_floatingip_v2.k8s_node[2] module.network.openstack_networking_network_v2.k8s[0] module.network.openstack_networking_router_interface_v2.k8s[0] module.network.openstack_networking_router_v2.k8s[0] module.network.openstack_networking_subnet_v2.k8s[0] $ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["1"]' Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"1\"]" Successfully moved 1 object(s). $ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["2"]' Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"2\"]" Successfully moved 1 object(s). $ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["3"]' Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"3\"]" Successfully moved 1 object(s). $ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[0]' 'module.compute.openstack_compute_instance_v2.k8s_node["1"]' Move "module.compute.openstack_compute_instance_v2.k8s_node[0]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"1\"]" Successfully moved 1 object(s). $ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[1]' 'module.compute.openstack_compute_instance_v2.k8s_node["2"]' Move "module.compute.openstack_compute_instance_v2.k8s_node[1]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"2\"]" Successfully moved 1 object(s). $ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[2]' 'module.compute.openstack_compute_instance_v2.k8s_node["3"]' Move "module.compute.openstack_compute_instance_v2.k8s_node[2]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"3\"]" Successfully moved 1 object(s). $ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[0]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["1"]' Move "module.ips.openstack_networking_floatingip_v2.k8s_node[0]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"1\"]" Successfully moved 1 object(s). $ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[1]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["2"]' Move "module.ips.openstack_networking_floatingip_v2.k8s_node[1]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"2\"]" Successfully moved 1 object(s). $ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[2]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["3"]' Move "module.ips.openstack_networking_floatingip_v2.k8s_node[2]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"3\"]" Successfully moved 1 object(s). ``` Of course for nodes without floating ips those steps can be omitted. ================================================ FILE: contrib/terraform/openstack/kubespray.tf ================================================ module "network" { source = "./modules/network" external_net = var.external_net network_name = var.network_name subnet_cidr = var.subnet_cidr cluster_name = var.cluster_name dns_nameservers = var.dns_nameservers network_dns_domain = var.network_dns_domain use_neutron = var.use_neutron port_security_enabled = var.port_security_enabled router_id = var.router_id } module "ips" { source = "./modules/ips" number_of_k8s_masters = var.number_of_k8s_masters number_of_k8s_masters_no_etcd = var.number_of_k8s_masters_no_etcd number_of_k8s_nodes = var.number_of_k8s_nodes floatingip_pool = var.floatingip_pool number_of_bastions = var.number_of_bastions external_net = var.external_net network_name = var.network_name router_id = module.network.router_id k8s_nodes = var.k8s_nodes k8s_masters = var.k8s_masters k8s_master_fips = var.k8s_master_fips bastion_fips = var.bastion_fips router_internal_port_id = module.network.router_internal_port_id } module "compute" { source = "./modules/compute" cluster_name = var.cluster_name az_list = var.az_list az_list_node = var.az_list_node number_of_k8s_masters = var.number_of_k8s_masters number_of_k8s_masters_no_etcd = var.number_of_k8s_masters_no_etcd number_of_etcd = var.number_of_etcd number_of_k8s_masters_no_floating_ip = var.number_of_k8s_masters_no_floating_ip number_of_k8s_masters_no_floating_ip_no_etcd = var.number_of_k8s_masters_no_floating_ip_no_etcd number_of_k8s_nodes = var.number_of_k8s_nodes number_of_bastions = var.number_of_bastions number_of_k8s_nodes_no_floating_ip = var.number_of_k8s_nodes_no_floating_ip number_of_gfs_nodes_no_floating_ip = var.number_of_gfs_nodes_no_floating_ip k8s_masters = var.k8s_masters k8s_nodes = var.k8s_nodes bastion_root_volume_size_in_gb = var.bastion_root_volume_size_in_gb etcd_root_volume_size_in_gb = var.etcd_root_volume_size_in_gb master_root_volume_size_in_gb = var.master_root_volume_size_in_gb node_root_volume_size_in_gb = var.node_root_volume_size_in_gb gfs_root_volume_size_in_gb = var.gfs_root_volume_size_in_gb gfs_volume_size_in_gb = var.gfs_volume_size_in_gb master_volume_type = var.master_volume_type node_volume_type = var.node_volume_type public_key_path = var.public_key_path image = var.image image_uuid = var.image_uuid image_gfs = var.image_gfs image_master = var.image_master image_master_uuid = var.image_master_uuid image_gfs_uuid = var.image_gfs_uuid ssh_user = var.ssh_user ssh_user_gfs = var.ssh_user_gfs flavor_k8s_master = var.flavor_k8s_master flavor_k8s_node = var.flavor_k8s_node flavor_etcd = var.flavor_etcd flavor_gfs_node = var.flavor_gfs_node network_name = var.network_name flavor_bastion = var.flavor_bastion k8s_master_fips = module.ips.k8s_master_fips k8s_master_no_etcd_fips = module.ips.k8s_master_no_etcd_fips k8s_masters_fips = module.ips.k8s_masters_fips k8s_node_fips = module.ips.k8s_node_fips k8s_nodes_fips = module.ips.k8s_nodes_fips bastion_fips = module.ips.bastion_fips bastion_allowed_remote_ips = var.bastion_allowed_remote_ips bastion_allowed_remote_ipv6_ips = var.bastion_allowed_remote_ipv6_ips master_allowed_remote_ips = var.master_allowed_remote_ips master_allowed_remote_ipv6_ips = var.master_allowed_remote_ipv6_ips k8s_allowed_remote_ips = var.k8s_allowed_remote_ips k8s_allowed_remote_ips_ipv6 = var.k8s_allowed_remote_ips_ipv6 k8s_allowed_egress_ips = var.k8s_allowed_egress_ips k8s_allowed_egress_ipv6_ips = var.k8s_allowed_egress_ipv6_ips supplementary_master_groups = var.supplementary_master_groups supplementary_node_groups = var.supplementary_node_groups master_allowed_ports = var.master_allowed_ports master_allowed_ports_ipv6 = var.master_allowed_ports_ipv6 worker_allowed_ports = var.worker_allowed_ports worker_allowed_ports_ipv6 = var.worker_allowed_ports_ipv6 bastion_allowed_ports = var.bastion_allowed_ports bastion_allowed_ports_ipv6 = var.bastion_allowed_ports_ipv6 use_access_ip = var.use_access_ip master_server_group_policy = var.master_server_group_policy node_server_group_policy = var.node_server_group_policy etcd_server_group_policy = var.etcd_server_group_policy extra_sec_groups = var.extra_sec_groups extra_sec_groups_name = var.extra_sec_groups_name group_vars_path = var.group_vars_path port_security_enabled = var.port_security_enabled force_null_port_security = var.force_null_port_security network_router_id = module.network.router_id network_id = module.network.network_id use_existing_network = var.use_existing_network private_subnet_id = module.network.subnet_id additional_server_groups = var.additional_server_groups depends_on = [ module.network.subnet_id ] } module "loadbalancer" { source = "./modules/loadbalancer" cluster_name = var.cluster_name subnet_id = module.network.subnet_id floatingip_pool = var.floatingip_pool k8s_master_ips = module.compute.k8s_master_ips k8s_master_loadbalancer_enabled = var.k8s_master_loadbalancer_enabled k8s_master_loadbalancer_listener_port = var.k8s_master_loadbalancer_listener_port k8s_master_loadbalancer_server_port = var.k8s_master_loadbalancer_server_port k8s_master_loadbalancer_public_ip = var.k8s_master_loadbalancer_public_ip depends_on = [ module.compute.k8s_master ] } output "private_subnet_id" { value = module.network.subnet_id } output "floating_network_id" { value = var.external_net } output "router_id" { value = module.network.router_id } output "k8s_master_fips" { value = var.number_of_k8s_masters + var.number_of_k8s_masters_no_etcd > 0 ? concat(module.ips.k8s_master_fips, module.ips.k8s_master_no_etcd_fips) : [for key, value in module.ips.k8s_masters_fips : value.address] } output "k8s_node_fips" { value = var.number_of_k8s_nodes > 0 ? module.ips.k8s_node_fips : [for key, value in module.ips.k8s_nodes_fips : value.address] } output "bastion_fips" { value = module.ips.bastion_fips } ================================================ FILE: contrib/terraform/openstack/modules/compute/ansible_bastion_template.txt ================================================ ansible_ssh_common_args: "-o ProxyCommand='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q USER@BASTION_ADDRESS {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}'" ================================================ FILE: contrib/terraform/openstack/modules/compute/main.tf ================================================ data "openstack_images_image_v2" "vm_image" { count = var.image_uuid == "" ? 1 : 0 most_recent = true name = var.image } data "openstack_images_image_v2" "gfs_image" { count = var.image_gfs_uuid == "" ? var.image_uuid == "" ? 1 : 0 : 0 most_recent = true name = var.image_gfs == "" ? var.image : var.image_gfs } data "openstack_images_image_v2" "image_master" { count = var.image_master_uuid == "" ? var.image_uuid == "" ? 1 : 0 : 0 name = var.image_master == "" ? var.image : var.image_master } data "cloudinit_config" "cloudinit" { part { content_type = "text/cloud-config" content = templatefile("${path.module}/templates/cloudinit.yaml.tmpl", { extra_partitions = [], netplan_critical_dhcp_interface = "" }) } } data "openstack_networking_network_v2" "k8s_network" { count = var.use_existing_network ? 1 : 0 name = var.network_name } resource "openstack_compute_keypair_v2" "k8s" { name = "kubernetes-${var.cluster_name}" public_key = chomp(file(var.public_key_path)) } resource "openstack_networking_secgroup_v2" "k8s_master" { name = "${var.cluster_name}-k8s-master" description = "${var.cluster_name} - Kubernetes Master" delete_default_rules = true } resource "openstack_networking_secgroup_v2" "k8s_master_extra" { count = "%{if var.extra_sec_groups}1%{else}0%{endif}" name = "${var.cluster_name}-k8s-master-${var.extra_sec_groups_name}" description = "${var.cluster_name} - Kubernetes Master nodes - rules not managed by terraform" delete_default_rules = true } resource "openstack_networking_secgroup_rule_v2" "k8s_master" { count = length(var.master_allowed_remote_ips) direction = "ingress" ethertype = "IPv4" protocol = "tcp" port_range_min = "6443" port_range_max = "6443" remote_ip_prefix = var.master_allowed_remote_ips[count.index] security_group_id = openstack_networking_secgroup_v2.k8s_master.id } resource "openstack_networking_secgroup_rule_v2" "k8s_master_ports" { count = length(var.master_allowed_ports) direction = "ingress" ethertype = "IPv4" protocol = lookup(var.master_allowed_ports[count.index], "protocol", "tcp") port_range_min = lookup(var.master_allowed_ports[count.index], "port_range_min") port_range_max = lookup(var.master_allowed_ports[count.index], "port_range_max") remote_ip_prefix = lookup(var.master_allowed_ports[count.index], "remote_ip_prefix", "0.0.0.0/0") security_group_id = openstack_networking_secgroup_v2.k8s_master.id } resource "openstack_networking_secgroup_rule_v2" "k8s_master_ipv6_ingress" { count = length(var.master_allowed_remote_ipv6_ips) direction = "ingress" ethertype = "IPv6" protocol = "tcp" port_range_min = "6443" port_range_max = "6443" remote_ip_prefix = var.master_allowed_remote_ipv6_ips[count.index] security_group_id = openstack_networking_secgroup_v2.k8s_master.id } resource "openstack_networking_secgroup_rule_v2" "k8s_master_ports_ipv6_ingress" { count = length(var.master_allowed_ports_ipv6) direction = "ingress" ethertype = "IPv6" protocol = lookup(var.master_allowed_ports_ipv6[count.index], "protocol", "tcp") port_range_min = lookup(var.master_allowed_ports_ipv6[count.index], "port_range_min") port_range_max = lookup(var.master_allowed_ports_ipv6[count.index], "port_range_max") remote_ip_prefix = lookup(var.master_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0") security_group_id = openstack_networking_secgroup_v2.k8s_master.id } resource "openstack_networking_secgroup_rule_v2" "master_egress_ipv6" { count = length(var.k8s_allowed_egress_ipv6_ips) direction = "egress" ethertype = "IPv6" remote_ip_prefix = var.k8s_allowed_egress_ipv6_ips[count.index] security_group_id = openstack_networking_secgroup_v2.k8s_master.id } resource "openstack_networking_secgroup_v2" "bastion" { name = "${var.cluster_name}-bastion" count = var.number_of_bastions != "" ? 1 : 0 description = "${var.cluster_name} - Bastion Server" delete_default_rules = true } resource "openstack_networking_secgroup_rule_v2" "bastion" { count = var.number_of_bastions != "" ? length(var.bastion_allowed_remote_ips) : 0 direction = "ingress" ethertype = "IPv4" protocol = "tcp" port_range_min = "22" port_range_max = "22" remote_ip_prefix = var.bastion_allowed_remote_ips[count.index] security_group_id = openstack_networking_secgroup_v2.bastion[0].id } resource "openstack_networking_secgroup_rule_v2" "k8s_bastion_ports" { count = length(var.bastion_allowed_ports) direction = "ingress" ethertype = "IPv4" protocol = lookup(var.bastion_allowed_ports[count.index], "protocol", "tcp") port_range_min = lookup(var.bastion_allowed_ports[count.index], "port_range_min") port_range_max = lookup(var.bastion_allowed_ports[count.index], "port_range_max") remote_ip_prefix = lookup(var.bastion_allowed_ports[count.index], "remote_ip_prefix", "0.0.0.0/0") security_group_id = openstack_networking_secgroup_v2.bastion[0].id } resource "openstack_networking_secgroup_rule_v2" "bastion_ipv6_ingress" { count = var.number_of_bastions != "" ? length(var.bastion_allowed_remote_ipv6_ips) : 0 direction = "ingress" ethertype = "IPv6" protocol = "tcp" port_range_min = "22" port_range_max = "22" remote_ip_prefix = var.bastion_allowed_remote_ipv6_ips[count.index] security_group_id = openstack_networking_secgroup_v2.bastion[0].id } resource "openstack_networking_secgroup_rule_v2" "k8s_bastion_ports_ipv6_ingress" { count = length(var.bastion_allowed_ports_ipv6) direction = "ingress" ethertype = "IPv6" protocol = lookup(var.bastion_allowed_ports_ipv6[count.index], "protocol", "tcp") port_range_min = lookup(var.bastion_allowed_ports_ipv6[count.index], "port_range_min") port_range_max = lookup(var.bastion_allowed_ports_ipv6[count.index], "port_range_max") remote_ip_prefix = lookup(var.bastion_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0") security_group_id = openstack_networking_secgroup_v2.bastion[0].id } resource "openstack_networking_secgroup_v2" "k8s" { name = "${var.cluster_name}-k8s" description = "${var.cluster_name} - Kubernetes" delete_default_rules = true } resource "openstack_networking_secgroup_rule_v2" "k8s" { direction = "ingress" ethertype = "IPv4" remote_group_id = openstack_networking_secgroup_v2.k8s.id security_group_id = openstack_networking_secgroup_v2.k8s.id } resource "openstack_networking_secgroup_rule_v2" "k8s_ipv6" { direction = "ingress" ethertype = "IPv6" remote_group_id = openstack_networking_secgroup_v2.k8s.id security_group_id = openstack_networking_secgroup_v2.k8s.id } resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips" { count = length(var.k8s_allowed_remote_ips) direction = "ingress" ethertype = "IPv4" protocol = "tcp" port_range_min = "22" port_range_max = "22" remote_ip_prefix = var.k8s_allowed_remote_ips[count.index] security_group_id = openstack_networking_secgroup_v2.k8s.id } resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips_ipv6" { count = length(var.k8s_allowed_remote_ips_ipv6) direction = "ingress" ethertype = "IPv6" protocol = "tcp" port_range_min = "22" port_range_max = "22" remote_ip_prefix = var.k8s_allowed_remote_ips_ipv6[count.index] security_group_id = openstack_networking_secgroup_v2.k8s.id } resource "openstack_networking_secgroup_rule_v2" "egress" { count = length(var.k8s_allowed_egress_ips) direction = "egress" ethertype = "IPv4" remote_ip_prefix = var.k8s_allowed_egress_ips[count.index] security_group_id = openstack_networking_secgroup_v2.k8s.id } resource "openstack_networking_secgroup_rule_v2" "egress_ipv6" { count = length(var.k8s_allowed_egress_ipv6_ips) direction = "egress" ethertype = "IPv6" remote_ip_prefix = var.k8s_allowed_egress_ipv6_ips[count.index] security_group_id = openstack_networking_secgroup_v2.k8s.id } resource "openstack_networking_secgroup_v2" "worker" { name = "${var.cluster_name}-k8s-worker" description = "${var.cluster_name} - Kubernetes worker nodes" delete_default_rules = true } resource "openstack_networking_secgroup_v2" "worker_extra" { count = "%{if var.extra_sec_groups}1%{else}0%{endif}" name = "${var.cluster_name}-k8s-worker-${var.extra_sec_groups_name}" description = "${var.cluster_name} - Kubernetes worker nodes - rules not managed by terraform" delete_default_rules = true } resource "openstack_networking_secgroup_rule_v2" "worker" { count = length(var.worker_allowed_ports) direction = "ingress" ethertype = "IPv4" protocol = lookup(var.worker_allowed_ports[count.index], "protocol", "tcp") port_range_min = lookup(var.worker_allowed_ports[count.index], "port_range_min") port_range_max = lookup(var.worker_allowed_ports[count.index], "port_range_max") remote_ip_prefix = lookup(var.worker_allowed_ports[count.index], "remote_ip_prefix", "0.0.0.0/0") security_group_id = openstack_networking_secgroup_v2.worker.id } resource "openstack_networking_secgroup_rule_v2" "worker_ipv6_ingress" { count = length(var.worker_allowed_ports_ipv6) direction = "ingress" ethertype = "IPv6" protocol = lookup(var.worker_allowed_ports_ipv6[count.index], "protocol", "tcp") port_range_min = lookup(var.worker_allowed_ports_ipv6[count.index], "port_range_min") port_range_max = lookup(var.worker_allowed_ports_ipv6[count.index], "port_range_max") remote_ip_prefix = lookup(var.worker_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0") security_group_id = openstack_networking_secgroup_v2.worker.id } resource "openstack_compute_servergroup_v2" "k8s_master" { count = var.master_server_group_policy != "" ? 1 : 0 name = "k8s-master-srvgrp" policies = [var.master_server_group_policy] } resource "openstack_compute_servergroup_v2" "k8s_node" { count = var.node_server_group_policy != "" ? 1 : 0 name = "k8s-node-srvgrp" policies = [var.node_server_group_policy] } resource "openstack_compute_servergroup_v2" "k8s_etcd" { count = var.etcd_server_group_policy != "" ? 1 : 0 name = "k8s-etcd-srvgrp" policies = [var.etcd_server_group_policy] } resource "openstack_compute_servergroup_v2" "k8s_node_additional" { for_each = var.additional_server_groups name = "k8s-${each.key}-srvgrp" policies = [each.value.policy] } locals { # master groups master_sec_groups = compact([ openstack_networking_secgroup_v2.k8s_master.id, openstack_networking_secgroup_v2.k8s.id, var.extra_sec_groups ?openstack_networking_secgroup_v2.k8s_master_extra[0].id : "", ]) # worker groups worker_sec_groups = compact([ openstack_networking_secgroup_v2.k8s.id, openstack_networking_secgroup_v2.worker.id, var.extra_sec_groups ? openstack_networking_secgroup_v2.worker_extra[0].id : "", ]) # bastion groups bastion_sec_groups = compact(concat([ openstack_networking_secgroup_v2.k8s.id, openstack_networking_secgroup_v2.bastion[0].id, ])) # etcd groups etcd_sec_groups = compact([openstack_networking_secgroup_v2.k8s.id]) # glusterfs groups gfs_sec_groups = compact([openstack_networking_secgroup_v2.k8s.id]) # Image uuid image_to_use_node = var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.vm_image[0].id # Image_gfs uuid image_to_use_gfs = var.image_gfs_uuid != "" ? var.image_gfs_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.gfs_image[0].id # image_master uuidimage_gfs_uuid image_to_use_master = var.image_master_uuid != "" ? var.image_master_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.image_master[0].id k8s_nodes_settings = { for name, node in var.k8s_nodes : name => { "use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb) == 0, "image_id" = node.image_id != null ? node.image_id : local.image_to_use_node, "volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb, "volume_type" = node.volume_type != null ? node.volume_type : var.node_volume_type, "network_id" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id) "server_group" = node.server_group != null ? [openstack_compute_servergroup_v2.k8s_node_additional[node.server_group].id] : (var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : []) } } k8s_masters_settings = { for name, node in var.k8s_masters : name => { "use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb) == 0, "image_id" = node.image_id != null ? node.image_id : local.image_to_use_master, "volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb, "volume_type" = node.volume_type != null ? node.volume_type : var.master_volume_type, "network_id" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id) } } } resource "openstack_networking_port_v2" "bastion_port" { count = var.number_of_bastions name = "${var.cluster_name}-bastion-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.bastion_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "bastion" { name = "${var.cluster_name}-bastion-${count.index + 1}" count = var.number_of_bastions image_id = var.bastion_root_volume_size_in_gb == 0 ? local.image_to_use_node : null flavor_id = var.flavor_bastion key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.bastion_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : [] content { uuid = local.image_to_use_node source_type = "image" volume_size = var.bastion_root_volume_size_in_gb boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.bastion_port.*.id, count.index) } metadata = { ssh_user = var.ssh_user kubespray_groups = "bastion" depends_on = var.network_router_id use_access_ip = var.use_access_ip } provisioner "local-exec" { command = "sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${var.bastion_fips[0]}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml" } } resource "openstack_networking_port_v2" "k8s_master_port" { count = var.number_of_k8s_masters name = "${var.cluster_name}-k8s-master-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_master" { name = "${var.cluster_name}-k8s-master-${count.index + 1}" count = var.number_of_k8s_masters availability_zone = element(var.az_list, count.index) image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null flavor_id = var.flavor_k8s_master key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] content { uuid = local.image_to_use_master source_type = "image" volume_size = var.master_root_volume_size_in_gb volume_type = var.master_volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_master[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "etcd,kube_control_plane,${var.supplementary_master_groups},k8s_cluster" depends_on = var.network_router_id use_access_ip = var.use_access_ip } provisioner "local-exec" { command = "sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml" } } resource "openstack_networking_port_v2" "k8s_masters_port" { for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {} name = "${var.cluster_name}-k8s-${each.key}" network_id = local.k8s_masters_settings[each.key].network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_masters" { for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {} name = "${var.cluster_name}-k8s-${each.key}" availability_zone = each.value.az image_id = local.k8s_masters_settings[each.key].use_local_disk ? local.k8s_masters_settings[each.key].image_id : null flavor_id = each.value.flavor key_pair = openstack_compute_keypair_v2.k8s.name dynamic "block_device" { for_each = !local.k8s_masters_settings[each.key].use_local_disk ? [local.k8s_masters_settings[each.key].image_id] : [] content { uuid = block_device.value source_type = "image" volume_size = local.k8s_masters_settings[each.key].volume_size volume_type = local.k8s_masters_settings[each.key].volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = openstack_networking_port_v2.k8s_masters_port[each.key].id } dynamic "scheduler_hints" { for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_master[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "%{if each.value.etcd == true}etcd,%{endif}kube_control_plane,${var.supplementary_master_groups},k8s_cluster%{if each.value.floating_ip == false},no_floating%{endif}" depends_on = var.network_router_id use_access_ip = var.use_access_ip } provisioner "local-exec" { command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.module}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_masters_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}" } } resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" { count = var.number_of_k8s_masters_no_etcd name = "${var.cluster_name}-k8s-master-ne-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_master_no_etcd" { name = "${var.cluster_name}-k8s-master-ne-${count.index + 1}" count = var.number_of_k8s_masters_no_etcd availability_zone = element(var.az_list, count.index) image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null flavor_id = var.flavor_k8s_master key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] content { uuid = local.image_to_use_master source_type = "image" volume_size = var.master_root_volume_size_in_gb volume_type = var.master_volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.k8s_master_no_etcd_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_master[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "kube_control_plane,${var.supplementary_master_groups},k8s_cluster" depends_on = var.network_router_id use_access_ip = var.use_access_ip } provisioner "local-exec" { command = "sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml" } } resource "openstack_networking_port_v2" "etcd_port" { count = var.number_of_etcd name = "${var.cluster_name}-etcd-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.etcd_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "etcd" { name = "${var.cluster_name}-etcd-${count.index + 1}" count = var.number_of_etcd availability_zone = element(var.az_list, count.index) image_id = var.etcd_root_volume_size_in_gb == 0 ? local.image_to_use_master : null flavor_id = var.flavor_etcd key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.etcd_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] content { uuid = local.image_to_use_master source_type = "image" volume_size = var.etcd_root_volume_size_in_gb boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.etcd_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.etcd_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_etcd[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_etcd[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "etcd,no_floating" depends_on = var.network_router_id use_access_ip = var.use_access_ip } } resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_port" { count = var.number_of_k8s_masters_no_floating_ip name = "${var.cluster_name}-k8s-master-nf-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" { name = "${var.cluster_name}-k8s-master-nf-${count.index + 1}" count = var.number_of_k8s_masters_no_floating_ip availability_zone = element(var.az_list, count.index) image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null flavor_id = var.flavor_k8s_master key_pair = openstack_compute_keypair_v2.k8s.name dynamic "block_device" { for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] content { uuid = local.image_to_use_master source_type = "image" volume_size = var.master_root_volume_size_in_gb volume_type = var.master_volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.k8s_master_no_floating_ip_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_master[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "etcd,kube_control_plane,${var.supplementary_master_groups},k8s_cluster,no_floating" depends_on = var.network_router_id use_access_ip = var.use_access_ip } } resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_no_etcd_port" { count = var.number_of_k8s_masters_no_floating_ip_no_etcd name = "${var.cluster_name}-k8s-master-ne-nf-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" { name = "${var.cluster_name}-k8s-master-ne-nf-${count.index + 1}" count = var.number_of_k8s_masters_no_floating_ip_no_etcd availability_zone = element(var.az_list, count.index) image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null flavor_id = var.flavor_k8s_master key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] content { uuid = local.image_to_use_master source_type = "image" volume_size = var.master_root_volume_size_in_gb volume_type = var.master_volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.k8s_master_no_floating_ip_no_etcd_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.master_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_master[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_master[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "kube_control_plane,${var.supplementary_master_groups},k8s_cluster,no_floating" depends_on = var.network_router_id use_access_ip = var.use_access_ip } } resource "openstack_networking_port_v2" "k8s_node_port" { count = var.number_of_k8s_nodes name = "${var.cluster_name}-k8s-node-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_node" { name = "${var.cluster_name}-k8s-node-${count.index + 1}" count = var.number_of_k8s_nodes availability_zone = element(var.az_list_node, count.index) image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null flavor_id = var.flavor_k8s_node key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : [] content { uuid = local.image_to_use_node source_type = "image" volume_size = var.node_root_volume_size_in_gb volume_type = var.node_volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_node[0].id } } metadata = { ssh_user = var.ssh_user kubespray_groups = "kube_node,k8s_cluster,${var.supplementary_node_groups}" depends_on = var.network_router_id use_access_ip = var.use_access_ip } provisioner "local-exec" { command = "sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml" } } resource "openstack_networking_port_v2" "k8s_node_no_floating_ip_port" { count = var.number_of_k8s_nodes_no_floating_ip name = "${var.cluster_name}-k8s-node-nf-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" { name = "${var.cluster_name}-k8s-node-nf-${count.index + 1}" count = var.number_of_k8s_nodes_no_floating_ip availability_zone = element(var.az_list_node, count.index) image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null flavor_id = var.flavor_k8s_node key_pair = openstack_compute_keypair_v2.k8s.name user_data = data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : [] content { uuid = local.image_to_use_node source_type = "image" volume_size = var.node_root_volume_size_in_gb volume_type = var.node_volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.k8s_node_no_floating_ip_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : [] content { group = scheduler_hints.value } } metadata = { ssh_user = var.ssh_user kubespray_groups = "kube_node,k8s_cluster,no_floating,${var.supplementary_node_groups}" depends_on = var.network_router_id use_access_ip = var.use_access_ip } } resource "openstack_networking_port_v2" "k8s_nodes_port" { for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {} name = "${var.cluster_name}-k8s-node-${each.key}" network_id = local.k8s_nodes_settings[each.key].network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } lifecycle { ignore_changes = [ allowed_address_pairs ] } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "k8s_nodes" { for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {} name = "${var.cluster_name}-k8s-node-${each.key}" availability_zone = each.value.az image_id = local.k8s_nodes_settings[each.key].use_local_disk ? local.k8s_nodes_settings[each.key].image_id : null flavor_id = each.value.flavor key_pair = openstack_compute_keypair_v2.k8s.name user_data = each.value.cloudinit != null ? templatefile("${path.module}/templates/cloudinit.yaml.tmpl", { extra_partitions = each.value.cloudinit.extra_partitions, netplan_critical_dhcp_interface = each.value.cloudinit.netplan_critical_dhcp_interface, }) : data.cloudinit_config.cloudinit.rendered dynamic "block_device" { for_each = !local.k8s_nodes_settings[each.key].use_local_disk ? [local.k8s_nodes_settings[each.key].image_id] : [] content { uuid = block_device.value source_type = "image" volume_size = local.k8s_nodes_settings[each.key].volume_size volume_type = local.k8s_nodes_settings[each.key].volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = openstack_networking_port_v2.k8s_nodes_port[each.key].id } dynamic "scheduler_hints" { for_each = local.k8s_nodes_settings[each.key].server_group content { group = scheduler_hints.value } } metadata = { ssh_user = var.ssh_user kubespray_groups = "kube_node,k8s_cluster,%{if !each.value.floating_ip}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}" depends_on = var.network_router_id use_access_ip = var.use_access_ip } provisioner "local-exec" { command = "%{if each.value.floating_ip}sed -e s/USER/${var.ssh_user}/ -e s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ ${path.module}/ansible_bastion_template.txt > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}" } } resource "openstack_networking_port_v2" "glusterfs_node_no_floating_ip_port" { count = var.number_of_gfs_nodes_no_floating_ip name = "${var.cluster_name}-gfs-node-nf-${count.index + 1}" network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id admin_state_up = "true" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.gfs_sec_groups : null no_security_groups = var.port_security_enabled ? null : false dynamic "fixed_ip" { for_each = var.private_subnet_id == "" ? [] : [true] content { subnet_id = var.private_subnet_id } } depends_on = [ var.network_router_id ] } resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" { name = "${var.cluster_name}-gfs-node-nf-${count.index + 1}" count = var.number_of_gfs_nodes_no_floating_ip availability_zone = element(var.az_list, count.index) image_id = var.gfs_root_volume_size_in_gb == 0 ? local.image_to_use_gfs : null flavor_id = var.flavor_gfs_node key_pair = openstack_compute_keypair_v2.k8s.name dynamic "block_device" { for_each = var.gfs_root_volume_size_in_gb > 0 ? [local.image_to_use_gfs] : [] content { uuid = local.image_to_use_gfs source_type = "image" volume_size = var.gfs_root_volume_size_in_gb boot_index = 0 destination_type = "volume" delete_on_termination = true } } network { port = element(openstack_networking_port_v2.glusterfs_node_no_floating_ip_port.*.id, count.index) } dynamic "scheduler_hints" { for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : [] content { group = openstack_compute_servergroup_v2.k8s_node[0].id } } metadata = { ssh_user = var.ssh_user_gfs kubespray_groups = "gfs-cluster,network-storage,no_floating" depends_on = var.network_router_id use_access_ip = var.use_access_ip } } resource "openstack_networking_floatingip_associate_v2" "bastion" { count = var.number_of_bastions floating_ip = var.bastion_fips[count.index] port_id = element(openstack_networking_port_v2.bastion_port.*.id, count.index) } resource "openstack_networking_floatingip_associate_v2" "k8s_master" { count = var.number_of_k8s_masters floating_ip = var.k8s_master_fips[count.index] port_id = element(openstack_networking_port_v2.k8s_master_port.*.id, count.index) } resource "openstack_networking_floatingip_associate_v2" "k8s_masters" { for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {} floating_ip = var.k8s_masters_fips[each.key].address port_id = openstack_networking_port_v2.k8s_masters_port[each.key].id } resource "openstack_networking_floatingip_associate_v2" "k8s_master_no_etcd" { count = var.master_root_volume_size_in_gb == 0 ? var.number_of_k8s_masters_no_etcd : 0 floating_ip = var.k8s_master_no_etcd_fips[count.index] port_id = element(openstack_networking_port_v2.k8s_master_no_etcd_port.*.id, count.index) } resource "openstack_networking_floatingip_associate_v2" "k8s_node" { count = var.node_root_volume_size_in_gb == 0 ? var.number_of_k8s_nodes : 0 floating_ip = var.k8s_node_fips[count.index] port_id = element(openstack_networking_port_v2.k8s_node_port.*.id, count.index) } resource "openstack_networking_floatingip_associate_v2" "k8s_nodes" { for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {} floating_ip = var.k8s_nodes_fips[each.key].address port_id = openstack_networking_port_v2.k8s_nodes_port[each.key].id } resource "openstack_blockstorage_volume_v3" "glusterfs_volume" { name = "${var.cluster_name}-glusterfs_volume-${count.index + 1}" count = var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0 description = "Non-ephemeral volume for GlusterFS" size = var.gfs_volume_size_in_gb } resource "openstack_compute_volume_attach_v2" "glusterfs_volume" { count = var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0 instance_id = element(openstack_compute_instance_v2.glusterfs_node_no_floating_ip.*.id, count.index) volume_id = element(openstack_blockstorage_volume_v3.glusterfs_volume.*.id, count.index) } ================================================ FILE: contrib/terraform/openstack/modules/compute/outputs.tf ================================================ output "k8s_master_ips" { value = concat(openstack_compute_instance_v2.k8s_master_no_floating_ip.*, openstack_compute_instance_v2.k8s_master_no_floating_ip_no_etcd.*) } ================================================ FILE: contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml.tmpl ================================================ %{~ if length(extra_partitions) > 0 || netplan_critical_dhcp_interface != "" } #cloud-config bootcmd: %{~ for idx, partition in extra_partitions } - [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, ${partition.volume_path} ] - [ cloud-init-per, once, create-part-${idx}, parted, --script, ${partition.volume_path}, 'mkpart extended ext4 ${partition.partition_start} ${partition.partition_end}' ] - [ cloud-init-per, once, create-fs-part-${idx}, mkfs.ext4, ${partition.partition_path} ] %{~ endfor } runcmd: %{~ if netplan_critical_dhcp_interface != "" } - netplan apply %{~ endif } %{~ for idx, partition in extra_partitions } - mkdir -p ${partition.mount_path} - chown nobody:nogroup ${partition.mount_path} - mount ${partition.partition_path} ${partition.mount_path} %{~ endfor ~} %{~ if netplan_critical_dhcp_interface != "" } write_files: - path: /etc/netplan/90-critical-dhcp.yaml content: | network: version: 2 ethernets: ${ netplan_critical_dhcp_interface }: dhcp4: true critical: true %{~ endif } mounts: %{~ for idx, partition in extra_partitions } - [ ${partition.partition_path}, ${partition.mount_path} ] %{~ endfor } %{~ else ~} # yamllint disable rule:comments #cloud-config ## in some cases novnc console access is required ## it requires ssh password to be set #ssh_pwauth: yes #chpasswd: # list: | # root:secret # expire: False ## in some cases direct root ssh access via ssh key is required #disable_root: false ## in some cases additional CA certs are required #ca-certs: # trusted: | # -----BEGIN CERTIFICATE----- %{~ endif } ================================================ FILE: contrib/terraform/openstack/modules/compute/variables.tf ================================================ variable "cluster_name" {} variable "az_list" { type = list(string) } variable "az_list_node" { type = list(string) } variable "number_of_k8s_masters" {} variable "number_of_k8s_masters_no_etcd" {} variable "number_of_etcd" {} variable "number_of_k8s_masters_no_floating_ip" {} variable "number_of_k8s_masters_no_floating_ip_no_etcd" {} variable "number_of_k8s_nodes" {} variable "number_of_k8s_nodes_no_floating_ip" {} variable "number_of_bastions" {} variable "number_of_gfs_nodes_no_floating_ip" {} variable "bastion_root_volume_size_in_gb" {} variable "etcd_root_volume_size_in_gb" {} variable "master_root_volume_size_in_gb" {} variable "node_root_volume_size_in_gb" {} variable "gfs_root_volume_size_in_gb" {} variable "gfs_volume_size_in_gb" {} variable "master_volume_type" {} variable "node_volume_type" {} variable "public_key_path" {} variable "image" {} variable "image_gfs" {} variable "ssh_user" {} variable "ssh_user_gfs" {} variable "flavor_k8s_master" {} variable "flavor_k8s_node" {} variable "flavor_etcd" {} variable "flavor_gfs_node" {} variable "network_name" {} variable "flavor_bastion" {} variable "network_id" { default = "" } variable "use_existing_network" { type = bool } variable "network_router_id" { default = "" } variable "k8s_master_fips" { type = list } variable "k8s_master_no_etcd_fips" { type = list } variable "k8s_node_fips" { type = list } variable "k8s_masters_fips" { type = map(object({ address = string })) } variable "k8s_nodes_fips" { type = map(object({ address = string })) } variable "bastion_fips" { type = list } variable "bastion_allowed_remote_ips" { type = list } variable "bastion_allowed_remote_ipv6_ips" { type = list } variable "master_allowed_remote_ips" { type = list } variable "master_allowed_remote_ipv6_ips" { type = list } variable "k8s_allowed_remote_ips" { type = list } variable "k8s_allowed_remote_ips_ipv6" { type = list } variable "k8s_allowed_egress_ips" { type = list } variable "k8s_allowed_egress_ipv6_ips" { type = list } variable "k8s_masters" { type = map(object({ az = string flavor = string etcd = bool floating_ip = bool reserved_floating_ip = optional(string) image_id = optional(string) root_volume_size_in_gb = optional(number) volume_type = optional(string) network_id = optional(string) })) } variable "k8s_nodes" { type = map(object({ az = string flavor = string floating_ip = bool reserved_floating_ip = optional(string) extra_groups = optional(string) image_id = optional(string) root_volume_size_in_gb = optional(number) volume_type = optional(string) network_id = optional(string) additional_server_groups = optional(list(string)) server_group = optional(string) cloudinit = optional(object({ extra_partitions = optional(list(object({ volume_path = string partition_path = string partition_start = string partition_end = string mount_path = string })), []) netplan_critical_dhcp_interface = optional(string, "") })) })) } variable "additional_server_groups" { type = map(object({ policy = string })) } variable "supplementary_master_groups" { default = "" } variable "supplementary_node_groups" { default = "" } variable "master_allowed_ports" { type = list } variable "master_allowed_ports_ipv6" { type = list } variable "worker_allowed_ports" { type = list } variable "worker_allowed_ports_ipv6" { type = list } variable "bastion_allowed_ports" { type = list } variable "bastion_allowed_ports_ipv6" { type = list } variable "use_access_ip" {} variable "master_server_group_policy" { type = string } variable "node_server_group_policy" { type = string } variable "etcd_server_group_policy" { type = string } variable "extra_sec_groups" { type = bool } variable "extra_sec_groups_name" { type = string } variable "image_uuid" { type = string } variable "image_gfs_uuid" { type = string } variable "image_master" { type = string } variable "image_master_uuid" { type = string } variable "group_vars_path" { type = string } variable "port_security_enabled" { type = bool } variable "force_null_port_security" { type = bool } variable "private_subnet_id" { type = string } ================================================ FILE: contrib/terraform/openstack/modules/compute/versions.tf ================================================ terraform { required_providers { openstack = { source = "terraform-provider-openstack/openstack" } } required_version = ">= 1.3.0" } ================================================ FILE: contrib/terraform/openstack/modules/ips/main.tf ================================================ resource "null_resource" "dummy_dependency" { triggers = { dependency_id = var.router_id } depends_on = [ var.router_internal_port_id ] } # If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones. resource "openstack_networking_floatingip_v2" "k8s_master" { count = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } resource "openstack_networking_floatingip_v2" "k8s_masters" { for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : tomap({}) pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } # If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones. resource "openstack_networking_floatingip_v2" "k8s_master_no_etcd" { count = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters_no_etcd pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } resource "openstack_networking_floatingip_v2" "k8s_node" { count = var.number_of_k8s_nodes pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } resource "openstack_networking_floatingip_v2" "bastion" { count = length(var.bastion_fips) > 0 ? 0 : var.number_of_bastions pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } resource "openstack_networking_floatingip_v2" "k8s_nodes" { for_each = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : tomap({}) pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } ================================================ FILE: contrib/terraform/openstack/modules/ips/outputs.tf ================================================ locals { k8s_masters_reserved_fips = { for key, value in var.k8s_masters : key => { address = value.reserved_floating_ip } if value.floating_ip && (lookup(value, "reserved_floating_ip", "") != "") } k8s_masters_create_fips = { for key, value in openstack_networking_floatingip_v2.k8s_masters : key => { address = value.address } } k8s_nodes_reserved_fips = { for key, value in var.k8s_nodes : key => { address = value.reserved_floating_ip } if value.floating_ip && (lookup(value, "reserved_floating_ip", "") != "") } k8s_nodes_create_fips = { for key, value in openstack_networking_floatingip_v2.k8s_nodes : key => { address = value.address } } } # If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created. output "k8s_master_fips" { value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master[*].address } output "k8s_masters_fips" { value = merge(local.k8s_masters_create_fips, local.k8s_masters_reserved_fips) } # If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created. output "k8s_master_no_etcd_fips" { value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master_no_etcd[*].address } output "k8s_node_fips" { value = openstack_networking_floatingip_v2.k8s_node[*].address } output "k8s_nodes_fips" { value = merge(local.k8s_nodes_create_fips, local.k8s_nodes_reserved_fips) } output "bastion_fips" { value = length(var.bastion_fips) > 0 ? var.bastion_fips : openstack_networking_floatingip_v2.bastion[*].address } ================================================ FILE: contrib/terraform/openstack/modules/ips/variables.tf ================================================ variable "number_of_k8s_masters" {} variable "number_of_k8s_masters_no_etcd" {} variable "number_of_k8s_nodes" {} variable "floatingip_pool" {} variable "number_of_bastions" {} variable "external_net" {} variable "network_name" {} variable "router_id" { default = "" } variable "k8s_masters" {} variable "k8s_nodes" {} variable "k8s_master_fips" {} variable "bastion_fips" {} variable "router_internal_port_id" {} ================================================ FILE: contrib/terraform/openstack/modules/ips/versions.tf ================================================ terraform { required_providers { null = { source = "hashicorp/null" } openstack = { source = "terraform-provider-openstack/openstack" } } required_version = ">= 0.12.26" } ================================================ FILE: contrib/terraform/openstack/modules/loadbalancer/main.tf ================================================ resource "openstack_lb_loadbalancer_v2" "k8s_lb" { count = var.k8s_master_loadbalancer_enabled ? 1 : 0 name = "${var.cluster_name}-api-loadbalancer" vip_subnet_id = var.subnet_id } resource "openstack_lb_listener_v2" "api_listener"{ count = var.k8s_master_loadbalancer_enabled ? 1 : 0 name = "api-listener" protocol = "TCP" protocol_port = var.k8s_master_loadbalancer_listener_port loadbalancer_id = openstack_lb_loadbalancer_v2.k8s_lb[0].id depends_on = [ openstack_lb_loadbalancer_v2.k8s_lb ] } resource "openstack_lb_pool_v2" "api_pool" { count = var.k8s_master_loadbalancer_enabled ? 1 : 0 name = "api-pool" protocol = "TCP" lb_method = "ROUND_ROBIN" listener_id = openstack_lb_listener_v2.api_listener[0].id depends_on = [ openstack_lb_listener_v2.api_listener ] } resource "openstack_lb_member_v2" "lb_member" { count = var.k8s_master_loadbalancer_enabled ? length(var.k8s_master_ips) : 0 name = var.k8s_master_ips[count.index].name pool_id = openstack_lb_pool_v2.api_pool[0].id address = var.k8s_master_ips[count.index].access_ip_v4 protocol_port = var.k8s_master_loadbalancer_server_port depends_on = [ openstack_lb_pool_v2.api_pool ] } resource "openstack_lb_monitor_v2" "monitor" { count = var.k8s_master_loadbalancer_enabled ? 1 : 0 name = "Api Monitor" pool_id = openstack_lb_pool_v2.api_pool[0].id type = "TCP" delay = 10 timeout = 5 max_retries = 5 } resource "openstack_networking_floatingip_v2" "floatip_1" { count = var.k8s_master_loadbalancer_enabled && var.k8s_master_loadbalancer_public_ip == "" ? 1 : 0 pool = var.floatingip_pool } resource "openstack_networking_floatingip_associate_v2" "public_ip" { count = var.k8s_master_loadbalancer_enabled ? 1 : 0 floating_ip = var.k8s_master_loadbalancer_public_ip != "" ? var.k8s_master_loadbalancer_public_ip : openstack_networking_floatingip_v2.floatip_1[0].address port_id = openstack_lb_loadbalancer_v2.k8s_lb[0].vip_port_id depends_on = [ openstack_lb_loadbalancer_v2.k8s_lb ] } ================================================ FILE: contrib/terraform/openstack/modules/loadbalancer/variables.tf ================================================ variable "cluster_name" {} variable "subnet_id" {} variable "floatingip_pool" {} variable "k8s_master_ips" {} variable "k8s_master_loadbalancer_enabled" {} variable "k8s_master_loadbalancer_listener_port" {} variable "k8s_master_loadbalancer_server_port" {} variable "k8s_master_loadbalancer_public_ip" {} ================================================ FILE: contrib/terraform/openstack/modules/loadbalancer/versions.tf ================================================ terraform { required_providers { openstack = { source = "terraform-provider-openstack/openstack" } } required_version = ">= 0.12.26" } ================================================ FILE: contrib/terraform/openstack/modules/network/main.tf ================================================ resource "openstack_networking_router_v2" "k8s" { name = "${var.cluster_name}-router" count = var.use_neutron == 1 && var.router_id == null ? 1 : 0 admin_state_up = "true" external_network_id = var.external_net } data "openstack_networking_router_v2" "k8s" { router_id = var.router_id count = var.use_neutron == 1 && var.router_id != null ? 1 : 0 } resource "openstack_networking_network_v2" "k8s" { name = var.network_name count = var.use_neutron dns_domain = var.network_dns_domain != null ? var.network_dns_domain : null admin_state_up = "true" port_security_enabled = var.port_security_enabled } resource "openstack_networking_subnet_v2" "k8s" { name = "${var.cluster_name}-internal-network" count = var.use_neutron network_id = openstack_networking_network_v2.k8s[count.index].id cidr = var.subnet_cidr ip_version = 4 dns_nameservers = var.dns_nameservers } resource "openstack_networking_router_interface_v2" "k8s" { count = var.use_neutron router_id = "%{if openstack_networking_router_v2.k8s != []}${openstack_networking_router_v2.k8s[count.index].id}%{else}${var.router_id}%{endif}" subnet_id = openstack_networking_subnet_v2.k8s[count.index].id } ================================================ FILE: contrib/terraform/openstack/modules/network/outputs.tf ================================================ output "router_id" { value = "%{if var.use_neutron == 1} ${var.router_id == null ? element(concat(openstack_networking_router_v2.k8s.*.id, [""]), 0) : var.router_id} %{else} %{endif}" } output "network_id" { value = element(concat(openstack_networking_network_v2.k8s.*.id, [""]),0) } output "router_internal_port_id" { value = element(concat(openstack_networking_router_interface_v2.k8s.*.id, [""]), 0) } output "subnet_id" { value = element(concat(openstack_networking_subnet_v2.k8s.*.id, [""]), 0) } ================================================ FILE: contrib/terraform/openstack/modules/network/variables.tf ================================================ variable "external_net" {} variable "network_name" {} variable "network_dns_domain" {} variable "cluster_name" {} variable "dns_nameservers" { type = list } variable "port_security_enabled" { type = bool } variable "subnet_cidr" {} variable "use_neutron" {} variable "router_id" {} ================================================ FILE: contrib/terraform/openstack/modules/network/versions.tf ================================================ terraform { required_providers { openstack = { source = "terraform-provider-openstack/openstack" } } required_version = ">= 0.12.26" } ================================================ FILE: contrib/terraform/openstack/variables.tf ================================================ variable "cluster_name" { default = "example" } variable "az_list" { description = "List of Availability Zones to use for masters in your OpenStack cluster" type = list(string) default = ["nova"] } variable "az_list_node" { description = "List of Availability Zones to use for nodes in your OpenStack cluster" type = list(string) default = ["nova"] } variable "number_of_bastions" { default = 1 } variable "number_of_k8s_masters" { default = 2 } variable "number_of_k8s_masters_no_etcd" { default = 2 } variable "number_of_etcd" { default = 2 } variable "number_of_k8s_masters_no_floating_ip" { default = 2 } variable "number_of_k8s_masters_no_floating_ip_no_etcd" { default = 2 } variable "number_of_k8s_nodes" { default = 1 } variable "number_of_k8s_nodes_no_floating_ip" { default = 1 } variable "number_of_gfs_nodes_no_floating_ip" { default = 0 } variable "bastion_root_volume_size_in_gb" { default = 0 } variable "etcd_root_volume_size_in_gb" { default = 0 } variable "master_root_volume_size_in_gb" { default = 0 } variable "node_root_volume_size_in_gb" { default = 0 } variable "gfs_root_volume_size_in_gb" { default = 0 } variable "gfs_volume_size_in_gb" { default = 75 } variable "master_volume_type" { default = "Default" } variable "node_volume_type" { default = "Default" } variable "public_key_path" { description = "The path of the ssh pub key" default = "~/.ssh/id_rsa.pub" } variable "image" { description = "the image to use" default = "" } variable "image_gfs" { description = "Glance image to use for GlusterFS" default = "" } variable "ssh_user" { description = "used to fill out tags for ansible inventory" default = "ubuntu" } variable "ssh_user_gfs" { description = "used to fill out tags for ansible inventory" default = "ubuntu" } variable "flavor_bastion" { description = "Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs" default = 3 } variable "flavor_k8s_master" { description = "Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs" default = 3 } variable "flavor_k8s_node" { description = "Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs" default = 3 } variable "flavor_etcd" { description = "Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs" default = 3 } variable "flavor_gfs_node" { description = "Use 'openstack flavor list' command to see what your OpenStack instance uses for IDs" default = 3 } variable "network_name" { description = "name of the internal network to use" default = "internal" } variable "use_existing_network" { description = "Use an existing network" type = bool default = "false" } variable "network_dns_domain" { description = "dns_domain for the internal network" type = string default = null } variable "use_neutron" { description = "Use neutron" default = 1 } variable "port_security_enabled" { description = "Enable port security on the internal network" type = bool default = "true" } variable "force_null_port_security" { description = "Force port security to be null. Some providers does not allow setting port security" type = bool default = "false" } variable "subnet_cidr" { description = "Subnet CIDR block." type = string default = "10.0.0.0/24" } variable "dns_nameservers" { description = "An array of DNS name server names used by hosts in this subnet." type = list(string) default = [] } variable "k8s_master_fips" { description = "specific pre-existing floating IPs to use for master nodes" type = list(string) default = [] } variable "bastion_fips" { description = "specific pre-existing floating IPs to use for bastion node" type = list(string) default = [] } variable "floatingip_pool" { description = "name of the floating ip pool to use" default = "external" } variable "wait_for_floatingip" { description = "Terraform will poll the instance until the floating IP has been associated." default = "false" } variable "external_net" { description = "uuid of the external/public network" } variable "supplementary_master_groups" { description = "supplementary kubespray ansible groups for masters, such kube_node" default = "" } variable "supplementary_node_groups" { description = "supplementary kubespray ansible groups for worker nodes, such as kube_ingress" default = "" } variable "bastion_allowed_remote_ips" { description = "An array of CIDRs allowed to SSH to hosts" type = list(string) default = ["0.0.0.0/0"] } variable "bastion_allowed_remote_ipv6_ips" { description = "An array of IPv6 CIDRs allowed to SSH to hosts" type = list(string) default = ["::/0"] } variable "master_allowed_remote_ips" { description = "An array of CIDRs allowed to access API of masters" type = list(string) default = ["0.0.0.0/0"] } variable "master_allowed_remote_ipv6_ips" { description = "An array of IPv6 CIDRs allowed to access API of masters" type = list(string) default = ["::/0"] } variable "k8s_allowed_remote_ips" { description = "An array of CIDRs allowed to SSH to hosts" type = list(string) default = [] } variable "k8s_allowed_remote_ips_ipv6" { description = "An array of IPv6 CIDRs allowed to SSH to hosts" type = list(string) default = [] } variable "k8s_allowed_egress_ips" { description = "An array of CIDRs allowed for egress traffic" type = list(string) default = ["0.0.0.0/0"] } variable "k8s_allowed_egress_ipv6_ips" { description = "An array of CIDRs allowed for egress IPv6 traffic" type = list(string) default = ["::/0"] } variable "master_allowed_ports" { type = list(any) default = [] } variable "master_allowed_ports_ipv6" { type = list(any) default = [ { "protocol" = "ipv6-icmp" "port_range_min" = 0 "port_range_max" = 0 "remote_ip_prefix" = "::/0" }, ] } variable "worker_allowed_ports" { type = list(any) default = [ { "protocol" = "tcp" "port_range_min" = 30000 "port_range_max" = 32767 "remote_ip_prefix" = "0.0.0.0/0" }, ] } variable "worker_allowed_ports_ipv6" { type = list(any) default = [ { "protocol" = "tcp" "port_range_min" = 30000 "port_range_max" = 32767 "remote_ip_prefix" = "::/0" }, { "protocol" = "ipv6-icmp" "port_range_min" = 0 "port_range_max" = 0 "remote_ip_prefix" = "::/0" }, ] } variable "bastion_allowed_ports" { type = list(any) default = [] } variable "bastion_allowed_ports_ipv6" { type = list(any) default = [] } variable "use_access_ip" { default = 1 } variable "master_server_group_policy" { description = "desired server group policy, e.g. anti-affinity" default = "" } variable "node_server_group_policy" { description = "desired server group policy, e.g. anti-affinity" default = "" } variable "etcd_server_group_policy" { description = "desired server group policy, e.g. anti-affinity" default = "" } variable "router_id" { description = "uuid of an externally defined router to use" default = null } variable "router_internal_port_id" { description = "uuid of the port connection our router to our network" default = null } variable "k8s_masters" { default = {} } variable "k8s_nodes" { default = {} } variable "additional_server_groups" { default = {} type = map(object({ policy = string })) } variable "extra_sec_groups" { default = false } variable "extra_sec_groups_name" { default = "custom" } variable "image_uuid" { description = "uuid of image inside openstack to use" default = "" } variable "image_gfs_uuid" { description = "uuid of image to be used on gluster fs nodes. If empty defaults to image_uuid" default = "" } variable "image_master" { description = "uuid of image inside openstack to use" default = "" } variable "image_master_uuid" { description = "uuid of image to be used on master nodes. If empty defaults to image_uuid" default = "" } variable "group_vars_path" { description = "path to the inventory group vars directory" type = string default = "./group_vars" } variable "k8s_master_loadbalancer_enabled" { type = bool default = "false" } variable "k8s_master_loadbalancer_listener_port" { type = string default = "6443" } variable "k8s_master_loadbalancer_server_port" { type = string default = 6443 } variable "k8s_master_loadbalancer_public_ip" { type = string default = "" } ================================================ FILE: contrib/terraform/openstack/versions.tf ================================================ terraform { required_providers { openstack = { source = "terraform-provider-openstack/openstack" version = "~> 1.17" } } required_version = ">= 1.3.0" } ================================================ FILE: contrib/terraform/terraform.py ================================================ #!/usr/bin/env python3 # # Copyright 2015 Cisco Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # original: https://github.com/CiscoCloud/terraform.py """\ Dynamic inventory for Terraform - finds all `.tfstate` files below the working directory and generates an inventory based on them. """ import argparse from collections import defaultdict import random from functools import wraps import json import os import re VERSION = '0.4.0pre' def tfstates(root=None): root = root or os.getcwd() for dirpath, _, filenames in os.walk(root): for name in filenames: if os.path.splitext(name)[-1] == '.tfstate': yield os.path.join(dirpath, name) def convert_to_v3_structure(attributes, prefix=''): """ Convert the attributes from v4 to v3 Receives a dict and return a dictionary """ result = {} if isinstance(attributes, str): # In the case when we receive a string (e.g. values for security_groups) return {'{}{}'.format(prefix, random.randint(1,10**10)): attributes} for key, value in attributes.items(): if isinstance(value, list): if len(value): result['{}{}.#'.format(prefix, key, hash)] = len(value) for i, v in enumerate(value): result.update(convert_to_v3_structure(v, '{}{}.{}.'.format(prefix, key, i))) elif isinstance(value, dict): result['{}{}.%'.format(prefix, key)] = len(value) for k, v in value.items(): result['{}{}.{}'.format(prefix, key, k)] = v else: result['{}{}'.format(prefix, key)] = value return result def iterresources(filenames): for filename in filenames: with open(filename, 'r') as json_file: state = json.load(json_file) tf_version = state['version'] if tf_version == 3: for module in state['modules']: name = module['path'][-1] for key, resource in module['resources'].items(): yield name, key, resource elif tf_version == 4: # In version 4 the structure changes so we need to iterate # each instance inside the resource branch. for resource in state['resources']: name = resource['provider'].split('.')[-1] for instance in resource['instances']: key = "{}.{}".format(resource['type'], resource['name']) if 'index_key' in instance: key = "{}.{}".format(key, instance['index_key']) data = {} data['type'] = resource['type'] data['provider'] = resource['provider'] data['depends_on'] = instance.get('depends_on', []) data['primary'] = {'attributes': convert_to_v3_structure(instance['attributes'])} if 'id' in instance['attributes']: data['primary']['id'] = instance['attributes']['id'] data['primary']['meta'] = instance['attributes'].get('meta',{}) yield name, key, data else: raise KeyError('tfstate version %d not supported' % tf_version) ## READ RESOURCES PARSERS = {} def _clean_dc(dcname): # Consul DCs are strictly alphanumeric with underscores and hyphens - # ensure that the consul_dc attribute meets these requirements. return re.sub(r'[^\w_\-]', '-', dcname) def iterhosts(resources): '''yield host tuples of (name, attributes, groups)''' for module_name, key, resource in resources: resource_type, name = key.split('.', 1) try: parser = PARSERS[resource_type] except KeyError: continue yield parser(resource, module_name) def iterips(resources): '''yield ip tuples of (port_id, ip)''' for module_name, key, resource in resources: resource_type, name = key.split('.', 1) if resource_type == 'openstack_networking_floatingip_associate_v2': yield openstack_floating_ips(resource) def parses(prefix): def inner(func): PARSERS[prefix] = func return func return inner def calculate_mantl_vars(func): """calculate Mantl vars""" @wraps(func) def inner(*args, **kwargs): name, attrs, groups = func(*args, **kwargs) # attrs if attrs.get('role', '') == 'control': attrs['consul_is_server'] = True else: attrs['consul_is_server'] = False # groups if attrs.get('publicly_routable', False): groups.append('publicly_routable') return name, attrs, groups return inner def _parse_prefix(source, prefix, sep='.'): for compkey, value in list(source.items()): try: curprefix, rest = compkey.split(sep, 1) except ValueError: continue if curprefix != prefix or rest == '#': continue yield rest, value def parse_attr_list(source, prefix, sep='.'): attrs = defaultdict(dict) for compkey, value in _parse_prefix(source, prefix, sep): idx, key = compkey.split(sep, 1) attrs[idx][key] = value return list(attrs.values()) def parse_dict(source, prefix, sep='.'): return dict(_parse_prefix(source, prefix, sep)) def parse_list(source, prefix, sep='.'): return [value for _, value in _parse_prefix(source, prefix, sep)] def parse_bool(string_form): if type(string_form) is bool: return string_form token = string_form.lower()[0] if token == 't': return True elif token == 'f': return False else: raise ValueError('could not convert %r to a bool' % string_form) def sanitize_groups(groups): _groups = [] chars_to_replace = ['+', '-', '=', '.', '/', ' '] for i in groups: _i = i for char in chars_to_replace: _i = _i.replace(char, '_') _groups.append(_i) groups.clear() groups.extend(_groups) @parses('equinix_metal_device') def equinix_metal_device(resource, tfvars=None): raw_attrs = resource['primary']['attributes'] name = raw_attrs['hostname'] groups = [] attrs = { 'id': raw_attrs['id'], 'facilities': parse_list(raw_attrs, 'facilities'), 'hostname': raw_attrs['hostname'], 'operating_system': raw_attrs['operating_system'], 'locked': parse_bool(raw_attrs['locked']), 'tags': parse_list(raw_attrs, 'tags'), 'plan': raw_attrs['plan'], 'project_id': raw_attrs['project_id'], 'state': raw_attrs['state'], # ansible 'ansible_host': raw_attrs['network.0.address'], 'ansible_ssh_user': 'root', # Use root by default in metal # generic 'ipv4_address': raw_attrs['network.0.address'], 'public_ipv4': raw_attrs['network.0.address'], 'ipv6_address': raw_attrs['network.1.address'], 'public_ipv6': raw_attrs['network.1.address'], 'private_ipv4': raw_attrs['network.2.address'], 'provider': 'equinix', } if raw_attrs['operating_system'] == 'flatcar_stable': # For Flatcar set the ssh_user to core attrs.update({'ansible_ssh_user': 'core'}) # add groups based on attrs groups.append('equinix_metal_operating_system_%s' % attrs['operating_system']) groups.append('equinix_metal_locked_%s' % attrs['locked']) groups.append('equinix_metal_state_%s' % attrs['state']) groups.append('equinix_metal_plan_%s' % attrs['plan']) # groups specific to kubespray groups = groups + attrs['tags'] sanitize_groups(groups) return name, attrs, groups def openstack_floating_ips(resource): raw_attrs = resource['primary']['attributes'] attrs = { 'ip': raw_attrs['floating_ip'], 'port_id': raw_attrs['port_id'], } return attrs def openstack_floating_ips(resource): raw_attrs = resource['primary']['attributes'] return raw_attrs['port_id'], raw_attrs['floating_ip'] @parses('openstack_compute_instance_v2') @calculate_mantl_vars def openstack_host(resource, module_name): raw_attrs = resource['primary']['attributes'] name = raw_attrs['name'] groups = [] attrs = { 'access_ip_v4': raw_attrs['access_ip_v4'], 'access_ip_v6': raw_attrs['access_ip_v6'], 'access_ip': raw_attrs['access_ip_v4'], 'access_ip6': raw_attrs['access_ip_v6'], 'ip': raw_attrs['network.0.fixed_ip_v4'], 'flavor': parse_dict(raw_attrs, 'flavor', sep='_'), 'id': raw_attrs['id'], 'image': parse_dict(raw_attrs, 'image', sep='_'), 'key_pair': raw_attrs['key_pair'], 'metadata': parse_dict(raw_attrs, 'metadata'), 'network': parse_attr_list(raw_attrs, 'network'), 'region': raw_attrs.get('region', ''), 'security_groups': parse_list(raw_attrs, 'security_groups'), # workaround for an OpenStack bug where hosts have a different domain # after they're restarted 'host_domain': 'novalocal', 'use_host_domain': True, # generic 'public_ipv4': raw_attrs['access_ip_v4'], 'private_ipv4': raw_attrs['access_ip_v4'], 'port_id' : raw_attrs['network.0.port'], 'provider': 'openstack', } if 'floating_ip' in raw_attrs: attrs['private_ipv4'] = raw_attrs['network.0.fixed_ip_v4'] if 'metadata.use_access_ip' in raw_attrs and raw_attrs['metadata.use_access_ip'] == "0": attrs.pop('access_ip') try: if 'metadata.prefer_ipv6' in raw_attrs and raw_attrs['metadata.prefer_ipv6'] == "1": attrs.update({ 'ansible_host': re.sub(r"[\[\]]", "", raw_attrs['access_ip_v6']), 'publicly_routable': True, }) else: attrs.update({ 'ansible_host': raw_attrs['access_ip_v4'], 'publicly_routable': True, }) except (KeyError, ValueError): attrs.update({'ansible_host': '', 'publicly_routable': False}) # Handling of floating IPs has changed: https://github.com/terraform-providers/terraform-provider-openstack/blob/master/CHANGELOG.md#010-june-21-2017 # attrs specific to Ansible if 'metadata.ssh_user' in raw_attrs: attrs['ansible_user'] = raw_attrs['metadata.ssh_user'] if 'metadata.ssh_port' in raw_attrs: attrs['ansible_port'] = raw_attrs['metadata.ssh_port'] if 'volume.#' in list(raw_attrs.keys()) and int(raw_attrs['volume.#']) > 0: device_index = 1 for key, value in list(raw_attrs.items()): match = re.search("^volume.*.device$", key) if match: attrs['disk_volume_device_'+str(device_index)] = value device_index += 1 # attrs specific to Mantl attrs.update({ 'role': attrs['metadata'].get('role', 'none') }) # add groups based on attrs groups.append('os_image=' + str(attrs['image']['id'])) groups.append('os_flavor=' + str(attrs['flavor']['name'])) groups.extend('os_metadata_%s=%s' % item for item in list(attrs['metadata'].items())) groups.append('os_region=' + str(attrs['region'])) # groups specific to kubespray for group in attrs['metadata'].get('kubespray_groups', "").split(","): groups.append(group) sanitize_groups(groups) return name, attrs, groups def iter_host_ips(hosts, ips): '''Update hosts that have an entry in the floating IP list''' for host in hosts: port_id = host[1]['port_id'] if port_id in ips: ip = ips[port_id] host[1].update({ 'access_ip_v4': ip, 'access_ip': ip, 'public_ipv4': ip, 'ansible_host': ip, }) if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == "0" and 'access_ip' in host[1]: host[1].pop('access_ip') yield host ## QUERY TYPES def query_host(hosts, target): for name, attrs, _ in hosts: if name == target: return attrs return {} def query_list(hosts): groups = defaultdict(dict) meta = {} for name, attrs, hostgroups in hosts: for group in set(hostgroups): # Ansible 2.6.2 stopped supporting empty group names: https://github.com/ansible/ansible/pull/42584/commits/d4cd474b42ed23d8f8aabb2a7f84699673852eaf # Empty group name defaults to "all" in Ansible < 2.6.2 so we alter empty group names to "all" if not group: group = "all" groups[group].setdefault('hosts', []) groups[group]['hosts'].append(name) meta[name] = attrs groups['_meta'] = {'hostvars': meta} return groups def query_hostfile(hosts): out = ['## begin hosts generated by terraform.py ##'] out.extend( '{}\t{}'.format(attrs['ansible_host'].ljust(16), name) for name, attrs, _ in hosts ) out.append('## end hosts generated by terraform.py ##') return '\n'.join(out) def main(): parser = argparse.ArgumentParser( __file__, __doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) modes = parser.add_mutually_exclusive_group(required=True) modes.add_argument('--list', action='store_true', help='list all variables') modes.add_argument('--host', help='list variables for a single host') modes.add_argument('--version', action='store_true', help='print version and exit') modes.add_argument('--hostfile', action='store_true', help='print hosts as a /etc/hosts snippet') parser.add_argument('--pretty', action='store_true', help='pretty-print output JSON') parser.add_argument('--nometa', action='store_true', help='with --list, exclude hostvars') default_root = os.environ.get('TERRAFORM_STATE_ROOT', os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', ))) parser.add_argument('--root', default=default_root, help='custom root to search for `.tfstate`s in') args = parser.parse_args() if args.version: print('%s %s' % (__file__, VERSION)) parser.exit() hosts = iterhosts(iterresources(tfstates(args.root))) # Perform a second pass on the file to pick up floating_ip entries to update the ip address of referenced hosts ips = dict(iterips(iterresources(tfstates(args.root)))) if ips: hosts = iter_host_ips(hosts, ips) if args.list: output = query_list(hosts) if args.nometa: del output['_meta'] print(json.dumps(output, indent=4 if args.pretty else None)) elif args.host: output = query_host(hosts, args.host) print(json.dumps(output, indent=4 if args.pretty else None)) elif args.hostfile: output = query_hostfile(hosts) print(output) parser.exit() if __name__ == '__main__': main() ================================================ FILE: contrib/terraform/upcloud/README.md ================================================ # Kubernetes on UpCloud with Terraform Provision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray ## Requirements * Terraform 0.13.0 or newer ## Quickstart NOTE: Assumes you are at the root of the kubespray repo. For authentication in your cluster you can use the environment variables. ```bash export TF_VAR_UPCLOUD_USERNAME=username export TF_VAR_UPCLOUD_PASSWORD=password ``` To allow API access to your UpCloud account, you need to allow API connections by visiting [Account-page](https://hub.upcloud.com/account) in your UpCloud Hub. Copy the cluster configuration file. ```bash CLUSTER=my-upcloud-cluster cp -r inventory/sample inventory/$CLUSTER cp contrib/terraform/upcloud/cluster-settings.tfvars inventory/$CLUSTER/ export ANSIBLE_CONFIG=ansible.cfg cd inventory/$CLUSTER ``` Edit `cluster-settings.tfvars` to match your requirement. Run Terraform to create the infrastructure. ```bash terraform init ../../contrib/terraform/upcloud terraform apply --var-file cluster-settings.tfvars \ -state=tfstate-$CLUSTER.tfstate \ ../../contrib/terraform/upcloud/ ``` You should now have a inventory file named `inventory.ini` that you can use with kubespray. You can use the inventory file with kubespray to set up a cluster. It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by: ```bash ansible -i inventory.ini -m ping all ``` You can setup Kubernetes with kubespray using the generated inventory: ```bash ansible-playbook -i inventory.ini ../../cluster.yml -b -v ``` ## Teardown You can teardown your infrastructure using the following Terraform command: ```bash terraform destroy --var-file cluster-settings.tfvars \ -state=tfstate-$CLUSTER.tfstate \ ../../contrib/terraform/upcloud/ ``` ## Variables * `prefix`: Prefix to add to all resources, if set to "" don't set any prefix * `template_name`: The name or UUID of a base image * `username`: a user to access the nodes, defaults to "ubuntu" * `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24" * `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]` * `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true` * `ssh_public_keys`: List of public SSH keys to install on all machines * `zone`: The zone where to run the cluster * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* * `plan`: Preconfigured cpu/mem plan to use (disables `cpu` and `mem` attributes below) * `cpu`: number of cpu cores * `mem`: memory size in MB * `disk_size`: The size of the storage in GB * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false` * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null` * `additional_disks`: Additional disks to attach to the node. * `size`: The size of the additional disk in GB * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm) * `firewall_enabled`: Enable firewall rules * `firewall_default_deny_in`: Set the firewall to deny inbound traffic by default. Automatically adds UpCloud DNS server and NTP port allowlisting. * `firewall_default_deny_out`: Set the firewall to deny outbound traffic by default. * `master_allowed_remote_ips`: List of IP ranges that should be allowed to access API of masters * `start_address`: Start of address range to allow * `end_address`: End of address range to allow * `k8s_allowed_remote_ips`: List of IP ranges that should be allowed SSH access to all nodes * `start_address`: Start of address range to allow * `end_address`: End of address range to allow * `master_allowed_ports`: List of port ranges that should be allowed to access the masters * `protocol`: Protocol *(tcp|udp|icmp)* * `port_range_min`: Start of port range to allow * `port_range_max`: End of port range to allow * `start_address`: Start of address range to allow * `end_address`: End of address range to allow * `worker_allowed_ports`: List of port ranges that should be allowed to access the workers * `protocol`: Protocol *(tcp|udp|icmp)* * `port_range_min`: Start of port range to allow * `port_range_max`: End of port range to allow * `start_address`: Start of address range to allow * `end_address`: End of address range to allow * `loadbalancer_enabled`: Enable managed load balancer * `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)* * `loadbalancer_legacy_network`: If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false (default value) * `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends * `port`: Port to load balance. * `target_port`: Port to the backend servers. * `backend_servers`: List of servers that traffic to the port should be forwarded to. * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol. * `router_enable`: If a router should be connected to the private network or not * `gateways`: Gateways that should be connected to the router, requires router_enable is set to true * `features`: List of features for the gateway * `plan`: Plan to use for the gateway * `connections`: The connections and tunnel to create for the gateway * `type`: What type of connection * `local_routes`: Map of local routes for the connection * `type`: Type of route * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix * `remote_routes`: Map of local routes for the connection * `type`: Type of route * `static_network`: Destination prefix of the route; needs to be a valid IPv4 prefix * `tunnels`: The tunnels to create for this connection * `remote_address`: The remote address for the tunnel * `ipsec_properties`: Set properties of IPSec, if not set, defaults will be used * `child_rekey_time`: IKE child SA rekey time in seconds * `dpd_delay`: Delay before sending Dead Peer Detection packets if no traffic is detected, in seconds * `dpd_timeout`: Timeout period for DPD reply before considering the peer to be dead, in seconds * `ike_lifetime`: Maximum IKE SA lifetime in seconds() * `rekey_time`: IKE SA rekey time in seconds * `phase1_algorithms`: List of Phase 1: Proposal algorithms * `phase1_dh_group_numbers`: List of Phase 1 Diffie-Hellman group numbers * `phase1_integrity_algorithms`: List of Phase 1 integrity algorithms * `phase2_algorithms`: List of Phase 2: Security Association algorithms * `phase2_dh_group_numbers`: List of Phase 2 Diffie-Hellman group numbers * `phase2_integrity_algorithms`: List of Phase 2 integrity algorithms * `gateway_vpn_psks`: Separate variable for providing psks for connection tunnels. Environment variable can be exported in the following format `export TF_VAR_gateway_vpn_psks='{"${gateway-name}-${connecton-name}-tunnel":{psk:"..."}}'` * `static_routes`: Static routes to apply to the router, requires `router_enable` is set to true * `network_peerings`: Other UpCloud private networks to peer with, requires `router_enable` is set to true * `server_groups`: Group servers together * `servers`: The servers that should be included in the group. * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed. ## Migration When `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state. The error messages you'll see if you encounter this is: ```text Error: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider "registry.terraform.io/hashicorp/null" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/null" Error: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider "registry.terraform.io/hashicorp/template" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/template" ``` This can be fixed with the following lines ```bash terraform state rm -state=terraform.tfstate null_resource.inventories terraform state rm -state=terraform.tfstate data.template_file.inventory ``` ### Public to Private only migration Since there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs. The way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`. After that you can add new nodes without `force_public_ip` and `dns_servers` set and create them. Add the new nodes into the cluster and when all of them are added, remove the old nodes. ================================================ FILE: contrib/terraform/upcloud/cluster-settings.tfvars ================================================ # See: https://developers.upcloud.com/1.3/5-zones/ zone = "fi-hel1" private_cloud = false # Only used if private_cloud = true, public zone equivalent # For example use finnish public zone for finnish private zone public_zone = "fi-hel2" username = "ubuntu" # Prefix to use for all resources to separate them from other resources prefix = "kubespray" inventory_file = "inventory.ini" # Set the operating system using UUID or exact name template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)" ssh_public_keys = [ # Put your public SSH key here "ssh-rsa public key 1", "ssh-rsa public key 2", ] # check list of available plan https://developers.upcloud.com/1.3/7-plans/ machines = { "control-plane-0" : { "node_type" : "master", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : {} }, "worker-0" : { "node_type" : "worker", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : { # "some-disk-name-1": { # "size": 100, # "tier": "maxiops", # }, # "some-disk-name-2": { # "size": 100, # "tier": "maxiops", # } } }, "worker-1" : { "node_type" : "worker", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : { # "some-disk-name-1": { # "size": 100, # "tier": "maxiops", # }, # "some-disk-name-2": { # "size": 100, # "tier": "maxiops", # } } }, "worker-2" : { "node_type" : "worker", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : { # "some-disk-name-1": { # "size": 100, # "tier": "maxiops", # }, # "some-disk-name-2": { # "size": 100, # "tier": "maxiops", # } } } } firewall_enabled = false firewall_default_deny_in = false firewall_default_deny_out = false master_allowed_remote_ips = [ { "start_address" : "0.0.0.0" "end_address" : "255.255.255.255" } ] k8s_allowed_remote_ips = [ { "start_address" : "0.0.0.0" "end_address" : "255.255.255.255" } ] master_allowed_ports = [] worker_allowed_ports = [] loadbalancer_enabled = false loadbalancer_plan = "development" loadbalancers = { # "http" : { # "proxy_protocol" : false # "port" : 80, # "target_port" : 80, # "backend_servers" : [ # "worker-0", # "worker-1", # "worker-2" # ] # } } server_groups = { # "control-plane" = { # servers = [ # "control-plane-0" # ] # anti_affinity_policy = "strict" # }, # "workers" = { # servers = [ # "worker-0", # "worker-1", # "worker-2" # ] # anti_affinity_policy = "yes" # } } router_enable = false gateways = { # "gateway" : { # features: [ "vpn" ] # plan = "production" # connections = { # "connection" = { # name = "connection" # type = "ipsec" # remote_routes = { # "them" = { # type = "static" # static_network = "1.2.3.4/24" # } # } # local_routes = { # "me" = { # type = "static" # static_network = "4.3.2.1/24" # } # } # tunnels = { # "tunnel1" = { # remote_address = "1.2.3.4" # } # } # } # } # } } # gateway_vpn_psks = {} # Should be loaded as an environment variable static_routes = { # "route": { # route: "1.2.3.4/24" # nexthop: "4.3.2.1" # } } network_peerings = { # "peering": { # remote_network: "uuid" # } } ================================================ FILE: contrib/terraform/upcloud/main.tf ================================================ terraform { required_version = ">= 0.13.0" } provider "upcloud" { # Your UpCloud credentials are read from environment variables: username = var.UPCLOUD_USERNAME password = var.UPCLOUD_PASSWORD } module "kubernetes" { source = "./modules/kubernetes-cluster" prefix = var.prefix zone = var.zone private_cloud = var.private_cloud public_zone = var.public_zone template_name = var.template_name username = var.username private_network_cidr = var.private_network_cidr dns_servers = var.dns_servers use_public_ips = var.use_public_ips machines = var.machines ssh_public_keys = var.ssh_public_keys firewall_enabled = var.firewall_enabled firewall_default_deny_in = var.firewall_default_deny_in firewall_default_deny_out = var.firewall_default_deny_out master_allowed_remote_ips = var.master_allowed_remote_ips k8s_allowed_remote_ips = var.k8s_allowed_remote_ips bastion_allowed_remote_ips = var.bastion_allowed_remote_ips master_allowed_ports = var.master_allowed_ports worker_allowed_ports = var.worker_allowed_ports loadbalancer_enabled = var.loadbalancer_enabled loadbalancer_plan = var.loadbalancer_plan loadbalancer_legacy_network = var.loadbalancer_legacy_network loadbalancers = var.loadbalancers router_enable = var.router_enable gateways = var.gateways gateway_vpn_psks = var.gateway_vpn_psks static_routes = var.static_routes network_peerings = var.network_peerings server_groups = var.server_groups } # # Generate ansible inventory # resource "local_file" "inventory" { content = templatefile("${path.module}/templates/inventory.tpl", { master_ip = module.kubernetes.master_ip worker_ip = module.kubernetes.worker_ip bastion_ip = module.kubernetes.bastion_ip username = var.username }) filename = var.inventory_file } ================================================ FILE: contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf ================================================ locals { # Create a list of all disks to create disks = flatten([ for node_name, machine in var.machines : [ for disk_name, disk in machine.additional_disks : { disk = disk disk_name = disk_name node_name = node_name } ] ]) lb_backend_servers = flatten([ for lb_name, loadbalancer in var.loadbalancers : [ for backend_server in loadbalancer.backend_servers : { port = loadbalancer.target_port lb_name = lb_name server_name = backend_server } ] ]) gateway_connections = flatten([ for gateway_name, gateway in var.gateways : [ for connection_name, connection in gateway.connections : { "gateway_id" = upcloud_gateway.gateway[gateway_name].id "gateway_name" = gateway_name "connection_name" = connection_name "type" = connection.type "local_routes" = connection.local_routes "remote_routes" = connection.remote_routes } ] ]) gateway_connection_tunnels = flatten([ for gateway_name, gateway in var.gateways : [ for connection_name, connection in gateway.connections : [ for tunnel_name, tunnel in connection.tunnels : { "gateway_id" = upcloud_gateway.gateway[gateway_name].id "gateway_name" = gateway_name "connection_id" = upcloud_gateway_connection.gateway_connection["${gateway_name}-${connection_name}"].id "connection_name" = connection_name "tunnel_name" = tunnel_name "local_address_name" = tolist(upcloud_gateway.gateway[gateway_name].address).0.name "remote_address" = tunnel.remote_address "ipsec_properties" = tunnel.ipsec_properties } ] ] ]) # If prefix is set, all resources will be prefixed with "${var.prefix}-" # Else don't prefix with anything resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}" master_ip = { for instance in upcloud_server.master : instance.hostname => { for nic in instance.network_interface : nic.type => nic.ip_address if nic.ip_address != null } } worker_ip = { for instance in upcloud_server.worker : instance.hostname => { for nic in instance.network_interface : nic.type => nic.ip_address if nic.ip_address != null } } bastion_ip = { for instance in upcloud_server.bastion : instance.hostname => { for nic in instance.network_interface : nic.type => nic.ip_address if nic.ip_address != null } } node_user_data = { for name, machine in var.machines : name => < 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~} #!/bin/bash echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf systemctl restart systemd-resolved %{ endif ~} EOF } } resource "upcloud_network" "private" { name = "${local.resource-prefix}k8s-network" zone = var.zone ip_network { address = var.private_network_cidr dhcp_default_route = var.router_enable # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here. # See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562 # dhcp_dns = length(var.private_network_dns) > 0 ? var.private_network_dns : null dhcp = true family = "IPv4" } router = var.router_enable ? upcloud_router.router[0].id : null } resource "upcloud_storage" "additional_disks" { for_each = { for disk in local.disks : "${disk.node_name}_${disk.disk_name}" => disk.disk } size = each.value.size tier = each.value.tier title = "${local.resource-prefix}${each.key}" zone = var.zone } resource "upcloud_server" "master" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "master" } hostname = "${local.resource-prefix}${each.key}" plan = each.value.plan cpu = each.value.cpu mem = each.value.mem zone = var.zone server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id template { storage = var.template_name size = each.value.disk_size } dynamic "network_interface" { for_each = each.value.force_public_ip || var.use_public_ips ? [1] : [] content { type = "public" } } # Private network interface network_interface { type = "private" network = upcloud_network.private.id } # Ignore volumes created by csi-driver lifecycle { ignore_changes = [storage_devices] } firewall = var.firewall_enabled dynamic "storage_devices" { for_each = { for disk_key_name, disk in upcloud_storage.additional_disks : disk_key_name => disk # Only add the disk if it matches the node name in the start of its name if length(regexall("^${each.key}_.+", disk_key_name)) > 0 } content { storage = storage_devices.value.id } } # Include at least one public SSH key login { user = var.username keys = var.ssh_public_keys create_password = false } metadata = local.node_user_data[each.key] != "" ? true : null user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null } resource "upcloud_server" "worker" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "worker" } hostname = "${local.resource-prefix}${each.key}" plan = each.value.plan cpu = each.value.cpu mem = each.value.mem zone = var.zone server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id template { storage = var.template_name size = each.value.disk_size } dynamic "network_interface" { for_each = each.value.force_public_ip || var.use_public_ips ? [1] : [] content { type = "public" } } # Private network interface network_interface { type = "private" network = upcloud_network.private.id } # Ignore volumes created by csi-driver lifecycle { ignore_changes = [storage_devices] } firewall = var.firewall_enabled dynamic "storage_devices" { for_each = { for disk_key_name, disk in upcloud_storage.additional_disks : disk_key_name => disk # Only add the disk if it matches the node name in the start of its name if length(regexall("^${each.key}_.+", disk_key_name)) > 0 } content { storage = storage_devices.value.id } } # Include at least one public SSH key login { user = var.username keys = var.ssh_public_keys create_password = false } metadata = local.node_user_data[each.key] != "" ? true : null user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null } resource "upcloud_server" "bastion" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "bastion" } hostname = "${local.resource-prefix}${each.key}" plan = each.value.plan cpu = each.value.cpu mem = each.value.mem zone = var.zone server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id template { storage = var.template_name size = each.value.disk_size } # Private network interface network_interface { type = "private" network = upcloud_network.private.id } # Private network interface network_interface { type = "public" } firewall = var.firewall_enabled dynamic "storage_devices" { for_each = { for disk_key_name, disk in upcloud_storage.additional_disks : disk_key_name => disk # Only add the disk if it matches the node name in the start of its name if length(regexall("^${each.key}_.+", disk_key_name)) > 0 } content { storage = storage_devices.value.id } } # Include at least one public SSH key login { user = var.username keys = var.ssh_public_keys create_password = false } } resource "upcloud_firewall_rules" "master" { for_each = upcloud_server.master server_id = each.value.id dynamic "firewall_rule" { for_each = var.master_allowed_remote_ips content { action = "accept" comment = "Allow master API access from this network" destination_port_end = "6443" destination_port_start = "6443" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = firewall_rule.value.end_address source_address_start = firewall_rule.value.start_address } } dynamic "firewall_rule" { for_each = length(var.master_allowed_remote_ips) > 0 ? [1] : [] content { action = "drop" comment = "Deny master API access from other networks" destination_port_end = "6443" destination_port_start = "6443" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = "255.255.255.255" source_address_start = "0.0.0.0" } } dynamic "firewall_rule" { for_each = var.k8s_allowed_remote_ips content { action = "accept" comment = "Allow SSH from this network" destination_port_end = "22" destination_port_start = "22" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = firewall_rule.value.end_address source_address_start = firewall_rule.value.start_address } } dynamic "firewall_rule" { for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : [] content { action = "drop" comment = "Deny SSH from other networks" destination_port_end = "22" destination_port_start = "22" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = "255.255.255.255" source_address_start = "0.0.0.0" } } dynamic "firewall_rule" { for_each = var.master_allowed_ports content { action = "accept" comment = "Allow access on this port" destination_port_end = firewall_rule.value.port_range_max destination_port_start = firewall_rule.value.port_range_min direction = "in" family = "IPv4" protocol = firewall_rule.value.protocol source_address_end = firewall_rule.value.end_address source_address_start = firewall_rule.value.start_address } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv4" protocol = firewall_rule.value source_address_end = "94.237.40.9" source_address_start = "94.237.40.9" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv4" protocol = firewall_rule.value source_address_end = "94.237.127.9" source_address_start = "94.237.127.9" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv6" protocol = firewall_rule.value source_address_end = "2a04:3540:53::1" source_address_start = "2a04:3540:53::1" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv6" protocol = firewall_rule.value source_address_end = "2a04:3544:53::1" source_address_start = "2a04:3544:53::1" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["udp"] : [] content { action = "accept" comment = "NTP Port" source_port_end = "123" source_port_start = "123" direction = "in" family = "IPv4" protocol = firewall_rule.value source_address_end = "255.255.255.255" source_address_start = "0.0.0.0" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["udp"] : [] content { action = "accept" comment = "NTP Port" source_port_end = "123" source_port_start = "123" direction = "in" family = "IPv6" protocol = firewall_rule.value } } firewall_rule { action = var.firewall_default_deny_in ? "drop" : "accept" direction = "in" } firewall_rule { action = var.firewall_default_deny_out ? "drop" : "accept" direction = "out" } } resource "upcloud_firewall_rules" "k8s" { for_each = upcloud_server.worker server_id = each.value.id dynamic "firewall_rule" { for_each = var.k8s_allowed_remote_ips content { action = "accept" comment = "Allow SSH from this network" destination_port_end = "22" destination_port_start = "22" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = firewall_rule.value.end_address source_address_start = firewall_rule.value.start_address } } dynamic "firewall_rule" { for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : [] content { action = "drop" comment = "Deny SSH from other networks" destination_port_end = "22" destination_port_start = "22" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = "255.255.255.255" source_address_start = "0.0.0.0" } } dynamic "firewall_rule" { for_each = var.worker_allowed_ports content { action = "accept" comment = "Allow access on this port" destination_port_end = firewall_rule.value.port_range_max destination_port_start = firewall_rule.value.port_range_min direction = "in" family = "IPv4" protocol = firewall_rule.value.protocol source_address_end = firewall_rule.value.end_address source_address_start = firewall_rule.value.start_address } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv4" protocol = firewall_rule.value source_address_end = "94.237.40.9" source_address_start = "94.237.40.9" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv4" protocol = firewall_rule.value source_address_end = "94.237.127.9" source_address_start = "94.237.127.9" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv6" protocol = firewall_rule.value source_address_end = "2a04:3540:53::1" source_address_start = "2a04:3540:53::1" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : [] content { action = "accept" comment = "UpCloud DNS" source_port_end = "53" source_port_start = "53" direction = "in" family = "IPv6" protocol = firewall_rule.value source_address_end = "2a04:3544:53::1" source_address_start = "2a04:3544:53::1" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["udp"] : [] content { action = "accept" comment = "NTP Port" source_port_end = "123" source_port_start = "123" direction = "in" family = "IPv4" protocol = firewall_rule.value source_address_end = "255.255.255.255" source_address_start = "0.0.0.0" } } dynamic "firewall_rule" { for_each = var.firewall_default_deny_in ? ["udp"] : [] content { action = "accept" comment = "NTP Port" source_port_end = "123" source_port_start = "123" direction = "in" family = "IPv6" protocol = firewall_rule.value } } firewall_rule { action = var.firewall_default_deny_in ? "drop" : "accept" direction = "in" } firewall_rule { action = var.firewall_default_deny_out ? "drop" : "accept" direction = "out" } } resource "upcloud_firewall_rules" "bastion" { for_each = upcloud_server.bastion server_id = each.value.id dynamic "firewall_rule" { for_each = var.bastion_allowed_remote_ips content { action = "accept" comment = "Allow bastion SSH access from this network" destination_port_end = "22" destination_port_start = "22" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = firewall_rule.value.end_address source_address_start = firewall_rule.value.start_address } } dynamic "firewall_rule" { for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : [] content { action = "drop" comment = "Drop bastion SSH access from other networks" destination_port_end = "22" destination_port_start = "22" direction = "in" family = "IPv4" protocol = "tcp" source_address_end = "255.255.255.255" source_address_start = "0.0.0.0" } } firewall_rule { action = var.firewall_default_deny_in ? "drop" : "accept" direction = "in" } firewall_rule { action = var.firewall_default_deny_out ? "drop" : "accept" direction = "out" } } resource "upcloud_loadbalancer" "lb" { count = var.loadbalancer_enabled ? 1 : 0 configured_status = "started" name = "${local.resource-prefix}lb" plan = var.loadbalancer_plan zone = var.private_cloud ? var.public_zone : var.zone network = var.loadbalancer_legacy_network ? upcloud_network.private.id : null dynamic "networks" { for_each = var.loadbalancer_legacy_network ? [] : [1] content { name = "Private-Net" type = "private" family = "IPv4" network = upcloud_network.private.id } } dynamic "networks" { for_each = var.loadbalancer_legacy_network ? [] : [1] content { name = "Public-Net" type = "public" family = "IPv4" } } lifecycle { ignore_changes = [ maintenance_dow, maintenance_time ] } } resource "upcloud_loadbalancer_backend" "lb_backend" { for_each = var.loadbalancer_enabled ? var.loadbalancers : {} loadbalancer = upcloud_loadbalancer.lb[0].id name = "lb-backend-${each.key}" properties { outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : "" } } resource "upcloud_loadbalancer_frontend" "lb_frontend" { for_each = var.loadbalancer_enabled ? var.loadbalancers : {} loadbalancer = upcloud_loadbalancer.lb[0].id name = "lb-frontend-${each.key}" mode = "tcp" port = each.value.port default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name dynamic "networks" { for_each = var.loadbalancer_legacy_network ? [] : [1] content { name = "Public-Net" } } dynamic "networks" { for_each = each.value.allow_internal_frontend ? [1] : [] content{ name = "Private-Net" } } } resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" { for_each = { for be_server in local.lb_backend_servers : "${be_server.server_name}-lb-backend-${be_server.lb_name}" => be_server if var.loadbalancer_enabled } backend = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id name = "${local.resource-prefix}${each.key}" ip = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private port = each.value.port weight = 100 max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000 enabled = true } resource "upcloud_server_group" "server_groups" { for_each = var.server_groups title = each.key anti_affinity_policy = each.value.anti_affinity_policy labels = {} # Managed upstream via upcloud_server resource members = [] lifecycle { ignore_changes = [members] } } resource "upcloud_router" "router" { count = var.router_enable ? 1 : 0 name = "${local.resource-prefix}router" dynamic "static_route" { for_each = var.static_routes content { name = static_route.key nexthop = static_route.value["nexthop"] route = static_route.value["route"] } } } resource "upcloud_gateway" "gateway" { for_each = var.router_enable ? var.gateways : {} name = "${local.resource-prefix}${each.key}-gateway" zone = var.private_cloud ? var.public_zone : var.zone features = each.value.features plan = each.value.plan router { id = upcloud_router.router[0].id } } resource "upcloud_gateway_connection" "gateway_connection" { for_each = { for gc in local.gateway_connections : "${gc.gateway_name}-${gc.connection_name}" => gc } gateway = each.value.gateway_id name = "${local.resource-prefix}${each.key}-gateway-connection" type = each.value.type dynamic "local_route" { for_each = each.value.local_routes content { name = local_route.key type = local_route.value["type"] static_network = local_route.value["static_network"] } } dynamic "remote_route" { for_each = each.value.remote_routes content { name = remote_route.key type = remote_route.value["type"] static_network = remote_route.value["static_network"] } } } resource "upcloud_gateway_connection_tunnel" "gateway_connection_tunnel" { for_each = { for gct in local.gateway_connection_tunnels : "${gct.gateway_name}-${gct.connection_name}-${gct.tunnel_name}-tunnel" => gct } connection_id = each.value.connection_id name = each.key local_address_name = each.value.local_address_name remote_address = each.value.remote_address ipsec_auth_psk { psk = var.gateway_vpn_psks[each.key].psk } dynamic "ipsec_properties" { for_each = each.value.ipsec_properties != null ? { "ip": each.value.ipsec_properties } : {} content { child_rekey_time = ipsec_properties.value["child_rekey_time"] dpd_delay = ipsec_properties.value["dpd_delay"] dpd_timeout = ipsec_properties.value["dpd_timeout"] ike_lifetime = ipsec_properties.value["ike_lifetime"] rekey_time = ipsec_properties.value["rekey_time"] phase1_algorithms = ipsec_properties.value["phase1_algorithms"] phase1_dh_group_numbers = ipsec_properties.value["phase1_dh_group_numbers"] phase1_integrity_algorithms = ipsec_properties.value["phase1_integrity_algorithms"] phase2_algorithms = ipsec_properties.value["phase2_algorithms"] phase2_dh_group_numbers = ipsec_properties.value["phase2_dh_group_numbers"] phase2_integrity_algorithms = ipsec_properties.value["phase2_integrity_algorithms"] } } } resource "upcloud_network_peering" "peering" { for_each = var.network_peerings name = "${local.resource-prefix}${each.key}" network { uuid = upcloud_network.private.id } peer_network { uuid = each.value.remote_network } } ================================================ FILE: contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf ================================================ output "master_ip" { value = local.master_ip } output "worker_ip" { value = local.worker_ip } output "bastion_ip" { value = local.bastion_ip } output "loadbalancer_domain" { value = var.loadbalancer_enabled ? upcloud_loadbalancer.lb[0].dns_name : null } ================================================ FILE: contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf ================================================ variable "prefix" { type = string } variable "zone" { type = string } variable "private_cloud" { type = bool } variable "public_zone" { type = string } variable "template_name" {} variable "username" {} variable "private_network_cidr" {} variable "dns_servers" {} variable "use_public_ips" {} variable "machines" { description = "Cluster machines" type = map(object({ node_type = string plan = string cpu = optional(number) mem = optional(number) disk_size = number server_group : string force_public_ip : optional(bool, false) dns_servers : optional(set(string)) additional_disks = map(object({ size = number tier = string })) })) } variable "ssh_public_keys" { type = list(string) } variable "firewall_enabled" { type = bool } variable "master_allowed_remote_ips" { type = list(object({ start_address = string end_address = string })) } variable "k8s_allowed_remote_ips" { type = list(object({ start_address = string end_address = string })) } variable "bastion_allowed_remote_ips" { type = list(object({ start_address = string end_address = string })) } variable "master_allowed_ports" { type = list(object({ protocol = string port_range_min = number port_range_max = number start_address = string end_address = string })) } variable "worker_allowed_ports" { type = list(object({ protocol = string port_range_min = number port_range_max = number start_address = string end_address = string })) } variable "firewall_default_deny_in" { type = bool } variable "firewall_default_deny_out" { type = bool } variable "loadbalancer_enabled" { type = bool } variable "loadbalancer_plan" { type = string } variable "loadbalancer_legacy_network" { type = bool default = false } variable "loadbalancers" { description = "Load balancers" type = map(object({ proxy_protocol = bool port = number target_port = number allow_internal_frontend = optional(bool) backend_servers = list(string) })) } variable "server_groups" { description = "Server groups" type = map(object({ anti_affinity_policy = string })) } variable "router_enable" { description = "If a router should be enabled and connected to the private network or not" type = bool } variable "gateways" { description = "Gateways that should be connected to the router, requires router_enable is set to true" type = map(object({ features = list(string) plan = optional(string) connections = optional(map(object({ type = string local_routes = optional(map(object({ type = string static_network = string }))) remote_routes = optional(map(object({ type = string static_network = string }))) tunnels = optional(map(object({ remote_address = string ipsec_properties = optional(object({ child_rekey_time = number dpd_delay = number dpd_timeout = number ike_lifetime = number rekey_time = number phase1_algorithms = set(string) phase1_dh_group_numbers = set(string) phase1_integrity_algorithms = set(string) phase2_algorithms = set(string) phase2_dh_group_numbers = set(string) phase2_integrity_algorithms = set(string) })) }))) }))) })) } variable "gateway_vpn_psks" { description = "Separate variable for providing psks for connection tunnels" type = map(object({ psk = string })) default = {} sensitive = true } variable "static_routes" { description = "Static routes to apply to the router, requires router_enable is set to true" type = map(object({ nexthop = string route = string })) } variable "network_peerings" { description = "Other UpCloud private networks to peer with, requires router_enable is set to true" type = map(object({ remote_network = string })) } ================================================ FILE: contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf ================================================ terraform { required_providers { upcloud = { source = "UpCloudLtd/upcloud" version = "~>5.29.1" } } required_version = ">= 0.13" } ================================================ FILE: contrib/terraform/upcloud/output.tf ================================================ output "master_ip" { value = module.kubernetes.master_ip } output "worker_ip" { value = module.kubernetes.worker_ip } output "bastion_ip" { value = module.kubernetes.bastion_ip } output "loadbalancer_domain" { value = module.kubernetes.loadbalancer_domain } ================================================ FILE: contrib/terraform/upcloud/sample-inventory/cluster.tfvars ================================================ # See: https://developers.upcloud.com/1.3/5-zones/ zone = "fi-hel1" username = "ubuntu" # Prefix to use for all resources to separate them from other resources prefix = "kubespray" inventory_file = "inventory.ini" # Set the operating system using UUID or exact name template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)" ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] # check list of available plan https://developers.upcloud.com/1.3/7-plans/ machines = { "control-plane-0" : { "node_type" : "master", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : {} }, "worker-0" : { "node_type" : "worker", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : { # "some-disk-name-1": { # "size": 100, # "tier": "maxiops", # }, # "some-disk-name-2": { # "size": 100, # "tier": "maxiops", # } } }, "worker-1" : { "node_type" : "worker", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : { # "some-disk-name-1": { # "size": 100, # "tier": "maxiops", # }, # "some-disk-name-2": { # "size": 100, # "tier": "maxiops", # } } }, "worker-2" : { "node_type" : "worker", # plan to use instead of custom cpu/mem "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB "mem" : "4096" # The size of the storage in GB "disk_size" : 250 "additional_disks" : { # "some-disk-name-1": { # "size": 100, # "tier": "maxiops", # }, # "some-disk-name-2": { # "size": 100, # "tier": "maxiops", # } } } } firewall_enabled = false firewall_default_deny_in = false firewall_default_deny_out = false master_allowed_remote_ips = [ { "start_address" : "0.0.0.0" "end_address" : "255.255.255.255" } ] k8s_allowed_remote_ips = [ { "start_address" : "0.0.0.0" "end_address" : "255.255.255.255" } ] master_allowed_ports = [] worker_allowed_ports = [] loadbalancer_enabled = false loadbalancer_plan = "development" loadbalancers = { # "http" : { # "port" : 80, # "target_port" : 80, # "backend_servers" : [ # "worker-0", # "worker-1", # "worker-2" # ] # } } server_groups = { # "control-plane" = { # servers = [ # "control-plane-0" # ] # anti_affinity_policy = "strict" # }, # "workers" = { # servers = [ # "worker-0", # "worker-1", # "worker-2" # ] # anti_affinity_policy = "yes" # } } ================================================ FILE: contrib/terraform/upcloud/templates/inventory.tpl ================================================ [all] %{ for name, ips in master_ip ~} ${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private} %{ endfor ~} %{ for name, ips in worker_ip ~} ${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private} %{ endfor ~} [kube_control_plane] %{ for name, ips in master_ip ~} ${name} %{ endfor ~} [etcd] %{ for name, ips in master_ip ~} ${name} %{ endfor ~} [kube_node] %{ for name, ips in worker_ip ~} ${name} %{ endfor ~} [k8s_cluster:children] kube_control_plane kube_node %{ if length(bastion_ip) > 0 ~} [bastion] %{ for name, ips in bastion_ip ~} bastion ansible_user=${username} ansible_host=${ips.public} %{ endfor ~} %{ endif ~} ================================================ FILE: contrib/terraform/upcloud/variables.tf ================================================ variable "prefix" { type = string default = "kubespray" description = "Prefix that is used to distinguish these resources from others" } variable "zone" { description = "The zone where to run the cluster" } variable "private_cloud" { description = "Whether the environment is in the private cloud region" default = false } variable "public_zone" { description = "The public zone equivalent if the cluster is running in a private cloud zone" } variable "template_name" { description = "Block describing the preconfigured operating system" } variable "username" { description = "The username to use for the nodes" default = "ubuntu" } variable "private_network_cidr" { description = "CIDR to use for the private network" default = "172.16.0.0/24" } variable "dns_servers" { description = "DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved" type = set(string) default = [] } variable "use_public_ips" { description = "If all nodes should get a public IP" type = bool default = true } variable "machines" { description = "Cluster machines" type = map(object({ node_type = string plan = string cpu = optional(number) mem = optional(number) disk_size = number server_group : string force_public_ip : optional(bool, false) dns_servers : optional(set(string)) additional_disks = map(object({ size = number tier = string })) })) } variable "ssh_public_keys" { description = "List of public SSH keys which are injected into the VMs." type = list(string) } variable "inventory_file" { description = "Where to store the generated inventory file" } variable "UPCLOUD_USERNAME" { description = "UpCloud username with API access" } variable "UPCLOUD_PASSWORD" { description = "Password for UpCloud API user" } variable "firewall_enabled" { description = "Enable firewall rules" default = false } variable "master_allowed_remote_ips" { description = "List of IP start/end addresses allowed to access API of masters" type = list(object({ start_address = string end_address = string })) default = [] } variable "k8s_allowed_remote_ips" { description = "List of IP start/end addresses allowed to SSH to hosts" type = list(object({ start_address = string end_address = string })) default = [] } variable "bastion_allowed_remote_ips" { description = "List of IP start/end addresses allowed to SSH to bastion" type = list(object({ start_address = string end_address = string })) default = [] } variable "master_allowed_ports" { description = "List of ports to allow on masters" type = list(object({ protocol = string port_range_min = number port_range_max = number start_address = string end_address = string })) } variable "worker_allowed_ports" { description = "List of ports to allow on workers" type = list(object({ protocol = string port_range_min = number port_range_max = number start_address = string end_address = string })) } variable "firewall_default_deny_in" { description = "Add firewall policies that deny all inbound traffic by default" default = false } variable "firewall_default_deny_out" { description = "Add firewall policies that deny all outbound traffic by default" default = false } variable "loadbalancer_enabled" { description = "Enable load balancer" default = false } variable "loadbalancer_plan" { description = "Load balancer plan (development/production-small)" default = "development" } variable "loadbalancer_legacy_network" { description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false" type = bool default = false } variable "loadbalancers" { description = "Load balancers" type = map(object({ proxy_protocol = bool port = number target_port = number allow_internal_frontend = optional(bool, false) backend_servers = list(string) })) default = {} } variable "server_groups" { description = "Server groups" type = map(object({ anti_affinity_policy = string })) default = {} } variable "router_enable" { description = "If a router should be enabled and connected to the private network or not" type = bool default = false } variable "gateways" { description = "Gateways that should be connected to the router, requires router_enable is set to true" type = map(object({ features = list(string) plan = optional(string) connections = optional(map(object({ type = string local_routes = optional(map(object({ type = string static_network = string })), {}) remote_routes = optional(map(object({ type = string static_network = string })), {}) tunnels = optional(map(object({ remote_address = string ipsec_properties = optional(object({ child_rekey_time = number dpd_delay = number dpd_timeout = number ike_lifetime = number rekey_time = number phase1_algorithms = set(string) phase1_dh_group_numbers = set(string) phase1_integrity_algorithms = set(string) phase2_algorithms = set(string) phase2_dh_group_numbers = set(string) phase2_integrity_algorithms = set(string) })) })), {}) })), {}) })) default = {} } variable "gateway_vpn_psks" { description = "Separate variable for providing psks for connection tunnels" type = map(object({ psk = string })) default = {} sensitive = true } variable "static_routes" { description = "Static routes to apply to the router, requires router_enable is set to true" type = map(object({ nexthop = string route = string })) default = {} } variable "network_peerings" { description = "Other UpCloud private networks to peer with, requires router_enable is set to true" type = map(object({ remote_network = string })) default = {} } ================================================ FILE: contrib/terraform/upcloud/versions.tf ================================================ terraform { required_providers { upcloud = { source = "UpCloudLtd/upcloud" version = "~>5.29.1" } } required_version = ">= 0.13" } ================================================ FILE: contrib/terraform/vsphere/README.md ================================================ # Kubernetes on vSphere with Terraform Provision a Kubernetes cluster on [vSphere](https://www.vmware.com/products/vsphere.html) using Terraform and Kubespray. ## Overview The setup looks like following. ```text Kubernetes cluster +-----------------------+ | +--------------+ | | | +--------------+ | | | | | | | | | Master/etcd | | | | | node(s) | | | +-+ | | | +--------------+ | | ^ | | | | | v | | +--------------+ | | | +--------------+ | | | | | | | | | Worker | | | | | node(s) | | | +-+ | | | +--------------+ | +-----------------------+ ``` ## Warning This setup assumes that the DHCP is disabled in the vSphere cluster and IP addresses have to be provided in the configuration file. ## Requirements * Terraform 0.13.0 or newer (0.12 also works if you modify the provider block to include version and remove all `versions.tf` files) ## Quickstart NOTE: *Assumes you are at the root of the kubespray repo* Copy the sample inventory for your cluster and copy the default terraform variables. ```bash CLUSTER=my-vsphere-cluster cp -r inventory/sample inventory/$CLUSTER cp contrib/terraform/vsphere/default.tfvars inventory/$CLUSTER/ cd inventory/$CLUSTER ``` Edit `default.tfvars` to match your setup. You MUST set values specific for you network and vSphere cluster. ```bash # Ensure $EDITOR points to your favorite editor, e.g., vim, emacs, VS Code, etc. $EDITOR default.tfvars ``` For authentication in your vSphere cluster you can use the environment variables. ```bash export TF_VAR_vsphere_user=username export TF_VAR_vsphere_password=password ``` Run Terraform to create the infrastructure. ```bash terraform init ../../contrib/terraform/vsphere terraform apply \ -var-file default.tfvars \ -state=tfstate-$CLUSTER.tfstate \ ../../contrib/terraform/vsphere ``` You should now have a inventory file named `inventory.ini` that you can use with kubespray. You can now copy your inventory file and use it with kubespray to set up a cluster. You can type `terraform output` to find out the IP addresses of the nodes. It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by: ```bash ansible -i inventory.ini -m ping all ``` Example to use this with the default sample inventory: ```bash ansible-playbook -i inventory.ini ../../cluster.yml -b -v ``` ## Variables ### Required * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* * `ip`: The IP address of the machine * `netmask`: The netmask to use (to be used on the right hand side in CIDR notation, e.g., `24`) * `network`: The name of the network to attach the machines to * `gateway`: The IP address of the network gateway * `vsphere_datacenter`: The identifier of vSphere data center * `vsphere_compute_cluster`: The identifier of vSphere compute cluster * `vsphere_datastore`: The identifier of vSphere data store * `vsphere_server`: This is the vCenter server name or address for vSphere API operations. * `ssh_public_keys`: List of public SSH keys to install on all machines * `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand) ### Optional * `folder`: Name of the folder to put all machines in (default: `""`) * `prefix`: Prefix to use for all resources, required to be unique for all clusters in the same project (default: `"k8s"`) * `inventory_file`: Name of the generated inventory file for Kubespray to use in the Ansible step (default: `inventory.ini`) * `dns_primary`: The IP address of primary DNS server (default: `8.8.4.4`) * `dns_secondary`: The IP address of secondary DNS server (default: `8.8.8.8`) * `firmware`: Firmware to use (default: `bios`) * `hardware_version`: The version of the hardware (default: `15`) * `master_cores`: The number of CPU cores for the master nodes (default: 4) * `master_memory`: The amount of RAM for the master nodes in MB (default: 4096) * `master_disk_size`: The amount of disk space for the master nodes in GB (default: 20) * `worker_cores`: The number of CPU cores for the worker nodes (default: 16) * `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192) * `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100) * `vapp`: Boolean to set the template type to vapp. (Default: false) * `interface_name`: Name of the interface to configure. (Default: ens192) An example variables file can be found `default.tfvars` ================================================ FILE: contrib/terraform/vsphere/default.tfvars ================================================ prefix = "k8s" inventory_file = "inventory.ini" network = "VM Network" machines = { "master-0" : { "node_type" : "master", "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.10 "netmask" : "24" }, "worker-0" : { "node_type" : "worker", "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.20 "netmask" : "24" }, "worker-1" : { "node_type" : "worker", "ip" : "i-did-not-read-the-docs", # e.g. 192.168.0.21 "netmask" : "24" } } gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.1 ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] vsphere_datacenter = "i-did-not-read-the-docs" vsphere_compute_cluster = "i-did-not-read-the-docs" # e.g. Cluster vsphere_datastore = "i-did-not-read-the-docs" # e.g. ssd-000000 vsphere_server = "i-did-not-read-the-docs" # e.g. vsphere.server.com template_name = "i-did-not-read-the-docs" # e.g. ubuntu-bionic-18.04-cloudimg ================================================ FILE: contrib/terraform/vsphere/main.tf ================================================ provider "vsphere" { # Username and password set through env vars VSPHERE_USER and VSPHERE_PASSWORD user = var.vsphere_user password = var.vsphere_password vsphere_server = var.vsphere_server # If you have a self-signed cert allow_unverified_ssl = true } data "vsphere_datacenter" "dc" { name = var.vsphere_datacenter } data "vsphere_datastore" "datastore" { name = var.vsphere_datastore datacenter_id = data.vsphere_datacenter.dc.id } data "vsphere_network" "network" { name = var.network datacenter_id = data.vsphere_datacenter.dc.id } data "vsphere_virtual_machine" "template" { name = var.template_name datacenter_id = data.vsphere_datacenter.dc.id } data "vsphere_compute_cluster" "compute_cluster" { name = var.vsphere_compute_cluster datacenter_id = data.vsphere_datacenter.dc.id } resource "vsphere_resource_pool" "pool" { name = "${var.prefix}-cluster-pool" parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id } module "kubernetes" { source = "./modules/kubernetes-cluster" prefix = var.prefix machines = var.machines ## Master ## master_cores = var.master_cores master_memory = var.master_memory master_disk_size = var.master_disk_size ## Worker ## worker_cores = var.worker_cores worker_memory = var.worker_memory worker_disk_size = var.worker_disk_size ## Global ## gateway = var.gateway dns_primary = var.dns_primary dns_secondary = var.dns_secondary pool_id = vsphere_resource_pool.pool.id datastore_id = data.vsphere_datastore.datastore.id folder = var.folder guest_id = data.vsphere_virtual_machine.template.guest_id scsi_type = data.vsphere_virtual_machine.template.scsi_type network_id = data.vsphere_network.network.id adapter_type = data.vsphere_virtual_machine.template.network_interface_types[0] interface_name = var.interface_name firmware = var.firmware hardware_version = var.hardware_version disk_thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned template_id = data.vsphere_virtual_machine.template.id vapp = var.vapp ssh_public_keys = var.ssh_public_keys } # # Generate ansible inventory # resource "local_file" "inventory" { content = templatefile("${path.module}/templates/inventory.tpl", { connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d", keys(module.kubernetes.master_ip), values(module.kubernetes.master_ip), range(1, length(module.kubernetes.master_ip) + 1))), connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s", keys(module.kubernetes.worker_ip), values(module.kubernetes.worker_ip))), list_master = join("\n", formatlist("%s", keys(module.kubernetes.master_ip))), list_worker = join("\n", formatlist("%s", keys(module.kubernetes.worker_ip))) }) filename = var.inventory_file } ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf ================================================ resource "vsphere_virtual_machine" "worker" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "worker" } name = "${var.prefix}-${each.key}" resource_pool_id = var.pool_id datastore_id = var.datastore_id num_cpus = var.worker_cores memory = var.worker_memory memory_reservation = var.worker_memory guest_id = var.guest_id enable_disk_uuid = "true" # needed for CSI provider scsi_type = var.scsi_type folder = var.folder firmware = var.firmware hardware_version = var.hardware_version wait_for_guest_net_routable = false wait_for_guest_net_timeout = 0 network_interface { network_id = var.network_id adapter_type = var.adapter_type } disk { label = "disk0" size = var.worker_disk_size thin_provisioned = var.disk_thin_provisioned } lifecycle { ignore_changes = [disk] } clone { template_uuid = var.template_id } cdrom { client_device = true } dynamic "vapp" { for_each = var.vapp ? [1] : [] content { properties = { "user-data" = base64encode(templatefile("${path.module}/templates/vapp-cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys })) } } } extra_config = { "isolation.tools.copy.disable" = "FALSE" "isolation.tools.paste.disable" = "FALSE" "isolation.tools.setGUIOptions.enable" = "TRUE" "guestinfo.userdata" = base64encode(templatefile("${path.module}/templates/cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys })) "guestinfo.userdata.encoding" = "base64" "guestinfo.metadata" = base64encode(templatefile("${path.module}/templates/metadata.tpl", { hostname = "${var.prefix}-${each.key}", interface_name = var.interface_name ip = each.value.ip, netmask = each.value.netmask, gw = var.gateway, dns = var.dns_primary, ssh_public_keys = var.ssh_public_keys })) "guestinfo.metadata.encoding" = "base64" } } resource "vsphere_virtual_machine" "master" { for_each = { for name, machine in var.machines : name => machine if machine.node_type == "master" } name = "${var.prefix}-${each.key}" resource_pool_id = var.pool_id datastore_id = var.datastore_id num_cpus = var.master_cores memory = var.master_memory memory_reservation = var.master_memory guest_id = var.guest_id enable_disk_uuid = "true" # needed for CSI provider scsi_type = var.scsi_type folder = var.folder firmware = var.firmware hardware_version = var.hardware_version wait_for_guest_net_routable = false wait_for_guest_net_timeout = 0 network_interface { network_id = var.network_id adapter_type = var.adapter_type } disk { label = "disk0" size = var.master_disk_size thin_provisioned = var.disk_thin_provisioned } lifecycle { ignore_changes = [disk] } clone { template_uuid = var.template_id } cdrom { client_device = true } dynamic "vapp" { for_each = var.vapp ? [1] : [] content { properties = { "user-data" = base64encode(templatefile("${path.module}/templates/vapp-cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys })) } } } extra_config = { "isolation.tools.copy.disable" = "FALSE" "isolation.tools.paste.disable" = "FALSE" "isolation.tools.setGUIOptions.enable" = "TRUE" "guestinfo.userdata" = base64encode(templatefile("${path.module}/templates/cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys })) "guestinfo.userdata.encoding" = "base64" "guestinfo.metadata" = base64encode(templatefile("${path.module}/templates/metadata.tpl", { hostname = "${var.prefix}-${each.key}", interface_name = var.interface_name ip = each.value.ip, netmask = each.value.netmask, gw = var.gateway, dns = var.dns_primary, ssh_public_keys = var.ssh_public_keys })) "guestinfo.metadata.encoding" = "base64" } } ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf ================================================ output "master_ip" { value = { for name, machine in var.machines : "${var.prefix}-${name}" => machine.ip if machine.node_type == "master" } } output "worker_ip" { value = { for name, machine in var.machines : "${var.prefix}-${name}" => machine.ip if machine.node_type == "worker" } } ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tpl ================================================ #cloud-config ssh_authorized_keys: %{ for ssh_public_key in ssh_public_keys ~} - ${ssh_public_key} %{ endfor ~} ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/templates/metadata.tpl ================================================ instance-id: ${hostname} local-hostname: ${hostname} network: version: 2 ethernets: ${interface_name}: match: name: ${interface_name} dhcp4: false addresses: - ${ip}/${netmask} gateway4: ${gw} nameservers: addresses: [${dns}] ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/templates/vapp-cloud-init.tpl ================================================ #cloud-config ssh_authorized_keys: %{ for ssh_public_key in ssh_public_keys ~} - ${ssh_public_key} %{ endfor ~} write_files: - path: /etc/netplan/10-user-network.yaml content: |. network: version: 2 ethernets: ${interface_name}: dhcp4: false #true to use dhcp addresses: - ${ip}/${netmask} gateway4: ${gw} # Set gw here nameservers: addresses: - ${dns} # Set DNS ip address here runcmd: - netplan apply ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf ================================================ ## Global ## variable "prefix" {} variable "machines" { description = "Cluster machines" type = map(object({ node_type = string ip = string netmask = string })) } variable "gateway" {} variable "dns_primary" {} variable "dns_secondary" {} variable "pool_id" {} variable "datastore_id" {} variable "guest_id" {} variable "scsi_type" {} variable "network_id" {} variable "interface_name" {} variable "adapter_type" {} variable "disk_thin_provisioned" {} variable "template_id" {} variable "vapp" { type = bool } variable "firmware" {} variable "folder" {} variable "ssh_public_keys" { type = list(string) } variable "hardware_version" {} ## Master ## variable "master_cores" {} variable "master_memory" {} variable "master_disk_size" {} ## Worker ## variable "worker_cores" {} variable "worker_memory" {} variable "worker_disk_size" {} ================================================ FILE: contrib/terraform/vsphere/modules/kubernetes-cluster/versions.tf ================================================ terraform { required_providers { vsphere = { source = "hashicorp/vsphere" version = ">= 1.24.3" } } required_version = ">= 0.13" } ================================================ FILE: contrib/terraform/vsphere/output.tf ================================================ output "master_ip_addresses" { value = module.kubernetes.master_ip } output "worker_ip_addresses" { value = module.kubernetes.worker_ip } output "vsphere_datacenter" { value = var.vsphere_datacenter } output "vsphere_server" { value = var.vsphere_server } output "vsphere_datastore" { value = var.vsphere_datastore } output "vsphere_network" { value = var.network } output "vsphere_folder" { value = var.folder } output "vsphere_pool" { value = "${terraform.workspace}-cluster-pool" } ================================================ FILE: contrib/terraform/vsphere/sample-inventory/cluster.tfvars ================================================ prefix = "default" inventory_file = "inventory.ini" machines = { "master-0" : { "node_type" : "master", "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 }, "worker-0" : { "node_type" : "worker", "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 }, "worker-1" : { "node_type" : "worker", "ip" : "i-did-not-read-the-docs" # e.g. 192.168.0.2/24 } } gateway = "i-did-not-read-the-docs" # e.g. 192.168.0.2 ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] vsphere_datacenter = "i-did-not-read-the-docs" vsphere_compute_cluster = "i-did-not-read-the-docs" # e.g. Cluster vsphere_datastore = "i-did-not-read-the-docs" # e.g. ssd-000000 vsphere_server = "i-did-not-read-the-docs" # e.g. vsphere.server.com template_name = "i-did-not-read-the-docs" # e.g. ubuntu-bionic-18.04-cloudimg ================================================ FILE: contrib/terraform/vsphere/templates/inventory.tpl ================================================ [all] ${connection_strings_master} ${connection_strings_worker} [kube_control_plane] ${list_master} [etcd] ${list_master} [kube_node] ${list_worker} [k8s_cluster:children] kube_control_plane kube_node ================================================ FILE: contrib/terraform/vsphere/variables.tf ================================================ ## Global ## # Required variables variable "machines" { description = "Cluster machines" type = map(object({ node_type = string ip = string netmask = string })) } variable "network" {} variable "gateway" {} variable "vsphere_datacenter" {} variable "vsphere_compute_cluster" {} variable "vsphere_datastore" {} variable "vsphere_user" {} variable "vsphere_password" { sensitive = true } variable "vsphere_server" {} variable "ssh_public_keys" { description = "List of public SSH keys which are injected into the VMs." type = list(string) } variable "template_name" {} # Optional variables (ones where reasonable defaults exist) variable "vapp" { default = false } variable "interface_name" { default = "ens192" } variable "folder" { default = "" } variable "prefix" { default = "k8s" } variable "inventory_file" { default = "inventory.ini" } variable "dns_primary" { default = "8.8.4.4" } variable "dns_secondary" { default = "8.8.8.8" } variable "firmware" { default = "bios" } variable "hardware_version" { default = "15" } ## Master ## variable "master_cores" { default = 4 } variable "master_memory" { default = 4096 } variable "master_disk_size" { default = "20" } ## Worker ## variable "worker_cores" { default = 16 } variable "worker_memory" { default = 8192 } variable "worker_disk_size" { default = "100" } ================================================ FILE: contrib/terraform/vsphere/versions.tf ================================================ terraform { required_providers { vsphere = { source = "hashicorp/vsphere" version = ">= 1.24.3" } } required_version = ">= 0.13" } ================================================ FILE: docs/CNI/calico.md ================================================ # Calico Check if the calico-node container is running ```ShellSession docker ps | grep calico ``` The **calicoctl.sh** is wrap script with configured access credentials for command calicoctl allows to check the status of the network workloads. * Check the status of Calico nodes ```ShellSession calicoctl.sh node status ``` * Show the configured network subnet for containers ```ShellSession calicoctl.sh get ippool -o wide ``` * Show the workloads (ip addresses of containers and their location) ```ShellSession calicoctl.sh get workloadEndpoint -o wide ``` and ```ShellSession calicoctl.sh get hostEndpoint -o wide ``` ## Configuration ### Optional : Define datastore type The default datastore, Kubernetes API datastore is recommended for on-premises deployments, and supports only Kubernetes workloads; etcd is the best datastore for hybrid deployments. Allowed values are `kdd` (default) and `etcd`. Note: using kdd and more than 50 nodes, consider using the `typha` daemon to provide scaling. To re-define you need to edit the inventory and add a group variable `calico_datastore` ```yml calico_datastore: kdd ``` ### Optional : Define network backend In some cases you may want to define Calico network backend. Allowed values are `bird`, `vxlan` or `none`. `vxlan` is the default value. To re-define you need to edit the inventory and add a group variable `calico_network_backend` ```yml calico_network_backend: none ``` ### Optional : Define the default pool CIDRs By default, `kube_pods_subnet` is used as the IP range CIDR for the default IP Pool, and `kube_pods_subnet_ipv6` for IPv6. In some cases you may want to add several pools and not have them considered by Kubernetes as external (which means that they must be within or equal to the range defined in `kube_pods_subnet` and `kube_pods_subnet_ipv6` ), it starts with the default IP Pools of which IP range CIDRs can by defined in group_vars (k8s_cluster/k8s-net-calico.yml): ```ShellSession calico_pool_cidr: 10.233.64.0/20 calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 ``` ### Optional : BGP Peering with border routers In some cases you may want to route the pods subnet and so NAT is not needed on the nodes. For instance if you have a cluster spread on different locations and you want your pods to talk each other no matter where they are located. The following variables need to be set as follow: ```yml peer_with_router: true # enable the peering with the datacenter's border router (default value: false). nat_outgoing: false # (optional) NAT outgoing (default value: true). ``` And you'll need to edit the inventory and add a hostvar `local_as` by node. ```ShellSession node1 ansible_ssh_host=95.54.0.12 local_as=xxxxxx ``` ### Optional : Defining BGP peers Peers can be defined using the `peers` variable (see docs/calico_peer_example examples). In order to define global peers, the `peers` variable can be defined in group_vars with the "scope" attribute of each global peer set to "global". In order to define peers on a per node basis, the `peers` variable must be defined in hostvars or group_vars with the "scope" attribute unset or set to "node". NB: Ansible's `hash_behaviour` is by default set to "replace", thus defining both global and per node peers would end up with having only per node peers. If having both global and per node peers defined was meant to happen, global peers would have to be defined in hostvars for each host (as well as per node peers) NB²: Peers definition at node scope can be customized with additional fields `filters`, `sourceAddress` and `numAllowedLocalASNumbers` (see for details) Since calico 3.4, Calico supports advertising Kubernetes service cluster IPs over BGP, just as it advertises pod IPs. This can be enabled by setting the following variable as follow in group_vars (k8s_cluster/k8s-net-calico.yml) ```yml calico_advertise_cluster_ips: true ``` Since calico 3.10, Calico supports advertising Kubernetes service ExternalIPs over BGP in addition to cluster IPs advertising. This can be enabled by setting the following variable in group_vars (k8s_cluster/k8s-net-calico.yml) ```yml calico_advertise_service_external_ips: - x.x.x.x/24 - y.y.y.y/32 ``` ### Optional : Define global AS number Optional parameter `global_as_num` defines Calico global AS number (`/calico/bgp/v1/global/as_num` etcd key). It defaults to "64512". ### Optional : BGP Peering with route reflectors At large scale you may want to disable full node-to-node mesh in order to optimize your BGP topology and improve `calico-node` containers' start times. To do so you can deploy BGP route reflectors and peer `calico-node` with them as recommended here: * * You need to edit your inventory and add: * `calico_rr` group with nodes in it. `calico_rr` can be combined with `kube_node` and/or `kube_control_plane`. * `cluster_id` by route reflector node/group (see details [here](https://hub.docker.com/r/calico/routereflector/)) Here's an example of Kubespray inventory with standalone route reflectors: ```ini [all] rr0 ansible_ssh_host=10.210.1.10 ip=10.210.1.10 rr1 ansible_ssh_host=10.210.1.11 ip=10.210.1.11 node2 ansible_ssh_host=10.210.1.12 ip=10.210.1.12 node3 ansible_ssh_host=10.210.1.13 ip=10.210.1.13 node4 ansible_ssh_host=10.210.1.14 ip=10.210.1.14 node5 ansible_ssh_host=10.210.1.15 ip=10.210.1.15 [kube_control_plane] node2 node3 [etcd] node2 node3 node4 [kube_node] node2 node3 node4 node5 [calico_rr] rr0 rr1 [rack0] rr0 rr1 node2 node3 node4 node5 [rack0:vars] cluster_id="1.0.0.1" calico_rr_id=rr1 calico_group_id=rr1 ``` The inventory above will deploy the following topology assuming that calico's `global_as_num` is set to `65400`: ![Image](../figures/kubespray-calico-rr.png?raw=true) ### Optional : Define default endpoint to host action By default Calico blocks traffic from endpoints to the host itself by using an iptables DROP action. When using it in kubernetes the action has to be changed to RETURN (default in kubespray) or ACCEPT (see ) Otherwise all network packets from pods (with hostNetwork=False) to services endpoints (with hostNetwork=True) within the same node are dropped. To re-define default action please set the following variable in your inventory: ```yml calico_endpoint_to_host_action: "ACCEPT" ``` ### Optional : Define address on which Felix will respond to health requests Since Calico 3.2.0, HealthCheck default behavior changed from listening on all interfaces to just listening on localhost. To re-define health host please set the following variable in your inventory: ```yml calico_healthhost: "0.0.0.0" ``` ### Optional : Configure VXLAN hardware Offload The VXLAN Offload is disable by default. It can be configured like this to enabled it: ```yml calico_feature_detect_override: "ChecksumOffloadBroken=false" # The vxlan offload will enabled (It may cause problem on buggy NIC driver) ``` ### Optional : Configure Calico Node probe timeouts Under certain conditions a deployer may need to tune the Calico liveness and readiness probes timeout settings. These can be configured like this: ```yml calico_node_livenessprobe_timeout: 10 calico_node_readinessprobe_timeout: 10 ``` ### Optional : Enable NAT with IPv6 To allow outgoing IPv6 traffic going from pods to the Internet, enable the following: ```yml nat_outgoing_ipv6: true # NAT outgoing ipv6 (default value: false). ``` ## Config encapsulation for cross server traffic Calico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is the more mature implementation and enabled by default, please check your environment if you need *IP in IP* encapsulation. *IP in IP* and *VXLAN* is mutually exclusive modes. Kubespray defaults have changed after version 2.18 from auto-enabling `ipip` mode to auto-enabling `vxlan`. This was done to facilitate wider deployment scenarios including those where vxlan acceleration is provided by the underlying network devices. If you are running your cluster with the default calico settings and are upgrading to a release post 2.18.x (i.e. 2.19 and later or `master` branch) then you have two options: * perform a manual migration to vxlan before upgrading kubespray (see migrating from IP in IP to VXLAN below) * pin the pre-2.19 settings in your ansible inventory (see IP in IP mode settings below) **Note:**: Vxlan in ipv6 only supported when kernel >= 3.12. So if your kernel version < 3.12, Please don't set `calico_vxlan_mode_ipv6: Always`. More details see [#Issue 6877](https://github.com/projectcalico/calico/issues/6877). ### IP in IP mode To configure Ip in Ip mode you need to use the bird network backend. ```yml calico_ipip_mode: 'Always' # Possible values is `Always`, `CrossSubnet`, `Never` calico_vxlan_mode: 'Never' calico_network_backend: 'bird' ``` ### BGP mode To enable BGP no-encapsulation mode: ```yml calico_ipip_mode: 'Never' calico_vxlan_mode: 'Never' calico_network_backend: 'bird' ``` ### Migrating from IP in IP to VXLAN If you would like to migrate from the old IP in IP with `bird` network backends default to the new VXLAN based encapsulation you need to perform this change before running an upgrade of your cluster; the `cluster.yml` and `upgrade-cluster.yml` playbooks will refuse to continue if they detect incompatible settings. Execute the following steps on one of the control plane nodes, ensure the cluster in healthy before proceeding. ```shell calicoctl.sh patch felixconfig default -p '{"spec":{"vxlanEnabled":true}}' calicoctl.sh patch ippool default-pool -p '{"spec":{"ipipMode":"Never", "vxlanMode":"Always"}}' ``` **Note:** if you created multiple ippools you will need to patch all of them individually to change their encapsulation. The kubespray playbooks only handle the default ippool created by kubespray. Wait for the `vxlan.calico` interfaces to be created on all cluster nodes and traffic to be routed through it then you can disable `ipip`. ```shell calicoctl.sh patch felixconfig default -p '{"spec":{"ipipEnabled":false}}' ``` ## Configuring interface MTU This is an advanced topic and should usually not be modified unless you know exactly what you are doing. Calico is smart enough to deal with the defaults and calculate the proper MTU. If you do need to set up a custom MTU you can change `calico_veth_mtu` as follows: * If Wireguard is enabled, subtract 60 from your network MTU (i.e. 1500-60=1440) * If using VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450) * If using IPIP, subtract 20 from your network MTU (i.e. 1500-20=1480) * if not using any encapsulation, set to your network MTU (i.e. 1500 or 9000) ```yaml calico_veth_mtu: 1440 ``` ## Cloud providers configuration Please refer to the official documentation, for example [GCE configuration](http://docs.projectcalico.org/v1.5/getting-started/docker/installation/gce) requires a security rule for calico ip-ip tunnels. Note, calico is always configured with ``calico_ipip_mode: Always`` if the cloud provider was defined. ### Optional : Ignore kernel's RPF check setting By default the felix agent(calico-node) will abort if the Kernel RPF setting is not 'strict'. If you want Calico to ignore the Kernel setting: ```yml calico_node_ignorelooserpf: true ``` Note that in OpenStack you must allow `ipip` traffic in your security groups, otherwise you will experience timeouts. To do this you must add a rule which allows it, for example: ### Optional : Felix configuration via extraenvs of calico node Possible environment variable parameters for [configuring Felix](https://docs.projectcalico.org/reference/felix/configuration) ```yml calico_node_extra_envs: FELIX_DEVICEROUTESOURCEADDRESS: 172.17.0.1 ``` ```ShellSession neutron security-group-rule-create --protocol 4 --direction egress k8s-a0tp4t neutron security-group-rule-create --protocol 4 --direction igress k8s-a0tp4t ``` ### Optional : Use Calico CNI host-local IPAM plugin Calico currently supports two types of CNI IPAM plugins, `host-local` and `calico-ipam` (default). To allow Calico to determine the subnet to use from the Kubernetes API based on the `Node.podCIDR` field, enable the following setting. ```yml calico_ipam_host_local: true ``` Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information. ### Optional : Disable CNI logging to disk Calico CNI plugin logs to /var/log/calico/cni/cni.log and to stderr. stderr of CNI plugins can be found in the logs of container runtime. You can disable Calico CNI logging to disk by setting `calico_cni_log_file_path: false`. ## eBPF Support Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information. Note that it is advisable to always use the latest version of Calico when using the eBPF dataplane. ### Enabling eBPF support To enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system. ```yaml calico_bpf_enabled: true ``` **NOTE:** there is known incompatibility in using the `kernel-kvm` kernel package on Ubuntu OSes because it is missing support for `CONFIG_NET_SCHED` which is a requirement for Calico eBPF support. When using Calico eBPF with Ubuntu ensure you run the `-generic` kernel. ### Cleaning up after kube-proxy Calico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned. To check that kube-proxy was running in ipvs mode: ```ShellSession # ipvsadm -l ``` To clean up any ipvs leftovers: ```ShellSession # ipvsadm -C ``` ### Calico access to the kube-api Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this. Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md). If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. ### Tunneled versus Direct Server Return By default Calico uses Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service. To configure DSR: ```yaml calico_bpf_service_mode: "DSR" ``` ### eBPF Logging and Troubleshooting In order to enable Calico eBPF mode logging: ```yaml calico_bpf_log_level: "Debug" ``` To view the logs you need to use the `tc` command to read the kernel trace buffer: ```ShellSession tc exec bpf debug ``` Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs). ## Wireguard Encryption Calico supports using Wireguard for encryption. Please see the docs on [encrypt cluster pod traffic](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic). To enable wireguard support: ```yaml calico_wireguard_enabled: true ``` The following OSes will require enabling the EPEL repo in order to bring in wireguard tools: * CentOS 8 * AlmaLinux 8 * Rocky Linux 8 * Amazon Linux 2 ```yaml epel_enabled: true ``` ================================================ FILE: docs/CNI/cilium.md ================================================ # Cilium ## Unprivileged agent configuration By default, Cilium is installed with `securityContext.privileged: false`. You need to set the `kube_owner` variable to `root` in the inventory: ```yml kube_owner: root ``` ## IP Address Management (IPAM) IP Address Management (IPAM) is responsible for the allocation and management of IP addresses used by network endpoints (container and others) managed by Cilium. The default mode is "Cluster Scope". You can set the following parameters, for example: cluster-pool, kubernetes: ```yml cilium_ipam_mode: cluster-pool ``` ### Set the cluster Pod CIDRs Cluster Pod CIDRs use the kube_pods_subnet value by default. If your node network is in the same range you will lose connectivity to other nodes. Defaults to kube_pods_subnet if not set. You can set the following parameters: ```yml cilium_pool_cidr: 10.233.64.0/18 ``` When cilium_enable_ipv6 is used. Defaults to kube_pods_subnet_ipv6 if not set. you need to set the IPV6 value: ```yml cilium_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 ``` ### Set the Pod CIDR size of a node When cilium IPAM uses the "Cluster Scope" mode, it will pre-allocate a segment of IP to each node, schedule the Pod to this node, and then allocate IP from here. cilium_pool_mask_size Specifies the size allocated from cluster Pod CIDR to node.ipam.podCIDRs. Defaults to kube_network_node_prefix if not set. ```yml cilium_pool_mask_size: "24" ``` cilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cluster Pod IPV6 CIDR. Defaults to kube_network_node_prefix_ipv6 if not set. ```yml cilium_pool_mask_size_ipv6: "120" ``` ### IP Load Balancer Pools Cilium's IP Load Balancer Pools can be configured with the `cilium_loadbalancer_ip_pools` variable: ```yml cilium_loadbalancer_ip_pools: - name: "blue-pool" cidrs: - "10.0.10.0/24" ranges: - start: "20.0.20.100" stop: "20.0.20.200" - start: "1.2.3.4" ``` For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/) ### BGP Control Plane Cilium's BGP Control Plane can be enabled by setting `cilium_enable_bgp_control_plane` to `true`.: ```yml cilium_enable_bgp_control_plane: true ``` For further information, check [BGP Peering Policy documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/) ### BGP Control Plane Resources (New bgpv2 API v1.16+) Cilium BGP control plane is managed by a set of custom resources which provide a flexible way to configure BGP peers, policies, and advertisements. Cilium's BGP Instances can be configured with the `cilium_bgp_cluster_configs` variable: ```yml cilium_bgp_cluster_configs: - name: "cilium-bgp" spec: bgpInstances: - name: "instance-64512" localASN: 64512 peers: - name: "peer-64512-tor1" peerASN: 64512 peerAddress: '10.47.1.1' peerConfigRef: name: "cilium-peer" nodeSelector: matchExpressions: - {key: somekey, operator: NotIn, values: ['never-used-value']} ``` Cillium's BGP Peers can be configured with the `cilium_bgp_peer_configs` variable: ```yml cilium_bgp_peer_configs: - name: cilium-peer spec: # authSecretRef: bgp-auth-secret gracefulRestart: enabled: true restartTimeSeconds: 15 families: - afi: ipv4 safi: unicast advertisements: matchLabels: advertise: "bgp" - afi: ipv6 safi: unicast advertisements: matchLabels: advertise: "bgp" ``` Cillium's BGP Advertisements can be configured with the `cilium_bgp_advertisements` variable: ```yml cilium_bgp_advertisements: - name: bgp-advertisements labels: advertise: bgp spec: advertisements: - advertisementType: "PodCIDR" attributes: communities: standard: [ "64512:99" ] - advertisementType: "Service" service: addresses: - ClusterIP - ExternalIP - LoadBalancerIP selector: matchExpressions: - {key: somekey, operator: NotIn, values: ['never-used-value']} ``` Cillium's BGP Node Config Overrides can be configured with the `cilium_bgp_node_config_overrides` variable: ```yml cilium_bgp_node_config_overrides: - name: bgpv2-cplane-dev-multi-homing-worker spec: bgpInstances: - name: "instance-65000" routerID: "192.168.10.1" localPort: 1790 peers: - name: "peer-65000-tor1" localAddress: fd00:10:0:2::2 - name: "peer-65000-tor2" localAddress: fd00:11:0:2::2 ``` For further information, check [BGP Control Plane Resources documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/) ### BGP Peering Policies (Legacy < v1.16) Cilium's BGP Peering Policies can be configured with the `cilium_bgp_peering_policies` variable: ```yml cilium_bgp_peering_policies: - name: "01-bgp-peering-policy" spec: virtualRouters: - localASN: 64512 exportPodCIDR: false neighbors: - peerAddress: '10.47.1.1/24' peerASN: 64512 serviceSelector: matchExpressions: - {key: somekey, operator: NotIn, values: ['never-used-value']} ``` For further information, check [BGP Peering Policy documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/#bgp-peering-policy-legacy) ## Kube-proxy replacement with Cilium Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement` to `strict` (< v1.16) or `true` (Cilium v1.16+ no longer accepts `strict`, however this is converted to `true` by kubespray when running v1.16+). Without kube-proxy, cilium needs to know the address of the kube-apiserver and this must be set globally for all Cilium components (agents and operators). We can only use the localhost apiserver loadbalancer in this mode whenever it uses the same port as the kube-apiserver (by default it does). ## Cilium Operator Unlike some operators, Cilium Operator does not exist for installation purposes. > The Cilium Operator is responsible for managing duties in the cluster which should logically be handled once for the entire cluster, rather than once for each node in the cluster. ### Adding custom flags to the Cilium Operator You can set additional cilium-operator container arguments using `cilium_operator_custom_args`. This is an advanced option, and you should only use it if you know what you are doing. Accepts an array or a string. ```yml cilium_operator_custom_args: ["--foo=bar", "--baz=qux"] ``` or ```yml cilium_operator_custom_args: "--foo=bar" ``` You do not need to add a custom flag to enable debugging. Instead, feel free to use the `CILIUM_DEBUG` variable. ### Adding extra volumes and mounting them You can use `cilium_operator_extra_volumes` to add extra volumes to the Cilium Operator, and use `cilium_operator_extra_volume_mounts` to mount those volumes. This is an advanced option, and you should only use it if you know what you are doing. ```yml cilium_operator_extra_volumes: - configMap: name: foo name: foo-mount-path cilium_operator_extra_volume_mounts: - mountPath: /tmp/foo/bar name: foo-mount-path readOnly: true ``` ## Choose Cilium version ```yml cilium_version: "1.19.1" ``` ## Add variable to config Use following variables: Example: ```yml cilium_config_extra_vars: enable-endpoint-routes: true ``` ## Change Identity Allocation Mode Cilium assigns an identity for each endpoint. This identity is used to enforce basic connectivity between endpoints. Cilium currently supports two different identity allocation modes: - "crd" stores identities in kubernetes as CRDs (custom resource definition). - These can be queried with `kubectl get ciliumid` - "kvstore" stores identities in an etcd kvstore. ## Enable Transparent Encryption Cilium supports the transparent encryption of Cilium-managed host traffic and traffic between Cilium-managed endpoints either using IPsec or Wireguard. Wireguard option is only available in Cilium 1.10.0 and newer. ### IPsec Encryption For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/) To enable IPsec encryption, you just need to set three variables. ```yml cilium_encryption_enabled: true cilium_encryption_type: "ipsec" ``` The third variable is `cilium_ipsec_key`. You need to create a secret key string for this variable. Kubespray does not automate this process. Cilium documentation currently recommends creating a key using the following command: ```shell echo "3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128" ``` Note that Kubespray handles secret creation. So you only need to pass the key as the `cilium_ipsec_key` variable, base64 encoded: ```shell echo "cilium_ipsec_key: "$(echo -n "3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128" | base64 -w0) ``` ### Wireguard Encryption For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/security/network/encryption-wireguard/) To enable Wireguard encryption, you just need to set two variables. ```yml cilium_encryption_enabled: true cilium_encryption_type: "wireguard" ``` Kubespray currently supports Linux distributions with Wireguard Kernel mode on Linux 5.6 and newer. ## Bandwidth Manager Cilium's bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies. In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods. Bandwidth Manager requires a v5.1.x or more recent Linux kernel. For further information, make sure to check the official [Cilium documentation](https://docs.cilium.io/en/latest/network/kubernetes/bandwidth-manager/) To use this function, set the following parameters ```yml cilium_enable_bandwidth_manager: true ``` ## Host Firewall Host Firewall enforces security policies for Kubernetes nodes. It is disable by default, since it can break the cluster connectivity. ```yaml cilium_enable_host_firewall: true ``` For further information, check [host firewall documentation](https://docs.cilium.io/en/latest/security/host-firewall/) ## Policy Audit Mode When _Policy Audit Mode_ is enabled, no network policy is enforced. This feature helps to validate the impact of host policies before enforcing them. ```yaml cilium_policy_audit_mode: true ``` It is disable by default, and should not be enabled in production. ## Install Cilium Hubble k8s-net-cilium.yml: ```yml cilium_enable_hubble: true ## enable support hubble in cilium cilium_hubble_install: true ## install hubble-relay, hubble-ui cilium_hubble_tls_generate: true ## install hubble-certgen and generate certificates ``` To validate that Hubble UI is properly configured, set up a port forwarding for hubble-ui service: ```shell script kubectl port-forward -n kube-system svc/hubble-ui 12000:80 ``` and then open [http://localhost:12000/](http://localhost:12000/). ## Hubble metrics ```yml cilium_enable_hubble_metrics: true cilium_hubble_metrics: - dns - drop - tcp - flow - icmp - http ``` [More](https://docs.cilium.io/en/v1.9/operations/metrics/#hubble-exported-metrics) ## Upgrade considerations ### Rolling-restart timeouts Cilium relies on the kernel's BPF support, which is extremely fast at runtime but incurs a compilation penalty on initialization and update. As a result, the Cilium DaemonSet pods can take a significant time to start, which scales with the number of nodes and endpoints in your cluster. As part of cluster.yml, this DaemonSet is restarted, and Kubespray's [default timeouts for this operation](../roles/network_plugin/cilium/defaults/main.yml) are not appropriate for large clusters. This means that you will likely want to update these timeouts to a value more in-line with your cluster's number of nodes and their respective CPU performance. This is configured by the following values: ```yaml # Configure how long to wait for the Cilium DaemonSet to be ready again cilium_rolling_restart_wait_retries_count: 30 cilium_rolling_restart_wait_retries_delay_seconds: 10 ``` The total time allowed (count * delay) should be at least `($number_of_nodes_in_cluster * $cilium_pod_start_time)` for successful rolling updates. There are no drawbacks to making it higher and giving yourself a time buffer to accommodate transient slowdowns. Note: To find the `$cilium_pod_start_time` for your cluster, you can simply restart a Cilium pod on a node of your choice and look at how long it takes for it to become ready. Note 2: The default CPU requests/limits for Cilium pods is set to a very conservative 100m:500m which will likely yield very slow startup for Cilium pods. You probably want to significantly increase the CPU limit specifically if short bursts of CPU from Cilium are acceptable to you. ================================================ FILE: docs/CNI/cni.md ================================================ CNI ============== This network plugin only unpacks CNI plugins version `cni_version` into `/opt/cni/bin` and instructs implementation of container runtime cri to use cni. It's intended usage is for custom CNI configuration, e.g. manual routing tables + bridge + loopback CNI plugin outside kubespray scope. Furthermore, it's used for non-kubespray supported CNI plugins which you can install afterward. You are required to fill `/etc/cni/net.d` with valid CNI configuration after using kubespray. ================================================ FILE: docs/CNI/flannel.md ================================================ # Flannel Flannel is a network fabric for containers, designed for Kubernetes Supported [backends](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard): `vxlan`, `host-gw` and `wireguard` **Warning:** You may encounter this [bug](https://github.com/coreos/flannel/pull/1282) with `VXLAN` backend, while waiting on a newer Flannel version the current workaround (`ethtool --offload flannel.1 rx off tx off`) is showcase in kubespray [networking test](tests/testcases/040_check-network-adv.yml:31). ## Verifying flannel install * Flannel configuration file should have been created there ```ShellSession cat /run/flannel/subnet.env FLANNEL_NETWORK=10.233.0.0/18 FLANNEL_SUBNET=10.233.16.1/24 FLANNEL_MTU=1450 FLANNEL_IPMASQ=false ``` * Check if the network interface has been created ```ShellSession ip a show dev flannel.1 4: flannel.1: mtu 1450 qdisc noqueue state UNKNOWN group default link/ether e2:f3:a7:0f:bf:cb brd ff:ff:ff:ff:ff:ff inet 10.233.16.0/18 scope global flannel.1 valid_lft forever preferred_lft forever inet6 fe80::e0f3:a7ff:fe0f:bfcb/64 scope link valid_lft forever preferred_lft forever ``` * Try to run a container and check its ip address ```ShellSession kubectl run test --image=busybox --command -- tail -f /dev/null replicationcontroller "test" created kubectl describe po test-34ozs | grep ^IP IP: 10.233.16.2 ``` ```ShellSession kubectl exec test-34ozs -- ip a show dev eth0 8: eth0@if9: mtu 1450 qdisc noqueue link/ether 02:42:0a:e9:2b:03 brd ff:ff:ff:ff:ff:ff inet 10.233.16.2/24 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::42:aff:fee9:2b03/64 scope link tentative flags 08 valid_lft forever preferred_lft forever ``` ================================================ FILE: docs/CNI/kube-ovn.md ================================================ # Kube-OVN Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises. For more information please check [Kube-OVN documentation](https://github.com/alauda/kube-ovn) **Warning:** Kernel version (`cat /proc/version`) needs to be different from `3.10.0-862` or kube-ovn won't start and will print this message: ```bash kernel version 3.10.0-862 has a nat related bug that will affect ovs function, please update to a version greater than 3.10.0-898 ``` ## How to use it Enable kube-ovn in `group_vars/k8s_cluster/k8s_cluster.yml` ```yml ... kube_network_plugin: kube-ovn ... ``` ## Verifying kube-ovn install Kube-OVN run ovn and controller in `kube-ovn` namespace * Check the status of kube-ovn pods ```ShellSession # From the CLI kubectl get pod -n kube-ovn # Output NAME READY STATUS RESTARTS AGE kube-ovn-cni-49lsm 1/1 Running 0 2d20h kube-ovn-cni-9db8f 1/1 Running 0 2d20h kube-ovn-cni-wftdk 1/1 Running 0 2d20h kube-ovn-controller-68d7bb48bd-7tnvg 1/1 Running 0 2d21h ovn-central-6675dbb7d9-d7z8m 1/1 Running 0 4d16h ovs-ovn-hqn8p 1/1 Running 0 4d16h ovs-ovn-hvpl8 1/1 Running 0 4d16h ovs-ovn-r5frh 1/1 Running 0 4d16h ``` * Check the default and node subnet ```ShellSession # From the CLI kubectl get subnet # Output NAME PROTOCOL CIDR PRIVATE NAT join IPv4 100.64.0.0/16 false false ovn-default IPv4 10.16.0.0/16 false true ``` ================================================ FILE: docs/CNI/kube-router.md ================================================ # Kube-router Kube-router is a L3 CNI provider, as such it will setup IPv4 routing between nodes to provide Pods' networks reachability. See [kube-router documentation](https://www.kube-router.io/). ## Verifying kube-router install Kube-router runs its pods as a `DaemonSet` in the `kube-system` namespace: * Check the status of kube-router pods ```ShellSession # From the CLI kubectl get pod --namespace=kube-system -l k8s-app=kube-router -owide # output NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE kube-router-4f679 1/1 Running 0 2d 192.168.186.4 mykube-k8s-node-nf-2 kube-router-5slf8 1/1 Running 0 2d 192.168.186.11 mykube-k8s-node-nf-3 kube-router-lb6k2 1/1 Running 0 20h 192.168.186.14 mykube-k8s-node-nf-6 kube-router-rzvrb 1/1 Running 0 20h 192.168.186.17 mykube-k8s-node-nf-4 kube-router-v6n56 1/1 Running 0 2d 192.168.186.6 mykube-k8s-node-nf-1 kube-router-wwhg8 1/1 Running 0 20h 192.168.186.16 mykube-k8s-node-nf-5 kube-router-x2xs7 1/1 Running 0 2d 192.168.186.10 mykube-k8s-master-1 ``` * Peek at kube-router container logs: ```ShellSession # From the CLI kubectl logs --namespace=kube-system -l k8s-app=kube-router | grep Peer.Up # output time="2018-09-17T16:47:14Z" level=info msg="Peer Up" Key=192.168.186.6 State=BGP_FSM_OPENCONFIRM Topic=Peer time="2018-09-17T16:47:16Z" level=info msg="Peer Up" Key=192.168.186.11 State=BGP_FSM_OPENCONFIRM Topic=Peer time="2018-09-17T16:47:46Z" level=info msg="Peer Up" Key=192.168.186.10 State=BGP_FSM_OPENCONFIRM Topic=Peer time="2018-09-18T19:12:24Z" level=info msg="Peer Up" Key=192.168.186.14 State=BGP_FSM_OPENCONFIRM Topic=Peer time="2018-09-18T19:12:28Z" level=info msg="Peer Up" Key=192.168.186.17 State=BGP_FSM_OPENCONFIRM Topic=Peer time="2018-09-18T19:12:38Z" level=info msg="Peer Up" Key=192.168.186.16 State=BGP_FSM_OPENCONFIRM Topic=Peer [...] ``` ## Gathering kube-router state Kube-router Pods come bundled with a "Pod Toolbox" which provides very useful internal state views for: * IPVS: via `ipvsadm` * BGP peering and routing info: via `gobgp` You need to `kubectl exec -it ...` into a kube-router container to use these, see for details. ## Kube-router configuration You can change the default configuration by overriding `kube_router_...` variables (as found at `roles/network_plugin/kube-router/defaults/main.yml`), these are named to follow `kube-router` command-line options as per . ## Advanced BGP Capabilities If you have other networking devices or SDN systems that talk BGP, kube-router will fit in perfectly. From a simple full node-to-node mesh to per-node peering configurations, most routing needs can be attained. The configuration is Kubernetes native (annotations) just like the rest of kube-router. For more details please refer to the . Next options will set up annotations for kube-router, using `kubectl annotate` command. ```yml kube_router_annotations_master: [] kube_router_annotations_node: [] kube_router_annotations_all: [] ``` ================================================ FILE: docs/CNI/macvlan.md ================================================ # Macvlan ## How to use it * Enable macvlan in `group_vars/k8s_cluster/k8s_cluster.yml` ```yml ... kube_network_plugin: macvlan ... ``` * Adjust the `macvlan_interface` in `group_vars/k8s_cluster/k8s-net-macvlan.yml` or by host in the `host.yml` file: ```yml all: hosts: node1: ip: 10.2.2.1 access_ip: 10.2.2.1 ansible_host: 10.2.2.1 macvlan_interface: ens5 ``` ## Issue encountered * Service DNS reply from unexpected source: add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml` * Disable nodelocaldns The nodelocal dns IP is not reachable. Disable it in `sample/group_vars/k8s_cluster/k8s_cluster.yml` ```yml enable_nodelocaldns: false ``` ================================================ FILE: docs/CNI/multus.md ================================================ # Multus Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface, Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. See [multus documentation](https://github.com/k8snetworkplumbingwg/multus-cni). ## Multus installation Since Multus itself does not implement networking, it requires a master plugin, which is specified through the variable `kube_network_plugin`. To enable Multus an additional variable `kube_network_plugin_multus` must be set to `true`. For example, ```yml kube_network_plugin: calico kube_network_plugin_multus: true ``` will install Multus and Calico and configure Multus to use Calico as the primary network plugin. Namespace isolation enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. To enable namespace isolation: ```yml multus_namespace_isolation: true ``` ### Cilium compatibility If you are using `cilium` as the primary CNI you'll have to set `cilium_cni_exclusive` to `false` to avoid cillium reverting multus config. ```yml kube_network_plugin: cilium kube_network_plugin_multus: true cilium_cni_exclusive: false ``` ## Using Multus Once Multus is installed, you can create CNI configurations (as a CRD objects) for additional networks, in this case a macvlan CNI configuration is defined. You may replace the config field with any valid CNI configuration where the CNI binary is available on the nodes. ```ShellSession cat < ================================================ FILE: docs/CRI/containerd.md ================================================ # containerd [containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability Kubespray supports basic functionality for using containerd as the default container runtime in a cluster. _To use the containerd container runtime set the following variables:_ ## k8s_cluster.yml When kube_node contains etcd, you define your etcd cluster to be as well schedulable for Kubernetes workloads. Thus containerd and dockerd can not run at same time, must be set to bellow for running etcd cluster with only containerd. ```yaml container_manager: containerd ``` ## etcd.yml ```yaml etcd_deployment_type: host ``` ## Containerd config Example: define registry mirror for docker hub ```yaml containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - host: https://registry-1.docker.io capabilities: ["pull", "resolve"] skip_verify: false ``` containerd falls back to `https://{{ prefix }}` when none of the mirrors have the image. This can be changed with the [`server` field](https://github.com/containerd/containerd/blob/main/docs/hosts.md#server-field): ```yaml containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - host: https://registry-1.docker.io capabilities: ["pull", "resolve"] skip_verify: false server: https://mirror.example.org ``` The `containerd_registries` and `containerd_insecure_registries` configs are deprecated. ### Containerd Runtimes Containerd supports multiple runtime configurations that can be used with [RuntimeClass] Kubernetes feature. See [runtime classes in containerd] for the details of containerd configuration. In kubespray, the default runtime name is "runc", and it can be configured with the `containerd_runc_runtime` dictionary: ```yaml containerd_runc_runtime: name: runc type: "io.containerd.runc.v2" options: Root: "" SystemdCgroup: "false" BinaryName: /usr/local/bin/my-runc base_runtime_spec: cri-base.json ``` Further runtimes can be configured with `containerd_additional_runtimes`, which is a list of such dictionaries. Default runtime can be changed by setting `containerd_default_runtime`. #### Base runtime specs and limiting number of open files `base_runtime_spec` key in a runtime dictionary is used to explicitly specify a runtime spec json file. `runc` runtime has it set to `cri-base.json`, which is generated with `ctr oci spec > /etc/containerd/cri-base.json` and updated to include a custom setting for maximum number of file descriptors per container. You can change maximum number of file descriptors per container for the default `runc` runtime by setting the `containerd_base_runtime_spec_rlimit_nofile` variable. You can tune many more [settings][runtime-spec] by supplying your own file name and content with `containerd_base_runtime_specs`: ```yaml containerd_base_runtime_specs: cri-spec-custom.json: | { "ociVersion": "1.1.0", "process": { "user": { "uid": 0, ... ``` The files in this dict will be placed in containerd config directory, `/etc/containerd` by default. The files can then be referenced by filename in a runtime: ```yaml containerd_runc_runtime: name: runc base_runtime_spec: cri-spec-custom.json ... ``` Config insecure-registry access to self hosted registries. ```yaml containerd_registries_mirrors: - prefix: test.registry.io mirrors: - host: http://test.registry.io capabilities: ["pull", "resolve"] skip_verify: true - prefix: 172.19.16.11:5000 mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve"] skip_verify: true - prefix: repo:5000 mirrors: - host: http://repo:5000 capabilities: ["pull", "resolve"] skip_verify: true ``` [containerd]: https://containerd.io/ [RuntimeClass]: https://kubernetes.io/docs/concepts/containers/runtime-class/ [runtime classes in containerd]: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#runtime-classes [runtime-spec]: https://github.com/opencontainers/runtime-spec ### Optional : NRI [Node Resource Interface](https://github.com/containerd/nri) (NRI) is disabled by default for the containerd. If you are using contained version v1.7.0 or above, then you can enable it with the following configuration: ```yaml nri_enabled: true ``` ### Optional : Static Binary To ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary. ```yaml containerd_static_binary: true ``` ================================================ FILE: docs/CRI/cri-o.md ================================================ # CRI-O [CRI-O] is a lightweight container runtime for Kubernetes. Kubespray supports basic functionality for using CRI-O as the default container runtime in a cluster. * Kubernetes supports CRI-O on v1.11.1 or later. * etcd: configure either kubeadm managed etcd or host deployment _To use the CRI-O container runtime set the following variables:_ ## all/all.yml ```yaml download_container: false skip_downloads: false etcd_deployment_type: host # optionally kubeadm ``` ## k8s_cluster/k8s_cluster.yml ```yaml container_manager: crio ``` ## all/crio.yml Enable docker hub registry mirrors ```yaml crio_registries: - prefix: docker.io insecure: false blocked: false location: registry-1.docker.io unqualified: false mirrors: - location: 192.168.100.100:5000 insecure: true - location: mirror.gcr.io insecure: false ``` [CRI-O]: https://cri-o.io/ The following is a method to enable insecure registries. ```yaml crio_insecure_registries: - 10.0.0.2:5000 ``` And you can config authentication for these registries after `crio_insecure_registries`. ```yaml crio_registry_auth: - registry: 10.0.0.2:5000 username: user password: pass ``` ## Note about user namespaces CRI-O has support for user namespaces. This feature is optional and can be enabled by setting the following two variables. ```yaml crio_runtimes: - name: runc path: /usr/bin/runc type: oci root: /run/runc allowed_annotations: - "io.kubernetes.cri-o.userns-mode" crio_remap_enable: true ``` The `allowed_annotations` configures `crio.conf` accordingly. The `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to add an entry for the **containers** user. By default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space. The `crio_default_capabilities` configure the default containers capabilities for the crio. Defaults capabilities are: ```yaml crio_default_capabilities: - CHOWN - DAC_OVERRIDE - FSETID - FOWNER - SETGID - SETUID - SETPCAP - NET_BIND_SERVICE - KILL ``` You can add MKNOD to the list for a rancher deployment ## Optional : NRI [Node Resource Interface](https://github.com/containerd/nri) (NRI) is disabled by default for the CRI-O. If you are using CRI-O version v1.26.0 or above, then you can enable it with the following configuration: ```yaml nri_enabled: true ``` ================================================ FILE: docs/CRI/docker.md ================================================ # Docker support The docker runtime is supported by kubespray and while the `dockershim` is deprecated to be removed in kubernetes 1.24+ there are alternative ways to use docker such as through the [cri-dockerd](https://github.com/Mirantis/cri-dockerd) project supported by Mirantis. Using the docker container manager: ```yaml container_manager: docker ``` *Note:* `cri-dockerd` has replaced `dockershim` across supported kubernetes version in kubespray 2.20. Enabling the `overlay2` graph driver: ```yaml docker_storage_options: -s overlay2 ``` Changing the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`: ```yaml docker_cgroup_driver: systemd ``` If you have more than 3 nameservers kubespray will only use the first 3 else it will fail. Set the `docker_dns_servers_strict` to `false` to prevent deployment failure. ```yaml docker_dns_servers_strict: false ``` Set the path used to store Docker data: ```yaml docker_daemon_graph: "/var/lib/docker" ``` Changing the docker daemon iptables support: ```yaml docker_iptables_enabled: "false" ``` Docker log options: ```yaml # Rotate container stderr/stdout logs at 50m and keep last 5 docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" ``` Change the docker `bin_dir`, this should not be changed unless you use a custom docker package: ```yaml docker_bin_dir: "/usr/bin" ``` To keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'. kubespray deletes the docker package on each run, so caching the package makes sense: ```yaml docker_rpm_keepcache: 1 ``` Allowing insecure-registry access to self hosted registries. Can be ipaddress and domain_name. ```yaml ## example define 172.19.16.11 or mirror.registry.io docker_insecure_registries: - mirror.registry.io - 172.19.16.11 ``` Adding other registry, i.e. China registry mirror: ```yaml docker_registry_mirrors: - https://registry.docker-cn.com - https://mirror.aliyuncs.com ``` Overriding default system MountFlags value. This option takes a mount propagation flag: `shared`, `slave` or `private`, which control whether mounts in the file system namespace set up for docker will receive or propagate mounts and unmounts. Leave empty for system default: ```yaml docker_mount_flags: ``` Adding extra options to pass to the docker daemon: ```yaml ## This string should be exactly as you wish it to appear. docker_options: "" ``` For Debian based distributions, set the path to store the GPG key to avoid using the default one used in `apt_key` module (e.g. /etc/apt/trusted.gpg) ```yaml docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg ``` ================================================ FILE: docs/CRI/gvisor.md ================================================ # gVisor [gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system. gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers. ## Usage To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`. Containerd support: ```yaml container_manager: containerd gvisor_enabled: true ``` ================================================ FILE: docs/CRI/kata-containers.md ================================================ # Kata Containers [Kata Containers](https://katacontainers.io) is a secure container runtime with lightweight virtual machines that supports multiple hypervisor solutions. ## Hypervisors _Qemu_ is the only hypervisor supported by Kubespray. ## Installation To enable Kata Containers, set the following variables: **k8s-cluster.yml**: ```yaml container_manager: containerd kata_containers_enabled: true ``` **etcd.yml**: ```yaml etcd_deployment_type: host ``` ## Usage By default, runc is used for pods. Kubespray generates the runtimeClass kata-qemu, and it is necessary to specify it as the runtimeClassName of a pod spec to use Kata Containers: ```shell $ kubectl get runtimeclass NAME HANDLER AGE kata-qemu kata-qemu 3m34s $ $ cat nginx.yaml apiVersion: v1 kind: Pod metadata: name: mypod spec: runtimeClassName: kata-qemu containers: - name: nginx image: nginx:1.14.2 $ $ kubectl apply -f nginx.yaml ``` ## Configuration ### Recommended : Pod Overhead [Pod Overhead](https://kubernetes.io/docs/concepts/configuration/pod-overhead/) is a feature for accounting for the resources consumed by the Runtime Class used by the Pod. When this feature is enabled, Kubernetes will count the fixed amount of CPU and memory set in the configuration as used by the virtual machine and not by the containers running in the Pod. Pod Overhead is mandatory if you run Pods with Kata Containers that use [resources limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits). **Set cgroup driver**: To enable Pod Overhead feature you have to configure Kubelet with the appropriate cgroup driver, using the following configuration: `cgroupfs` works best: ```yaml kubelet_cgroup_driver: cgroupfs ``` ... but when using `cgroups v2` (see ) you can use systemd as well: ```yaml kubelet_cgroup_driver: systemd ``` **Qemu hypervisor configuration**: The configuration for the Qemu hypervisor uses the following values: ```yaml kata_containers_qemu_overhead: true kata_containers_qemu_overhead_fixed_cpu: 10m kata_containers_qemu_overhead_fixed_memory: 290Mi ``` ### Optional : Select Kata Containers version Optionally you can select the Kata Containers release version to be installed. The available releases are published in [GitHub](https://github.com/kata-containers/kata-containers/releases). ```yaml kata_containers_version: 2.2.2 ``` ### Optional : Debug Debug is disabled by default for all the components of Kata Containers. You can change this behaviour with the following configuration: ```yaml kata_containers_qemu_debug: 'false' ``` ================================================ FILE: docs/CSI/aws-ebs-csi.md ================================================ # AWS EBS CSI Driver AWS EBS CSI driver allows you to provision EBS volumes for pods in EC2 instances. The old in-tree AWS cloud provider is deprecated and will be removed in future versions of Kubernetes. So transitioning to the CSI driver is advised. To enable AWS EBS CSI driver, uncomment the `aws_ebs_csi_enabled` option in `group_vars/all/aws.yml` and set it to `true`. To set the number of replicas for the AWS CSI controller, you can change `aws_ebs_csi_controller_replicas` option in `group_vars/all/aws.yml`. Make sure to add a role, for your EC2 instances hosting Kubernetes, that allows it to do the actions necessary to request a volume and attach it: [AWS CSI Policy](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json) If you want to deploy the AWS EBS storage class used with the CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`. You can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over AWS EC2 with EBS CSI Driver enabled. ## Usage example To check if AWS EBS CSI Driver is deployed properly, check that the ebs-csi pods are running: ```ShellSession $ kubectl -n kube-system get pods | grep ebs ebs-csi-controller-85d86bccc5-8gtq5 4/4 Running 4 40s ebs-csi-node-n4b99 3/3 Running 3 40s ``` Check the associated storage class (if you enabled persistent_volumes): ```ShellSession $ kubectl get storageclass NAME PROVISIONER AGE ebs-sc ebs.csi.aws.com 45s ``` You can run a PVC and an example Pod using this file `ebs-pod.yml`: ```yml -- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ebs-claim spec: accessModes: - ReadWriteOnce storageClassName: ebs-sc resources: requests: storage: 1Gi --- apiVersion: v1 kind: Pod metadata: name: app spec: containers: - name: app image: centos command: ["/bin/sh"] args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"] volumeMounts: - name: persistent-storage mountPath: /data volumes: - name: persistent-storage persistentVolumeClaim: claimName: ebs-claim ``` Apply this conf to your cluster: ```kubectl apply -f ebs-pod.yml``` You should see the PVC provisioned and bound: ```ShellSession $ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ebs-claim Bound pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c 1Gi RWO ebs-sc 50s ``` And the volume mounted to the example Pod (wait until the Pod is Running): ```ShellSession $ kubectl exec -it app -- df -h | grep data /dev/nvme1n1 1014M 34M 981M 4% /data ``` ## More info For further information about the AWS EBS CSI Driver, you can refer to this page: [AWS EBS Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/). ================================================ FILE: docs/CSI/azure-csi.md ================================================ # Azure Disk CSI Driver The Azure Disk CSI driver allows you to provision volumes for pods with a Kubernetes deployment over Azure Cloud. The CSI driver replaces to volume provisioning done by the in-tree azure cloud provider which is deprecated. This documentation is an updated version of the in-tree Azure cloud provider documentation (azure.md). To deploy Azure Disk CSI driver, uncomment the `azure_csi_enabled` option in `group_vars/all/azure.yml` and set it to `true`. ## Azure Disk CSI Storage Class If you want to deploy the Azure Disk storage class to provision volumes dynamically, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`. ## Parameters Before creating the instances you must first set the `azure_csi_` variables in the `group_vars/all.yml` file. All values can be retrieved using the azure cli tool which can be downloaded here: After installation you have to run `az login` to get access to your account. ### azure\_csi\_tenant\_id + azure\_csi\_subscription\_id Run `az account show` to retrieve your subscription id and tenant id: `azure_csi_tenant_id` -> tenantId field `azure_csi_subscription_id` -> id field ### azure\_csi\_location The region your instances are located in, it can be something like `francecentral` or `norwayeast`. A full list of region names can be retrieved via `az account list-locations` ### azure\_csi\_resource\_group The name of the resource group your instances are in, a list of your resource groups can be retrieved via `az group list` Or you can do `az vm list | grep resourceGroup` and get the resource group corresponding to the VMs of your cluster. The resource group name is not case-sensitive. ### azure\_csi\_vnet\_name The name of the virtual network your instances are in, can be retrieved via `az network vnet list` ### azure\_csi\_vnet\_resource\_group The name of the resource group your vnet is in, can be retrieved via `az network vnet list | grep resourceGroup` and get the resource group corresponding to the vnet of your cluster. ### azure\_csi\_subnet\_name The name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME` ### azure\_csi\_security\_group\_name The name of the network security group your instances are in, can be retrieved via `az network nsg list` ### azure\_csi\_aad\_client\_id + azure\_csi\_aad\_client\_secret These will have to be generated first: - Create an Azure AD Application with: ```ShellSession az ad app create --display-name kubespray --identifier-uris http://kubespray --homepage http://kubespray.com --password CLIENT_SECRET ``` Display name, identifier-uri, homepage and the password can be chosen Note the AppId in the output. - Create Service principal for the application with: ```ShellSession az ad sp create --id AppId ``` This is the AppId from the last command - Create the role assignment with: ```ShellSession az role assignment create --role "Owner" --assignee http://kubespray --subscription SUBSCRIPTION_ID ``` azure\_csi\_aad\_client\_id must be set to the AppId, azure\_csi\_aad\_client\_secret is your chosen secret. ### azure\_csi\_use\_instance\_metadata Use instance metadata service where possible. Boolean value. ## Test the Azure Disk CSI driver To test the dynamic provisioning using Azure CSI driver, make sure to have the storage class deployed (through persistent volumes), and apply the following manifest: ```yml --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-azuredisk spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi storageClassName: disk.csi.azure.com --- kind: Pod apiVersion: v1 metadata: name: nginx-azuredisk spec: nodeSelector: kubernetes.io/os: linux containers: - image: nginx name: nginx-azuredisk command: - "/bin/sh" - "-c" - while true; do echo $(date) >> /mnt/azuredisk/outfile; sleep 1; done volumeMounts: - name: azuredisk mountPath: "/mnt/azuredisk" volumes: - name: azuredisk persistentVolumeClaim: claimName: pvc-azuredisk ``` ================================================ FILE: docs/CSI/cinder-csi.md ================================================ # Cinder CSI Driver Cinder CSI driver allows you to provision volumes over an OpenStack deployment. The Kubernetes historic in-tree cloud provider is deprecated and will be removed in future versions. To enable Cinder CSI driver, uncomment the `cinder_csi_enabled` option in `group_vars/all/openstack.yml` and set it to `true`. To set the number of replicas for the Cinder CSI controller, you can change `cinder_csi_controller_replicas` option in `group_vars/all/openstack.yml`. You need to source the OpenStack credentials you use to deploy your machines that will host Kubernetes: `source path/to/your/openstack-rc` or `. path/to/your/openstack-rc`. Make sure the hostnames in your `inventory` file are identical to your instance names in OpenStack. Otherwise [cinder](https://docs.openstack.org/cinder/latest/) won't work as expected. If you want to deploy the cinder provisioner used with Cinder CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`. You can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over OpenStack with Cinder CSI Driver enabled. ## Usage example To check if Cinder CSI Driver works properly, see first that the cinder-csi pods are running: ```ShellSession $ kubectl -n kube-system get pods | grep cinder csi-cinder-controllerplugin-7f8bf99785-cpb5v 5/5 Running 0 100m csi-cinder-nodeplugin-rm5x2 2/2 Running 0 100m ``` Check the associated storage class (if you enabled persistent_volumes): ```ShellSession $ kubectl get storageclass NAME PROVISIONER AGE cinder-csi cinder.csi.openstack.org 100m ``` You can run a PVC and an Nginx Pod using this file `nginx.yaml`: ```yml --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: csi-pvc-cinderplugin spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi storageClassName: cinder-csi --- apiVersion: v1 kind: Pod metadata: name: nginx spec: containers: - image: nginx imagePullPolicy: IfNotPresent name: nginx ports: - containerPort: 80 protocol: TCP volumeMounts: - mountPath: /var/lib/www/html name: csi-data-cinderplugin volumes: - name: csi-data-cinderplugin persistentVolumeClaim: claimName: csi-pvc-cinderplugin readOnly: false ``` Apply this conf to your cluster: ```kubectl apply -f nginx.yml``` You should see the PVC provisioned and bound: ```ShellSession $ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE csi-pvc-cinderplugin Bound pvc-f21ad0a1-5b7b-405e-a462-48da5cb76beb 1Gi RWO cinder-csi 8s ``` And the volume mounted to the Nginx Pod (wait until the Pod is Running): ```ShellSession kubectl exec -it nginx -- df -h | grep /var/lib/www/html /dev/vdb 976M 2.6M 958M 1% /var/lib/www/html ``` ## Compatibility with in-tree cloud provider It is not necessary to enable OpenStack as a cloud provider for Cinder CSI Driver to work. Though, you can run both the in-tree openstack cloud provider and the Cinder CSI Driver at the same time. The storage class provisioners associated to each one of them are differently named. ## Cinder v2 support For the moment, only Cinder v3 is supported by the CSI Driver. ## More info For further information about the Cinder CSI Driver, you can refer to this page: [Cloud Provider OpenStack](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md). ================================================ FILE: docs/CSI/gcp-pd-csi.md ================================================ # GCP Persistent Disk CSI Driver The GCP Persistent Disk CSI driver allows you to provision volumes for pods with a Kubernetes deployment over Google Cloud Platform. The CSI driver replaces to volume provioning done by the in-tree azure cloud provider which is deprecated. To deploy GCP Persistent Disk CSI driver, uncomment the `gcp_pd_csi_enabled` option in `group_vars/all/gcp.yml` and set it to `true`. ## GCP Persistent Disk Storage Class If you want to deploy the GCP Persistent Disk storage class to provision volumes dynamically, you should set `persistent_volumes_enabled` in `group_vars/k8s_cluster/k8s_cluster.yml` to `true`. ## GCP credentials In order for the CSI driver to provision disks, you need to create for it a service account on GCP with the appropriate permissions. Follow these steps to configure it: ```ShellSession # This will open a web page for you to authenticate gcloud auth login export PROJECT=nameofmyproject gcloud config set project $PROJECT git clone https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver export GCE_PD_SA_NAME=my-gce-pd-csi-sa export GCE_PD_SA_DIR=/my/safe/credentials/directory ./deploy/setup-project.sh ``` The above will create a file named `cloud-sa.json` in the specified `GCE_PD_SA_DIR`. This file contains the service account with the appropriate credentials for the CSI driver to perform actions on GCP to request disks for pods. You need to provide this file's path through the variable `gcp_pd_csi_sa_cred_file` in `inventory/mycluster/group_vars/all/gcp.yml` You can now deploy Kubernetes with Kubespray over GCP. ## GCP PD CSI Driver test To test the dynamic provisioning using GCP PD CSI driver, make sure to have the storage class deployed (through persistent volumes), and apply the following manifest: ```yml --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: podpvc spec: accessModes: - ReadWriteOnce storageClassName: csi-gce-pd resources: requests: storage: 1Gi --- apiVersion: v1 kind: Pod metadata: name: web-server spec: containers: - name: web-server image: nginx volumeMounts: - mountPath: /var/lib/www/html name: mypvc volumes: - name: mypvc persistentVolumeClaim: claimName: podpvc readOnly: false ``` ## GCP PD documentation You can find the official GCP Persistent Disk CSI driver installation documentation here: [GCP PD CSI Driver](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/docs/kubernetes/user-guides/driver-install.md ) ================================================ FILE: docs/CSI/vsphere-csi.md ================================================ # vSphere CSI Driver vSphere CSI driver allows you to provision volumes over a vSphere deployment. The Kubernetes historic in-tree cloud provider is deprecated and will be removed in future versions. ## Prerequisites The vSphere user for CSI driver requires a set of privileges to perform Cloud Native Storage operations. Follow the [official guide](https://vsphere-csi-driver.sigs.k8s.io/driver-deployment/prerequisites.html#roles_and_privileges) to configure those. ## Kubespray configuration To enable vSphere CSI driver, uncomment the `vsphere_csi_enabled` option in `group_vars/all/vsphere.yml` and set it to `true`. To set the number of replicas for the vSphere CSI controller, you can change `vsphere_csi_controller_replicas` option in `group_vars/all/vsphere.yml`. You need to source the vSphere credentials you use to deploy your machines that will host Kubernetes. | Variable | Required | Type | Choices | Default | Comment | |-------------------------------------------------|----------|---------|-----------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------| | external_vsphere_vcenter_ip | TRUE | string | | | IP/URL of the vCenter | | external_vsphere_vcenter_port | TRUE | string | | "443" | Port of the vCenter API | | external_vsphere_insecure | TRUE | string | "true", "false" | "true" | set to "true" if the host above uses a self-signed cert | | external_vsphere_user | TRUE | string | | | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) | | external_vsphere_password | TRUE | string | | | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable) | | external_vsphere_datacenter | TRUE | string | | | Datacenter name to use | | external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use | | external_vsphere_version | TRUE | string | | "7.0u1" | Vmware Vsphere version where located all VMs | | external_vsphere_cloud_controller_image_tag | TRUE | string | | "v1.31.0" | CPI manager image tag to use | | vsphere_syncer_image_tag | TRUE | string | | "v3.3.1" | Syncer image tag to use | | vsphere_csi_attacher_image_tag | TRUE | string | | "v4.3.0" | CSI attacher image tag to use | | vsphere_csi_controller | TRUE | string | | "v3.3.1" | CSI controller image tag to use | | vsphere_csi_controller_replicas | TRUE | integer | | 1 | Number of pods Kubernetes should deploy for the CSI controller | | vsphere_csi_liveness_probe_image_tag | TRUE | string | | "v2.10.0" | CSI liveness probe image tag to use | | vsphere_csi_provisioner_image_tag | TRUE | string | | "v2.1.0" | CSI provisioner image tag to use | | vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v3.5.0" | CSI node driver registrar image tag to use | | vsphere_csi_driver_image_tag | TRUE | string | | "v3.3.1" | CSI driver image tag to use | | vsphere_csi_resizer_tag | TRUE | string | | "v1.8.0" | CSI resizer image tag to use | | vsphere_csi_aggressive_node_drain | FALSE | boolean | | false | Enable aggressive node drain strategy | | vsphere_csi_aggressive_node_unreachable_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in an unreachable state | | vsphere_csi_aggressive_node_not_ready_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in not-ready state | | vsphere_csi_namespace | TRUE | string | | "kube-system" | vSphere CSI namespace to use; kube-system for backward compatibility, should be change to vmware-system-csi on the long run | ## Usage example To test the dynamic provisioning using vSphere CSI driver, make sure to create a [storage policy](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#create-a-storage-policy) and [storage class](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#create-a-storageclass), then apply the following manifest: ```yml --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: csi-pvc-vsphere spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi storageClassName: mongodb-sc --- apiVersion: v1 kind: Pod metadata: name: nginx spec: containers: - image: nginx imagePullPolicy: IfNotPresent name: nginx ports: - containerPort: 80 protocol: TCP volumeMounts: - mountPath: /usr/share/nginx/html name: csi-data-vsphere volumes: - name: csi-data-vsphere persistentVolumeClaim: claimName: csi-pvc-vsphere readOnly: false ``` Apply this conf to your cluster: ```kubectl apply -f nginx.yml``` You should see the PVC provisioned and bound: ```ShellSession $ kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE csi-pvc-vsphere Bound pvc-dc7b1d21-ee41-45e1-98d9-e877cc1533ac 1Gi RWO mongodb-sc 10s ``` And the volume mounted to the Nginx Pod (wait until the Pod is Running): ```ShellSession kubectl exec -it nginx -- df -h | grep /usr/share/nginx/html /dev/sdb 976M 2.6M 907M 1% /usr/share/nginx/html ``` ## More info For further information about the vSphere CSI Driver, you can refer to the official [vSphere Cloud Provider documentation](https://cloud-provider-vsphere.sigs.k8s.io/container_storage_interface.html). ================================================ FILE: docs/_sidebar.md ================================================ * [Readme](/) * Advanced * [Arch](/docs/advanced/arch.md) * [Cert Manager](/docs/advanced/cert_manager.md) * [Dns-stack](/docs/advanced/dns-stack.md) * [Downloads](/docs/advanced/downloads.md) * [Gcp-lb](/docs/advanced/gcp-lb.md) * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md) * [Netcheck](/docs/advanced/netcheck.md) * [Ntp](/docs/advanced/ntp.md) * [Proxy](/docs/advanced/proxy.md) * [Registry](/docs/advanced/registry.md) * Ansible * [Ansible](/docs/ansible/ansible.md) * [Ansible Collection](/docs/ansible/ansible_collection.md) * [Inventory](/docs/ansible/inventory.md) * [Vars](/docs/ansible/vars.md) * Cloud Controllers * [Openstack](/docs/cloud_controllers/openstack.md) * [Vsphere](/docs/cloud_controllers/vsphere.md) * Cloud Providers * [Aws](/docs/cloud_providers/aws.md) * [Azure](/docs/cloud_providers/azure.md) * [Cloud](/docs/cloud_providers/cloud.md) * CNI * [Calico](/docs/CNI/calico.md) * [Cilium](/docs/CNI/cilium.md) * [Cni](/docs/CNI/cni.md) * [Flannel](/docs/CNI/flannel.md) * [Kube-ovn](/docs/CNI/kube-ovn.md) * [Kube-router](/docs/CNI/kube-router.md) * [Macvlan](/docs/CNI/macvlan.md) * [Multus](/docs/CNI/multus.md) * CRI * [Containerd](/docs/CRI/containerd.md) * [Cri-o](/docs/CRI/cri-o.md) * [Docker](/docs/CRI/docker.md) * [Gvisor](/docs/CRI/gvisor.md) * [Kata-containers](/docs/CRI/kata-containers.md) * CSI * [Aws-ebs-csi](/docs/CSI/aws-ebs-csi.md) * [Azure-csi](/docs/CSI/azure-csi.md) * [Cinder-csi](/docs/CSI/cinder-csi.md) * [Gcp-pd-csi](/docs/CSI/gcp-pd-csi.md) * [Vsphere-csi](/docs/CSI/vsphere-csi.md) * Developers * [Ci-setup](/docs/developers/ci-setup.md) * [Ci](/docs/developers/ci.md) * [Test Cases](/docs/developers/test_cases.md) * [Vagrant](/docs/developers/vagrant.md) * External Storage Provisioners * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md) * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md) * Getting Started * [Comparisons](/docs/getting_started/comparisons.md) * [Getting-started](/docs/getting_started/getting-started.md) * [Setting-up-your-first-cluster](/docs/getting_started/setting-up-your-first-cluster.md) * Ingress * [Alb Ingress Controller](/docs/ingress/alb_ingress_controller.md) * [Kube-vip](/docs/ingress/kube-vip.md) * [Metallb](/docs/ingress/metallb.md) * Operating Systems * [Amazonlinux](/docs/operating_systems/amazonlinux.md) * [Bootstrap-os](/docs/operating_systems/bootstrap-os.md) * [Fcos](/docs/operating_systems/fcos.md) * [Flatcar](/docs/operating_systems/flatcar.md) * [Kylinlinux](/docs/operating_systems/kylinlinux.md) * [Openeuler](/docs/operating_systems/openeuler.md) * [Opensuse](/docs/operating_systems/opensuse.md) * [Rhel](/docs/operating_systems/rhel.md) * [Uoslinux](/docs/operating_systems/uoslinux.md) * Operations * [Cgroups](/docs/operations/cgroups.md) * [Encrypting-secret-data-at-rest](/docs/operations/encrypting-secret-data-at-rest.md) * [Etcd](/docs/operations/etcd.md) * [Ha-mode](/docs/operations/ha-mode.md) * [Hardening](/docs/operations/hardening.md) * [Integration](/docs/operations/integration.md) * [Kernel-requirements](/docs/operations/kernel-requirements.md) * [Large-deployments](/docs/operations/large-deployments.md) * [Mirror](/docs/operations/mirror.md) * [Nodes](/docs/operations/nodes.md) * [Offline-environment](/docs/operations/offline-environment.md) * [Port-requirements](/docs/operations/port-requirements.md) * [Recover-control-plane](/docs/operations/recover-control-plane.md) * [Upgrades](/docs/operations/upgrades.md) * Roadmap * [Roadmap](/docs/roadmap/roadmap.md) * Upgrades * [Migrate Docker2containerd](/docs/upgrades/migrate_docker2containerd.md) ================================================ FILE: docs/advanced/arch.md ================================================ # Architecture compatibility The following table shows the impact of the CPU architecture on compatible features: - amd64: Cluster using only x86/amd64 CPUs - arm64: Cluster using only arm64 CPUs - amd64 + arm64: Cluster with a mix of x86/amd64 and arm64 CPUs | kube_network_plugin | amd64 | arm64 | amd64 + arm64 | |---------------------|-------|-------|---------------| | Calico | Y | Y | Y | | Flannel | Y | N | N | | Canal | Y | N | N | | Cilium | Y | Y | N | | Contib | Y | N | N | | kube-router | Y | N | N | ================================================ FILE: docs/advanced/cert_manager.md ================================================ # Installation Guide - [Installation Guide](#installation-guide) - [Kubernetes TLS Root CA Certificate/Key Secret](#kubernetes-tls-root-ca-certificatekey-secret) - [Securing Ingress Resources](#securing-ingress-resources) - [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key) - [Install Cloudflare PKI/TLS `cfssl` Toolkit.](#install-cloudflare-pkitls-cfssl-toolkit) - [Create Root Certificate Authority (CA) Configuration File](#create-root-certificate-authority-ca-configuration-file) - [Create Certificate Signing Request (CSR) Configuration File](#create-certificate-signing-request-csr-configuration-file) - [Create TLS Root CA Certificate and Key](#create-tls-root-ca-certificate-and-key) Cert-Manager is a native Kubernetes certificate management controller. It can help with issuing certificates from a variety of sources, such as Let’s Encrypt, HashiCorp Vault, Venafi, a simple signing key pair, or self signed. It will ensure certificates are valid and up to date, and attempt to renew certificates at a configured time before expiry. ## Kubernetes TLS Root CA Certificate/Key Secret If you're planning to secure your ingress resources using TLS client certificates, you'll need to create and deploy the Kubernetes `ca-key-pair` secret consisting of the Root CA certificate and key to your K8s cluster. For further information, read the official [Cert-Manager CA Configuration](https://cert-manager.io/docs/configuration/ca/) doc. `cert-manager` can now be enabled by editing your K8s cluster addons inventory e.g. `inventory\sample\group_vars\k8s_cluster\addons.yml` and setting `cert_manager_enabled` to true. ```ini # Cert manager deployment cert_manager_enabled: true ``` If you don't have a TLS Root CA certificate and key available, you can create these by following the steps outlined in section [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key) using the Cloudflare PKI/TLS `cfssl` toolkit. TLS Root CA certificates and keys can also be created using `ssh-keygen` and OpenSSL, if `cfssl` is not available. ## Securing Ingress Resources A common use-case for cert-manager is requesting TLS signed certificates to secure your ingress resources. This can be done by simply adding annotations to your Ingress resources and cert-manager will facilitate creating the Certificate resource for you. A small sub-component of cert-manager, ingress-shim, is responsible for this. For example, if you're using the Traefik ingress controller, you can secure the Prometheus ingress by adding the annotation `cert-manager.io/cluster-issuer: ca-issuer` and the `spec.tls` section to the `Ingress` resource definition. ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: prometheus-k8s namespace: monitoring labels: prometheus: k8s annotations: cert-manager.io/cluster-issuer: ca-issuer spec: ingressClassName: "traefik" tls: - hosts: - prometheus.example.com secretName: prometheus-dashboard-certs rules: - host: prometheus.example.com http: paths: - path: / pathType: ImplementationSpecific backend: service: name: prometheus-k8s port: name: web ``` Once deployed to your K8s cluster, every 3 months cert-manager will automatically rotate the Prometheus `prometheus.example.com` TLS client certificate and key, and store these as the Kubernetes `prometheus-dashboard-certs` secret. Please consult the official upstream documentation: - [cert-manager Ingress Usage](https://cert-manager.io/usage/ingress/) - [cert-manager Ingress Tutorial](https://cert-manager.io/tutorials/acme/ingress/#step-3-assign-a-dns-name) ### ACME The ACME Issuer type represents a single account registered with the Automated Certificate Management Environment (ACME) Certificate Authority server. When you create a new ACME Issuer, cert-manager will generate a private key which is used to identify you with the ACME server. Certificates issued by public ACME servers are typically trusted by client’s computers by default. This means that, for example, visiting a website that is backed by an ACME certificate issued for that URL, will be trusted by default by most client’s web browsers. ACME certificates are typically free. - [ACME Configuration](https://cert-manager.io/docs/configuration/acme/) - [ACME HTTP Validation](https://cert-manager.io/docs/tutorials/acme/http-validation/) - [HTTP01 Challenges](https://cert-manager.io/docs/configuration/acme/http01/) - [ACME DNS Validation](https://cert-manager.io/docs/tutorials/acme/dns-validation/) - [DNS01 Challenges](https://cert-manager.io/docs/configuration/acme/dns01/) - [ACME FAQ](https://cert-manager.io/docs/troubleshooting/acme/) #### ACME With An Internal Certificate Authority The ACME Issuer with an internal certificate authority requires cert-manager to trust the certificate authority. This trust must be done at the cert-manager deployment level. To add a trusted certificate authority to cert-manager, add it's certificate to `group_vars/k8s-cluster/addons.yml`: ```yaml cert_manager_trusted_internal_ca: | -----BEGIN CERTIFICATE----- [REPLACE with your CA certificate] -----END CERTIFICATE----- ``` Once the CA is trusted, you can define your issuer normally. ### Create New TLS Root CA Certificate and Key #### Install Cloudflare PKI/TLS `cfssl` Toolkit e.g. For Ubuntu/Debian distributions, the toolkit is part of the `golang-cfssl` package. ```shell sudo apt-get install -y golang-cfssl ``` #### Create Root Certificate Authority (CA) Configuration File The default TLS certificate expiry time period is `8760h` which is 1 years from the date the certificate is created. ```shell $ cat > ca-config.json < ca-csr.json < for details). These are configurable in inventory in as a dictionary in the `dns_upstream_forward_extra_opts` variable. By default, no other option than the ones hardcoded (see `roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2` and `roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2`). ### coredns_kubernetes_extra_opts Custom options to be added to the kubernetes coredns plugin. ### coredns_kubernetes_extra_domains Extra domains to be forwarded to the kubernetes coredns plugin. ### coredns_additional_configs Extra configuration to be added to CoreDNS configuration ### coredns_rewrite_block [Rewrite](https://coredns.io/plugins/rewrite/) plugin block to perform internal message rewriting. ### coredns_external_zones Array of optional external zones to coredns forward queries to. It's injected into `coredns`' config file before default kubernetes zone. Use it as an optimization for well-known zones and/or internal-only domains, i.e. VPN for internal networks (default is unset) Example: ```yaml coredns_external_zones: - zones: - example.com - example.io:1053 nameservers: - 1.1.1.1 - 2.2.2.2 cache: 5 - zones: - https://mycompany.local:4453 nameservers: - 192.168.0.53 cache: 0 - zones: - mydomain.tld nameservers: - 10.233.0.3 cache: 5 rewrite: - name stop website.tld website.namespace.svc.cluster.local ``` or as INI ```ini coredns_external_zones='[{"cache": 30,"zones":["example.com","example.io:453"],"nameservers":["1.1.1.1","2.2.2.2"]}]' ``` ### dns_etchosts (coredns) Optional hosts file content to coredns use as /etc/hosts file. This will also be used by nodelocaldns, if enabled. Example: ```yaml dns_etchosts: | 192.168.0.100 api.example.com 192.168.0.200 ingress.example.com ``` ### enable_coredns_reverse_dns_lookups Whether reverse DNS lookups are enabled in the coredns config. Defaults to `true`. ### CoreDNS default zone cache plugin If you wish to configure the caching behaviour of CoreDNS on the default zone, you can do so using the `coredns_default_zone_cache_block` string block. An example value (more information on the [plugin's documentation](https://coredns.io/plugins/cache/)) to: * raise the max cache TTL to 3600 seconds * raise the max amount of success responses to cache to 3000 * disable caching of denial responses altogether * enable pre-fetching of lookups with at least 10 lookups per minute before they expire Would be as follows: ```yaml coredns_default_zone_cache_block: | cache 3600 { success 3000 denial 0 prefetch 10 1m } ``` ### Handle old/extra dns_domains If you need to change the dns_domain of your cluster for whatever reason (switching to or from `cluster.local` for example), and you have workloads that embed it in their configuration you can use the variable `old_dns_domains`. This will add some configuration to coredns and nodelocaldns to ensure the DNS requests using the old domain are handled correctly. Example: ```yaml old_dns_domains: - example1.com - example2.com dns_domain: cluster.local ``` will make `my-svc.my-ns.svc.example1.com`, `my-svc.my-ns.svc.example2.com` and `my-svc.my-ns.svc.cluster.local` have the same DNS answer. ### systemd_resolved_disable_stub_listener Whether or not to set `DNSStubListener=no` when using systemd-resolved. Defaults to `true` on Flatcar. You might need to set it to `true` if CoreDNS fails to start with `address already in use` errors. ## DNS modes supported by Kubespray You can modify how Kubespray sets up DNS for your cluster with the variables ``dns_mode`` and ``resolvconf_mode``. ### dns_mode ``dns_mode`` configures how Kubespray will setup cluster DNS. There are four modes available: #### dns_mode: coredns (default) This installs CoreDNS as the default cluster DNS for all queries. #### dns_mode: coredns_dual This installs CoreDNS as the default cluster DNS for all queries, plus a secondary CoreDNS stack. #### dns_mode: manual This does not install coredns, but allows you to specify `manual_dns_server`, which will be configured on nodes for handling Pod DNS. Use this method if you plan to install your own DNS server in the cluster after initial deployment. #### dns_mode: none This does not install any of DNS solution at all. This basically disables cluster DNS completely and leaves you with a non functional cluster. ## resolvconf_mode ``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers. There are three modes available: ### resolvconf_mode: host_resolvconf (default) This activates the classic Kubespray behavior that modifies the hosts ``/etc/resolv.conf`` file and dhclient configuration to point to the cluster dns server (either coredns or coredns_dual, depending on dns_mode). As cluster DNS is not available on early deployment stage, this mode is split into 2 stages. In the first stage (``dns_early: true``), ``/etc/resolv.conf`` is configured to use the DNS servers found in ``upstream_dns_servers`` and ``nameservers``. Later, ``/etc/resolv.conf`` is reconfigured to use the cluster DNS server first, leaving the other nameservers as backups. Also note, existing records will be purged from the `/etc/resolv.conf`, including resolvconf's base/head/cloud-init config files and those that come from dhclient. ### resolvconf_mode: docker_dns This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags. The following nameservers are added to the docker daemon (in the same order as listed here): * cluster nameserver (depends on dns_mode) * content of optional upstream_dns_servers variable * host system nameservers (read from hosts /etc/resolv.conf) The following search domains are added to the docker daemon (in the same order as listed here): * cluster domains (``default.svc.{{ dns_domain }}``, ``svc.{{ dns_domain }}``) * content of optional searchdomains variable * host system search domains (read from hosts /etc/resolv.conf) The following dns options are added to the docker daemon * ndots:{{ ndots }} * timeout:2 * attempts:2 These dns options can be overridden by setting a different list: ```yaml docker_dns_options: - ndots:{{ ndots }} - timeout:2 - attempts:2 - rotate ``` For normal PODs, k8s will ignore these options and setup its own DNS settings for the PODs, taking the --cluster_dns (either coredns or coredns_dual, depending on dns_mode) kubelet option into account. For ``hostNetwork: true`` PODs however, k8s will let docker setup DNS settings. Docker containers which are not started/managed by k8s will also use these docker options. The host system name servers are added to ensure name resolution is also working while cluster DNS is not running yet. This is especially important in early stages of cluster deployment. In this early stage, DNS queries to the cluster DNS will timeout after a few seconds, resulting in the system nameserver being used as a backup nameserver. After cluster DNS is running, all queries will be answered by the cluster DNS servers, which in turn will forward queries to the system nameserver if required. ### resolvconf_mode: none Does nothing regarding ``/etc/resolv.conf``. This leaves you with a cluster that works as expected in most cases. The only exception is that ``hostNetwork: true`` PODs and non-k8s managed containers will not be able to resolve cluster service names. ## Nodelocal DNS cache Setting ``enable_nodelocaldns`` to ``true`` will make pods reach out to the dns (core-dns) caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query core-dns (depending on what main DNS plugin is configured in your cluster) for cache misses of cluster hostnames(cluster.local suffix by default). More information on the rationale behind this implementation can be found [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/1024-nodelocal-cache-dns/README.md). **As per the 2.10 release, Nodelocal DNS cache is enabled by default.** ### External zones It's possible to extent the `nodelocaldns`' configuration by adding an array of external zones. For example: ```yaml nodelocaldns_external_zones: - zones: - example.com - example.io:1053 nameservers: - 1.1.1.1 - 2.2.2.2 cache: 5 - zones: - https://mycompany.local:4453 nameservers: - 192.168.0.53 ``` ### dns_etchosts (nodelocaldns) See [dns_etchosts](#dns_etchosts-coredns) above. ### nodelocaldns_additional_configs Extra configuration to be added to CoreDNS configuration ### Nodelocal DNS HA Under some circumstances the single POD nodelocaldns implementation may not be able to be replaced soon enough and a cluster upgrade or a nodelocaldns upgrade can cause DNS requests to time out for short intervals. If for any reason your applications cannot tolerate this behavior you can enable a redundant nodelocal DNS pod on each node: ```yaml enable_nodelocaldns_secondary: true ``` **Note:** when the nodelocaldns secondary is enabled, the primary is instructed to no longer tear down the iptables rules it sets up to direct traffic to itself. In case both daemonsets have failing pods on the same node, this can cause a DNS blackout with traffic no longer being forwarded to the coredns central service as a fallback. Please ensure you account for this also if you decide to disable the nodelocaldns cache. There is a time delta (in seconds) allowed for the secondary nodelocaldns to survive in case both primary and secondary daemonsets are updated at the same time. It is advised to tune this variable after you have performed some tests in your own environment. ```yaml nodelocaldns_secondary_skew_seconds: 5 ``` ## Limitations * Kubespray has yet ways to configure Kubedns addon to forward requests SkyDns can not answer with authority to arbitrary recursive resolvers. This task is left for future. See [official SkyDns docs](https://github.com/skynetservices/skydns) for details. * There is [no way to specify a custom value](https://github.com/kubernetes/kubernetes/issues/33554) for the SkyDNS ``ndots`` param. * the ``searchdomains`` have a limitation of a 6 names and 256 chars length. Due to default ``svc, default.svc`` subdomains, the actual limits are a 4 names and 239 chars respectively. If `remove_default_searchdomains: true` added you are back to 6 names. * the ``nameservers`` have a limitation of a 3 servers, although there is a way to mitigate that with the ``upstream_dns_servers``, see below. Anyway, the ``nameservers`` can take no more than a two custom DNS servers because of one slot is reserved for a Kubernetes cluster needs. ================================================ FILE: docs/advanced/downloads.md ================================================ # Downloading binaries and containers Kubespray supports several download/upload modes. The default is: * Each node downloads binaries and container images on its own, which is ``download_run_once: False``. * For K8s apps, pull policy is ``k8s_image_pull_policy: IfNotPresent``. * For system managed containers, like kubelet or etcd, pull policy is ``download_always_pull: False``, which is pull if only the wanted repo and tag/sha256 digest differs from that the host has. There is also a "pull once, push many" mode as well: * Setting ``download_run_once: True`` will make kubespray download container images and binaries only once and then push them to the cluster nodes. The default download delegate node is the first `kube_control_plane`. * Set ``download_localhost: True`` to make localhost the download delegate. This can be useful if cluster nodes cannot access external addresses. To use this requires that the container runtime is installed and running on the Ansible master and that the current user is either in the docker group or can do passwordless sudo, to be able to use the container runtime. Note: even if `download_localhost` is false, files will still be copied to the Ansible server (local host) from the delegated download node, and then distributed from the Ansible server to all cluster nodes. NOTE: When `download_run_once` is true and `download_localhost` is false, all downloads will be done on the delegate node, including downloads for container images that are not required on that node. As a consequence, the storage required on that node will probably be more than if download_run_once was false, because all images will be loaded into the storage of the container runtime on that node, instead of just the images required for that node. On caching: * When `download_run_once` is `True`, all downloaded files will be cached locally in `download_cache_dir`, which defaults to `/tmp/kubespray_cache`. On subsequent provisioning runs, this local cache will be used to provision the nodes, minimizing bandwidth usage and improving provisioning time. Expect about 800MB of disk space to be used on the ansible node for the cache. Disk space required for the image cache on the kubernetes nodes is a much as is needed for the largest image, which is currently slightly less than 150MB. * By default, if `download_run_once` is false, kubespray will not retrieve the downloaded images and files from the download delegate node to the local cache, or use that cache to pre-provision those nodes. If you have a full cache with container images and files and you don’t need to download anything, but want to use a cache - set `download_force_cache` to `True`. * By default, cached images that are used to pre-provision the remote nodes will be deleted from the remote nodes after use, to save disk space. Setting `download_keep_remote_cache` will prevent the files from being deleted. This can be useful while developing kubespray, as it can decrease provisioning times. As a consequence, the required storage for images on the remote nodes will increase from 150MB to about 550MB, which is currently the combined size of all required container images. Container images and binary files are described by the vars like ``foo_version``, ``foo_download_url``, ``foo_checksum`` for binaries and ``foo_image_repo``, ``foo_image_tag`` or optional ``foo_digest_checksum`` for containers. Container images may be defined by its repo and tag, for example: `andyshinn/dnsmasq:2.72`. Or by repo and tag and sha256 digest: `andyshinn/dnsmasq@sha256:7c883354f6ea9876d176fe1d30132515478b2859d6fc0cbf9223ffdc09168193`. Note, the SHA256 digest and the image tag must be both specified and correspond to each other. The given example above is represented by the following vars: ```yaml dnsmasq_digest_checksum: 7c883354f6ea9876d176fe1d30132515478b2859d6fc0cbf9223ffdc09168193 dnsmasq_image_repo: andyshinn/dnsmasq dnsmasq_image_tag: '2.72' ``` The full list of available vars may be found in the download's ansible role defaults. Those also allow to specify custom urls and local repositories for binaries and container images as well. See also the DNS stack docs for the related intranet configuration, so the hosts can resolve those urls and repos. ================================================ FILE: docs/advanced/gcp-lb.md ================================================ # GCP Load Balancers for type=LoadBalancer of Kubernetes Services > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) Google Cloud Platform can be used for creation of Kubernetes Service Load Balancer. This feature is able to deliver by adding parameters to `kube-controller-manager` and `kubelet`. You need specify: ```ShellSession --cloud-provider=gce --cloud-config=/etc/kubernetes/cloud-config ``` To get working it in kubespray, you need to add tag to GCE instances and specify it in kubespray group vars and also set `cloud_provider` to `gce`. So for example, in file `group_vars/all/gcp.yml`: ```yaml cloud_provider: gce gce_node_tags: k8s-lb ``` When you will setup it and create SVC in Kubernetes with `type=LoadBalancer`, cloud provider will create public IP and will set firewall. Note: Cloud provider run under VM service account, so this account needs to have correct permissions to be able to create all GCP resources. ================================================ FILE: docs/advanced/kubernetes-reliability.md ================================================ # Overview Distributed system such as Kubernetes are designed to be resilient to the failures. More details about Kubernetes High-Availability (HA) may be found at [Building High-Availability Clusters](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) To have a simple view the most of the parts of HA will be skipped to describe Kubelet<->Controller Manager communication only. By default the normal behavior looks like: 1. Kubelet updates it status to apiserver periodically, as specified by `--node-status-update-frequency`. The default value is **10s**. 2. Kubernetes controller manager checks the statuses of Kubelet every `–-node-monitor-period`. The default value is **5s**. 3. In case the status is updated within `--node-monitor-grace-period` of time, Kubernetes controller manager considers healthy status of Kubelet. The default value is **40s**. > Kubernetes controller manager and Kubelet work asynchronously. It means that > the delay may include any network latency, API Server latency, etcd latency, > latency caused by load on one's control plane nodes and so on. So if > `--node-status-update-frequency` is set to 5s in reality it may appear in > etcd in 6-7 seconds or even longer when etcd cannot commit data to quorum > nodes. ## Failure Kubelet will try to make `nodeStatusUpdateRetry` post attempts. Currently `nodeStatusUpdateRetry` is constantly set to 5 in [kubelet.go](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet.go#L102). Kubelet will try to update the status in [tryUpdateNodeStatus](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet_node_status.go#L312) function. Kubelet uses `http.Client()` Golang method, but has no specified timeout. Thus there may be some glitches when API Server is overloaded while TCP connection is established. So, there will be `nodeStatusUpdateRetry` * `--node-status-update-frequency` attempts to set a status of node. At the same time Kubernetes controller manager will try to check `nodeStatusUpdateRetry` times every `--node-monitor-period` of time. After `--node-monitor-grace-period` it will consider node unhealthy. Pods will then be rescheduled based on the [Taint Based Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions) timers that you set on them individually, or the API Server's global timers:`--default-not-ready-toleration-seconds` & ``--default-unreachable-toleration-seconds``. Kube proxy has a watcher over API. Once pods are evicted, Kube proxy will notice and will update iptables of the node. It will remove endpoints from services so pods from failed node won't be accessible anymore. ## Recommendations for different cases ## Fast Update and Fast Reaction If `--node-status-update-frequency` is set to **4s** (10s is default). `--node-monitor-period` to **2s** (5s is default). `--node-monitor-grace-period` to **20s** (40s is default). `--default-not-ready-toleration-seconds` and ``--default-unreachable-toleration-seconds`` are set to **30** (300 seconds is default). Note these two values should be integers representing the number of seconds ("s" or "m" for seconds\minutes are not specified). In such scenario, pods will be evicted in **50s** because the node will be considered as down after **20s**, and `--default-not-ready-toleration-seconds` or ``--default-unreachable-toleration-seconds`` occur after **30s** more. However, this scenario creates an overhead on etcd as every node will try to update its status every 2 seconds. If the environment has 1000 nodes, there will be 15000 node updates per minute which may require large etcd containers or even dedicated nodes for etcd. > If we calculate the number of tries, the division will give 5, but in reality > it will be from 3 to 5 with `nodeStatusUpdateRetry` attempts of each try. The > total number of attempts will vary from 15 to 25 due to latency of all > components. ## Medium Update and Average Reaction Let's set `--node-status-update-frequency` to **20s** `--node-monitor-grace-period` to **2m** and `--default-not-ready-toleration-seconds` and ``--default-unreachable-toleration-seconds`` to **60**. In that case, Kubelet will try to update status every 20s. So, it will be 6 * 5 = 30 attempts before Kubernetes controller manager will consider unhealthy status of node. After 1m it will evict all pods. The total time will be 3m before eviction process. Such scenario is good for medium environments as 1000 nodes will require 3000 etcd updates per minute. > In reality, there will be from 4 to 6 node update tries. The total number of > of attempts will vary from 20 to 30. ## Low Update and Slow reaction Let's set `--node-status-update-frequency` to **1m**. `--node-monitor-grace-period` will set to **5m** and `--default-not-ready-toleration-seconds` and ``--default-unreachable-toleration-seconds`` to **60**. In this scenario, every kubelet will try to update the status every minute. There will be 5 * 5 = 25 attempts before unhealthy status. After 5m, Kubernetes controller manager will set unhealthy status. This means that pods will be evicted after 1m after being marked unhealthy. (6m in total). > In reality, there will be from 3 to 5 tries. The total number of attempt will > vary from 15 to 25. There can be different combinations such as Fast Update with Slow reaction to satisfy specific cases. ================================================ FILE: docs/advanced/netcheck.md ================================================ # Network Checker Application With the ``deploy_netchecker`` var enabled (defaults to false), Kubespray deploys a Network Checker Application from the 3rd side `mirantis/k8s-netchecker` docker images. It consists of the server and agents trying to reach the server by usual for Kubernetes applications network connectivity meanings. Therefore, this automatically verifies a pod to pod connectivity via the cluster IP and checks if DNS resolve is functioning as well. The checks are run by agents on a periodic basis and cover standard and host network pods as well. The history of performed checks may be found in the agents' application logs. To get the most recent and cluster-wide network connectivity report, run from any of the cluster nodes: ```ShellSession curl http://localhost:31081/api/v1/connectivity_check ``` Note that Kubespray does not invoke the check but only deploys the application, if requested. There are related application specific variables: ```yml netchecker_port: 31081 agent_report_interval: 15 netcheck_namespace: default ``` Note that the application verifies DNS resolve for FQDNs comprising only the combination of the ``netcheck_namespace.dns_domain`` vars, for example the ``netchecker-service.default.svc.cluster.local``. If you want to deploy the application to the non default namespace, make sure as well to adjust the ``searchdomains`` var so the resulting search domain records to contain that namespace, like: ```yml search: foospace.cluster.local default.cluster.local ... nameserver: ... ``` ================================================ FILE: docs/advanced/ntp.md ================================================ # NTP synchronization The Network Time Protocol (NTP) is a networking protocol for clock synchronization between computer systems. Time synchronization is important to Kubernetes and Etcd. ## Enable the NTP To start the ntpd(or chrony) service and enable it at system boot. There are related specific variables: ```ShellSession ntp_enabled: true ``` The NTP service would be enabled and sync time automatically. ## Customize the NTP configure file In the Air-Gap environment, the node cannot access the NTP server by internet. So the node can use the customized ntp server by configuring ntp file. ```ShellSession ntp_enabled: true ntp_manage_config: true ntp_servers: - "0.your-ntp-server.org iburst" - "1.your-ntp-server.org iburst" - "2.your-ntp-server.org iburst" - "3.your-ntp-server.org iburst" ``` ## Setting the TimeZone The timezone can also be set by the `ntp_timezone` , eg: "Etc/UTC","Asia/Shanghai". If not set, the timezone will not change. ```ShellSession ntp_enabled: true ntp_timezone: Etc/UTC ``` ## Advanced Configure Enable `tinker panic` is useful when running NTP in a VM environment to avoiding clock drift on VMs. It only takes effect when ntp_manage_config is true. ```ShellSession ntp_tinker_panic: true ``` Force sync time immediately by NTP after the ntp installed, which is useful in newly installed system. ```ShellSession ntp_force_sync_immediately: true ``` When using Ubuntu 24.04 or a distribution that already has `systemd-timesyncd` installed, use the `ntpsec` package. ```ShellSession ntp_package: ntpsec ``` ================================================ FILE: docs/advanced/proxy.md ================================================ # Setting up Environment Proxy If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy. ## Set proxy for http and https `http_proxy:"http://example.proxy.tld:port"` `https_proxy:"http://example.proxy.tld:port"` ## Set custom CA CA must be already on each target nodes `https_proxy_cert_file: /path/to/host/custom/ca.crt` ## Set default no_proxy (this will override default no_proxy generation) `no_proxy: "node1,node1_ip,node2,node2_ip...additional_host"` ## Set additional addresses to default no_proxy (all cluster nodes and loadbalancer) `additional_no_proxy: "additional_host1,additional_host2"` ## Exclude workers from no_proxy Since workers are included in the no_proxy variable, by default, docker engine will be restarted on all nodes (all pods will restart) when adding or removing workers. To override this behaviour by only including control plane nodes in the no_proxy variable, set: `no_proxy_exclude_workers: true` ================================================ FILE: docs/advanced/registry.md ================================================ # Private Docker Registry in Kubernetes Kubernetes offers an optional private Docker registry addon, which you can turn on when you bring up a cluster or install later. This gives you a place to store truly private Docker images for your cluster. ## How it works The private registry runs as a `Pod` in your cluster. It does not currently support SSL or authentication, which triggers Docker's "insecure registry" logic. To work around this, we run a proxy on each node in the cluster, exposing a port onto the node (via a hostPort), which Docker accepts as "secure", since it is accessed by `localhost`. ## Turning it on Some cluster installs (e.g. GCE) support this as a cluster-birth flag. The `ENABLE_CLUSTER_REGISTRY` variable in `cluster/gce/config-default.sh` governs whether the registry is run or not. To set this flag, you can specify `KUBE_ENABLE_CLUSTER_REGISTRY=true` when running `kube-up.sh`. If your cluster does not include this flag, the following steps should work. Note that some of this is cloud-provider specific, so you may have to customize it a bit. ### Make some storage The primary job of the registry is to store data. To do that we have to decide where to store it. For cloud environments that have networked storage, we can use Kubernetes's `PersistentVolume` abstraction. The following template is expanded by `salt` in the GCE cluster turnup, but can easily be adapted to other situations: ```yaml kind: PersistentVolume apiVersion: v1 metadata: name: kube-system-kube-registry-pv spec: {% if pillar.get('cluster_registry_disk_type', '') == 'gce' %} capacity: storage: {{ pillar['cluster_registry_disk_size'] }} accessModes: - ReadWriteOnce gcePersistentDisk: pdName: "{{ pillar['cluster_registry_disk_name'] }}" fsType: "ext4" {% endif %} ``` If, for example, you wanted to use NFS you would just need to change the `gcePersistentDisk` block to `nfs`. See [here](https://kubernetes.io/docs/concepts/storage/volumes/) for more details on volumes. Note that in any case, the storage (in the case the GCE PersistentDisk) must be created independently - this is not something Kubernetes manages for you (yet). ### I don't want or don't have persistent storage If you are running in a place that doesn't have networked storage, or if you just want to kick the tires on this without committing to it, you can easily adapt the `ReplicationController` specification below to use a simple `emptyDir` volume instead of a `persistentVolumeClaim`. ## Claim the storage Now that the Kubernetes cluster knows that some storage exists, you can put a claim on that storage. As with the `PersistentVolume` above, you can start with the `salt` template: ```yaml kind: PersistentVolumeClaim apiVersion: v1 metadata: name: kube-registry-pvc namespace: kube-system spec: accessModes: - ReadWriteOnce resources: requests: storage: {{ pillar['cluster_registry_disk_size'] }} ``` This tells Kubernetes that you want to use storage, and the `PersistentVolume` you created before will be bound to this claim (unless you have other `PersistentVolumes` in which case those might get bound instead). This claim gives you the right to use this storage until you release the claim. ## Run the registry Now we can run a Docker registry: ```yaml apiVersion: v1 kind: ReplicationController metadata: name: kube-registry-v0 namespace: kube-system labels: k8s-app: registry version: v0 spec: replicas: 1 selector: k8s-app: registry version: v0 template: metadata: labels: k8s-app: registry version: v0 spec: containers: - name: registry image: registry:2 resources: limits: cpu: 100m memory: 100Mi env: - name: REGISTRY_HTTP_ADDR value: :5000 - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY value: /var/lib/registry volumeMounts: - name: image-store mountPath: /var/lib/registry ports: - containerPort: 5000 name: registry protocol: TCP volumes: - name: image-store persistentVolumeClaim: claimName: kube-registry-pvc ``` *Note:* that if you have set multiple replicas, make sure your CSI driver has support for the `ReadWriteMany` accessMode. ## Expose the registry in the cluster Now that we have a registry `Pod` running, we can expose it as a Service: ```yaml apiVersion: v1 kind: Service metadata: name: kube-registry namespace: kube-system labels: k8s-app: registry kubernetes.io/name: "KubeRegistry" spec: selector: k8s-app: registry ports: - name: registry port: 5000 protocol: TCP ``` ## Expose the registry on each node Now that we have a running `Service`, we need to expose it onto each Kubernetes `Node` so that Docker will see it as `localhost`. We can load a `Pod` on every node by creating following daemonset. ```yaml apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-registry-proxy namespace: kube-system labels: k8s-app: kube-registry-proxy version: v0.4 spec: template: metadata: labels: k8s-app: kube-registry-proxy kubernetes.io/name: "kube-registry-proxy" version: v0.4 spec: containers: - name: kube-registry-proxy image: gcr.io/google_containers/kube-registry-proxy:0.4 resources: limits: cpu: 100m memory: 50Mi env: - name: REGISTRY_HOST value: kube-registry.kube-system.svc.cluster.local - name: REGISTRY_PORT value: "5000" ports: - name: registry containerPort: 80 hostPort: 5000 ``` When modifying replication-controller, service and daemon-set definitions, take care to ensure *unique* identifiers for the rc-svc couple and the daemon-set. Failing to do so will have register the localhost proxy daemon-sets to the upstream service. As a result they will then try to proxy themselves, which will, for obvious reasons, not work. This ensures that port 5000 on each node is directed to the registry `Service`. You should be able to verify that it is running by hitting port 5000 with a web browser and getting a 404 error: ```ShellSession $ curl localhost:5000 404 page not found ``` ## Using the registry To use an image hosted by this registry, simply say this in your `Pod`'s `spec.containers[].image` field: ```yaml image: localhost:5000/user/container ``` Before you can use the registry, you have to be able to get images into it, though. If you are building an image on your Kubernetes `Node`, you can spell out `localhost:5000` when you build and push. More likely, though, you are building locally and want to push to your cluster. You can use `kubectl` to set up a port-forward from your local node to a running Pod: ```ShellSession $ POD=$(kubectl get pods --namespace kube-system -l k8s-app=registry \ -o template --template '{{range .items}}{{.metadata.name}} {{.status.phase}}{{"\n"}}{{end}}' \ | grep Running | head -1 | cut -f1 -d' ') $ kubectl port-forward --namespace kube-system $POD 5000:5000 & ``` Now you can build and push images on your local computer as `localhost:5000/yourname/container` and those images will be available inside your kubernetes cluster with the same name. ================================================ FILE: docs/ansible/ansible.md ================================================ # Ansible ## Installing Ansible Kubespray supports multiple ansible versions and ships different `requirements.txt` files for them. Depending on your available python version you may be limited in choosing which ansible version to use. It is recommended to deploy the ansible version used by kubespray into a python virtual environment. ```ShellSession VENVDIR=kubespray-venv KUBESPRAYDIR=kubespray python3 -m venv $VENVDIR source $VENVDIR/bin/activate cd $KUBESPRAYDIR pip install -r requirements.txt ``` In case you have a similar message when installing the requirements: ```ShellSession ERROR: Could not find a version that satisfies the requirement ansible==7.6.0 (from -r requirements.txt (line 1)) (from versions: [...], 6.7.0) ERROR: No matching distribution found for ansible==7.6.0 (from -r requirements.txt (line 1)) ``` It means that the version of Python you are running is not compatible with the version of Ansible that Kubespray supports. If the latest version supported according to pip is 6.7.0 it means you are running Python 3.8 or lower while you need at least Python 3.9 (see the table below). ### Ansible Python Compatibility Based on the table below and the available python version for your ansible host you should choose the appropriate ansible version to use with kubespray. | Ansible Version | Python Version | |-------------------|----------------| | >=2.18.0, <2.19.0 | 3.11-3.13 | ## Customize Ansible vars Kubespray expects users to use one of the following variables sources for settings and customization: | Layer | Comment | |----------------------------------------|------------------------------------------------------------------------------| | inventory vars | | | - **inventory group_vars** | most used | | - inventory host_vars | host specific vars overrides, group_vars is usually more practical | | **extra vars** (always win precedence) | override with ``ansible-playbook -e @foo.yml`` | > Extra vars are best used to override kubespray internal variables, for instances, roles/vars/. Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray interface. Thus they can change, disappear, or break stuff unexpectedly. ## Ansible tags The following tags are defined in playbooks: | Tag name | Used for | |--------------------------------|-------------------------------------------------------| | annotate | Create kube-router annotation | | apps | K8s apps definitions | | asserts | Check tasks for download role | | aws-ebs-csi-driver | Configuring csi driver: aws-ebs | | azure-csi-driver | Configuring csi driver: azure | | bastion | Setup ssh config for bastion | | bootstrap_os | Anything related to host OS configuration | | calico | Network plugin Calico | | calico_rr | Configuring Calico route reflector | | cert-manager | Configuring certificate manager for K8s | | cilium | Network plugin Cilium | | cinder-csi-driver | Configuring csi driver: cinder | | client | Kubernetes clients role | | cloud-provider | Cloud-provider related tasks | | cluster-roles | Configuring cluster wide application (psp ...) | | cni | CNI plugins for Network Plugins | | containerd | Configuring containerd engine runtime for hosts | | container_engine_accelerator | Enable nvidia accelerator for runtimes | | container-engine | Configuring container engines | | container-runtimes | Configuring container runtimes | | control-plane | Configuring K8s control plane node role | | coredns | Configuring coredns deployment | | crio | Configuring crio container engine for hosts | | crun | Configuring crun runtime | | csi-driver | Configuring csi driver | | dns | Remove dns entries when resetting | | docker | Configuring docker engine runtime for hosts | | download | Fetching container images to a delegate host | | etcd | Configuring etcd cluster | | etcd-secrets | Configuring etcd certs/keys | | etchosts | Configuring /etc/hosts entries for hosts | | external-cloud-controller | Configure cloud controllers | | external-openstack | Cloud controller : openstack | | external-provisioner | Configure external provisioners | | external-vsphere | Cloud controller : vsphere | | facts | Gathering facts and misc check results | | files | Remove files when resetting | | flannel | Network plugin flannel | | gce | Cloud-provider GCP | | gcp-pd-csi-driver | Configuring csi driver: gcp-pd | | gvisor | Configuring gvisor runtime | | helm | Installing and configuring Helm | | ingress-controller | Configure ingress controllers | | ingress_alb | AWS ALB Ingress Controller | | init | Windows kubernetes init nodes | | iptables | Flush and clear iptable when resetting | | k8s-pre-upgrade | Upgrading K8s cluster | | kata-containers | Configuring kata-containers runtime | | kubeadm | Roles linked to kubeadm tasks | | kube-apiserver | Configuring static pod kube-apiserver | | kube-controller-manager | Configuring static pod kube-controller-manager | | kube-vip | Installing and configuring kube-vip | | kubectl | Installing kubectl and bash completion | | kubelet | Configuring kubelet service | | kube-ovn | Network plugin kube-ovn | | kube-router | Network plugin kube-router | | kube-proxy | Configuring static pod kube-proxy | | localhost | Special steps for the localhost (ansible runner) | | local-path-provisioner | Configure External provisioner: local-path | | local-volume-provisioner | Configure External provisioner: local-volume | | macvlan | Network plugin macvlan | | metallb | Installing and configuring metallb | | metrics_server | Configuring metrics_server | | netchecker | Installing netchecker K8s app | | network | Configuring networking plugins for K8s | | mounts | Umount kubelet dirs when resetting | | multus | Network plugin multus | | nginx | Configuring LB for kube-apiserver instances | | node | Configuring K8s minion (compute) node role | | nodelocaldns | Configuring nodelocaldns daemonset | | node-label | Tasks linked to labeling of nodes | | node-webhook | Tasks linked to webhook (granting access to resources)| | nvidia_gpu | Enable nvidia accelerator for runtimes | | oci | Cloud provider: oci | | persistent_volumes | Configure csi volumes | | persistent_volumes_aws_ebs_csi | Configuring csi driver: aws-ebs | | persistent_volumes_cinder_csi | Configuring csi driver: cinder | | persistent_volumes_gcp_pd_csi | Configuring csi driver: gcp-pd | | persistent_volumes_openstack | Configuring csi driver: openstack | | policy-controller | Configuring Calico policy controller | | post-remove | Tasks running post-remove operation | | post-upgrade | Tasks running post-upgrade operation | | pre-remove | Tasks running pre-remove operation | | pre-upgrade | Tasks running pre-upgrade operation | | preinstall | Preliminary configuration steps | | registry | Configuring local docker registry | | reset | Tasks running doing the node reset | | resolvconf | Configuring /etc/resolv.conf for hosts/apps | | services | Remove services (etcd, kubelet etc...) when resetting | | snapshot | Enabling csi snapshot | | snapshot-controller | Configuring csi snapshot controller | | system-packages | Install packages using OS package manager | | upgrade | Upgrading, f.e. container images/binaries | | upload | Distributing images/binaries across hosts | | vsphere-csi-driver | Configuring csi driver: vsphere | | win_nodes | Running windows specific tasks | | youki | Configuring youki runtime | ## Example commands Example command to filter and apply only DNS configuration tasks and skip everything else related to host OS configuration and downloading images of containers: ```ShellSession ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os ``` And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files: ```ShellSession ansible-playbook -i inventory/sample/hosts.ini -e dns_mode='none' cluster.yml --tags resolvconf ``` And this prepares all container images locally (at the ansible runner node) without installing or upgrading related stuff or trying to upload container to K8s cluster nodes: ```ShellSession ansible-playbook -i inventory/sample/hosts.ini cluster.yml \ -e download_run_once=true -e download_localhost=true \ --tags download --skip-tags upload,upgrade ``` Note: use `--tags` and `--skip-tags` wisely and only if you're 100% sure what you're doing. ## Troubleshooting Ansible issues Having the wrong version of ansible, ansible collections or python dependencies can cause issue. In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRARY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection) ```ShellSession export ANSIBLE_LIBRARY=/library` ``` A simple way to ensure you get all the correct version of Ansible is to use the [pre-built docker image from Quay](https://quay.io/repository/kubespray/kubespray?tab=tags). You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) to access the inventory and SSH key in the container, like this: ```ShellSession git checkout v2.30.0 docker pull quay.io/kubespray/kubespray:v2.30.0 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ quay.io/kubespray/kubespray:v2.30.0 bash # Inside the container you may now run the kubespray playbooks: ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` ================================================ FILE: docs/ansible/ansible_collection.md ================================================ # Ansible collection Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html). ## Usage 1. Set up an inventory with the appropriate host groups and required group vars. See also the documentation on [kubespray inventories](./inventory.md) and the general ["Getting started" documentation](../getting_started/getting-started.md#building-your-own-inventory). 2. Add Kubespray to your requirements.yml file ```yaml collections: - name: https://github.com/kubernetes-sigs/kubespray type: git version: master # use the appropriate tag or branch for the version you need ``` 3. Install your collection ```ShellSession ansible-galaxy install -r requirements.yml ``` 4. Create a playbook to install your Kubernetes cluster ```yaml - name: Install Kubernetes ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster ``` 5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray ```ShellSession ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK ``` ================================================ FILE: docs/ansible/inventory.md ================================================ # Inventory The inventory is composed of 3 groups: * **kube_node** : list of kubernetes nodes where the pods will run. * **kube_control_plane** : list of servers where kubernetes control plane components (apiserver, scheduler, controller) will run. * **etcd**: list of servers to compose the etcd server. You should have at least 3 servers for failover purpose. When _kube_node_ contains _etcd_, you define your etcd cluster to be as well schedulable for Kubernetes workloads. If you want it a standalone, make sure those groups do not intersect. If you want the server to act both as control-plane and node, the server must be defined on both groups _kube_control_plane_ and _kube_node_. If you want a standalone and unschedulable control plane, the server must be defined only in the _kube_control_plane_ and not _kube_node_. There are also two special groups: * **calico_rr** : explained for [advanced Calico networking cases](/docs/CNI/calico.md) * **bastion** : configure a bastion host if your nodes are not directly reachable Lastly, the **k8s_cluster** is dynamically defined as the union of **kube_node**, **kube_control_plane** and **calico_rr**. This is used internally and for the purpose of defining whole cluster variables (`/group_vars/k8s_cluster/*.yml`) Below is a complete inventory example: ```ini ## Configure 'ip' variable to bind kubernetes services on a ## different ip than the default iface node1 ansible_host=95.54.0.12 ip=10.3.0.1 node2 ansible_host=95.54.0.13 ip=10.3.0.2 node3 ansible_host=95.54.0.14 ip=10.3.0.3 node4 ansible_host=95.54.0.15 ip=10.3.0.4 node5 ansible_host=95.54.0.16 ip=10.3.0.5 node6 ansible_host=95.54.0.17 ip=10.3.0.6 [kube_control_plane] node1 node2 [etcd] node1 node2 node3 [kube_node] node2 node3 node4 node5 node6 ``` ## Inventory customization See [Customize Ansible vars](/docs/ansible/ansible.md#customize-ansible-vars) and [Ansible documentation on group_vars](https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html#assigning-a-variable-to-many-machines-group-variables) ## Bastion host If you prefer to not make your nodes publicly accessible (nodes with private IPs only), you can use a so-called _bastion_ host to connect to your nodes. To specify and use a bastion, simply add a line to your inventory, where you have to replace x.x.x.x with the public IP of the bastion host. ```ShellSession [bastion] bastion ansible_host=x.x.x.x ``` For more information about Ansible and bastion hosts, read [Running Ansible Through an SSH Bastion Host](https://blog.scottlowe.org/2015/12/24/running-ansible-through-ssh-bastion-host/) ================================================ FILE: docs/ansible/vars.md ================================================ # Configurable Parameters in Kubespray ## Generic Ansible variables You can view facts gathered by Ansible automatically [here](https://docs.ansible.com/ansible/latest/user_guide/playbooks_vars_facts.html#ansible-facts). Some variables of note include: * *ansible_user*: user to connect to via SSH * *ansible_default_ipv4.address*: IP address Ansible automatically chooses. Generated based on the output from the command ``ip -4 route get 8.8.8.8`` ## Common vars that are used in Kubespray * *calico_version* - Specify version of Calico to use * *calico_cni_version* - Specify version of Calico CNI plugin to use * *docker_version* - Specify version of Docker to use (should be quoted string). Must match one of the keys defined for *docker_versioned_pkg* in `roles/container-engine/docker/vars/*.yml`. * *containerd_version* - Specify version of containerd to use when setting `container_manager` to `containerd` * *docker_containerd_version* - Specify which version of containerd to use when setting `container_manager` to `docker` * *etcd_version* - Specify version of ETCD to use * *calico_ipip_mode* - Configures Calico ipip encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Never') * *calico_vxlan_mode* - Configures Calico vxlan encapsulation - valid values are 'Never', 'Always' and 'CrossSubnet' (default 'Always') * *calico_network_backend* - Configures Calico network backend - valid values are 'none', 'bird' and 'vxlan' (default 'vxlan') * *kube_network_plugin* - Sets k8s network plugin (default Calico) * *kube_proxy_mode* - Changes k8s proxy mode to iptables, ipvs, nftables mode * *kube_version* - Specify a given Kubernetes version * *searchdomains* - Array of DNS domains to search when looking up hostnames * *remove_default_searchdomains* - Boolean that removes the default searchdomain * *nameservers* - Array of nameservers to use for DNS lookup * *preinstall_selinux_state* - Set selinux state, permitted values are permissive, enforcing and disabled. ## Addressing variables * *ip* - IP to use for binding services (host var). This would **usually** be the public ip. * *access_ip* - IP to use from other hosts to connect to this host. Often required when deploying from a cloud, such as OpenStack or GCE and you have separate public/floating and private IPs. This would **usually** be the private ip. * *ansible_default_ipv4.address* - Not Kubespray-specific, but it is used if ip and access_ip are undefined * *ip6* - IPv6 address to use for binding services. (host var) If *ipv6_stack*(*enable_dual_stack_networks* deprecated) is set to ``true`` and *ip6* is defined, kubelet's ``--node-ip`` and node's ``InternalIP`` will be the combination of *ip* and *ip6*. Similarly used for ipv6only scheme. * *access_ip6* - similarly ``access_ip`` but IPv6 * *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6 and access_ip6 are undefined * *loadbalancer_apiserver* - If defined, all hosts will connect to this address instead of localhost for kube_control_planes and kube_control_plane[0] for kube_nodes. See more details in the [HA guide](/docs/operations/ha-mode.md). * *loadbalancer_apiserver_localhost* - makes all hosts to connect to the apiserver internally load balanced endpoint. Mutual exclusive to the `loadbalancer_apiserver`. See more details in the [HA guide](/docs/operations/ha-mode.md). ## Special network variables These variables help avoid a large number of if/else constructs throughout the code associated with enabling different network stack. These variables are used in all templates. By default, only ipv4_stack is enabled, so it is given priority in dualstack mode. Don't change these variables if you don't understand what you're doing. * *main_access_ip* - equal to ``access_ip`` when ipv4_stack is enabled(even in case of dualstack), and ``access_ip6`` for IPv6 only clusters * *main_ip* - equal to ``ip`` when ipv4_stack is enabled(even in case of dualstack), and ``ip6`` for IPv6 only clusters * *main_access_ips* - list of ``access_ip`` and ``access_ip6`` for dualstack and one corresponding variable for single * *main_ips* - list of ``ip`` and ``ip6`` for dualstack and one corresponding variable for single ## Cluster variables Kubernetes needs some parameters in order to get deployed. These are the following default cluster parameters: * *cluster_name* - Name of cluster (default is cluster.local) * *container_manager* - Container Runtime to install in the nodes (default is containerd) * *image_command_tool* - Tool used to pull images (default depends on `container_manager` and is `nerdctl` for `containerd`, `crictl` for `crio`, `docker` for `docker`) * *image_command_tool_on_localhost* - Tool used to pull images on localhost (default is equal to `image_command_tool`) * *dns_domain* - Name of cluster DNS domain (default is cluster.local) * *kube_network_plugin* - Plugin to use for container networking * *kube_service_addresses* - Subnet for cluster IPs (default is 10.233.0.0/18). Must not overlap with kube_pods_subnet * *kube_pods_subnet* - Subnet for Pod IPs (default is 10.233.64.0/18). Must not overlap with kube_service_addresses. * *kube_network_node_prefix* - Subnet allocated per-node for pod IPs. Remaining bits in kube_pods_subnet dictates how many kube_nodes can be in cluster. Setting this > 25 will raise an assertion in playbooks if the `kubelet_max_pods` var also isn't adjusted accordingly (assertion not applicable to calico which doesn't use this as a hard limit, see [Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes)). * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``. * *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options), for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`). It is not recommended to change this variable directly. * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``. * *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options), for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`). It is not recommended to change this variable directly. * *kube_network_node_prefix_ipv6* - Subnet allocated per-node for pod IPv6 IPs. Remaining bits in ``kube_pods_subnet_ipv6`` dictates how many kube_nodes can be in cluster. * *skydns_server* - Cluster IP for DNS (default is 10.233.0.3) * *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4) * *enable_coredns_k8s_external* - If enabled, it configures the [k8s_external plugin](https://coredns.io/plugins/k8s_external/) on the CoreDNS service. * *coredns_k8s_external_zone* - Zone that will be used when CoreDNS k8s_external plugin is enabled (default is k8s_external.local) * *enable_coredns_k8s_endpoint_pod_names* - If enabled, it configures endpoint_pod_names option for kubernetes plugin. on the CoreDNS service. * *cloud_provider* - The provider for cloud services. (default is unset, Set to `external` for running with an external cloud provider) * *kube_feature_gates* - A list of key=value pairs that describe feature gates for alpha/experimental Kubernetes features. (defaults is `[]`). Additionally, you can use also the following variables to individually customize your kubernetes components installation (they works exactly like `kube_feature_gates`): * *kube_apiserver_feature_gates* * *kube_controller_feature_gates* * *kube_scheduler_feature_gates* * *kube_proxy_feature_gates* * *kubelet_feature_gates* * *kubeadm_feature_gates* - A list of key=value pairs that describe feature gates for alpha/experimental Kubeadm features. (defaults is `[]`) * *authorization_modes* - A list of [authorization mode]( https://kubernetes.io/docs/reference/access-authn-authz/authorization/#using-flags-for-your-authorization-module) that the cluster should be configured for. Defaults to `['Node', 'RBAC']` (Node and RBAC authorizers). Note: `Node` and `RBAC` are enabled by default. Previously deployed clusters can be converted to RBAC mode. However, your apps which rely on Kubernetes API will require a service account and cluster role bindings. You can override this setting by setting authorization_modes to `[]`. * *kube_apiserver_admission_control_config_file* - Enable configuration for `kube-apiserver` admission plugins. Currently this variable allow you to configure the `EventRateLimit` admission plugin. To configure the **EventRateLimit** plugin you have to define a data structure like this: ```yml kube_apiserver_admission_event_rate_limits: limit_1: type: Namespace qps: 50 burst: 100 cache_size: 2000 limit_2: type: User qps: 50 burst: 100 ... ``` * *kube_apiserver_service_account_lookup* - Enable validation service account before validating token. Default `true`. Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances' private addresses, make sure to pick another values for ``kube_service_addresses`` and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``. ## Enabling Dual Stack (IPV4 + IPV6) or IPV6 only networking IPv4 stack enable by *ipv4_stack* is set to ``true``, by default. IPv6 stack enable by *ipv6_stack* is set to ``false`` by default. This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services. Set both variables to ``true`` for Dual Stack mode. IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other). You can also make IPv6 only clusters with ``false`` in *ipv4_stack*. ## DNS variables By default, hosts are set up with 8.8.8.8 as an upstream DNS server and all other settings from your existing /etc/resolv.conf are lost. Set the following variables to match your requirements. * *upstream_dns_servers* - Array of upstream DNS servers configured on host in addition to Kubespray deployed DNS * *nameservers* - Array of DNS servers configured for use by hosts * *searchdomains* - Array of up to 4 search domains * *remove_default_searchdomains* - Boolean. If enabled, `searchdomains` variable can hold 6 search domains. * *dns_etchosts* - Content of hosts file for coredns and nodelocaldns * *dns_upstream_forward_extra_opts* - Options to add in the forward section of coredns/nodelocaldns related to upstream DNS servers For more information, see [DNS Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/advanced/dns-stack.md). ## Other service variables * *docker_options* - Commonly used to set ``--insecure-registry=myregistry.mydomain:5000`` * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install. * *containerd_default_runtime* - If defined, changes the default Containerd runtime used by the Kubernetes CRI plugin. * *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin. [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overridden in inventory vars. * *crio_criu_support_enabled* - When set to `true`, enables the container checkpoint/restore in CRI-O. It's required to install [CRIU](https://criu.org/Installation) on the host when dumping/restoring checkpoints. And it's recommended to enable the feature gate `ContainerCheckpoint` so that the kubelet get a higher level API to simplify the operations (**Note**: It's still in experimental stage, just for container analytics so far). You can follow the [documentation](https://kubernetes.io/blog/2022/12/05/forensic-container-checkpointing-alpha/). * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames that correspond to each node. * *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet. By default autodetection is used to match container manager configuration. `systemd` is the preferred driver for `containerd` though it can have issues with `cgroups v1` and `kata-containers` in which case you may want to change to `cgroupfs`. * *kubelet_rotate_certificates* - Auto rotate the kubelet client certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches. * *kubelet_rotate_server_certificates* - Auto rotate the kubelet server certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches. Note that enabling this also activates *kubelet_csr_approver* which approves automatically the CSRs. To customize its behavior, you can override the Helm values via *kubelet_csr_approver_values*. See [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver) for more information. * *kubelet_streaming_connection_idle_timeout* - Set the maximum time a streaming connection can be idle before the connection is automatically closed. * *kubelet_image_gc_high_threshold* - Set the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than imageGCLowThresholdPercent. Default: 85 * *kubelet_image_gc_low_threshold* - Set the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent. Default: 80 * *kubelet_max_parallel_image_pulls* - Sets the maximum number of image pulls in parallel. The value is `1` by default which means the default is serial image pulling, set it to a integer great than `1` to enable image pulling in parallel. * *kubelet_make_iptables_util_chains* - If `true`, causes the kubelet ensures a set of `iptables` rules are present on host. * *kubelet_cpu_manager_policy* - If set to `static`, allows pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node. And it should be set with `kube_reserved` or `system-reserved`, enable this with the following guide:[Control CPU Management Policies on the Node](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) * *kubelet_cpu_manager_policy_options* - A dictionary of cpuManagerPolicyOptions to enable. Keep in mind to enable the corresponding feature gates and make sure to pass the booleans as string (i.e. don't forget the quotes)! ```yml kubelet_cpu_manager_policy_options: distribute-cpus-across-numa: "true" full-pcpus-only: "true" ``` * *kubelet_topology_manager_policy* - Control the behavior of the allocation of CPU and Memory from different [NUMA](https://en.wikipedia.org/wiki/Non-uniform_memory_access) Nodes. Enable this with the following guide: [Control Topology Management Policies on a node](https://kubernetes.io/docs/tasks/administer-cluster/topology-manager). * *kubelet_topology_manager_scope* - The Topology Manager can deal with the alignment of resources in a couple of distinct scopes: `container` and `pod`. See [Topology Manager Scopes](https://kubernetes.io/docs/tasks/administer-cluster/topology-manager/#topology-manager-scopes). * *kubelet_systemd_hardening* - If `true`, provides kubelet systemd service with security features for isolation. **N.B.** To enable this feature, ensure you are using the **`cgroup v2`** on your system. Check it out with command: `sudo ls -l /sys/fs/cgroup/*.slice`. If directory does not exist, enable this with the following guide: [enable cgroup v2](https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cgroup-v2). * *kubelet_secure_addresses* - By default *kubelet_systemd_hardening* set the **control plane** `ansible_host` IPs as the `kubelet_secure_addresses`. In case you have multiple interfaces in your control plane nodes and the `kube-apiserver` is not bound to the default interface, you can override them with this variable. Example: The **control plane** node may have 2 interfaces with the following IP addresses: `eth0:10.0.0.110`, `eth1:192.168.1.110`. By default the `kubelet_secure_addresses` is set with the `10.0.0.110` the ansible control host uses `eth0` to connect to the machine. In case you want to use `eth1` as the outgoing interface on which `kube-apiserver` connects to the `kubelet`s, you should override the variable in this way: `kubelet_secure_addresses: "192.168.1.110"`. * *kubelet_systemd_wants_dependencies* - List of kubelet service dependencies, other than container runtime. If you use nfs dynamically mounted volumes, sometimes rpc-statd does not start within the kubelet. You can fix it with this parameter : `kubelet_systemd_wants_dependencies: ["rpc-statd.service"]` This will add `Wants=rpc-statd.service` in `[Unit]` section of /etc/systemd/system/kubelet.service * *node_labels* - Labels applied to nodes via `kubectl label node`. For example, labels can be set in the inventory as variables or more widely in group_vars. *node_labels* can only be defined as a dict: ```yml node_labels: label1_name: label1_value label2_name: label2_value ``` * *node_taints* - Taints applied to nodes via `kubectl taint node`. For example, taints can be set in the inventory as variables or more widely in group_vars. *node_taints* has to be defined as a list of strings in format `key=value:effect`, e.g.: ```yml node_taints: - "node.example.com/external=true:NoSchedule" ``` * *kubernetes_audit* - When set to `true`, enables Auditing. The auditing parameters can be tuned via the following variables (which default values are shown below): * `audit_log_path`: /var/log/audit/kube-apiserver-audit.log * `audit_log_maxage`: 30 * `audit_log_maxbackups`: 10 * `audit_log_maxsize`: 100 * `audit_policy_file`: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" By default, the `audit_policy_file` contains [default rules](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2) that can be overridden with the `audit_policy_custom_rules` variable. * *kubernetes_audit_webhook* - When set to `true`, enables the webhook audit backend. The webhook parameters can be tuned via the following variables (which default values are shown below): * `audit_webhook_config_file`: "{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml" * `audit_webhook_server_url`: `"https://audit.app"` * `audit_webhook_server_extra_args`: {} * `audit_webhook_mode`: batch * `audit_webhook_batch_max_size`: 100 * `audit_webhook_batch_max_wait`: 1s * *kubectl_alias* - Bash alias of kubectl to interact with Kubernetes cluster much easier. * *remove_anonymous_access* - When set to `true`, removes the `kubeadm:bootstrap-signer-clusterinfo` rolebinding created by kubeadm. By default, kubeadm creates a rolebinding in the `kube-public` namespace which grants permissions to anonymous users. This rolebinding allows kubeadm to discover and validate cluster information during the join phase. In a nutshell, this option removes the rolebinding after the init phase of the first control plane node and then configures kubeadm to use file discovery for the join phase of other nodes. This option does not remove the anonymous authentication feature of the API server. ### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. Extra flags for the kubelet can be specified using these variables, in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. Example: ```yml kubelet_config_extra_args: evictionHard: memory.available: "100Mi" evictionSoftGracePeriod: memory.available: "30s" evictionSoft: memory.available: "300Mi" ``` The possible vars are: * *kubelet_config_extra_args* Previously, the same parameters could be passed as flags to kubelet binary with the following vars: * *kubelet_custom_flags* ```yml kubelet_custom_flags: - "--eviction-hard=memory.available<100Mi" - "--eviction-soft-grace-period=memory.available=30s" - "--eviction-soft=memory.available<300Mi" ``` This alternative is deprecated and will remain until the flags are completely removed from kubelet Extra flags for the API server, controller, and scheduler components can be specified using these variables, in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubeadm YAML config file: * *kube_kubeadm_apiserver_extra_args* * *kube_kubeadm_controller_extra_args* * *kube_kubeadm_scheduler_extra_args* ### Kubeadm patches When extra flags are not sufficient and there is a need to further customize kubernetes components, [kubeadm patches](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches) can be used. You should use the [`kubeadm_patches` variable](../../roles/kubernetes/kubeadm_common/defaults/main.yml) for that purpose. ## App variables * *helm_version* - Only supports v3.x. Existing v2 installs (with Tiller) will not be modified and need to be removed manually. ================================================ FILE: docs/calico_peer_example/new-york.yml ================================================ # --- # peers: # - router_id: "10.99.0.34" # as: "65xxx" # filters: [] # numallowedlocalasnumbers: 0 # sourceaddress: "None" # - router_id: "10.99.0.35" # as: "65xxx" # filters: [] # numallowedlocalasnumbers: 0 # sourceaddress: "None" # loadbalancer_apiserver: # address: "10.99.0.44" # port: "8383" ================================================ FILE: docs/calico_peer_example/paris.yml ================================================ # --- # peers: # - router_id: "10.99.0.2" # as: "65xxx" # filters: [] # numallowedlocalasnumbers: 0 # sourceaddress: "None" # - router_id: "10.99.0.3" # as: "65xxx" # filters: [] # numallowedlocalasnumbers: 0 # sourceaddress: "None" # loadbalancer_apiserver: # address: "10.99.0.21" # port: "8383" ================================================ FILE: docs/cloud_controllers/openstack.md ================================================ # OpenStack ## Known compatible public clouds Kubespray has been tested on a number of OpenStack Public Clouds including (in alphabetical order): - [Auro](https://auro.io/) - [Betacloud](https://www.betacloud.io/) - [CityCloud](https://www.citycloud.com/) - [DreamHost](https://www.dreamhost.com/cloud/computing/) - [ELASTX](https://elastx.se/) - [EnterCloudSuite](https://www.entercloudsuite.com/) - [FugaCloud](https://fuga.cloud/) - [Infomaniak](https://infomaniak.com) - [Open Telekom Cloud](https://cloud.telekom.de/) : requires to set the variable `wait_for_floatingip = "true"` in your cluster.tfvars - [OVHcloud](https://www.ovhcloud.com/) - [Rackspace](https://www.rackspace.com/) - [Ultimum](https://ultimum.io/) - [VexxHost](https://vexxhost.com/) - [Zetta](https://www.zetta.io/) ## The OpenStack cloud provider The cloud provider is configured to have Octavia by default in Kubespray. - Enable the external OpenStack cloud provider in `group_vars/all/all.yml`: ```yaml cloud_provider: external external_cloud_provider: openstack ``` - Enable Cinder CSI in `group_vars/all/openstack.yml`: ```yaml cinder_csi_enabled: true ``` - Enable topology support (optional), if your openstack provider has custom Zone names you can override the default "nova" zone by setting the variable `cinder_topology_zones` ```yaml cinder_topology: true ``` - Enabling `cinder_csi_ignore_volume_az: true`, ignores volumeAZ and schedules on any of the available node AZ. ```yaml cinder_csi_ignore_volume_az: true ``` - If you are using OpenStack loadbalancer(s) replace the `openstack_lbaas_subnet_id` with the new `external_openstack_lbaas_subnet_id`. **Note** The new cloud provider is using Octavia instead of Neutron LBaaS by default! - If you are in a case of a multi-nic OpenStack VMs (see [kubernetes/cloud-provider-openstack#407](https://github.com/kubernetes/cloud-provider-openstack/issues/407) and [#6083](https://github.com/kubernetes-sigs/kubespray/issues/6083) for explanation), you should override the default OpenStack networking configuration: ```yaml external_openstack_network_ipv6_disabled: false external_openstack_network_internal_networks: [] external_openstack_network_public_networks: [] ``` - You can override the default OpenStack metadata configuration (see [#6338](https://github.com/kubernetes-sigs/kubespray/issues/6338) for explanation): ```yaml external_openstack_metadata_search_order: "configDrive,metadataService" ``` - Available variables for configuring lbaas: ```yaml external_openstack_lbaas_enabled: true external_openstack_lbaas_floating_network_id: "Neutron network ID to get floating IP from" external_openstack_lbaas_floating_subnet_id: "Neutron subnet ID to get floating IP from" external_openstack_lbaas_method: ROUND_ROBIN external_openstack_lbaas_provider: amphora external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP" external_openstack_lbaas_member_subnet_id: "Neutron subnet ID on which to create the members of the load balancer" external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP" external_openstack_lbaas_manage_security_groups: false external_openstack_lbaas_create_monitor: false external_openstack_lbaas_monitor_delay: 5s external_openstack_lbaas_monitor_max_retries: 1 external_openstack_lbaas_monitor_timeout: 3s external_openstack_lbaas_internal_lb: false ``` - Run `source path/to/your/openstack-rc` to read your OpenStack credentials like `OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, etc. Those variables are used for accessing OpenStack from the external cloud provider. - Run the `cluster.yml` playbook ## Additional step needed when using calico or kube-router Being L3 CNI, calico and kube-router do not encapsulate all packages with the hosts' ip addresses. Instead the packets will be routed with the PODs ip addresses directly. OpenStack will filter and drop all packets from ips it does not know to prevent spoofing. In order to make L3 CNIs work on OpenStack you will need to tell OpenStack to allow pods packets by allowing the network they use. First you will need the ids of your OpenStack instances that will run kubernetes: ```bash openstack server list --project YOUR_PROJECT +--------------------------------------+--------+----------------------------------+--------+-------------+ | ID | Name | Tenant ID | Status | Power State | +--------------------------------------+--------+----------------------------------+--------+-------------+ | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | k8s-1 | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running | | 725cd548-6ea3-426b-baaa-e7306d3c8052 | k8s-2 | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running | ``` Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports (though they are now configured through using OpenStack): ```bash openstack port list -c id -c device_id --project YOUR_PROJECT +--------------------------------------+--------------------------------------+ | id | device_id | +--------------------------------------+--------------------------------------+ | 5662a4e0-e646-47f0-bf88-d80fbd2d99ef | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | | e5ae2045-a1e1-4e99-9aac-4353889449a7 | 725cd548-6ea3-426b-baaa-e7306d3c8052 | ``` Given the port ids on the left, you can set the two `allowed-address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.) ```bash # allow kube_service_addresses and kube_pods_subnet network openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18 openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18 ``` If all the VMs in the tenant correspond to Kubespray deployment, you can "sweep run" above with: ```bash openstack port list --device-owner=compute:nova -c ID -f value | xargs -tI@ openstack port set @ --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18 ``` Now you can finally run the playbook. ================================================ FILE: docs/cloud_controllers/vsphere.md ================================================ # vSphere Kubespray can be deployed with vSphere as Cloud provider. This feature supports: - Volumes - Persistent Volumes - Storage Classes and provisioning of volumes - vSphere Storage Policy Based Management for Containers orchestrated by Kubernetes ## Out-of-tree vSphere cloud provider ### Prerequisites You need at first to configure your vSphere environment by following the [official documentation](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#prerequisites). After this step you should have: - vSphere upgraded to 6.7 U3 or later - VM hardware upgraded to version 15 or higher - UUID activated for each VM where Kubernetes will be deployed ### Kubespray configuration First in `inventory/sample/group_vars/all/all.yml` you must set the `cloud_provider` to `external` and `external_cloud_provider` to `vsphere`. ```yml cloud_provider: "external" external_cloud_provider: "vsphere" ``` Then, `inventory/sample/group_vars/all/vsphere.yml`, you need to declare your vCenter credentials and enable the vSphere CSI following the description below. | Variable | Required | Type | Choices | Default | Comment | |----------------------------------------|----------|---------|----------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------| | external_vsphere_vcenter_ip | TRUE | string | | | IP/URL of the vCenter | | external_vsphere_vcenter_port | TRUE | string | | "443" | Port of the vCenter API | | external_vsphere_insecure | TRUE | string | "true", "false" | "true" | set to "true" if the host above uses a self-signed cert | | external_vsphere_user | TRUE | string | | | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) | | external_vsphere_password | TRUE | string | | | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable) | | external_vsphere_datacenter | TRUE | string | | | Datacenter name to use | | external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use | | vsphere_csi_enabled | TRUE | boolean | | false | Enable vSphere CSI | Example configuration: ```yml external_vsphere_vcenter_ip: "myvcenter.domain.com" external_vsphere_vcenter_port: "443" external_vsphere_insecure: "true" external_vsphere_user: "administrator@vsphere.local" external_vsphere_password: "K8s_admin" external_vsphere_datacenter: "DATACENTER_name" external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" vsphere_csi_enabled: true ``` For a more fine-grained CSI setup, refer to the [vsphere-csi](/docs/CSI/vsphere-csi.md) documentation. ### Deployment Once the configuration is set, you can execute the playbook again to apply the new configuration: ```ShellSession cd kubespray ansible-playbook -i inventory/sample/hosts.ini -b -v cluster.yml ``` You'll find some useful examples [here](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md#sample-manifests-to-test-csi-driver-functionality) to test your configuration. ## In-tree vSphere cloud provider ([deprecated](https://cloud-provider-vsphere.sigs.k8s.io/concepts/in_tree_vs_out_of_tree.html)) ### Prerequisites (deprecated) You need at first to configure your vSphere environment by following the [official documentation](https://kubernetes.io/docs/getting-started-guides/vsphere/#vsphere-cloud-provider). After this step you should have: - UUID activated for each VM where Kubernetes will be deployed - A vSphere account with required privileges If you intend to leverage the [zone and region node labeling](https://kubernetes.io/docs/reference/kubernetes-api/labels-annotations-taints/#failure-domain-beta-kubernetes-io-region), create a tag category for both the zone and region in vCenter. The tags can then be applied at the host, cluster, datacenter, or folder level, and the cloud provider will walk the hierarchy to extract and apply the labels to the Kubernetes nodes. ### Kubespray configuration (deprecated) First you must define the cloud provider in `inventory/sample/group_vars/all.yml` and set it to `vsphere`. ```yml cloud_provider: vsphere ``` Then, in the same file, you need to declare your vCenter credentials following the description below. | Variable | Required | Type | Choices | Default | Comment | |------------------------------|----------|---------|----------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | vsphere_vcenter_ip | TRUE | string | | | IP/URL of the vCenter | | vsphere_vcenter_port | TRUE | integer | | | Port of the vCenter API. Commonly 443 | | vsphere_insecure | TRUE | integer | 1, 0 | | set to 1 if the host above uses a self-signed cert | | vsphere_user | TRUE | string | | | User name for vCenter with required privileges | | vsphere_password | TRUE | string | | | Password for vCenter | | vsphere_datacenter | TRUE | string | | | Datacenter name to use | | vsphere_datastore | TRUE | string | | | Datastore name to use | | vsphere_working_dir | TRUE | string | | | Working directory from the view "VMs and template" in the vCenter where VM are placed | | vsphere_scsi_controller_type | TRUE | string | buslogic, pvscsi, parallel | pvscsi | SCSI controller name. Commonly "pvscsi". | | vsphere_vm_uuid | FALSE | string | | | VM Instance UUID of virtual machine that host K8s master. Can be retrieved from instanceUuid property in VmConfigInfo, or as vc.uuid in VMX file or in `/sys/class/dmi/id/product_serial` (Optional, only used for Kubernetes <= 1.9.2) | | vsphere_public_network | FALSE | string | | Blank | Name of the network the VMs are joined to | | vsphere_resource_pool | FALSE | string | | Blank | Name of the Resource pool where the VMs are located (Optional, only used for Kubernetes >= 1.9.2) | | vsphere_zone_category | FALSE | string | | | Name of the tag category used to set the `failure-domain.beta.kubernetes.io/zone` label on nodes (Optional, only used for Kubernetes >= 1.12.0) | | vsphere_region_category | FALSE | string | | | Name of the tag category used to set the `failure-domain.beta.kubernetes.io/region` label on nodes (Optional, only used for Kubernetes >= 1.12.0) | Example configuration: ```yml vsphere_vcenter_ip: "myvcenter.domain.com" vsphere_vcenter_port: 443 vsphere_insecure: 1 vsphere_user: "k8s@vsphere.local" vsphere_password: "K8s_admin" vsphere_datacenter: "DATACENTER_name" vsphere_datastore: "DATASTORE_name" vsphere_working_dir: "Docker_hosts" vsphere_scsi_controller_type: "pvscsi" vsphere_resource_pool: "K8s-Pool" ``` ### Deployment (deprecated) Once the configuration is set, you can execute the playbook again to apply the new configuration: ```ShellSession cd kubespray ansible-playbook -i inventory/sample/hosts.ini -b -v cluster.yml ``` You'll find some useful examples [here](https://github.com/kubernetes/examples/tree/master/staging/volumes/vsphere) to test your configuration. ================================================ FILE: docs/cloud_providers/aws.md ================================================ # AWS > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`. Refer to the [Kubespray Configuration](#kubespray-configuration) for customizing the provider. Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/aws_iam/). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role. You would also need to tag the resources in your VPC accordingly for the aws provider to utilize them. Tag the subnets, route tables and all instances that kubernetes will be run on with key `kubernetes.io/cluster/$cluster_name` (`$cluster_name` must be a unique identifier for the cluster). Tag the subnets that must be targeted by external ELBs with the key `kubernetes.io/role/elb` and internal ELBs with the key `kubernetes.io/role/internal-elb`. Make sure your VPC has both DNS Hostnames support and Private DNS enabled. The next step is to make sure the hostnames in your `inventory` file are identical to your internal hostnames in AWS. This may look something like `ip-111-222-333-444.us-west-2.compute.internal`. You can then specify how Ansible connects to these instances with `ansible_ssh_host` and `ansible_ssh_user`. You can now create your cluster! ## Dynamic Inventory There is also a dynamic inventory script for AWS that can be used if desired. However, be aware that it makes some certain assumptions about how you'll create your inventory. It also does not handle all use cases and groups that we may use as part of more advanced deployments. Additions welcome. This will produce an inventory that is passed into Ansible that looks like the following: ```json { "_meta": { "hostvars": { "ip-172-31-3-xxx.us-east-2.compute.internal": { "ansible_ssh_host": "172.31.3.xxx" }, "ip-172-31-8-xxx.us-east-2.compute.internal": { "ansible_ssh_host": "172.31.8.xxx" } } }, "etcd": [ "ip-172-31-3-xxx.us-east-2.compute.internal" ], "k8s_cluster": { "children": [ "kube_control_plane", "kube_node" ] }, "kube_control_plane": [ "ip-172-31-3-xxx.us-east-2.compute.internal" ], "kube_node": [ "ip-172-31-8-xxx.us-east-2.compute.internal" ] } ``` Guide: - Create instances in AWS as needed. - Either during or after creation, add tags to the instances with a key of `kubespray-role` and a value of `kube_control_plane`, `etcd`, or `kube_node`. You can also share roles like `kube_control_plane, etcd` - Copy the `kubespray-aws-inventory.py` script from `kubespray/contrib/aws_inventory` to the `kubespray/inventory` directory. - Set the following AWS credentials and info as environment variables in your terminal: ```ShellSession export AWS_ACCESS_KEY_ID="xxxxx" export AWS_SECRET_ACCESS_KEY="yyyyy" export AWS_REGION="us-east-2" ``` - We will now create our cluster. There will be either one or two small changes. The first is that we will specify `-i inventory/kubespray-aws-inventory.py` as our inventory script. The other is conditional. If your AWS instances are public facing, you can set the `VPC_VISIBILITY` variable to `public` and that will result in public IP and DNS names being passed into the inventory. This causes your cluster.yml command to look like `VPC_VISIBILITY="public" ansible-playbook ... cluster.yml` **Optional** Using labels and taints To add labels to your kubernetes node, add the following tag to your instance: - Key: `kubespray-node-labels` - Value: `node-role.kubernetes.io/ingress=` To add taints to your kubernetes node, add the following tag to your instance: - Key: `kubespray-node-taints` - Value: `node-role.kubernetes.io/ingress=:NoSchedule` ## Kubespray configuration Declare the cloud config variables for the `aws` provider as follows. Setting these variables are optional and depend on your use case. | Variable | Type | Comment | |------------------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | aws_zone | string | Force set the AWS zone. Recommended to leave blank. | | aws_vpc | string | The AWS VPC flag enables the possibility to run the master components on a different aws account, on a different cloud provider or on-premise. If the flag is set also the KubernetesClusterTag must be provided | | aws_subnet_id | string | SubnetID enables using a specific subnet to use for ELB's | | aws_route_table_id | string | RouteTableID enables using a specific RouteTable | | aws_role_arn | string | RoleARN is the IAM role to assume when interaction with AWS APIs | | aws_kubernetes_cluster_tag | string | KubernetesClusterTag is the legacy cluster id we'll use to identify our cluster resources | | aws_kubernetes_cluster_id | string | KubernetesClusterID is the cluster id we'll use to identify our cluster resources | | aws_disable_security_group_ingress | bool | The aws provider creates an inbound rule per load balancer on the node security group. However, this can run into the AWS security group rule limit of 50 if many LoadBalancers are created. This flag disables the automatic ingress creation. It requires that the user has setup a rule that allows inbound traffic on kubelet ports from the local VPC subnet (so load balancers can access it). E.g. 10.82.0.0/16 30000-32000. | | aws_elb_security_group | string | Only in Kubelet version >= 1.7 : AWS has a hard limit of 500 security groups. For large clusters creating a security group for each ELB can cause the max number of security groups to be reached. If this is set instead of creating a new Security group for each ELB this security group will be used instead. | | aws_disable_strict_zone_check | bool | During the instantiation of an new AWS cloud provider, the detected region is validated against a known set of regions. In a non-standard, AWS like environment (e.g. Eucalyptus), this check may be undesirable. Setting this to true will disable the check and provide a warning that the check was skipped. Please note that this is an experimental feature and work-in-progress for the moment. | ================================================ FILE: docs/cloud_providers/azure.md ================================================ # Azure > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) To deploy Kubernetes on [Azure](https://azure.microsoft.com) uncomment the `cloud_provider` option in `group_vars/all/all.yml` and set it to `'azure'`. All your instances are required to run in a resource group and a routing table has to be attached to the subnet your instances are in. Not all features are supported yet though, for a list of the current status have a look [here](https://github.com/Azure/AKS) ## Parameters Before creating the instances you must first set the `azure_` variables in the `group_vars/all/all.yml` file. All values can be retrieved using the Azure CLI tool which can be downloaded here: After installation you have to run `az login` to get access to your account. ### azure_cloud Azure Stack has different API endpoints, depending on the Azure Stack deployment. These need to be provided to the Azure SDK. Possible values are: `AzureChinaCloud`, `AzureGermanCloud`, `AzurePublicCloud` and `AzureUSGovernmentCloud`. The full list of existing settings for the AzureChinaCloud, AzureGermanCloud, AzurePublicCloud and AzureUSGovernmentCloud is available in the source code [here](https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/docs/cloud-provider-config.md) ### azure\_tenant\_id + azure\_subscription\_id run `az account show` to retrieve your subscription id and tenant id: `azure_tenant_id` -> Tenant ID field `azure_subscription_id` -> ID field ### azure\_location The region your instances are located, can be something like `westeurope` or `westcentralus`. A full list of region names can be retrieved via `az account list-locations` ### azure\_resource\_group The name of the resource group your instances are in, can be retrieved via `az group list` ### azure\_vmtype The type of the vm. Supported values are `standard` or `vmss`. If vm is type of `Virtual Machines` then value is `standard`. If vm is part of `Virtual Machine Scale Sets` then value is `vmss` ### azure\_vnet\_name The name of the virtual network your instances are in, can be retrieved via `az network vnet list` ### azure\_vnet\_resource\_group The name of the resource group that contains the vnet. ### azure\_subnet\_name The name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME` ### azure\_security\_group\_name The name of the network security group your instances are in, can be retrieved via `az network nsg list` ### azure\_security\_group\_resource\_group The name of the resource group that contains the network security group. Defaults to `azure_vnet_resource_group` ### azure\_route\_table\_name The name of the route table used with your instances. ### azure\_route\_table\_resource\_group The name of the resource group that contains the route table. Defaults to `azure_vnet_resource_group` ### azure\_aad\_client\_id + azure\_aad\_client\_secret These will have to be generated first: - Create an Azure AD Application with: ```ShellSession az ad app create --display-name kubernetes --identifier-uris http://kubernetes --homepage http://example.com --password CLIENT_SECRET ``` display name, identifier-uri, homepage and the password can be chosen Note the AppId in the output. - Create Service principal for the application with: ```ShellSession az ad sp create --id AppId ``` This is the AppId from the last command - Create the role assignment with: ```ShellSession az role assignment create --role "Owner" --assignee http://kubernetes --subscription SUBSCRIPTION_ID ``` azure\_aad\_client\_id must be set to the AppId, azure\_aad\_client\_secret is your chosen secret. ### azure\_loadbalancer\_sku Sku of Load Balancer and Public IP. Candidate values are: basic and standard. ### azure\_exclude\_master\_from\_standard\_lb azure\_exclude\_master\_from\_standard\_lb excludes master nodes from `standard` load balancer. ### azure\_disable\_outbound\_snat azure\_disable\_outbound\_snat disables the outbound SNAT for public load balancer rules. It should only be set when azure\_exclude\_master\_from\_standard\_lb is `standard`. ### azure\_primary\_availability\_set\_name (Optional) The name of the availability set that should be used as the load balancer backend .If this is set, the Azure cloudprovider will only add nodes from that availability set to the load balancer backend pool. If this is not set, and multiple agent pools (availability sets) are used, then the cloudprovider will try to add all nodes to a single backend pool which is forbidden. In other words, if you use multiple agent pools (availability sets), you MUST set this field. ### azure\_use\_instance\_metadata Use instance metadata service where possible ## Provisioning Azure with Resource Group Templates You'll find Resource Group Templates and scripts to provision the required infrastructure to Azure in [*contrib/azurerm*](../contrib/azurerm/README.md) ================================================ FILE: docs/cloud_providers/cloud.md ================================================ # Cloud providers > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) ## Provisioning You can deploy instances in your cloud environment in several ways. Examples include Terraform, Ansible (ec2 and gce modules), and manual creation. ## Deploy kubernetes With ansible-playbook command ```ShellSession ansible-playbook -u smana -e ansible_ssh_user=admin -e cloud_provider=[aws|gce] -b --become-user=root -i inventory/single.cfg cluster.yml ``` ================================================ FILE: docs/developers/ci-setup.md ================================================ # CI Setup ## Pipeline See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview. ## Runners Kubespray has 2 types of GitLab runners, both deployed on the Kubespray CI cluster (hosted on Oracle Cloud Infrastructure): - pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/) - vagrant: custom executor running in pods with access to the libvirt socket on the nodes ## Vagrant Vagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docker/Dockerfile) docker image with `/var/run/libvirt/libvirt-sock` exposed from the host, allowing the container to boot VMs on the host. ## CI Variables In CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on. ## CI clusters DISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix. The cluster is deployed with kubespray itself and maintained by the kubespray maintainers. The following files are used for that inventory: ### cluster.tfvars (OBSOLETE: this section is no longer accurate) ```ini # your Kubernetes cluster name here cluster_name = "ci" # Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/ equinix_metal_project_id = "_redacted_" # The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned # leave this value blank if the public key is already setup in the Equinix Metal project # Terraform will complain if the public key is setup in Equinix Metal public_key_path = "~/.ssh/id_rsa.pub" # cluster location metro = "da" # standalone etcds number_of_etcd = 0 plan_etcd = "t1.small.x86" # masters number_of_k8s_masters = 1 number_of_k8s_masters_no_etcd = 0 plan_k8s_masters = "c3.small.x86" plan_k8s_masters_no_etcd = "t1.small.x86" # nodes number_of_k8s_nodes = 1 plan_k8s_nodes = "c3.medium.x86" ``` ### group_vars/all/mirrors.yml ```yaml --- docker_registry_mirrors: - "https://mirror.gcr.io" containerd_grpc_max_recv_message_size: 16777216 containerd_grpc_max_send_message_size: 16777216 containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - host: https://registry-1.docker.io capabilities: ["pull", "resolve"] skip_verify: false containerd_max_container_log_line_size: 16384 crio_registries_mirrors: - prefix: docker.io insecure: false blocked: false location: registry-1.docker.io mirrors: - location: mirror.gcr.io insecure: false netcheck_agent_image_repo: "{{ quay_image_repo }}/kubespray/k8s-netchecker-agent" netcheck_server_image_repo: "{{ quay_image_repo }}/kubespray/k8s-netchecker-server" nginx_image_repo: "{{ quay_image_repo }}/kubespray/nginx" ``` ### group_vars/all/settings.yml ```yaml --- # Networking setting kube_service_addresses: 172.30.0.0/18 kube_pods_subnet: 172.30.64.0/18 kube_network_plugin: calico # avoid overlap with CI jobs deploying nodelocaldns nodelocaldns_ip: 169.254.255.100 # ipip: False calico_ipip_mode: "Never" calico_vxlan_mode: "Never" calico_network_backend: "bird" calico_wireguard_enabled: True # Cluster settings upgrade_cluster_setup: True force_certificate_regeneration: True # Etcd settings etcd_deployment_type: "host" # Kubernetes settings kube_controller_terminated_pod_gc_threshold: 100 kubelet_enforce_node_allocatable: pods kubelet_preferred_address_types: 'InternalIP,ExternalIP,Hostname' kubelet_custom_flags: - "--serialize-image-pulls=true" - "--eviction-hard=memory.available<1Gi" - "--eviction-soft-grace-period=memory.available=30s" - "--eviction-soft=memory.available<2Gi" - "--system-reserved cpu=100m,memory=4Gi" - "--eviction-minimum-reclaim=memory.available=2Gi" # DNS settings resolvconf_mode: none dns_min_replicas: 1 upstream_dns_servers: - 1.1.1.1 - 1.0.0.1 # Extensions helm_enabled: True cert_manager_enabled: True metrics_server_enabled: True # Enable ZSWAP kubelet_fail_swap_on: False kube_feature_gates: - "NodeSwap=True" ``` ## Additional files This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster. ### /root/path-calico.sh ```bash #!/bin/bash calicoctl patch felixconfig default -p '{"spec":{"allowIPIPPacketsFromWorkloads":true, "allowVXLANPacketsFromWorkloads": true}}' ``` ### /root/kubevirt/kubevirt.sh ```bash #!/bin/bash export VERSION=$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases | grep tag_name | grep -v -- '-rc' | sort -r | head -1 | awk -F': ' '{print $2}' | sed 's/,//' | xargs) echo $VERSION kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml ``` ### /root/kubevirt/virtctl.sh ```bash #!/bin/bash VERSION=$(kubectl get kubevirt.kubevirt.io/kubevirt -n kubevirt -o=jsonpath="{.status.observedKubeVirtVersion}") ARCH=$(uname -s | tr A-Z a-z)-$(uname -m | sed 's/x86_64/amd64/') || windows-amd64.exe echo ${ARCH} curl -L -o virtctl https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/virtctl-${VERSION}-${ARCH} chmod +x virtctl sudo install virtctl /usr/local/bin ``` ================================================ FILE: docs/developers/ci.md ================================================ # CI test coverage To generate this Matrix run `./tests/scripts/md-table/main.py` ## containerd | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | |---| --- | --- | --- | --- | --- | --- | --- | almalinux9 | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | amazon | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | debian11 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | debian12 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | debian13 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | fedora39 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | fedora40 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | fedora41 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | fedora42 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | flatcar4081 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | openeuler24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | rockylinux10 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | rockylinux9 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | ubuntu24 | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | ## crio | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | |---| --- | --- | --- | --- | --- | --- | --- | almalinux9 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | debian11 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | debian12 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | debian13 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | fedora39 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | fedora40 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | fedora41 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | fedora42 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | flatcar4081 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | openeuler24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | rockylinux10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | ubuntu24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | ## docker | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | |---| --- | --- | --- | --- | --- | --- | --- | almalinux9 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | debian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | debian12 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | debian13 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | fedora39 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | fedora40 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | fedora41 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | fedora42 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | flatcar4081 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | openeuler24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | rockylinux10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | ubuntu24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | ================================================ FILE: docs/developers/test_cases.md ================================================ # Node Layouts There are five node layout types: `default`, `separate`, `ha`, `all-in-one`, and `node-etcd-client`. `default` is a non-HA two nodes setup with one separate `kube_node` and the `etcd` group merged with the `kube_control_plane`. `separate` layout is when there is only node of each type, which includes a kube_control_plane, kube_node, and etcd cluster member. `ha` layout consists of two etcd nodes, two control planes and a single worker node, with role intersection. `all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged. `node-etcd-client` layout consists of a 4 nodes cluster, all of them in `kube_node`, first 3 in `etcd` and only one `kube_control_plane`. This is necessary to tests setups requiring that nodes are etcd clients (use of cilium as `network_plugin` for instance) Note, the canal network plugin deploys flannel as well plus calico policy controller. ## Test cases The [CI Matrix](/docs/developers/ci.md) displays OS, Network Plugin and Container Manager tested. All tests are breakdown into 3 "stages" ("Stage" means a build step of the build pipeline) as follows: - _unit_tests_: Linting, markdown, vagrant & terraform validation etc... - _part1_: Molecule and AIO tests - _part2_: Standard tests with different layouts and OS/Runtime/Network - _part3_: Upgrade jobs, terraform jobs and recover control plane tests - _special_: Other jobs (manuals) The steps are ordered as `unit_tests->part1->part2->part3->special`. ================================================ FILE: docs/developers/vagrant.md ================================================ # Vagrant Assuming you have Vagrant 2.0+ installed with virtualbox or libvirt/qemu (vmware may work, but is untested) you should be able to launch a 3 node Kubernetes cluster by simply running `vagrant up`. This will spin up 3 VMs and install kubernetes on them. Once they are completed you can connect to any of them by running `vagrant ssh k8s-[1..3]`. To give an estimate of the expected duration of a provisioning run: On a dual core i5-6300u laptop with an SSD, provisioning takes around 13 to 15 minutes, once the container images and other files are cached. Note that libvirt/qemu is recommended over virtualbox as it is quite a bit faster, especially during boot-up time. For proper performance a minimum of 12GB RAM is recommended. It is possible to run a 3 node cluster on a laptop with 8GB of RAM using the default Vagrantfile, provided you have 8GB zram swap configured and not much more than a browser and a mail client running. If you decide to run on such a machine, then also make sure that any tmpfs devices, that are mounted, are mostly empty and disable any swapfiles mounted on HDD/SSD or you will be in for some serious swap-madness. Things can get a bit sluggish during provisioning, but when that's done, the system will actually be able to perform quite well. ## Customize Vagrant You can override the default settings in the `Vagrantfile` either by directly modifying the `Vagrantfile` or through an override file. In the same directory as the `Vagrantfile`, create a folder called `vagrant` and create `config.rb` file in it. Example: ```ruby # vagrant/config.rb $instance_name_prefix = "kub" $vm_cpus = 1 $num_instances = 3 $os = "centos8-bento" $subnet = "10.0.20" $network_plugin = "flannel" $extra_vars = { dns_domain: my.custom.domain } # or $extra_vars = "path/to/extra/vars/file.yml" ``` For all available options look at the Vagrantfile (search for "CONFIG") ## Use alternative OS for Vagrant By default, Vagrant uses Ubuntu 18.04 box to provision a local cluster. You may use an alternative supported operating system for your local cluster. Customize `$os` variable in `Vagrantfile` or as override, e.g.,: ```ShellSession echo '$os = "flatcar-stable"' >> vagrant/config.rb ``` The supported operating systems for vagrant are defined in the `SUPPORTED_OS` constant in the `Vagrantfile`. ## File and image caching Kubespray can take quite a while to start on a laptop. To improve provisioning speed, the variable 'download_run_once' is set. This will make kubespray download all files and containers just once and then redistributes them to the other nodes and as a bonus, also cache all downloads locally and re-use them on the next provisioning run. For more information on download settings see [download documentation](/docs/advanced/downloads.md). ## Example use of Vagrant The following is an example of setting up and running kubespray using `vagrant`. Customize your settings as shown, above, then run the commands: ```ShellSession # use virtualenv to install all python requirements VENVDIR=venv $ virtualenv --python=/usr/bin/python3.7 $VENVDIR $ source $VENVDIR/bin/activate $ pip install -r requirements.txt $ vagrant up # Access the cluster $ export INV=.vagrant/provisioners/ansible/inventory $ export KUBECONFIG=${INV}/artifacts/admin.conf # make the kubectl binary available $ export PATH=$PATH:$PWD/$INV/artifacts ``` If a vagrant run failed and you've made some changes to fix the issue causing the fail, here is how you would re-run ansible: ```ShellSession vagrant provision ``` If all went well, you check if it's all working as expected: ```ShellSession $ kubectl get nodes NAME STATUS ROLES AGE VERSION kub-1 Ready control-plane,master 4m37s v1.22.5 kub-2 Ready control-plane,master 4m7s v1.22.5 kub-3 Ready 3m7s v1.22.5 ``` Another nice test is the following: ```ShellSession $ kubectl get pods --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system coredns-8474476ff8-m2469 1/1 Running 0 2m45s 10.233.65.2 kub-2 kube-system coredns-8474476ff8-v5wzj 1/1 Running 0 2m41s 10.233.64.3 kub-1 kube-system dns-autoscaler-5ffdc7f89d-76tnv 1/1 Running 0 2m43s 10.233.64.2 kub-1 kube-system kube-apiserver-kub-1 1/1 Running 1 4m54s 10.0.20.101 kub-1 kube-system kube-apiserver-kub-2 1/1 Running 1 4m33s 10.0.20.102 kub-2 kube-system kube-controller-manager-kub-1 1/1 Running 1 5m1s 10.0.20.101 kub-1 kube-system kube-controller-manager-kub-2 1/1 Running 1 4m33s 10.0.20.102 kub-2 kube-system kube-flannel-9xgf5 1/1 Running 0 3m10s 10.0.20.102 kub-2 kube-system kube-flannel-l8jbl 1/1 Running 0 3m10s 10.0.20.101 kub-1 kube-system kube-flannel-zss4t 1/1 Running 0 3m10s 10.0.20.103 kub-3 kube-system kube-multus-ds-amd64-bhpc9 1/1 Running 0 3m2s 10.0.20.103 kub-3 kube-system kube-multus-ds-amd64-n6vl8 1/1 Running 0 3m2s 10.0.20.102 kub-2 kube-system kube-multus-ds-amd64-qttgs 1/1 Running 0 3m2s 10.0.20.101 kub-1 kube-system kube-proxy-2x4jl 1/1 Running 0 3m33s 10.0.20.101 kub-1 kube-system kube-proxy-d48r7 1/1 Running 0 3m33s 10.0.20.103 kub-3 kube-system kube-proxy-f45lp 1/1 Running 0 3m33s 10.0.20.102 kub-2 kube-system kube-scheduler-kub-1 1/1 Running 1 4m54s 10.0.20.101 kub-1 kube-system kube-scheduler-kub-2 1/1 Running 1 4m33s 10.0.20.102 kub-2 kube-system nginx-proxy-kub-3 1/1 Running 0 3m33s 10.0.20.103 kub-3 kube-system nodelocaldns-cg9tz 1/1 Running 0 2m41s 10.0.20.102 kub-2 kube-system nodelocaldns-htswt 1/1 Running 0 2m41s 10.0.20.103 kub-3 kube-system nodelocaldns-nsp7s 1/1 Running 0 2m41s 10.0.20.101 kub-1 local-path-storage local-path-provisioner-66df45bfdd-km4zg 1/1 Running 0 2m54s 10.233.66.2 kub-3 ``` ================================================ FILE: docs/external_storage_provisioners/local_volume_provisioner.md ================================================ # Local Static Storage Provisioner The [local static storage provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) is NOT a dynamic storage provisioner as you would expect from a cloud provider. Instead, it simply creates PersistentVolumes for all mounts under the `host_dir` of the specified storage class. These storage classes are specified in the `local_volume_provisioner_storage_classes` nested dictionary. Example: ```yaml local_volume_provisioner_storage_classes: local-storage: host_dir: /mnt/disks mount_dir: /mnt/disks fast-disks: host_dir: /mnt/fast-disks mount_dir: /mnt/fast-disks block_cleaner_command: - "/scripts/shred.sh" - "2" volume_mode: Filesystem fs_type: ext4 ``` For each key in `local_volume_provisioner_storage_classes` a "storage class" with the same name is created in the entry `storageClassMap` of the ConfigMap `local-volume-provisioner`. The subkeys of each storage class in `local_volume_provisioner_storage_classes` are converted to camelCase and added as attributes to the storage class in the ConfigMap. The result of the above example is: ```yaml data: storageClassMap: | local-storage: hostDir: /mnt/disks mountDir: /mnt/disks fast-disks: hostDir: /mnt/fast-disks mountDir: /mnt/fast-disks blockCleanerCommand: - "/scripts/shred.sh" - "2" volumeMode: Filesystem fsType: ext4 ``` Additionally, a StorageClass object (`storageclasses.storage.k8s.io`) is also created for each storage class: ```bash $ kubectl get storageclasses.storage.k8s.io NAME PROVISIONER RECLAIMPOLICY fast-disks kubernetes.io/no-provisioner Delete local-storage kubernetes.io/no-provisioner Delete ``` The default StorageClass is `local-storage` on `/mnt/disks`; the rest of this documentation will use that path as an example. ## Examples to create local storage volumes 1. Using tmpfs ```bash for vol in vol1 vol2 vol3; do mkdir /mnt/disks/$vol mount -t tmpfs -o size=5G $vol /mnt/disks/$vol done ``` The tmpfs method is not recommended for production because the mounts are not persistent and data will be deleted on reboot. 1. Mount physical disks ```bash mkdir /mnt/disks/ssd1 mount /dev/vdb1 /mnt/disks/ssd1 ``` Physical disks are recommended for production environments because it offers complete isolation in terms of I/O and capacity. 1. Mount unpartitioned physical devices ```bash for disk in /dev/sdc /dev/sdd /dev/sde; do ln -s $disk /mnt/disks done ``` This saves time of precreating filesystems. Note that your storageclass must have `volume_mode` set to `"Filesystem"` and `fs_type` defined. If either is not set, the disk will be added as a raw block device. 1. PersistentVolumes with `volumeMode="Block"` Just like above, you can create PersistentVolumes with volumeMode `Block` by creating a symbolic link under discovery directory to the block device on the node, if you set `volume_mode` to `"Block"`. This will create a volume presented into a Pod as a block device, without any filesystem on it. 1. File-backed sparsefile method ```bash truncate /mnt/disks/disk5 --size 2G mkfs.ext4 /mnt/disks/disk5 mkdir /mnt/disks/vol5 mount /mnt/disks/disk5 /mnt/disks/vol5 ``` If you have a development environment and only one disk, this is the best way to limit the quota of persistent volumes. 1. Simple directories In a development environment, using `mount --bind` works also, but there is no capacity management. ## Usage notes Make sure to make any mounts persist via `/etc/fstab` or with systemd mounts (for Flatcar Container Linux or Fedora CoreOS). Pods with persistent volume claims will not be able to start if the mounts become unavailable. ## Further reading Refer to the upstream docs here: ================================================ FILE: docs/external_storage_provisioners/scheduler_plugins.md ================================================ # Scheduler plugins for Kubernetes [scheduler-plugins](https://github.com/kubernetes-sigs/scheduler-plugins) is out-of-tree scheduler plugins based on the [scheduler framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/). The kube-scheduler binary includes a list of plugins: - [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta] - [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta] - [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta] - [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta] - [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha] - [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha] - [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample] - [CrossNodePreemption](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/crossnodepreemption/README.md) [Sample] - [PodState](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/podstate/README.md) [Sample] - [QualityOfService](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/qos/README.md) [Sample] Currently, we use [helm chart](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#installing-the-chart) to install the scheduler plugins, so that a second scheduler would be created and running. **Note that running multi-scheduler will inevitably encounter resource conflicts when the cluster is short of resources**. ## Compatibility Matrix There are requirements for the version of Kubernetes, please see [Compatibility Matrix ](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master?tab=readme-ov-file#compatibility-matrix). It deserves our attention. | Scheduler Plugins | Compiled With K8s Version | | ----------------- | ------------------------- | | v0.28.9 | v1.28.9 | | v0.27.8 | v1.27.8 | ## Turning it on The `scheduler_plugins_enabled` option is used to enable the installation of scheduler plugins. You can enable or disable some plugins by setting the `scheduler_plugins_enabled_plugins` or `scheduler_plugins_disabled_plugins` option. They must be in the list we mentioned above. In addition, to use custom plugin configuration, set a value for `scheduler_plugins_plugin_config` option. For example, for Coscheduling plugin, you want to customize the permit waiting timeout to 10 seconds: ```yaml scheduler_plugins_plugin_config: - name: Coscheduling args: permitWaitingTimeSeconds: 10 # default is 60 ``` ## Leverage plugin Once the plugin is installed, we can apply CRs into cluster. For example, if using `CoScheduling`, we can apply the CR and test the deployment in the [example](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/doc/install.md#test-coscheduling). ================================================ FILE: docs/getting_started/comparisons.md ================================================ # Comparison ## Kubespray vs Kops Kubespray runs on bare metal and most clouds, using Ansible as its substrate for provisioning and orchestration. [Kops](https://github.com/kubernetes/kops) performs the provisioning and orchestration itself, and as such is less flexible in deployment platforms. For people with familiarity with Ansible, existing Ansible deployments or the desire to run a Kubernetes cluster across multiple platforms, Kubespray is a good choice. Kops, however, is more tightly integrated with the unique features of the clouds it supports so it could be a better choice if you know that you will only be using one platform for the foreseeable future. ## Kubespray vs Kubeadm [Kubeadm](https://github.com/kubernetes/kubeadm) provides domain Knowledge of Kubernetes clusters' life cycle management, including self-hosted layouts, dynamic discovery services and so on. Had it belonged to the new [operators world](https://coreos.com/blog/introducing-operators.html), it may have been named a "Kubernetes cluster operator". Kubespray however, does generic configuration management tasks from the "OS operators" ansible world, plus some initial K8s clustering (with networking plugins included) and control plane bootstrapping. Kubespray has started using `kubeadm` internally for cluster creation since v2.3 in order to consume life cycle management domain knowledge from it and offload generic OS configuration things from it, which hopefully benefits both sides. ================================================ FILE: docs/getting_started/getting-started.md ================================================ # Getting started ## Install ansible Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible). ## Building your own inventory Ansible inventory can be stored in 3 formats: YAML, JSON, or INI-like. See the [example inventory](/inventory/sample/inventory.ini) and [Ansible documentation on building your inventory](https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html), and [details on the inventory structure expected by Kubespray](/docs/ansible/inventory.md). ```ShellSession inventory/mycluster/inventory.ini # Review and change parameters under ``inventory/mycluster/group_vars`` inventory/mycluster/group_vars/all.yml # for every node, including etcd inventory/mycluster/group_vars/k8s_cluster.yml # for every node in the cluster (not etcd when it's separate) inventory/mycluster/group_vars/kube_control_plane.yml # for the control plane inventory/myclsuter/group_vars/kube_node.yml # for worker nodes ``` ## Installing the cluster ```ShellSession ansible-playbook -i inventory/mycluster/ cluster.yml -b -v \ --private-key=~/.ssh/private_key ``` ### Adding nodes You may want to add worker, control plane or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your control planes. This is especially helpful when doing something like autoscaling your clusters. - Add the new worker node to your inventory in the appropriate group (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html)). - Run the ansible-playbook command, substituting `cluster.yml` for `scale.yml`: ```ShellSession ansible-playbook -i inventory/mycluster/hosts.yml scale.yml -b -v \ --private-key=~/.ssh/private_key ``` ### Remove nodes You may want to remove **control plane**, **worker**, or **etcd** nodes from your existing cluster. This can be done by re-running the `remove-node.yml` playbook. First, all specified nodes will be drained, then stop some kubernetes services and delete some certificates, and finally execute the kubectl command to delete these nodes. This can be combined with the add node function. This is generally helpful when doing something like autoscaling your clusters. Of course, if a node is not working, you can remove the node and install it again. Use `--extra-vars "node=,"` to select the node(s) you want to delete. ```ShellSession ansible-playbook -i inventory/mycluster/hosts.yml remove-node.yml -b -v \ --private-key=~/.ssh/private_key \ --extra-vars "node=nodename,nodename2" ``` > Note: The playbook does not currently support the removal of the first control plane or etcd node. These nodes are essential for maintaining cluster operations and must remain intact. If a node is completely unreachable by ssh, add `--extra-vars reset_nodes=false` to skip the node reset step. If one node is unavailable, but others you wish to remove are able to connect via SSH, you could set `reset_nodes=false` as a host var in inventory. ## Connecting to Kubernetes By default, Kubespray configures kube_control_plane hosts with insecure access to kube-apiserver via port 8080. A kubeconfig file is not necessary in this case, because kubectl will use to connect. The kubeconfig files generated will point to localhost (on kube_control_planes) and kube_node hosts will connect either to a localhost nginx proxy or to a loadbalancer if configured. More details on this process are in the [HA guide](/docs/operations/ha-mode.md). Kubespray permits connecting to the cluster remotely on any IP of any kube_control_plane host on port 6443 by default. However, this requires authentication. One can get a kubeconfig from kube_control_plane hosts (see [below](#accessing-kubernetes-api)). For more information on kubeconfig and accessing a Kubernetes cluster, refer to the Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/). ## Accessing Kubernetes API The main client of Kubernetes is `kubectl`. It is installed on each kube_control_plane host and can optionally be configured on your ansible host by setting `kubectl_localhost: true` and `kubeconfig_localhost: true` in the configuration: - If `kubectl_localhost` enabled, `kubectl` will download onto `/usr/local/bin/` and setup with bash completion. A helper script `inventory/mycluster/artifacts/kubectl.sh` also created for setup with below `admin.conf`. - If `kubeconfig_localhost` enabled `admin.conf` will appear in the `inventory/mycluster/artifacts/` directory after deployment. - The location where these files are downloaded to can be configured via the `artifacts_dir` variable. NOTE: The controller host name in the admin.conf file might be a private IP. If so, change it to use the controller's public IP or the cluster's load balancer. You can see a list of nodes by running the following commands: ```ShellSession cd inventory/mycluster/artifacts ./kubectl.sh get nodes ``` If desired, copy admin.conf to ~/.kube/config. ## Setting up your first cluster [Setting up your first cluster](/docs/getting_started/setting-up-your-first-cluster.md) is an applied step-by-step guide for setting up your first cluster with Kubespray. ================================================ FILE: docs/getting_started/setting-up-your-first-cluster.md ================================================ # Setting up your first cluster with Kubespray This tutorial walks you through the detailed steps for setting up Kubernetes with [Kubespray](https://kubespray.io/). The guide is inspired on the tutorial [Kubernetes The Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way), with the difference that here we want to showcase how to spin up a Kubernetes cluster in a more managed fashion with Kubespray. ## Target Audience The target audience for this tutorial is someone looking for a hands-on guide to get started with Kubespray. ## Cluster Details * [kubespray](https://github.com/kubernetes-sigs/kubespray) * [kubernetes](https://github.com/kubernetes/kubernetes) ## Prerequisites * Google Cloud Platform: This tutorial leverages the [Google Cloud Platform](https://cloud.google.com/) to streamline provisioning of the compute infrastructure required to bootstrap a Kubernetes cluster from the ground up. [Sign up](https://cloud.google.com/free/) for $300 in free credits. * Google Cloud Platform SDK: Follow the Google Cloud SDK [documentation](https://cloud.google.com/sdk/) to install and configure the `gcloud` command line utility. Make sure to set a default compute region and compute zone. * The [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) command line utility is used to interact with the Kubernetes API Server. * Linux or Mac environment with Python 3 ## Provisioning Compute Resources Kubernetes requires a set of machines to host the Kubernetes control plane and the worker nodes where containers are ultimately run. In this lab you will provision the compute resources required for running a secure and highly available Kubernetes cluster across a single [compute zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones). ### Networking The Kubernetes [networking model](https://kubernetes.io/docs/concepts/cluster-administration/networking/#kubernetes-model) assumes a flat network in which containers and nodes can communicate with each other. In cases where this is not desired [network policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/) can limit how groups of containers are allowed to communicate with each other and external network endpoints. > Setting up network policies is out of scope for this tutorial. #### Virtual Private Cloud Network In this section a dedicated [Virtual Private Cloud](https://cloud.google.com/compute/docs/networks-and-firewalls#networks) (VPC) network will be setup to host the Kubernetes cluster. Create the `kubernetes-the-kubespray-way` custom VPC network: ```ShellSession gcloud compute networks create kubernetes-the-kubespray-way --subnet-mode custom ``` A [subnet](https://cloud.google.com/compute/docs/vpc/#vpc_networks_and_subnets) must be provisioned with an IP address range large enough to assign a private IP address to each node in the Kubernetes cluster. Create the `kubernetes` subnet in the `kubernetes-the-kubespray-way` VPC network: ```ShellSession gcloud compute networks subnets create kubernetes \ --network kubernetes-the-kubespray-way \ --range 10.240.0.0/24 ``` > The `10.240.0.0/24` IP address range can host up to 254 compute instances. #### Firewall Rules Create a firewall rule that allows internal communication across all protocols. It is important to note that the vxlan (udp) protocol has to be allowed in order for the calico (see later) networking plugin to work. ```ShellSession gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \ --allow tcp,udp,icmp \ --network kubernetes-the-kubespray-way \ --source-ranges 10.240.0.0/24 ``` Create a firewall rule that allows external SSH, ICMP, and HTTPS: ```ShellSession gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-external \ --allow tcp:80,tcp:6443,tcp:443,tcp:22,icmp \ --network kubernetes-the-kubespray-way \ --source-ranges 0.0.0.0/0 ``` It is not feasible to restrict the firewall to a specific IP address from where you are accessing the cluster as the nodes also communicate over the public internet and would otherwise run into this firewall. Technically you could limit the firewall to the (fixed) IP addresses of the cluster nodes and the remote IP addresses for accessing the cluster. ### Compute Instances The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04. Each compute instance will be provisioned with a fixed private IP address and a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)). Using fixed public IP addresses has the advantage that our cluster node configuration does not need to be updated with new public IP addresses every time the machines are shut down and later on restarted. Create three compute instances which will host the Kubernetes control plane: ```ShellSession for i in 0 1 2; do gcloud compute instances create controller-${i} \ --async \ --boot-disk-size 200GB \ --can-ip-forward \ --image-family ubuntu-2404-lts-amd64 \ --image-project ubuntu-os-cloud \ --machine-type e2-standard-2 \ --private-network-ip 10.240.0.1${i} \ --scopes compute-rw,storage-ro,service-management,service-control,logging-write,monitoring \ --subnet kubernetes \ --tags kubernetes-the-kubespray-way,controller done ``` > Do not forget to fix the IP addresses if you plan on re-using the cluster after temporarily shutting down the VMs - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address) Create three compute instances which will host the Kubernetes worker nodes: ```ShellSession for i in 0 1 2; do gcloud compute instances create worker-${i} \ --async \ --boot-disk-size 200GB \ --can-ip-forward \ --image-family ubuntu-2404-lts-amd64 \ --image-project ubuntu-os-cloud \ --machine-type e2-standard-2 \ --private-network-ip 10.240.0.2${i} \ --scopes compute-rw,storage-ro,service-management,service-control,logging-write,monitoring \ --subnet kubernetes \ --tags kubernetes-the-kubespray-way,worker done ``` > Do not forget to fix the IP addresses if you plan on re-using the cluster after temporarily shutting down the VMs - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address) List the compute instances in your default compute zone: ```ShellSession gcloud compute instances list --filter="tags.items=kubernetes-the-kubespray-way" ``` > Output ```ShellSession NAME ZONE MACHINE_TYPE PREEMPTIBLE INTERNAL_IP EXTERNAL_IP STATUS controller-0 us-west1-c e2-standard-2 10.240.0.10 XX.XX.XX.XXX RUNNING controller-1 us-west1-c e2-standard-2 10.240.0.11 XX.XXX.XXX.XX RUNNING controller-2 us-west1-c e2-standard-2 10.240.0.12 XX.XXX.XX.XXX RUNNING worker-0 us-west1-c e2-standard-2 10.240.0.20 XX.XX.XXX.XXX RUNNING worker-1 us-west1-c e2-standard-2 10.240.0.21 XX.XX.XX.XXX RUNNING worker-2 us-west1-c e2-standard-2 10.240.0.22 XX.XXX.XX.XX RUNNING ``` ### Configuring SSH Access Kubespray is relying on SSH to configure the controller and worker instances. Test SSH access to the `controller-0` compute instance: ```ShellSession IP_CONTROLLER_0=$(gcloud compute instances list --filter="tags.items=kubernetes-the-kubespray-way AND name:controller-0" --format="value(EXTERNAL_IP)") USERNAME=$(whoami) ssh $USERNAME@$IP_CONTROLLER_0 ``` If this is your first time connecting to a compute instance SSH keys will be generated for you. In this case you will need to enter a passphrase at the prompt to continue. > If you get a 'Remote host identification changed!' warning, you probably already connected to that IP address in the past with another host key. You can remove the old host key by running `ssh-keygen -R $IP_CONTROLLER_0` Please repeat this procedure for all the controller and worker nodes, to ensure that SSH access is properly functioning for all nodes. ## Set-up Kubespray The following set of instruction is based on the [Quick Start](https://github.com/kubernetes-sigs/kubespray) but slightly altered for our set-up. As Ansible is a python application, we will create a fresh virtual environment to install the dependencies for the Kubespray playbook: ```ShellSession python3 -m venv venv source venv/bin/activate ``` Next, we will git clone the Kubespray code into our working directory: ```ShellSession git clone https://github.com/kubernetes-sigs/kubespray.git cd kubespray git checkout release-2.17 ``` Now we need to install the dependencies for Ansible to run the Kubespray playbook: ```ShellSession pip install -r requirements.txt ``` Copy ``inventory/sample`` as ``inventory/mycluster``: ```ShellSession cp -rfp inventory/sample inventory/mycluster ``` Update the sample Ansible inventory file with ip given by gcloud: ```ShellSession gcloud compute instances list --filter="tags.items=kubernetes-the-kubespray-way" ``` Open `inventory/mycluster/inventory.ini` file and add it so that controller-0, controller-1 and controller-2 in the `kube_control_plane` group and worker-0, worker-1 and worker-2 in the `kube_node` group. Add respective `ip` to the respective local VPC IP for each node. The main configuration for the cluster is stored in `inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml`. In this file we will update the `supplementary_addresses_in_ssl_keys` with a list of the IP addresses of the controller nodes. In that way we can access the kubernetes API server as an administrator from outside the VPC network. You can also see that the `kube_network_plugin` is by default set to 'calico'. If you set this to 'cloud', it did not work on GCP at the time of testing. Kubespray also offers to easily enable popular kubernetes add-ons. You can modify the list of add-ons in `inventory/mycluster/group_vars/k8s_cluster/addons.yml`. Let's enable the metrics server as this is a crucial monitoring element for the kubernetes cluster, just change the 'false' to 'true' for `metrics_server_enabled`. Now we will deploy the configuration: ```ShellSession ansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa cluster.yml ``` Ansible will now execute the playbook, this can take up to 20 minutes. ## Access the kubernetes cluster We will leverage a kubeconfig file from one of the controller nodes to access the cluster as administrator from our local workstation. > In this simplified set-up, we did not include a load balancer that usually sits on top of the three controller nodes for a high available API server endpoint. In this simplified tutorial we connect directly to one of the three controllers. First, we need to edit the permission of the kubeconfig file on one of the controller nodes: ```ShellSession ssh $USERNAME@$IP_CONTROLLER_0 USERNAME=$(whoami) sudo chown -R $USERNAME:$USERNAME /etc/kubernetes/admin.conf exit ``` Now we will copy over the kubeconfig file: ```ShellSession scp $USERNAME@$IP_CONTROLLER_0:/etc/kubernetes/admin.conf kubespray-do.conf ``` This kubeconfig file uses the internal IP address of the controller node to access the API server. This kubeconfig file will thus not work of from outside the VPC network. We will need to change the API server IP address to the controller node his external IP address. The external IP address will be accepted in the TLS negotiation as we added the controllers external IP addresses in the SSL certificate configuration. Open the file and modify the server IP address from the local IP to the external IP address of controller-0, as stored in $IP_CONTROLLER_0. > Example ```ShellSession apiVersion: v1 clusters: - cluster: certificate-authority-data: XXX server: https://35.205.205.80:6443 name: cluster.local ... ``` Now, we load the configuration for `kubectl`: ```ShellSession export KUBECONFIG=$PWD/kubespray-do.conf ``` We should be all set to communicate with our cluster from our local workstation: ```ShellSession kubectl get nodes ``` > Output ```ShellSession NAME STATUS ROLES AGE VERSION controller-0 Ready master 47m v1.17.9 controller-1 Ready master 46m v1.17.9 controller-2 Ready master 46m v1.17.9 worker-0 Ready 45m v1.17.9 worker-1 Ready 45m v1.17.9 worker-2 Ready 45m v1.17.9 ``` ## Smoke tests ### Metrics Verify if the metrics server addon was correctly installed and works: ```ShellSession kubectl top nodes ``` > Output ```ShellSession NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% controller-0 191m 10% 1956Mi 26% controller-1 190m 10% 1828Mi 24% controller-2 182m 10% 1839Mi 24% worker-0 87m 4% 1265Mi 16% worker-1 102m 5% 1268Mi 16% worker-2 108m 5% 1299Mi 17% ``` Please note that metrics might not be available at first and need a couple of minutes before you can actually retrieve them. ### Network Let's verify if the network layer is properly functioning and pods can reach each other: ```ShellSession kubectl run myshell1 -it --rm --image busybox -- sh hostname -i # launch myshell2 in separate terminal (see next code block) and ping the hostname of myshell2 ping ``` ```ShellSession kubectl run myshell2 -it --rm --image busybox -- sh hostname -i ping ``` > Output ```ShellSession PING 10.233.108.2 (10.233.108.2): 56 data bytes 64 bytes from 10.233.108.2: seq=0 ttl=62 time=2.876 ms 64 bytes from 10.233.108.2: seq=1 ttl=62 time=0.398 ms 64 bytes from 10.233.108.2: seq=2 ttl=62 time=0.378 ms ^C --- 10.233.108.2 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max = 0.378/1.217/2.876 ms ``` ### Deployments In this section you will verify the ability to create and manage [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/). Create a deployment for the [nginx](https://nginx.org/en/) web server: ```ShellSession kubectl create deployment nginx --image=nginx ``` List the pod created by the `nginx` deployment: ```ShellSession kubectl get pods -l app=nginx ``` > Output ```ShellSession NAME READY STATUS RESTARTS AGE nginx-86c57db685-bmtt8 1/1 Running 0 18s ``` #### Port Forwarding In this section you will verify the ability to access applications remotely using [port forwarding](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/). Retrieve the full name of the `nginx` pod: ```ShellSession POD_NAME=$(kubectl get pods -l app=nginx -o jsonpath="{.items[0].metadata.name}") ``` Forward port `8080` on your local machine to port `80` of the `nginx` pod: ```ShellSession kubectl port-forward $POD_NAME 8080:80 ``` > Output ```ShellSession Forwarding from 127.0.0.1:8080 -> 80 Forwarding from [::1]:8080 -> 80 ``` In a new terminal make an HTTP request using the forwarding address: ```ShellSession curl --head http://127.0.0.1:8080 ``` > Output ```ShellSession HTTP/1.1 200 OK Server: nginx/1.19.1 Date: Thu, 13 Aug 2020 11:12:04 GMT Content-Type: text/html Content-Length: 612 Last-Modified: Tue, 07 Jul 2020 15:52:25 GMT Connection: keep-alive ETag: "5f049a39-264" Accept-Ranges: bytes ``` Switch back to the previous terminal and stop the port forwarding to the `nginx` pod: ```ShellSession Forwarding from 127.0.0.1:8080 -> 80 Forwarding from [::1]:8080 -> 80 Handling connection for 8080 ^C ``` #### Logs In this section you will verify the ability to [retrieve container logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/). Print the `nginx` pod logs: ```ShellSession kubectl logs $POD_NAME ``` > Output ```ShellSession ... 127.0.0.1 - - [13/Aug/2020:11:12:04 +0000] "HEAD / HTTP/1.1" 200 0 "-" "curl/7.64.1" "-" ``` #### Exec In this section you will verify the ability to [execute commands in a container](https://kubernetes.io/docs/tasks/debug/debug-application/get-shell-running-container/#running-individual-commands-in-a-container). Print the nginx version by executing the `nginx -v` command in the `nginx` container: ```ShellSession kubectl exec -ti $POD_NAME -- nginx -v ``` > Output ```ShellSession nginx version: nginx/1.19.1 ``` ### Kubernetes services #### Expose outside the cluster In this section you will verify the ability to expose applications using a [Service](https://kubernetes.io/docs/concepts/services-networking/service/). Expose the `nginx` deployment using a [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) service: ```ShellSession kubectl expose deployment nginx --port 80 --type NodePort ``` > The LoadBalancer service type can not be used because your cluster is not configured with [cloud provider integration](https://kubernetes.io/docs/getting-started-guides/scratch/#cloud-provider). Setting up cloud provider integration is out of scope for this tutorial. Retrieve the node port assigned to the `nginx` service: ```ShellSession NODE_PORT=$(kubectl get svc nginx \ --output=jsonpath='{range .spec.ports[0]}{.nodePort}') ``` Create a firewall rule that allows remote access to the `nginx` node port: ```ShellSession gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-nginx-service \ --allow=tcp:${NODE_PORT} \ --network kubernetes-the-kubespray-way ``` Retrieve the external IP address of a worker instance: ```ShellSession EXTERNAL_IP=$(gcloud compute instances describe worker-0 \ --format 'value(networkInterfaces[0].accessConfigs[0].natIP)') ``` Make an HTTP request using the external IP address and the `nginx` node port: ```ShellSession curl -I http://${EXTERNAL_IP}:${NODE_PORT} ``` > Output ```ShellSession HTTP/1.1 200 OK Server: nginx/1.19.1 Date: Thu, 13 Aug 2020 11:15:02 GMT Content-Type: text/html Content-Length: 612 Last-Modified: Tue, 07 Jul 2020 15:52:25 GMT Connection: keep-alive ETag: "5f049a39-264" Accept-Ranges: bytes ``` #### Local DNS We will now also verify that kubernetes built-in DNS works across namespaces. Create a namespace: ```ShellSession kubectl create namespace dev ``` Create an nginx deployment and expose it within the cluster: ```ShellSession kubectl create deployment nginx --image=nginx -n dev kubectl expose deployment nginx --port 80 --type ClusterIP -n dev ``` Run a temporary container to see if we can reach the service from the default namespace: ```ShellSession kubectl run curly -it --rm --image curlimages/curl:7.70.0 -- /bin/sh curl --head http://nginx.dev:80 ``` > Output ```ShellSession HTTP/1.1 200 OK Server: nginx/1.19.1 Date: Thu, 13 Aug 2020 11:15:59 GMT Content-Type: text/html Content-Length: 612 Last-Modified: Tue, 07 Jul 2020 15:52:25 GMT Connection: keep-alive ETag: "5f049a39-264" Accept-Ranges: bytes ``` Type `exit` to leave the shell. ## Cleaning Up ### Kubernetes resources Delete the dev namespace, the nginx deployment and service: ```ShellSession kubectl delete namespace dev kubectl delete deployment nginx kubectl delete svc/nginx ``` ### Kubernetes state Note: you can skip this step if you want to entirely remove the machines. If you want to keep the VMs and just remove the cluster state, you can simply run another Ansible playbook: ```ShellSession ansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa reset.yml ``` Resetting the cluster to the VMs original state usually takes about a couple of minutes. ### Compute instances Delete the controller and worker compute instances: ```ShellSession gcloud -q compute instances delete \ controller-0 controller-1 controller-2 \ worker-0 worker-1 worker-2 \ --zone $(gcloud config get-value compute/zone) ``` ### Network Delete the fixed IP addresses (assuming you named them equal to the VM names), if any: ```ShellSession gcloud -q compute addresses delete controller-0 controller-1 controller-2 \ worker-0 worker-1 worker-2 ``` Delete the `kubernetes-the-kubespray-way` firewall rules: ```ShellSession gcloud -q compute firewall-rules delete \ kubernetes-the-kubespray-way-allow-nginx-service \ kubernetes-the-kubespray-way-allow-internal \ kubernetes-the-kubespray-way-allow-external ``` Delete the `kubernetes-the-kubespray-way` network VPC: ```ShellSession gcloud -q compute networks subnets delete kubernetes gcloud -q compute networks delete kubernetes-the-kubespray-way ``` ================================================ FILE: docs/ingress/alb_ingress_controller.md ================================================ # AWS ALB Ingress Controller **NOTE:** The current image version is `v1.1.9`. Please file any issues you find and note the version used. The AWS ALB Ingress Controller satisfies Kubernetes [ingress resources](https://kubernetes.io/docs/concepts/services-networking/ingress/) by provisioning [Application Load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html). This project was originated by [Ticketmaster](https://github.com/ticketmaster) and [CoreOS](https://github.com/coreos) as part of Ticketmaster's move to AWS and CoreOS Tectonic. Learn more about Ticketmaster's Kubernetes initiative from Justin Dean's video at [Tectonic Summit](https://www.youtube.com/watch?v=wqXVKneP0Hg). This project was donated to Kubernetes SIG-AWS to allow AWS, CoreOS, Ticketmaster and other SIG-AWS contributors to officially maintain the project. SIG-AWS reached this consensus on June 1, 2018. ## Documentation Checkout our [Live Docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/#aws-alb-ingress-controller)! ## Getting started To get started with the controller, see our [walkthrough](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/guide/walkthrough/echoserver/). ## Setup - See [controller setup](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/guide/controller/setup/) on how to install ALB ingress controller - See [external-dns setup](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/guide/external-dns/setup/) for how to setup the external-dns to manage route 53 records. ## Building For details on building this project, see our [building guide](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/BUILDING/). ## Community, discussion, contribution, and support Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). You can reach the maintainers of this project at: - [Slack channel](https://kubernetes.slack.com/messages/sig-aws) - [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-aws) ### Code of conduct Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md). ## License [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fcoreos%2Falb-ingress-controller.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fcoreos%2Falb-ingress-controller?ref=badge_large) ================================================ FILE: docs/ingress/kube-vip.md ================================================ # kube-vip kube-vip provides Kubernetes clusters with a virtual IP and load balancer for both the control plane (for building a highly-available cluster) and Kubernetes Services of type LoadBalancer without relying on any external hardware or software. ## Prerequisites You have to configure `kube_proxy_strict_arp` when the kube_proxy_mode is `ipvs` and kube-vip ARP is enabled. ```yaml kube_proxy_strict_arp: true ``` ## Install You have to explicitly enable the kube-vip extension: ```yaml kube_vip_enabled: true ``` You also need to enable [kube-vip as HA, Load Balancer, or both](https://kube-vip.io/docs/installation/static/#kube-vip-as-ha-load-balancer-or-both): ```yaml # HA for control-plane, requires a VIP kube_vip_controlplane_enabled: true kube_vip_address: 10.42.42.42 loadbalancer_apiserver: address: "{{ kube_vip_address }}" port: 6443 # kube_vip_interface: ens160 # LoadBalancer for services kube_vip_services_enabled: false # kube_vip_services_interface: ens320 ``` > Note: When using `kube-vip` as LoadBalancer for services, [additional manual steps](https://kube-vip.io/docs/usage/cloud-provider/) are needed. If using [local traffic policy](https://kube-vip.io/docs/usage/kubernetes-services/#external-traffic-policy-kube-vip-v050): ```yaml kube_vip_enableServicesElection: true ``` If using [ARP mode](https://kube-vip.io/docs/installation/static/#arp) : ```yaml kube_vip_arp_enabled: true ``` If using [BGP mode](https://kube-vip.io/docs/installation/static/#bgp) : ```yaml kube_vip_bgp_enabled: true kube_vip_local_as: 65000 kube_vip_bgp_routerid: 192.168.0.2 kube_vip_bgppeers: - 192.168.0.10:65000::false - 192.168.0.11:65000::false # kube_vip_bgp_peeraddress: # kube_vip_bgp_peerpass: # kube_vip_bgp_peeras: # kube_vip_bgp_sourceip: # kube_vip_bgp_sourceif: ``` If using [control plane load-balancing](https://kube-vip.io/docs/about/architecture/#control-plane-load-balancing): ```yaml kube_vip_lb_enable: true ``` In addition, [load-balancing method](https://kube-vip.io/docs/installation/flags/#environment-variables) could be changed: ```yaml kube_vip_lb_fwdmethod: masquerade ``` If you want to adjust the parameters of [kube-vip LeaderElection](https://kube-vip.io/docs/installation/flags/#environment-variables): ```yaml kube_vip_leaseduration: 30 kube_vip_renewdeadline: 20 kube_vip_retryperiod: 4 ``` ================================================ FILE: docs/ingress/metallb.md ================================================ # MetalLB MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. It allows you to create Kubernetes services of type "LoadBalancer" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers. The default operating mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode. ## Prerequisites You have to configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface for MetalLB to work. ```yaml kube_proxy_strict_arp: true ``` ## Install You have to explicitly enable the MetalLB extension. ```yaml metallb_enabled: true metallb_speaker_enabled: true ``` By default, MetalLB resources are deployed into the `metallb-system` namespace. You can override this namespace using a variable. ```yaml metallb_namespace: woodenlb-system ``` By default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller: ```yaml metallb_config: controller: nodeselector: kubernetes.io/os: linux tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" effect: "NoSchedule" ``` If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fashion: ```yaml metallb_config: controller: nodeselector: kubernetes.io/os: linux tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" effect: "NoSchedule" speaker: nodeselector: kubernetes.io/os: linux tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Equal" value: "" effect: "NoSchedule" ``` ## Pools First you need to specify all of the pools you are going to use: ```yaml metallb_config: address_pools: primary: ip_range: - 192.0.1.0-192.0.1.254 pool1: ip_range: - 192.0.2.1-192.0.2.1 auto_assign: false # When set to false, you need to explicitly set the loadBalancerIP in the service! pool2: ip_range: - 192.0.3.0/24 avoid_buggy_ips: true # When set to true, .0 and .255 addresses will be avoided. ``` ## Layer2 Mode Pools that need to be configured in layer2 mode, need to be specified in a list: ```yaml metallb_config: layer2: - primary ``` ## BGP Mode When operating in BGP Mode MetalLB needs to have defined upstream peers and link the pool(s) specified above to the correct peer: ```yaml metallb_config: layer3: defaults: peer_port: 179 # The TCP port to talk to. Defaults to 179, you shouldn't need to set this in production. hold_time: 120s # Requested BGP hold time, per RFC4271. communities: vpn-only: "1234:1" NO_ADVERTISE: "65535:65282" metallb_peers: peer1: peer_address: 192.0.2.1 peer_asn: 64512 my_asn: 4200000000 communities: - vpn-only address_pool: - pool1 # (optional) The source IP address to use when establishing the BGP session. In most cases the source-address field should only be used with per-node peers, i.e. peers with node selectors which select only one node. CURRENTLY NOT SUPPORTED source_address: 192.0.2.2 # (optional) The router ID to use when connecting to this peer. Defaults to the node IP address. # Generally only useful when you need to peer with another BGP router running on the same machine as MetalLB. router_id: 1.2.3.4 # (optional) Password for TCPMD5 authenticated BGP sessions offered by some peers. password: "changeme" peer2: peer_address: 192.0.2.2 peer_asn: 64513 my_asn: 4200000000 communities: - NO_ADVERTISE address_pool: - pool2 # (optional) The source IP address to use when establishing the BGP session. In most cases the source-address field should only be used with per-node peers, i.e. peers with node selectors which select only one node. CURRENTLY NOT SUPPORTED source_address: 192.0.2.1 # (optional) The router ID to use when connecting to this peer. Defaults to the node IP address. # Generally only useful when you need to peer with another BGP router running on the same machine as MetalLB. router_id: 1.2.3.5 # (optional) Password for TCPMD5 authenticated BGP sessions offered by some peers. password: "changeme" ``` When using calico >= 3.18 you can replace MetalLB speaker by calico Service LoadBalancer IP advertisement. See [calico service IPs advertisement documentation](https://docs.projectcalico.org/archive/v3.18/networking/advertise-service-ips#advertise-service-load-balancer-ip-addresses). In this scenario you should disable the MetalLB speaker and configure the `calico_advertise_service_loadbalancer_ips` to match your `ip_range` ```yaml metallb_speaker_enabled: false metallb_config: address_pools: primary: ip_range: - 10.5.0.0/16 auto_assign: true layer2: - primary calico_advertise_service_loadbalancer_ips: "{{ metallb_config.address_pools.primary.ip_range }}" ``` If you have additional loadbalancer IP pool in `metallb_config.address_pools` , ensure to add them to the list. ```yaml metallb_speaker_enabled: false metallb_config: address_pools: primary: ip_range: - 10.5.0.0/16 auto_assign: true pool1: ip_range: - 10.6.0.0/16 auto_assign: true pool2: ip_range: - 10.10.0.0/16 auto_assign: true layer2: - primary layer3: defaults: peer_port: 179 hold_time: 120s communities: vpn-only: "1234:1" NO_ADVERTISE: "65535:65282" metallb_peers: peer1: peer_address: 10.6.0.1 peer_asn: 64512 my_asn: 4200000000 communities: - vpn-only address_pool: - pool1 peer2: peer_address: 10.10.0.1 peer_asn: 64513 my_asn: 4200000000 communities: - NO_ADVERTISE address_pool: - pool2 calico_advertise_service_loadbalancer_ips: - 10.5.0.0/16 - 10.6.0.0/16 - 10.10.0.0/16 ``` ================================================ FILE: docs/operating_systems/amazonlinux.md ================================================ # Amazon Linux 2 Amazon Linux is supported with docker,containerd and cri-o runtimes. **Note:** that Amazon Linux is not currently covered in kubespray CI and support for it is currently considered experimental. Amazon Linux 2, while derived from the Redhat OS family, does not keep in sync with RHEL upstream like CentOS/AlmaLinux/Oracle Linux. In order to use Amazon Linux as the ansible host for your kubespray deployments you need to manually install `python3` and deploy ansible and kubespray dependencies in a python virtual environment or use the official kubespray containers. There are no special considerations for using Amazon Linux as the target OS for Kubespray deployments. ================================================ FILE: docs/operating_systems/bootstrap-os.md ================================================ # bootstrap_os Bootstrap an Ansible host to be able to run Ansible modules. This role will: * configure the package manager (if applicable) to be able to fetch packages * install Python * install the necessary packages to use Ansible's package manager modules * set the hostname of the host to `{{ inventory_hostname }}` when requested ## Requirements A host running an operating system that is supported by Kubespray. See [Supported Linux Distributions](https://github.com/kubernetes-sigs/kubespray#supported-linux-distributions) for a current list. SSH access to the host. ## Role Variables Variables are listed with their default values, if applicable. ### General variables * `http_proxy`/`https_proxy` The role will configure the package manager (if applicable) to download packages via a proxy. * `override_system_hostname: true` The role will set the hostname of the machine to the name it has according to Ansible's inventory (the variable `{{ inventory_hostname }}`). ### Per distribution variables #### Flatcar Container Linux * `coreos_locksmithd_disable: false` Whether `locksmithd` (responsible for rolling restarts) should be disabled or be left alone. #### CentOS/RHEL/AlmaLinux/Rocky Linux * `centos_fastestmirror_enabled: false` Whether the [fastestmirror](https://wiki.centos.org/PackageManagement/Yum/FastestMirror) yum plugin should be enabled. ## Example Playbook Remember to disable fact gathering since Python might not be present on hosts. ```yaml - hosts: all gather_facts: false # not all hosts might be able to run modules yet roles: - kubespray_defaults - bootstrap_os ``` ## License Apache 2.0 ================================================ FILE: docs/operating_systems/fcos.md ================================================ # Fedora CoreOS Tested with stable version 37.20230322.3.0 Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap. Restart playbook again. ## Containers Tested with - containerd - crio ## Network ### calico To use calico create sysctl file with ignition: ```yaml files: - path: /etc/sysctl.d/reverse-path-filter.conf contents: inline: | net.ipv4.conf.all.rp_filter=1 ``` ## libvirt setup ### Prepare Prepare ignition and serve via http (a.e. python -m http.server ) ```json { "ignition": { "version": "3.0.0" }, "passwd": { "users": [ { "name": "ansibleUser", "sshAuthorizedKeys": [ "ssh-rsa ..publickey.." ], "groups": [ "wheel" ] } ] } } ``` ### create guest ```ShellSeasion machine_name=myfcos1 ignition_url=http://mywebserver/fcos.ign fcos_version=34.20210611.3.0 kernel=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-kernel-x86_64 initrd=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-initramfs.x86_64.img rootfs=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-rootfs.x86_64.img kernel_args="console=ttyS0 coreos.live.rootfs_url=${rootfs} coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}" sudo virt-install --name ${machine_name} --ram 4048 --graphics=none --vcpus 2 --disk size=20 \ --network bridge=virbr0 \ --install kernel=${kernel},initrd=${initrd},kernel_args_overwrite=yes,kernel_args="${kernel_args}" ``` ================================================ FILE: docs/operating_systems/flatcar.md ================================================ Flatcar Container Linux bootstrap =============== Example with Ansible: Before running the cluster playbook you must satisfy the following requirements: General Flatcar Pre-Installation Notes: - Ensure that the bin_dir is set to `/opt/bin` - ansible_python_interpreter should be `/opt/bin/python`. This will be laid down by the bootstrap task. - The resolvconf_mode setting of `docker_dns` **does not** work for Flatcar. This is because we do not edit the systemd service file for docker on Flatcar nodes. Instead, just use the default `host_resolvconf` mode. It should work out of the box. Then you can proceed to [cluster deployment](#run-deployment) ================================================ FILE: docs/operating_systems/kylinlinux.md ================================================ # Kylin Linux Kylin Linux is supported with docker and containerd runtimes. **Note:** that Kylin Linux is not currently covered in kubespray CI and support for it is currently considered experimental. At present, only `Kylin Linux Advanced Server V10 (Sword)` has been adapted, which can support the deployment of aarch64 and x86_64 platforms. There are no special considerations for using Kylin Linux as the target OS for Kubespray deployments. ================================================ FILE: docs/operating_systems/openeuler.md ================================================ # OpenEuler [OpenEuler](https://www.openeuler.org/en/) Linux is supported with docker and containerd runtimes. **Note:** that OpenEuler Linux is not currently covered in kubespray CI and support for it is currently considered experimental. At present, only `openEuler 22.03 LTS` has been adapted, which can support the deployment of aarch64 and x86_64 platforms. There are no special considerations for using OpenEuler Linux as the target OS for Kubespray deployments. ================================================ FILE: docs/operating_systems/opensuse.md ================================================ # openSUSE Leap 15.6 and Tumbleweed openSUSE Leap installation Notes: - Install Ansible ```ShellSession sudo zypper ref sudo zypper -n install ansible ``` - Install Jinja2 and Python-Netaddr ```sudo zypper -n install python-Jinja2 python-netaddr``` Now you can continue with [Preparing your deployment](getting-started.md#starting-custom-deployment) ================================================ FILE: docs/operating_systems/rhel.md ================================================ # Red Hat Enterprise Linux (RHEL) The documentation also applies to Red Hat derivatives, including Alma Linux, Rocky Linux, Oracle Linux, and CentOS. ## RHEL Support Subscription Registration The content of this section does not apply to open-source derivatives. In order to install packages via yum or dnf, RHEL hosts are required to be registered for a valid Red Hat support subscription. You can apply for a 1-year Development support subscription by creating a [Red Hat Developers](https://developers.redhat.com/) account. Be aware though that as the Red Hat Developers subscription is limited to only 1 year, it should not be used to register RHEL hosts provisioned in Production environments. Once you have a Red Hat support account, simply add the credentials to the Ansible inventory parameters `rh_subscription_username` and `rh_subscription_password` prior to deploying Kubespray. If your company has a Corporate Red Hat support account, then obtain an **Organization ID** and **Activation Key**, and add these to the Ansible inventory parameters `rh_subscription_org_id` and `rh_subscription_activation_key` instead of using your Red Hat support account credentials. ```ini rh_subscription_username: "" rh_subscription_password: "" # rh_subscription_org_id: "" # rh_subscription_activation_key: "" ``` Either the Red Hat support account username/password, or Organization ID/Activation Key combination must be specified in the Ansible inventory in order for the Red Hat subscription registration to complete successfully during the deployment of Kubespray. Update the Ansible inventory parameters `rh_subscription_usage`, `rh_subscription_role` and `rh_subscription_sla` if necessary to suit your specific requirements. ```ini rh_subscription_usage: "Development" rh_subscription_role: "Red Hat Enterprise Server" rh_subscription_sla: "Self-Support" ``` If the RHEL hosts are already registered to a valid Red Hat support subscription via an alternative configuration management approach prior to the deployment of Kubespray, the successful RHEL `subscription-manager` status check will simply result in the RHEL subscription registration tasks being skipped. ## Rocky Linux 10 (Experimental in Kubespray CI) The official Rocky Linux 10 cloud image does not include `kernel-module-extra`. Both Kube Proxy and CNI rely on this package, and since it relates to kernel version compatibility (which may require VM reboots, etc.), we haven't found an ideal solution. However, some users report that it doesn't affect them (minimal version). Therefore, the Kubespray CI Rocky Linux 10 image is built by Kubespray maintainers using `diskimage-builder`. For detailed methods, please refer to [the comments](https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093). ================================================ FILE: docs/operating_systems/uoslinux.md ================================================ # UOS Linux UOS Linux(UnionTech OS Server 20) is supported with docker and containerd runtimes. **Note:** that UOS Linux is not currently covered in kubespray CI and support for it is currently considered experimental. There are no special considerations for using UOS Linux as the target OS for Kubespray deployments. ================================================ FILE: docs/operations/cgroups.md ================================================ # cgroups To avoid resource contention between containers and host daemons in Kubernetes, the kubelet components can use cgroups to limit resource usage. ## Enforcing Node Allocatable You can use `kubelet_enforce_node_allocatable` to set node allocatable enforcement. ```yaml # A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. kubelet_enforce_node_allocatable: "pods" # kubelet_enforce_node_allocatable: "pods,kube-reserved" # kubelet_enforce_node_allocatable: "pods,kube-reserved,system-reserved" ``` Note that to enforce kube-reserved or system-reserved, `kube_reserved_cgroups` or `system_reserved_cgroups` needs to be specified respectively. Here is an example: ```yaml kubelet_enforce_node_allocatable: "pods,kube-reserved,system-reserved" # Set kube_reserved to true to run kubelet and container-engine daemons in a dedicated cgroup. # This is required if you want to enforce limits on the resource usage of these daemons. # It is not required if you just want to make resource reservations (kube_memory_reserved, kube_cpu_reserved, etc.) kube_reserved: true kube_reserved_cgroups_for_service_slice: kube.slice kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" kube_memory_reserved: 256Mi kube_cpu_reserved: 100m # kube_ephemeral_storage_reserved: 2Gi # kube_pid_reserved: "1000" # Set to true to reserve resources for system daemons system_reserved: true system_reserved_cgroups_for_service_slice: system.slice system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" system_memory_reserved: 512Mi system_cpu_reserved: 500m # system_ephemeral_storage_reserved: 2Gi # system_pid_reserved: "1000" ``` After the setup, the cgroups hierarchy is as follows: ```bash / (Cgroups Root) ├── kubepods.slice │ ├── ... │ ├── kubepods-besteffort.slice │ ├── kubepods-burstable.slice │ └── ... ├── kube.slice │ ├── ... │ ├── {{container_manager}}.service │ ├── kubelet.service │ └── ... ├── system.slice │ └── ... └── ... ``` You can learn more in the [official kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/). ================================================ FILE: docs/operations/encrypting-secret-data-at-rest.md ================================================ # Encrypting Secret Data at Rest Before enabling Encrypting Secret Data at Rest, please read the following documentation carefully. As you can see from the documentation above, 5 encryption providers are supported as of today (22.02.2022). As default value for the provider we have chosen `secretbox`. Alternatively you can use the values `identity`, `aesgcm`, `aescbc` or `kms`. | Provider | Why we have decided against the value as default | |----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | identity | no encryption | | aesgcm | Must be rotated every 200k writes | | aescbc | Not recommended due to CBC's vulnerability to padding oracle attacks. | | kms | Is the official recommended way, but assumes that a key management service independent of Kubernetes exists, we cannot assume this in all environments, so not a suitable default value. | ## Details about Secretbox Secretbox uses [Poly1305](https://cr.yp.to/mac.html) as message-authentication code and [XSalsa20](https://www.xsalsa20.com/) as secret-key authenticated encryption and secret-key encryption. ================================================ FILE: docs/operations/etcd.md ================================================ # etcd ## Deployment Types It is possible to deploy etcd with three methods. To change the default deployment method (host), use the `etcd_deployment_type` variable. Possible values are `host`, `kubeadm`, and `docker`. ### Host Host deployment is the default method. Using this method will result in etcd installed as a systemd service. ### Docker Installs docker in etcd group members and runs etcd on docker containers. Only usable when `container_manager` is set to `docker`. ### Kubeadm This deployment method is experimental and is only available for new deployments. This deploys etcd as a static pod on control plane hosts. ## Metrics To expose metrics on a separate HTTP port, define it in the inventory with: ```yaml etcd_metrics_port: 2381 ``` To create a service `etcd-metrics` and associated endpoints in the `kube-system` namespace, define its labels in the inventory with: ```yaml etcd_metrics_service_labels: k8s-app: etcd app.kubernetes.io/managed-by: Kubespray app: kube-prometheus-stack-kube-etcd release: kube-prometheus-stack ``` The last two labels in the above example allows to scrape the metrics from the [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart when it is installed with the release name `kube-prometheus-stack` and the following Helm `values.yaml`: ```yaml kubeEtcd: service: enabled: false ``` If your Helm release name is different, adjust the `release` label accordingly. To fully override metrics exposition URLs, define it in the inventory with: ```yaml etcd_listen_metrics_urls: "http://0.0.0.0:2381" ``` If you choose to expose metrics on specific node IPs (for example `10.141.4.22`, `10.141.4.23`, `10.141.4.24`) in `etcd_listen_metrics_urls`, you can configure kube-prometheus-stack to scrape those endpoints directly with: ```yaml kubeEtcd: enabled: true endpoints: - 10.141.4.22 - 10.141.4.23 - 10.141.4.24 ``` ================================================ FILE: docs/operations/ha-mode.md ================================================ # HA endpoints for K8s The following components require a highly available endpoints: * etcd cluster, * kube-apiserver service instances. The latter relies on a 3rd side reverse proxy, like Nginx or HAProxy, to achieve the same goal. ## Etcd The etcd clients (kube-api-masters) are configured with the list of all etcd peers. If the etcd-cluster has multiple instances, it's configured in HA already. ## Kube-apiserver K8s components require a loadbalancer to access the apiservers via a reverse proxy. Kubespray includes support for an nginx-based proxy that resides on each non-master Kubernetes node. This is referred to as localhost loadbalancing. It is less efficient than a dedicated load balancer because it creates extra health checks on the Kubernetes apiserver, but is more practical for scenarios where an external LB or virtual IP management is inconvenient. This option is configured by the variable `loadbalancer_apiserver_localhost` (defaults to `True`. Or `False`, if there is an external `loadbalancer_apiserver` defined). You may also define the port the local internal loadbalancer uses by changing, `loadbalancer_apiserver_port`. This defaults to the value of `kube_apiserver_port`. It is also important to note that Kubespray will only configure kubelet and kube-proxy on non-master nodes to use the local internal loadbalancer. If you wish to control the name of the loadbalancer container, you can set the variable `loadbalancer_apiserver_pod_name`. If you choose to NOT use the local internal loadbalancer, you will need to use the [kube-vip](/docs/ingress/kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the `access_ip` or IP address of the first server node in the `kube_control_plane` group. It can also configure clients to use endpoints for a given loadbalancer type. The following diagram shows how traffic to the apiserver is directed. ![Image](/docs/figures/loadbalancer_localhost.png?raw=true) A user may opt to use an external loadbalancer (LB) instead. An external LB provides access for external clients, while the internal LB accepts client connections only to the localhost. Given a frontend `VIP` address and `IP1, IP2` addresses of backends, here is an example configuration for a HAProxy service acting as an external LB: ```raw listen kubernetes-apiserver-https bind :8383 mode tcp option log-health-checks timeout client 3h timeout server 3h server master1 :6443 check check-ssl verify none inter 10000 server master2 :6443 check check-ssl verify none inter 10000 balance roundrobin ``` Note: That's an example config managed elsewhere outside Kubespray. And the corresponding example global vars for such a "cluster-aware" external LB with the cluster API access modes configured in Kubespray: ```yml apiserver_loadbalancer_domain_name: "my-apiserver-lb.example.com" loadbalancer_apiserver: address: port: 8383 ``` Note: The default kubernetes apiserver configuration binds to all interfaces, so you will need to use a different port for the vip from that the API is listening on, or set the `kube_apiserver_bind_address` so that the API only listens on a specific interface (to avoid conflict with haproxy binding the port on the VIP address) This domain name, or default "lb-apiserver.kubernetes.local", will be inserted into the `/etc/hosts` file of all servers in the `k8s_cluster` group and wired into the generated self-signed TLS/SSL certificates as well. Note that the HAProxy service should as well be HA and requires a VIP management, which is out of scope of this doc. There is a special case for an internal and an externally configured (not with Kubespray) LB used simultaneously. Keep in mind that the cluster is not aware of such an external LB and you need no to specify any configuration variables for it. Note: TLS/SSL termination for externally accessed API endpoints' will **not** be covered by Kubespray for that case. Make sure your external LB provides it. Alternatively you may specify an external load balanced VIPs in the `supplementary_addresses_in_ssl_keys` list. Then, kubespray will add them into the generated cluster certificates as well. Aside of that specific case, the `loadbalancer_apiserver` considered mutually exclusive to `loadbalancer_apiserver_localhost`. Access API endpoints are evaluated automatically, as the following: | Endpoint type | kube_control_plane | non-master | external | |------------------------------|------------------------------------------|-------------------------|-----------------------| | Local LB (default) | `https://dbip:sp` | `https://lc:nsp` | `https://m[0].aip:sp` | | Local LB (default) + cbip | `https://cbip:sp` and `https://lc:nsp` | `https://lc:nsp` | `https://m[0].aip:sp` | | Local LB + Unmanaged here LB | `https://dbip:sp` | `https://lc:nsp` | `https://ext` | | External LB, no internal | `https://dbip:sp` | `` | `https://lb:lp` | | No ext/int LB | `https://dbip:sp` | `` | `https://m[0].aip:sp` | Where: * `m[0]` - the first node in the `kube_control_plane` group; * `lb` - LB FQDN, `apiserver_loadbalancer_domain_name`; * `ext` - Externally load balanced VIP:port and FQDN, not managed by Kubespray; * `lc` - localhost; * `cbip` - a custom bind IP, `kube_apiserver_bind_address`; * `dbip` - localhost for the default bind IP '0.0.0.0'; * `nsp` - nginx secure port, `loadbalancer_apiserver_port`, defers to `sp`; * `sp` - secure port, `kube_apiserver_port`; * `lp` - LB port, `loadbalancer_apiserver.port`, defers to the secure port; * `ip` - the node IP, defers to the ansible IP; * `aip` - `access_ip`, defers to the ip. A second and a third column represent internal cluster access modes. The last column illustrates an example URI to access the cluster APIs externally. Kubespray has nothing to do with it, this is informational only. As you can see, the masters' internal API endpoints are always contacted via the local bind IP, which is `https://bip:sp`. ## Optional configurations ### ETCD with a LB In order to use an external loadbalancing (L4/TCP or L7 w/ SSL Passthrough VIP), the following variables need to be overridden in group_vars * `etcd_access_addresses` * `etcd_client_url` * `etcd_cert_alt_names` * `etcd_cert_alt_ips` #### Example of a VIP w/ FQDN ```yaml etcd_access_addresses: https://etcd.example.com:2379 etcd_client_url: https://etcd.example.com:2379 etcd_cert_alt_names: - "etcd.kube-system.svc.{{ dns_domain }}" - "etcd.kube-system.svc" - "etcd.kube-system" - "etcd" - "etcd.example.com" # This one needs to be added to the default etcd_cert_alt_names ``` #### Example of a VIP w/o FQDN (IP only) ```yaml etcd_access_addresses: https://2.3.7.9:2379 etcd_client_url: https://2.3.7.9:2379 etcd_cert_alt_ips: - "2.3.7.9" ``` ================================================ FILE: docs/operations/hardening.md ================================================ # Cluster Hardening If you want to improve the security on your cluster and make it compliant with the [CIS Benchmarks](https://learn.cisecurity.org/benchmarks), here you can find a configuration to harden your **kubernetes** installation. To apply the hardening configuration, create a file (eg. `hardening.yaml`) and paste the content of the following code snippet into that. ## Minimum Requirements The **kubernetes** version should be at least `v1.23.6` to have all the most recent security features (eg. the new `PodSecurity` admission plugin, etc). **N.B.** Some of these configurations have just been added to **kubespray**, so ensure that you have the latest version to make it works properly. Also, ensure that other configurations doesn't override these. `hardening.yaml`: ```yaml # Hardening --- ## kube-apiserver authorization_modes: ['Node', 'RBAC'] kube_apiserver_request_timeout: 120s kube_apiserver_service_account_lookup: true # enable kubernetes audit kubernetes_audit: true audit_log_path: "/var/log/kube-apiserver-log.json" audit_log_maxage: 30 audit_log_maxbackups: 10 audit_log_maxsize: 100 tls_min_version: VersionTLS12 tls_cipher_suites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 # enable encryption at rest kube_encrypt_secret_data: true kube_encryption_resources: [secrets] kube_encryption_algorithm: "secretbox" kube_apiserver_enable_admission_plugins: - EventRateLimit - AlwaysPullImages - ServiceAccount - NamespaceLifecycle - NodeRestriction - LimitRanger - ResourceQuota - MutatingAdmissionWebhook - ValidatingAdmissionWebhook - PodNodeSelector - PodSecurity kube_apiserver_admission_control_config_file: true # Creates config file for PodNodeSelector # kube_apiserver_admission_plugins_needs_configuration: [PodNodeSelector] # Define the default node selector, by default all the workloads will be scheduled on nodes # with label network=srv1 # kube_apiserver_admission_plugins_podnodeselector_default_node_selector: "network=srv1" # EventRateLimit plugin configuration kube_apiserver_admission_event_rate_limits: limit_1: type: Namespace qps: 50 burst: 100 cache_size: 2000 limit_2: type: User qps: 50 burst: 100 kube_profiling: false # Remove anonymous access to cluster remove_anonymous_access: true ## kube-controller-manager kube_controller_manager_bind_address: 127.0.0.1 kube_controller_terminated_pod_gc_threshold: 50 kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"] ## kube-scheduler kube_scheduler_bind_address: 127.0.0.1 ## etcd # Running etcd (on dedicated hosts) outside the Kubernetes cluster is the most secure deployment option, # as it isolates etcd from the cluster's CNI network and removes direct pod-level attack vectors. # This approach prevents RBAC misconfigurations that potentially compromise etcd, # creating an additional security boundary that protects the cluster's critical state store. etcd_deployment_type: host ## kubelet kubelet_authorization_mode_webhook: true kubelet_authentication_token_webhook: true kube_read_only_port: 0 kubelet_rotate_server_certificates: true kubelet_protect_kernel_defaults: true kubelet_event_record_qps: 1 kubelet_rotate_certificates: true kubelet_streaming_connection_idle_timeout: "5m" kubelet_make_iptables_util_chains: true kubelet_feature_gates: ["RotateKubeletServerCertificate=true"] kubelet_seccomp_default: true kubelet_systemd_hardening: true # In case you have multiple interfaces in your # control plane nodes and you want to specify the right # IP addresses, kubelet_secure_addresses allows you # to specify the IP from which the kubelet # will receive the packets. kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} 192.168.10.110 192.168.10.111 192.168.10.112" # additional configurations kube_owner: root kube_cert_group: root # create a default Pod Security Configuration and deny running of insecure pods # kube_system namespace is exempted by default kube_pod_security_use_default: true kube_pod_security_default_enforce: restricted ``` Let's take a deep look to the resultant **kubernetes** configuration: * The `anonymous-auth` (on `kube-apiserver`) is set to `true` by default. This is fine, because it is considered safe if you enable `RBAC` for the `authorization-mode`. * The `enable-admission-plugins` includes `PodSecurity` (for more details, please take a look here: ). Then, we set the `EventRateLimit` plugin, providing additional configuration files (that are automatically created under the hood and mounted inside the `kube-apiserver` container) to make it work. * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this). * The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`. See for more information on the subject. * The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image: ![kubelet hardening](../img/kubelet-hardening.png) Once you have the file properly filled, you can run the **Ansible** command to start the installation: ```bash ansible-playbook -v cluster.yml \ -i inventory.ini \ -b --become-user=root \ --private-key ~/.ssh/id_ecdsa \ -e "@vars.yaml" \ -e "@hardening.yaml" ``` **N.B.** The `vars.yaml` contains our general cluster information (SANs, load balancer, dns, etc..) and `hardening.yaml` is the file described above. ================================================ FILE: docs/operations/integration.md ================================================ # Kubespray (kubespray) in own ansible playbooks repo 1. Fork [kubespray repo](https://github.com/kubernetes-sigs/kubespray) to your personal/organisation account on github. Note: * All forked public repos at github will be also public, so **never commit sensitive data to your public forks**. * List of all forked repos could be retrieved from github page of original project. 2. Add **forked repo** as submodule to desired folder in your existent ansible repo (for example 3d/kubespray): ```ShellSession git submodule add https://github.com/YOUR_GITHUB/kubespray.git kubespray ``` Git will create `.gitmodules` file in your existent ansible repo: ```ini [submodule "3d/kubespray"] path = 3d/kubespray url = https://github.com/YOUR_GITHUB/kubespray.git ``` 3. Configure git to show submodule status: ```ShellSession git config --global status.submoduleSummary true ``` 4. Add *original* kubespray repo as upstream: ```ShellSession cd kubespray && git remote add upstream https://github.com/kubernetes-sigs/kubespray.git ``` 5. Sync your master branch with upstream: ```ShellSession git checkout master git fetch upstream git merge upstream/master git push origin master ``` 6. Create a new branch which you will use in your working environment: ```ShellSession git checkout -b work ``` ***Never*** use master branch of your repository for your commits. 7. Modify path to library and roles in your ansible.cfg file (role naming should be unique, you may have to rename your existent roles if they have same names as kubespray project), if you had roles in your existing ansible project before, you can add the path to those separated with `:`: ```ini ... library = ./library/:3d/kubespray/library/ roles_path = ./roles/:3d/kubespray/roles/ ... ``` 8. Copy and modify configs from kubespray `group_vars` folder to corresponding `group_vars` folder in your existent project. You could rename *all.yml* config to something else, i.e. *kubespray.yml* and create corresponding group in your inventory file, which will include all hosts groups related to kubernetes setup. 9. Modify your ansible inventory file by adding mapping of your existent groups (if any) to kubespray naming. For example: ```ini ... #Kubespray groups: [kube_node:children] kubenode [etcd:children] kubemaster kubemaster-ha [kube_control_plane:children] kubemaster kubemaster-ha ``` * Last entry here needed to apply kubespray.yml config file, renamed from all.yml of kubespray project. 10. Now you can include kubespray tasks in you existent playbooks by including cluster.yml file: ```yml - name: Import kubespray playbook ansible.builtin.import_playbook: 3d/kubespray/cluster.yml ``` Or you could copy separate tasks from cluster.yml into your ansible repository. 11. Commit changes to your ansible repo. Keep in mind, that submodule folder is just a link to the git commit hash of your forked repo. When you update your "work" branch you need to commit changes to ansible repo as well. Other members of your team should use ```git submodule sync```, ```git submodule update --init``` to get actual code from submodule. ## Contributing If you made useful changes or fixed a bug in existent kubespray repo, use this flow for PRs to original kubespray repo. 1. Sign the [CNCF CLA](https://git.k8s.io/community/CLA.md). 2. Change working directory to git submodule directory (3d/kubespray). 3. Setup desired user.name and user.email for submodule. If kubespray is only one submodule in your repo you could use something like: ```ShellSession git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-address@used.for.cncf"' ``` 4. Sync with upstream master: ```ShellSession git fetch upstream git merge upstream/master git push origin master ``` 5. Create new branch for the specific fixes that you want to contribute: ```ShellSession git checkout -b fixes-name-date-index ``` Branch name should be self explaining to you, adding date and/or index will help you to track/delete your old PRs. 6. Find git hash of your commit in "work" repo and apply it to newly created "fix" repo: ```ShellSession git cherry-pick ``` 7. If you have several temporary-stage commits - squash them using [git rebase -i](https://eli.thegreenplace.net/2014/02/19/squashing-github-pull-requests-into-a-single-commit) Also you could use interactive rebase ```ShellSession git rebase -i HEAD~10 ``` to delete commits which you don't want to contribute into original repo. 8. When your changes is in place, you need to check upstream repo one more time because it could be changed during your work. Check that you're on correct branch: ```ShellSession git status ``` And pull changes from upstream (if any): ```ShellSession git pull --rebase upstream master ``` 9. Now push your changes to your **fork** repo with ```ShellSession git push ``` If your branch doesn't exist on github, git will propose you to use something like ```ShellSession git push --set-upstream origin fixes-name-date-index ``` 10. Open you forked repo in browser, on the main page you will see proposition to create pull request for your newly created branch. Check proposed diff of your PR. If something is wrong you could safely delete "fix" branch on github using ```ShellSession git push origin --delete fixes-name-date-index git branch -D fixes-name-date-index ``` and start whole process from the beginning. If everything is fine - add description about your changes (what they do and why they're needed) and confirm pull request creation. ================================================ FILE: docs/operations/kernel-requirements.md ================================================ # Kernel Requirements For Kubernetes >=1.32.0, the recommended kernel LTS version from the 4.x series is 4.19. Any 5.x or 6.x versions are also supported. For cgroups v2 support, the minimum version is 4.15 and the recommended version is 5.8+. Refer to [this link](https://github.com/kubernetes/kubernetes/blob/v1.32.0/vendor/k8s.io/system-validators/validators/types_unix.go#L33). For more information, see [kernel version requirements](https://kubernetes.io/docs/reference/node/kernel-version-requirements). If the OS kernel version is lower than required, add the following configuration to ignore the kubeadm preflight errors: ```yaml kubeadm_ignore_preflight_errors: - SystemVerification ``` The Kernel Version Matrixs: | OS Version | Kernel Version | Kernel >=4.19 | |--- | --- | --- | | RHEL 9 | 5.14 | :white_check_mark: | | RHEL 8 | 4.18 | :x: | | Alma Linux 9 | 5.14 | :white_check_mark: | | Alma Linux 8 | 4.18 | :x: | | Rocky Linux 9 | 5.14 | :white_check_mark: | | Rocky Linux 8 | 4.18 | :x: | | Oracle Linux 9 | 5.14 | :white_check_mark: | | Oracle Linux 8 | 4.18 | :x: | | Ubuntu 24.04 | 6.6 | :white_check_mark: | | Ubuntu 22.04 | 5.15 | :white_check_mark: | | Ubuntu 20.04 | 5.4 | :white_check_mark: | | Debian 12 | 6.1 | :white_check_mark: | | Debian 11 | 5.10 | :white_check_mark: | | Fedora 40 | 6.8 | :white_check_mark: | | Fedora 39 | 6.5 | :white_check_mark: | | openSUSE Leap 15.5 | 5.14 | :white_check_mark: | | Amazon Linux 2 | 4.14 | :x: | | openEuler 24.03 | 6.6 | :white_check_mark: | | openEuler 22.03 | 5.10 | :white_check_mark: | | openEuler 20.03 | 4.19 | :white_check_mark: | ================================================ FILE: docs/operations/large-deployments.md ================================================ Large deployments of K8s ======================== For a large scaled deployments, consider the following configuration changes: * Tune [ansible settings](https://docs.ansible.com/ansible/latest/intro_configuration.html) for `forks` and `timeout` vars to fit large numbers of nodes being deployed. * Override containers' `foo_image_repo` vars to point to intranet registry. * Override the ``download_run_once: true`` and/or ``download_localhost: true``. See [Downloading binaries and containers](/docs/advanced/downloads.md) for details. * Adjust the `retry_stagger` global var as appropriate. It should provide sane load on a delegate (the first K8s control plane node) then retrying failed push or download operations. * Tune parameters for DNS related applications Those are ``dns_replicas``, ``dns_cpu_limit``, ``dns_cpu_requests``, ``dns_memory_limit``, ``dns_memory_requests``. Please note that limits must always be greater than or equal to requests. * Tune CPU/memory limits and requests. Those are located in roles' defaults and named like ``foo_memory_limit``, ``foo_memory_requests`` and ``foo_cpu_limit``, ``foo_cpu_requests``. Note that 'Mi' memory units for K8s will be submitted as 'M', if applied for ``docker run``, and cpu K8s units will end up with the 'm' skipped for docker as well. This is required as docker does not understand k8s units well. * Tune ``kubelet_status_update_frequency`` to increase reliability of kubelet. ``kube_controller_node_monitor_grace_period``, ``kube_controller_node_monitor_period``, ``kube_apiserver_pod_eviction_not_ready_timeout_seconds`` & ``kube_apiserver_pod_eviction_unreachable_timeout_seconds`` for better Kubernetes reliability. Check out [Kubernetes Reliability](/docs/advanced/kubernetes-reliability.md) * Tune network prefix sizes. Those are ``kube_network_node_prefix``, ``kube_service_addresses`` and ``kube_pods_subnet``. * Add calico_rr nodes if you are deploying with Calico or Canal. Nodes recover from host/network interruption much quicker with calico_rr. * Check out the [Inventory](/docs/getting_started/getting-started.md#building-your-own-inventory) section of the Getting started guide for tips on creating a large scale Ansible inventory. * Override the ``etcd_events_cluster_setup: true`` store events in a separate dedicated etcd instance. For example, when deploying 200 nodes, you may want to run ansible with ``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``. ================================================ FILE: docs/operations/mirror.md ================================================ # Public Download Mirror The public mirror is useful to make the public resources download quickly in some areas of the world. (such as China). ## Configuring Kubespray to use a mirror site You can follow the [offline](offline-environment.md) to config the image/file download configuration to the public mirror site. If you want to download quickly in China, the configuration can be like: ```shell # this should be in /group_vars/k8s_cluster.yml gcr_image_repo: "gcr.m.daocloud.io" kube_image_repo: "k8s.m.daocloud.io" docker_image_repo: "docker.m.daocloud.io" quay_image_repo: "quay.m.daocloud.io" github_image_repo: "ghcr.m.daocloud.io" files_repo: "https://files.m.daocloud.io" ``` Use mirror sites only if you trust the provider. The Kubespray team cannot verify their reliability or security. You can replace the `m.daocloud.io` with any site you want. ## Community-run mirror sites DaoCloud(China) * [image-mirror](https://github.com/DaoCloud/public-image-mirror) * [files-mirror](https://github.com/DaoCloud/public-binary-files-mirror) ================================================ FILE: docs/operations/nodes.md ================================================ # Adding/replacing a node Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/issues/3471#issuecomment-530036084) ## Adding/replacing a worker node This should be the easiest. ### 1) Add new node to the inventory ### 2) Run `scale.yml` You can use `--limit=NODE_NAME` to limit Kubespray to avoid disturbing other nodes in the cluster. Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes. ### 3) Remove an old node with remove-node.yml With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed. If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars: `-e node=NODE_NAME -e reset_nodes=false -e allow_ungraceful_removal=true`. Use this flag even when you remove other types of nodes like a control plane or etcd nodes. ### 4) Remove the node from the inventory That's it. ## Adding/replacing a control plane node ### 1) Run `cluster.yml` Append the new host to the inventory and run `cluster.yml`. You can NOT use `scale.yml` for that. **Note:** When adding new control plane nodes, always append them to the end of the `kube_control_plane` group in your inventory. Adding control plane nodes in the first position is not supported and will cause the playbook to fail. ### 2) Restart kube-system/nginx-proxy In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserver. Kubespray will update its static config, but it needs to be restarted in order to reload. ```sh # run in every host docker ps | grep k8s_nginx-proxy_nginx-proxy | awk '{print $1}' | xargs docker restart # or with containerd crictl ps | grep nginx-proxy | awk '{print $1}' | xargs crictl stop ``` ### 3) Remove old control plane nodes With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed. If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars. ## Adding/Removal of first `kube_control_plane` and etcd-master Currently you can't remove the first node in your `kube_control_plane` and etcd-master list. If you still want to remove this node you have to: ### 1) Change order of current control planes Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example: ```yaml all: hosts: children: kube_control_plane: hosts: node-1: node-2: node-3: kube_node: hosts: node-1: node-2: node-3: etcd: hosts: node-1: node-2: node-3: ``` change your inventory to: ```yaml all: hosts: children: kube_control_plane: hosts: node-2: node-3: node-1: kube_node: hosts: node-2: node-3: node-1: etcd: hosts: node-2: node-3: node-1: ``` ### 2) Upgrade the cluster run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal. ### 3) Remove old first control plane node from cluster With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed. If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars. ### 4) Edit cluster-info configmap in kube-public namespace `kubectl edit cm -n kube-public cluster-info` Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs. ### 5) Add new control plane node Update inventory (if needed) Run `cluster.yml` with `--limit=kube_control_plane` ## Adding an etcd node You need to make sure there are always an odd number of etcd nodes in the cluster. In such a way, this is always a replacement or scale up operation. Either add two new nodes or remove an old one. ### 1) Add the new node running cluster.yml Update the inventory and run `cluster.yml` passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`. If the node you want to add as an etcd node is already a worker or control plane node in your cluster, you have to remove him first using `remove-node.yml`. Run `upgrade-cluster.yml` also passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`. This is necessary to update all etcd configuration in the cluster. At this point, you will have an even number of nodes. Everything should still be working, and you should only have problems if the cluster decides to elect a new etcd leader before you remove a node. Even so, running applications should continue to be available. If you add multiple etcd nodes with one run, you might want to append `-e etcd_retries=10` to increase the amount of retries between each etcd node join. Otherwise the etcd cluster might still be processing the first join and fail on subsequent nodes. `etcd_retries=10` might work to join 3 new nodes. ### 2) Add the new node to apiserver config In every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure the new etcd nodes are present in the apiserver command line parameter `--etcd-servers=...`. ## Removing an etcd node ### 1) Remove an old etcd node With the node still in the inventory, run `remove-node.yml` passing `-e node=NODE_NAME` as the name of the node that should be removed. If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars. ### 2) Make sure only remaining nodes are in your inventory Remove `NODE_NAME` from your inventory file. ### 3) Update kubernetes and network configuration files with the valid list of etcd members Run `cluster.yml` to regenerate the configuration files on all remaining nodes. ### 4) Remove the old etcd node from apiserver config In every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure only active etcd nodes are still present in the apiserver command line parameter `--etcd-servers=...`. ### 5) Shutdown the old instance That's it. ================================================ FILE: docs/operations/offline-environment.md ================================================ # Offline environment In case your servers don't have access to the internet directly (for example when deploying on premises with security constraints), you need to get the following artifacts in advance from another environment where has access to the internet. * Some static files (zips and binaries) * OS packages (rpm/deb files) * Container images used by Kubespray. Exhaustive list depends on your setup * [Optional] Python packages used by Kubespray (only required if your OS doesn't provide all python packages/versions listed in `requirements.txt`) * [Optional] Helm chart files (only required if `helm_enabled=true`) Then you need to setup the following services on your offline environment: * an HTTP reverse proxy/cache/mirror to serve some static files (zips and binaries) * an internal Yum/Deb repository for OS packages * an internal container image registry that need to be populated with all container images used by Kubespray * [Optional] an internal PyPi server for python packages used by Kubespray * [Optional] an internal Helm registry for Helm chart files You can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script. In addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md). ## Access Control ### Note: access controlled files_repo To specify a username and password for "{{ files_repo }}", used to download the binaries, you can use url-encoding. Be aware that the Boolean `unsafe_show_logs` will show these credentials when `roles/download/tasks/download_file.yml` runs the task "Download_file | Show url of file to download". You can disable that Boolean in a job-template when running AWX/AAP/Semaphore. ```yaml files_repo_host: example.com files_repo_path: /repo files_repo_user: download files_repo_pass: !vault | $ANSIBLE_VAULT;1.1;AES256 61663232643236353864663038616361373739613338623338656434386662363539613462626661 6435333438313034346164313631303534346564316361370a306661393232626364376436386439 64653965663965356137333436616536643132336630313235333232336661373761643766356366 6232353233386534380a373262313634613833623537626132633033373064336261383166323230 3164 files_repo: "https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_repo_host ~ files_repo_path }}" ``` ### Note: access controlled registry To specify a username and password for "{{ registry_host }}", used to download the container images, you can use url-encoding too. ```yaml registry_pass: !vault | $ANSIBLE_VAULT;1.1;AES256 61663232643236353864663038616361373739613338623338656434386662363539613462626661 6435333438313034346164313631303534346564316361370a306661393232626364376436386439 64653965663965356137333436616536643132336630313235333232336661373761643766356366 6232353233386534380a373262313634613833623537626132633033373064336261383166323230 3164 containerd_registry_auth: - registry: "{{ registry_host }}" username: "{{ registry_user }}" password: "{{ registry_pass }}" ``` ## Configure Inventory Once all artifacts are accessible from your internal network, **adjust** the following variables in [your inventory](/inventory/sample/group_vars/all/offline.yml) to match your environment: ```yaml # Registry overrides kube_image_repo: "{{ registry_host }}" gcr_image_repo: "{{ registry_host }}" docker_image_repo: "{{ registry_host }}" quay_image_repo: "{{ registry_host }}" github_image_repo: "{{ registry_host }}" local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox" kubeadm_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm" kubectl_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl" kubelet_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet" # etcd is optional if you **DON'T** use etcd_deployment=host etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz" crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" # If using Calico calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" # If using Calico with kdd calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml" # Containerd containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}" nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" get_helm_url: "{{ files_repo }}/get.helm.sh" # Insecure registries for containerd containerd_registries_mirrors: - prefix: "{{ registry_addr }}" mirrors: - host: "{{ registry_host }}" capabilities: ["pull", "resolve"] skip_verify: true # CentOS/Redhat/AlmaLinux/Rocky Linux ## Docker / Containerd docker_rh_repo_base_url: "{{ yum_repo }}/docker-ce/$releasever/$basearch" docker_rh_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg" # Fedora ## Docker docker_fedora_repo_base_url: "{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}" docker_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg" ## Containerd containerd_fedora_repo_base_url: "{{ yum_repo }}/containerd" containerd_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg" # Debian ## Docker docker_debian_repo_base_url: "{{ debian_repo }}/docker-ce" docker_debian_repo_gpgkey: "{{ debian_repo }}/docker-ce/gpg" ## Containerd containerd_debian_repo_base_url: "{{ ubuntu_repo }}/containerd" containerd_debian_repo_gpgkey: "{{ ubuntu_repo }}/containerd/gpg" containerd_debian_repo_repokey: 'YOURREPOKEY' # Ubuntu ## Docker docker_ubuntu_repo_base_url: "{{ ubuntu_repo }}/docker-ce" docker_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/docker-ce/gpg" ## Containerd containerd_ubuntu_repo_base_url: "{{ ubuntu_repo }}/containerd" containerd_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/containerd/gpg" containerd_ubuntu_repo_repokey: 'YOURREPOKEY' ``` For the OS specific settings, just define the one matching your OS. If you use the settings like the one above, you'll need to define in your inventory the following variables: * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that the ones defined in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray_defaults/defaults/main/download.yml) , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the same repository path, you won't have to override anything else. * `registry_addr`: Container image registry, but only have [domain or ip]:[port]. * `files_repo`: HTTP webserver or reverse proxy that is able to serve the files listed above. Path is not important, you can store them anywhere as long as it's accessible by kubespray. It's recommended to use `*_version` in the path so that you don't need to modify this setting everytime kubespray upgrades one of these components. * `yum_repo`/`debian_repo`/`ubuntu_repo`: OS package repository depending on your OS, should point to your internal repository. Adjust the path accordingly. Used only for Docker/Containerd packages (if needed); other packages might be installed from other repositories. You might disable installing packages from other repositories by skipping the `system-packages` tag ## Install Kubespray Python Packages ### Recommended way: Kubespray Container Image The easiest way is to use [kubespray container image](https://quay.io/kubespray/kubespray) as all the required packages are baked in the image. Just copy the container image in your private container image registry and you are all set! ### Manual installation Look at the `requirements.txt` file and check if your OS provides all packages out-of-the-box (Using the OS package manager). For those missing, you need to either use a proxy that has Internet access (typically from a DMZ) or setup a PyPi server in your network that will host these packages. If you're using an HTTP(S) proxy to download your python packages: ```bash sudo pip install --proxy=https://[username:password@]proxyserver:port -r requirements.txt ``` When using an internal PyPi server: ```bash # If you host all required packages pip install -i https://pypiserver/pypi -r requirements.txt # If you only need the ones missing from the OS package manager pip install -i https://pypiserver/pypi package_you_miss ``` ## Run Kubespray as usual Once all artifacts are in place and your inventory properly set up, you can run kubespray with the regular `cluster.yaml` command: ```bash ansible-playbook -i inventory/my_airgap_cluster/hosts.yaml -b cluster.yml ``` If you use [Kubespray Container Image](#recommended-way:-kubespray-container-image), you can mount your inventory inside the container: ```bash docker run --rm -it -v path_to_inventory/my_airgap_cluster:inventory/my_airgap_cluster myprivateregisry.com/kubespray/kubespray:v2.14.0 ansible-playbook -i inventory/my_airgap_cluster/hosts.yaml -b cluster.yml ``` ================================================ FILE: docs/operations/port-requirements.md ================================================ # Port Requirements To operate properly, Kubespray requires some ports to be opened. If the network is configured with firewall rules, it is needed to ensure infrastructure components can communicate with each other through specific ports. Ensure the following ports required by Kubespray are open on the network and configured to allow access between hosts. Some ports are optional depending on the configuration and usage. ## Kubernetes ### Control plane | Protocol | Port | Description | |----------|--------| ------------ | | TCP | 22 | ssh for ansible | | TCP | 2379 | etcd client port| | TCP | 2380 | etcd peer port | | TCP | 6443 | kubernetes api | | TCP | 10250 | kubelet api | | TCP | 10257 | kube-scheduler | | TCP | 10259 | kube-controller-manager | ### Worker node(s) | Protocol | Port | Description | |----------|-------- | ------------ | | TCP | 22 | ssh for ansible | | TCP | 10250 | kubelet api | | TCP | 30000-32767| kube nodePort range | refers to: [Kubernetes Docs](https://kubernetes.io/docs/reference/networking/ports-and-protocols/) ## Calico If Calico is used, it requires: | Protocol | Port | Description | |----------|-------- | ------------ | | TCP | 179 | Calico networking (BGP) | | UDP | 4789 | Calico CNI with VXLAN enabled | | TCP | 5473 | Calico CNI with Typha enabled | | UDP | 51820 | Calico with IPv4 Wireguard enabled | | UDP | 51821 | Calico with IPv6 Wireguard enabled | | IPENCAP / IPIP | - | Calico CNI with IPIP enabled | refers to: [Calico Docs](https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#network-requirements) ## Cilium If Cilium is used, it requires: | Protocol | Port | Description | |----------|-------- | ------------ | | TCP | 4240 | Cilium Health checks (``cilium-health``) | | TCP | 4244 | Hubble server | | TCP | 4245 | Hubble Relay | | UDP | 8472 | VXLAN overlay | | TCP | 9962 | Cilium-agent Prometheus metrics | | TCP | 9963 | Cilium-operator Prometheus metrics | | TCP | 9964 | Cilium-proxy Prometheus metrics | | UDP | 51871 | WireGuard encryption tunnel endpoint | | ICMP | - | health checks | refers to: [Cilium Docs](https://docs.cilium.io/en/v1.13/operations/system_requirements/) ## Addons | Protocol | Port | Description | |----------|-------- | ------------ | | TCP | 9100 | node exporter | | TCP/UDP | 7472 | metallb metrics ports | | TCP/UDP | 7946 | metallb L2 operating mode | ================================================ FILE: docs/operations/recover-control-plane.md ================================================ # Recovering the control plane To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook. Examples of what broken means in this context: * One or more bare metal node(s) suffer from unrecoverable hardware failure * One or more node(s) fail during patching or upgrading * Etcd database corruption * Other node related failures leaving your control plane degraded or nonfunctional __Note that you need at least one functional node to be able to recover using this method.__ ## Runbook * Backup what you can * Provision new nodes to replace the broken ones * Copy any broken etcd nodes into the "broken\_etcd" group, make sure the "etcd\_member\_name" variable is set. * Copy any broken control plane nodes into the "broken\_kube\_control\_plane" group. * Place the surviving nodes of the control plane first in the "etcd" and "kube\_control\_plane" groups * Add the new nodes below the surviving control plane nodes in the "etcd" and "kube\_control\_plane" groups Then run the playbook with ```--limit etcd,kube_control_plane``` and increase the number of ETCD retries by setting ```-e etcd_retries=10``` or something even larger. The amount of retries required is difficult to predict. When finished you should have a fully working control plane again. ## Recover from lost quorum The playbook attempts to figure out it the etcd quorum is intact. If quorum is lost it will attempt to take a snapshot from the first node in the "etcd" group and restore from that. If you would like to restore from an alternate snapshot set the path to that snapshot in the "etcd\_snapshot" variable. ```-e etcd_snapshot=/tmp/etcd_snapshot``` ## Caveats * The playbook has only been tested with fairly small etcd databases. * There may be disruptions while running the playbook. * There are absolutely no guarantees. If possible try to break a cluster in the same way that your target cluster is broken and test to recover that before trying on the real target cluster. ================================================ FILE: docs/operations/upgrades.md ================================================ # Upgrading Kubernetes in Kubespray Kubespray handles upgrades the same way it handles initial deployment. That is to say that each component is laid down in a fixed order. You can also individually control versions of components by explicitly defining their versions. Here are all version vars for each component: * docker_version * docker_containerd_version (relevant when `container_manager` == `docker`) * containerd_version (relevant when `container_manager` == `containerd`) * kube_version * etcd_version * calico_version * calico_cni_version * flannel_version > **Warning** > [Attempting to upgrade from an older release straight to the latest release is unsupported and likely to break something](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515) See [Multiple Upgrades](#multiple-upgrades) for how to upgrade from older Kubespray release to the latest release ## Unsafe upgrade example If you wanted to upgrade just kube_version from v1.18.10 to v1.19.7, you could deploy the following way: ```ShellSession ansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=1.18.10 -e upgrade_cluster_setup=true ``` And then repeat with 1.19.7 as kube_version: ```ShellSession ansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=1.19.7 -e upgrade_cluster_setup=true ``` The var ```-e upgrade_cluster_setup=true``` is needed to be set in order to migrate the deploys of e.g kube-apiserver inside the cluster immediately which is usually only done in the graceful upgrade. (Refer to [#4139](https://github.com/kubernetes-sigs/kubespray/issues/4139) and [#4736](https://github.com/kubernetes-sigs/kubespray/issues/4736)) ## Graceful upgrade Kubespray also supports cordon, drain and uncordoning of nodes when performing a cluster upgrade. There is a separate playbook used for this purpose. It is important to note that upgrade-cluster.yml can only be used for upgrading an existing cluster. That means there must be at least 1 kube_control_plane already deployed. ```ShellSession ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.19.7 ``` After a successful upgrade, the Server Version should be updated: ```ShellSession $ kubectl version Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-13T13:23:52Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-13T13:15:20Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"} ``` You can control how many nodes are upgraded at the same time by modifying the ansible variable named `serial`, as explained [here](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_strategies.html#setting-the-batch-size-with-serial). If you don't set this variable, it will upgrade the cluster nodes in batches of 20% of the available nodes. Setting `serial=1` would mean upgrade one node at a time. ```ShellSession ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 -e "serial=1" ``` ### Pausing the upgrade If you want to manually control the upgrade procedure, you can set some variables to pause the upgrade playbook. Pausing *before* upgrading each upgrade may be useful for inspecting pods running on that node, or performing manual actions on the node: * `upgrade_node_confirm: true` - This will pause the playbook execution prior to upgrading each node. The play will resume when manually approved by typing "yes" at the terminal. * `upgrade_node_pause_seconds: 60` - This will pause the playbook execution for 60 seconds prior to upgrading each node. The play will resume automatically after 60 seconds. Pausing *after* upgrading each node may be useful for rebooting the node to apply kernel updates, or testing the still-cordoned node: * `upgrade_node_post_upgrade_confirm: true` - This will pause the playbook execution after upgrading each node, but before the node is uncordoned. The play will resume when manually approved by typing "yes" at the terminal. * `upgrade_node_post_upgrade_pause_seconds: 60` - This will pause the playbook execution for 60 seconds after upgrading each node, but before the node is uncordoned. The play will resume automatically after 60 seconds. ## Node-based upgrade If you don't want to upgrade all nodes in one run, you can use `--limit` [patterns](https://docs.ansible.com/ansible/latest/user_guide/intro_patterns.html#patterns-and-ansible-playbook-flags). Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes: ```ShellSession ansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini ``` After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147): ```ShellSession ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit "kube_control_plane:etcd" ``` Now you can upgrade other nodes in any order and quantity: ```ShellSession ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit "node4:node6:node7:node12" ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=1.20.7 --limit "node5*" ``` ## Multiple upgrades > **Warning** > [Do not skip minor releases (patches releases are ok) when upgrading--upgrade by one tag at a > time.](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515) For instances, given the tag list: ```console $ git tag v2.20.0 v2.21.0 v2.22.0 v2.22.1 v2.23.0 v2.23.1 v2.23.2 v2.24.0 ... ``` v2.22.0 -> v2.23.2 -> v2.24.0 : ✓ v.22.0 -> v2.24.0 : ✕ Assuming you don't explicitly define a kubernetes version in your k8s_cluster.yml, you simply check out the next tag and run the upgrade-cluster.yml playbook * If you do define kubernetes version in your inventory (e.g. group_vars/k8s_cluster.yml) then either make sure to update it before running upgrade-cluster, or specify the new version you're upgrading to: `ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml -e kube_version=1.11.3` Otherwise, the upgrade will leave your cluster at the same k8s version defined in your inventory vars. The below example shows taking a cluster that was set up for v2.6.0 up to v2.10.0 ```ShellSession $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 1h v1.10.4 boomer Ready master,node 42m v1.10.4 caprica Ready master,node 42m v1.10.4 $ git describe --tags v2.6.0 $ git tag ... v2.6.0 v2.7.0 v2.8.0 v2.8.1 v2.8.2 ... $ git checkout v2.7.0 Previous HEAD position was 8b3ce6e4 bump upgrade tests to v2.5.0 commit (#3087) HEAD is now at 05dabb7e Fix Bionic networking restart error #3430 (#3431) # NOTE: May need to `pip3 install -r requirements.txt` when upgrading. ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 1h v1.11.3 boomer Ready master,node 1h v1.11.3 caprica Ready master,node 1h v1.11.3 $ git checkout v2.8.0 Previous HEAD position was 05dabb7e Fix Bionic networking restart error #3430 (#3431) HEAD is now at 9051aa52 Fix ubuntu-contiv test failed (#3808) ``` > **Note** > Review changes between the sample inventory and your inventory when upgrading versions. Some deprecations between versions that mean you can't just upgrade straight from 2.7.0 to 2.8.0 if you started with the sample inventory. In this case, I set "kubeadm_enabled" to false, knowing that it is deprecated and removed by 2.9.0, to delay converting the cluster to kubeadm as long as I could. ```ShellSession $ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release." ... Are you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden): yes ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 114m v1.12.3 boomer Ready master,node 114m v1.12.3 caprica Ready master,node 114m v1.12.3 $ git checkout v2.8.1 Previous HEAD position was 9051aa52 Fix ubuntu-contiv test failed (#3808) HEAD is now at 2ac1c756 More Feature/2.8 backports for 2.8.1 (#3911) $ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release." ... Are you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden): yes ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 2h36m v1.12.4 boomer Ready master,node 2h36m v1.12.4 caprica Ready master,node 2h36m v1.12.4 $ git checkout v2.8.2 Previous HEAD position was 2ac1c756 More Feature/2.8 backports for 2.8.1 (#3911) HEAD is now at 4167807f Upgrade to 1.12.5 (#4066) $ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release." ... Are you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden): yes ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 3h3m v1.12.5 boomer Ready master,node 3h3m v1.12.5 caprica Ready master,node 3h3m v1.12.5 $ git checkout v2.8.3 Previous HEAD position was 4167807f Upgrade to 1.12.5 (#4066) HEAD is now at ea41fc5e backport cve-2019-5736 to release-2.8 (#4234) $ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release." ... Are you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden): yes ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 5h18m v1.12.5 boomer Ready master,node 5h18m v1.12.5 caprica Ready master,node 5h18m v1.12.5 $ git checkout v2.8.4 Previous HEAD position was ea41fc5e backport cve-2019-5736 to release-2.8 (#4234) HEAD is now at 3901480b go to k8s 1.12.7 (#4400) $ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release." ... Are you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden): yes ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 5h37m v1.12.7 boomer Ready master,node 5h37m v1.12.7 caprica Ready master,node 5h37m v1.12.7 $ git checkout v2.8.5 Previous HEAD position was 3901480b go to k8s 1.12.7 (#4400) HEAD is now at 6f97687d Release 2.8 robust san handling (#4478) $ ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... "msg": "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release." ... Are you sure you want to deploy cluster using the deprecated non-kubeadm mode. (output is hidden): yes ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 5h45m v1.12.7 boomer Ready master,node 5h45m v1.12.7 caprica Ready master,node 5h45m v1.12.7 $ git checkout v2.9.0 Previous HEAD position was 6f97687d Release 2.8 robust san handling (#4478) HEAD is now at a4e65c7c Upgrade to Ansible >2.7.0 (#4471) ``` > **Warning** > IMPORTANT: Some variable formats changed in the k8s_cluster.yml between 2.8.5 and 2.9.0 If you do not keep your inventory copy up to date, **your upgrade will fail** and your first master will be left non-functional until fixed and re-run. It is at this point the cluster was upgraded from non-kubeadm to kubeadm as per the deprecation warning. ```ShellSession ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 6h54m v1.13.5 boomer Ready master,node 6h55m v1.13.5 caprica Ready master,node 6h54m v1.13.5 # Watch out: 2.10.0 is hiding between 2.1.2 and 2.2.0 $ git tag ... v2.1.0 v2.1.1 v2.1.2 v2.10.0 v2.2.0 ... $ git checkout v2.10.0 Previous HEAD position was a4e65c7c Upgrade to Ansible >2.7.0 (#4471) HEAD is now at dcd9c950 Add etcd role dependency on kube user to avoid etcd role failure when running scale.yml with a fresh node. (#3240) (#4479) ansible-playbook -i inventory/mycluster/hosts.ini -b upgrade-cluster.yml ... $ kubectl get node NAME STATUS ROLES AGE VERSION apollo Ready master,node 7h40m v1.14.1 boomer Ready master,node 7h40m v1.14.1 caprica Ready master,node 7h40m v1.14.1 ``` ## Upgrading to v2.19 `etcd_kubeadm_enabled` is being deprecated at v2.19. The same functionality is achievable by setting `etcd_deployment_type` to `kubeadm`. Deploying etcd using kubeadm is experimental and is only available for either new or deployments where `etcd_kubeadm_enabled` was set to `true` while deploying the cluster. From 2.19 and onward `etcd_deployment_type` variable will be placed in `group_vars/all/etcd.yml` instead of `group_vars/etcd.yml`, due to scope issues. The placement of the variable is only important for `etcd_deployment_type: kubeadm` right now. However, since this might change in future updates, it is recommended to move the variable. Upgrading is straightforward; no changes are required if `etcd_kubeadm_enabled` was not set to `true` when deploying. If you have a cluster where `etcd` was deployed using `kubeadm`, you will need to remove `etcd_kubeadm_enabled` the variable. Then move `etcd_deployment_type` variable from `group_vars/etcd.yml` to `group_vars/all/etcd.yml` due to scope issues and set `etcd_deployment_type` to `kubeadm`. ## Upgrade order As mentioned above, components are upgraded in the order in which they were installed in the Ansible playbook. The order of component installation is as follows: * Docker * Containerd * etcd * kubelet and kube-proxy * network_plugin (such as Calico) * kube-apiserver, kube-scheduler, and kube-controller-manager * Add-ons (such as KubeDNS) ### Component-based upgrades A deployer may want to upgrade specific components in order to minimize risk or save time. This strategy is not covered by CI as of this writing, so it is not guaranteed to work. These commands are useful only for upgrading fully-deployed, healthy, existing hosts. This will definitely not work for undeployed or partially deployed hosts. Upgrade docker: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=docker ``` Upgrade etcd: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd ``` Upgrade etcd without rotating etcd certs: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd --limit=etcd --skip-tags=etcd-secrets ``` Upgrade kubelet: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs ``` Upgrade Kubernetes master components: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=master ``` Upgrade network plugins: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=network ``` Upgrade all add-ons: ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=apps ``` Upgrade just helm (assuming `helm_enabled` is true): ```ShellSession ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=helm ``` ## Migrate from Docker to Containerd Please note that **migrating container engines is not officially supported by Kubespray**. While this procedure can be used to migrate your cluster, it applies to one particular scenario and will likely evolve over time. At the moment, they are intended as an additional resource to provide insight into how these steps can be officially integrated into the Kubespray playbooks. As of Kubespray 2.18.0, containerd is already the default container engine. If you have the chance, it is advisable and safer to reset and redeploy the entire cluster with a new container engine. * [Migrating from Docker to Containerd](/docs/upgrades/migrate_docker2containerd.md) ## System upgrade If you want to upgrade the APT or YUM packages while the nodes are cordoned, you can use: ```ShellSession ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e system_upgrade=true ``` Nodes will be rebooted when there are package upgrades (`system_upgrade_reboot: on-upgrade`). This can be changed to `always` or `never`. Note: Downloads will happen twice unless `system_upgrade_reboot` is `never`. ================================================ FILE: docs/roadmap/roadmap.md ================================================ # Kubespray's roadmap We are tracking the evolution towards Kubespray 3.0 in [#6400](https://github.com/kubernetes-sigs/kubespray/issues/6400) as well as in other open issue in our [github issues](https://github.com/kubernetes-sigs/kubespray/issues/) section. ================================================ FILE: docs/upgrades/migrate_docker2containerd.md ================================================ # Migrating from Docker to Containerd ❗MAKE SURE YOU READ BEFORE PROCEEDING❗ **Migrating container engines is not officially supported by Kubespray**. The following procedure covers one particular scenario and involves manual steps, along with multiple runs of `cluster.yml`. It provides no guarantees that it will actually work or that any further action is needed. Please, consider these instructions as experimental guidelines. While they can be used to migrate your cluster, they will likely evolve over time. At the moment, they are intended as an additional resource to provide insight into how these steps can be officially integrated into the Kubespray playbooks. As of Kubespray 2.18.0, containerd is already the default container engine. If you have the chance, it is still advisable and safer to reset and redeploy the entire cluster with a new container engine. Input and feedback are always appreciated. ## Tested environment Nodes: Ubuntu 18.04 LTS\ Cloud Provider: None (baremetal or VMs)\ Kubernetes version: 1.21.5\ Kubespray version: 2.18.0 ## Important considerations If you require minimum downtime, nodes need to be cordoned and drained before being processed, one by one. If you wish to run `cluster.yml` only once and get it all done in one swoop, downtime will be significantly higher. Docker will need to be manually removed from all nodes before the playbook runs (see [#8431](https://github.com/kubernetes-sigs/kubespray/issues/8431)). For minimum downtime, the following steps will be executed multiple times, once for each node. Processing nodes one by one also means you will not be able to update any other cluster configuration using Kubespray before this procedure is finished and the cluster is fully migrated. Everything done here requires full root access to every node. ## Migration steps Before you begin, adjust your inventory: ```yaml # Filename: k8s_cluster/k8s-cluster.yml resolvconf_mode: host_resolvconf container_manager: containerd # Filename: etcd.yml etcd_deployment_type: host ``` ### 1) Pick one or more nodes for processing It is still unclear how the order might affect this procedure. So, to be sure, it might be best to start with the control plane and etcd nodes all together, followed by each worker node individually. ### 2) Cordon and drain the node ... because, downtime. ### 3) Stop docker and kubelet daemons ```commandline service kubelet stop service docker stop ``` ### 4) Uninstall docker + dependencies ```commandline apt-get remove -y --allow-change-held-packages containerd.io docker-ce docker-ce-cli docker-ce-rootless-extras ``` In some cases, there might a `pigz` missing dependency. Some image layers need this to be extracted. ```shell apt-get install pigz ``` ### 5) Run `cluster.yml` playbook with `--limit` ```commandline ansible-playbook -i inventory/sample/hosts.ini cluster.yml --limit=NODENAME ``` This effectively reinstalls containerd and seems to place all config files in the right place. When this completes, kubelet will immediately pick up the new container engine and start spinning up DaemonSets and kube-system Pods. Optionally, if you feel confident, you can remove `/var/lib/docker` anytime after this step. ```commandline rm -fr /var/lib/docker ``` You can watch new containers using `crictl`. ```commandline crictl ps -a ``` ### 6) Replace the cri-socket node annotation Node annotations need to be adjusted. Kubespray will not do this, but a simple kubectl is enough. ```commandline kubectl annotate node NODENAME --overwrite kubeadm.alpha.kubernetes.io/cri-socket=/var/run/containerd/containerd.sock ``` The annotation is required by kubeadm to follow through future cluster upgrades. ### 7) Reboot the node Reboot, just to make sure everything restarts fresh before the node is uncordoned. ## After thoughts If your cluster runs a log aggregator, like fluentd+Graylog, you will likely need to adjust collection filters and parsers. While docker generates Json logs, containerd has its own space delimited format. Example: ```text 2020-01-10T18:10:40.01576219Z stdout F application log message... ``` ================================================ FILE: extra_playbooks/files/get_cinder_pvs.sh ================================================ #!/bin/sh kubectl get pv -o go-template --template='{{ range .items }}{{ $metadata := .metadata }}{{ with $value := index .metadata.annotations "pv.kubernetes.io/provisioned-by" }}{{ if eq $value "kubernetes.io/cinder" }}{{printf "%s\n" $metadata.name}}{{ end }}{{ end }}{{ end }}' ================================================ FILE: extra_playbooks/migrate_openstack_provider.yml ================================================ --- - name: Remove old cloud provider config hosts: kube_node:kube_control_plane tasks: - name: Remove old cloud provider config file: path: "{{ item }}" state: absent with_items: - /etc/kubernetes/cloud_config - name: Migrate intree Cinder PV hosts: kube_control_plane[0] tasks: - name: Include kubespray-default variables include_vars: ../roles/kubespray_defaults/defaults/main/main.yml - name: Copy get_cinder_pvs.sh to first control plane node copy: src: get_cinder_pvs.sh dest: /tmp mode: u+rwx - name: Get PVs provisioned by in-tree cloud provider command: /tmp/get_cinder_pvs.sh register: pvs - name: Remove get_cinder_pvs.sh file: path: /tmp/get_cinder_pvs.sh state: absent - name: Rewrite the "pv.kubernetes.io/provisioned-by" annotation command: "{{ bin_dir }}/kubectl annotate --overwrite pv {{ item }} pv.kubernetes.io/provisioned-by=cinder.csi.openstack.org" loop: "{{ pvs.stdout_lines | list }}" ================================================ FILE: extra_playbooks/upgrade-only-k8s.yml ================================================ --- ### NOTE: This playbook cannot be used to deploy any new nodes to the cluster. ### Additional information: ### * Will not upgrade etcd ### * Will not upgrade network plugins ### * Will not upgrade Docker ### * Will not pre-download containers or kubeadm ### * Currently does not support Vault deployment. ### ### In most cases, you probably want to use upgrade-cluster.yml playbook and ### not this one. - name: Setup ssh config to use the bastion hosts: localhost gather_facts: false roles: - { role: kubespray_defaults} - { role: bastion-ssh-config, tags: ["localhost", "bastion"]} - name: Bootstrap hosts OS for Ansible hosts: k8s_cluster:etcd:calico_rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" gather_facts: false vars: # Need to disable pipelining for bootstrap_os as some systems have requiretty in sudoers set, which makes pipelining # fail. bootstrap_os fixes this on these systems, so in later plays it can be enabled. ansible_ssh_pipelining: false roles: - { role: kubespray_defaults} - { role: bootstrap_os, tags: bootstrap_os} - name: Preinstall hosts: k8s_cluster:etcd:calico_rr any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray_defaults} - { role: kubernetes/preinstall, tags: preinstall } - name: Handle upgrades to control plane components first to maintain backwards compat. hosts: kube_control_plane any_errors_fatal: "{{ any_errors_fatal | default(true) }}" serial: 1 roles: - { role: kubespray_defaults} - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true } - { role: kubernetes/client, tags: client } - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - { role: upgrade/post-upgrade, tags: post-upgrade } - name: Finally handle worker upgrades, based on given batch size hosts: kube_node:!kube_control_plane any_errors_fatal: "{{ any_errors_fatal | default(true) }}" serial: "{{ serial | default('20%') }}" roles: - { role: kubespray_defaults} - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - { role: upgrade/post-upgrade, tags: post-upgrade } - { role: kubespray_defaults} ================================================ FILE: extra_playbooks/wait-for-cloud-init.yml ================================================ --- - name: Wait for cloud-init to finish hosts: all tasks: - name: Wait for cloud-init to finish command: cloud-init status --wait ================================================ FILE: galaxy.yml ================================================ --- namespace: kubernetes_sigs description: Deploy a production ready Kubernetes cluster name: kubespray version: 2.31.0 readme: README.md authors: - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray) tags: - infrastructure repository: https://github.com/kubernetes-sigs/kubespray issues: https://github.com/kubernetes-sigs/kubespray/issues documentation: https://kubespray.io license_file: LICENSE dependencies: ansible.utils: '>=2.5.0' community.crypto: '>=2.22.3' community.general: '>=7.0.0' ansible.netcommon: '>=5.3.0' ansible.posix: '>=1.5.4' community.docker: '>=3.11.0' kubernetes.core: '>=2.4.2' manifest: directives: - recursive-exclude tests ** - recursive-include roles **/files/* ================================================ FILE: index.html ================================================ Kubespray - Deploy a Production Ready Kubernetes Cluster
================================================ FILE: inventory/local/hosts.ini ================================================ node1 ansible_connection=local local_release_dir={{ansible_env.HOME}}/releases [kube_control_plane] node1 [etcd] node1 [kube_node] node1 ================================================ FILE: inventory/sample/group_vars/all/all.yml ================================================ --- ## Directory where the binaries will be installed bin_dir: /usr/local/bin ## The access_ip variable is used to define how other nodes should access ## the node. This is used in flannel to allow other flannel nodes to see ## this node for example. The access_ip is really useful AWS and Google ## environments where the nodes are accessed remotely by the "public" ip, ## but don't know about that address themselves. # access_ip: 1.1.1.1 ## External LB example config ## apiserver_loadbalancer_domain_name: "elb.some.domain" # loadbalancer_apiserver: # address: 1.2.3.4 # port: 1234 ## Internal loadbalancers for apiservers # loadbalancer_apiserver_localhost: true # valid options are "nginx" or "haproxy" # loadbalancer_apiserver_type: nginx # valid values "nginx" or "haproxy" ## Local loadbalancer should use this port ## And must be set port 6443 loadbalancer_apiserver_port: 6443 ## If loadbalancer_apiserver_healthcheck_port variable defined, enables proxy liveness check for nginx. loadbalancer_apiserver_healthcheck_port: 8081 ### OTHER OPTIONAL VARIABLES ## By default, Kubespray collects nameservers on the host. It then adds the previously collected nameservers in nameserverentries. ## If true, Kubespray does not include host nameservers in nameserverentries in dns_late stage. However, It uses the nameserver to make sure cluster installed safely in dns_early stage. ## Use this option with caution, you may need to define your dns servers. Otherwise, the outbound queries such as www.google.com may fail. # disable_host_nameservers: false ## Upstream dns servers # upstream_dns_servers: # - 8.8.8.8 # - 8.8.4.4 ## There are some changes specific to the cloud providers ## for instance we need to encapsulate packets with some network plugins ## If set the possible values only 'external' after K8s v1.31. # cloud_provider: # External Cloud Controller Manager (Formerly known as cloud provider) # cloud_provider must be "external", otherwise this setting is invalid. # Supported external cloud controllers are: 'openstack', 'vsphere', 'oci', 'huaweicloud', 'hcloud' and 'manual' # 'manual' does not install the cloud controller manager used by Kubespray. # If you fill in a value other than the above, the check will fail. # external_cloud_provider: ## Set these proxy values in order to update package manager and docker daemon to use proxies and custom CA for https_proxy if needed # http_proxy: "" # https_proxy: "" # https_proxy_cert_file: "" ## Refer to roles/kubespray_defaults/defaults/main/main.yml before modifying no_proxy # no_proxy: "" ## Some problems may occur when downloading files over https proxy due to ansible bug ## https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable ## SSL validation of get_url module. Note that kubespray will still be performing checksum validation. # download_validate_certs: False ## If you need exclude all cluster nodes from proxy and other resources, add other resources here. # additional_no_proxy: "" ## If you need to disable proxying of os package repositories but are still behind an http_proxy set ## skip_http_proxy_on_os_packages to true ## This will cause kubespray not to set proxy environment in /etc/yum.conf for centos and in /etc/apt/apt.conf for debian/ubuntu ## Special information for debian/ubuntu - you have to set the no_proxy variable, then apt package will install from your source of wish # skip_http_proxy_on_os_packages: false ## Since workers are included in the no_proxy variable by default, docker engine will be restarted on all nodes (all ## pods will restart) when adding or removing workers. To override this behaviour by only including control plane nodes ## in the no_proxy variable, set below to true: no_proxy_exclude_workers: false ## Certificate Management ## This setting determines whether certs are generated via scripts. ## Chose 'none' if you provide your own certificates. ## Option is "script", "none" # cert_management: script ## Set to true to allow pre-checks to fail and continue deployment # ignore_assert_errors: false ## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable. # kube_read_only_port: 10255 ## Set true to download and cache container # download_container: true ## Deploy container engine # Set false if you want to deploy container engine manually. # deploy_container_engine: true ## Red Hat Enterprise Linux subscription registration ## Add either RHEL subscription Username/Password or Organization ID/Activation Key combination ## Update RHEL subscription purpose usage, role and SLA if necessary # rh_subscription_username: "" # rh_subscription_password: "" # rh_subscription_org_id: "" # rh_subscription_activation_key: "" # rh_subscription_usage: "Development" # rh_subscription_role: "Red Hat Enterprise Server" # rh_subscription_sla: "Self-Support" ## Check if access_ip responds to ping. Set false if your firewall blocks ICMP. # ping_access_ip: true # sysctl_file_path to add sysctl conf to # sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" # ignore sysctl errors about unknown keys # sysctl_ignore_unknown_keys: false ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication kube_webhook_token_auth: false kube_webhook_token_auth_url_skip_tls_verify: false # kube_webhook_token_auth_url: https://... ## base64-encoded string of the webhook's CA certificate # kube_webhook_token_auth_ca_data: "LS0t..." ## NTP Settings # Start the ntpd or chrony service and enable it at system boot. ntp_enabled: false ntp_manage_config: false ntp_servers: - "0.pool.ntp.org iburst" - "1.pool.ntp.org iburst" - "2.pool.ntp.org iburst" - "3.pool.ntp.org iburst" ## Used to control no_log attribute unsafe_show_logs: false ## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases. allow_unsupported_distribution_setup: false ================================================ FILE: inventory/sample/group_vars/all/aws.yml ================================================ ## To use AWS EBS CSI Driver to provision volumes, uncomment the first value ## and configure the parameters below # aws_ebs_csi_enabled: true # aws_ebs_csi_enable_volume_scheduling: true # aws_ebs_csi_enable_volume_snapshot: false # aws_ebs_csi_enable_volume_resizing: false # aws_ebs_csi_controller_replicas: 1 # aws_ebs_csi_plugin_image_tag: latest # aws_ebs_csi_extra_volume_tags: "Owner=owner,Team=team,Environment=environment' ================================================ FILE: inventory/sample/group_vars/all/azure.yml ================================================ ## When azure is used, you need to also set the following variables. ## see docs/azure.md for details on how to get these values # azure_cloud: # azure_tenant_id: # azure_subscription_id: # azure_aad_client_id: # azure_aad_client_secret: # azure_resource_group: # azure_location: # azure_subnet_name: # azure_security_group_name: # azure_security_group_resource_group: # azure_vnet_name: # azure_vnet_resource_group: # azure_route_table_name: # azure_route_table_resource_group: # supported values are 'standard' or 'vmss' # azure_vmtype: standard ## Azure Disk CSI credentials and parameters ## see docs/azure-csi.md for details on how to get these values # azure_csi_tenant_id: # azure_csi_subscription_id: # azure_csi_aad_client_id: # azure_csi_aad_client_secret: # azure_csi_location: # azure_csi_resource_group: # azure_csi_vnet_name: # azure_csi_vnet_resource_group: # azure_csi_subnet_name: # azure_csi_security_group_name: # azure_csi_use_instance_metadata: # azure_csi_tags: "Owner=owner,Team=team,Environment=environment' ## To enable Azure Disk CSI, uncomment below # azure_csi_enabled: true # azure_csi_controller_replicas: 1 # azure_csi_plugin_image_tag: latest ================================================ FILE: inventory/sample/group_vars/all/containerd.yml ================================================ --- # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options # containerd_storage_dir: "/var/lib/containerd" # containerd_state_dir: "/run/containerd" # containerd_oom_score: 0 # containerd_default_runtime: "runc" # containerd_snapshotter: "native" # containerd_runc_runtime: # name: runc # type: "io.containerd.runc.v2" # options: # Root: "" # containerd_additional_runtimes: # Example for Kata Containers as additional runtime: # - name: kata # type: "io.containerd.kata.v2" # options: # Root: "" # containerd_grpc_max_recv_message_size: 16777216 # containerd_grpc_max_send_message_size: 16777216 # Containerd debug socket location: unix or tcp format # containerd_debug_address: "" # Containerd log level # containerd_debug_level: "info" # Containerd logs format, supported values: text, json # containerd_debug_format: "" # Containerd debug socket UID # containerd_debug_uid: 0 # Containerd debug socket GID # containerd_debug_gid: 0 # containerd_metrics_address: "" # containerd_metrics_grpc_histogram: false # Registries defined within containerd. # containerd_registries_mirrors: # - prefix: docker.io # mirrors: # - host: https://registry-1.docker.io # capabilities: ["pull", "resolve"] # skip_verify: false # header: # Authorization: "Basic XXX" # containerd_max_container_log_line_size: 16384 # containerd_registry_auth: # - registry: 10.0.0.2:5000 # username: user # password: pass ================================================ FILE: inventory/sample/group_vars/all/coreos.yml ================================================ ## Does coreos need auto upgrade, default is true # coreos_auto_upgrade: true ================================================ FILE: inventory/sample/group_vars/all/cri-o.yml ================================================ # Registries defined within cri-o. # crio_insecure_registries: # - 10.0.0.2:5000 # Auth config for the registries # crio_registry_auth: # - registry: 10.0.0.2:5000 # username: user # password: pass ================================================ FILE: inventory/sample/group_vars/all/docker.yml ================================================ --- ## Uncomment this if you want to force overlay/overlay2 as docker storage driver ## Please note that overlay2 is only supported on newer kernels # docker_storage_options: -s overlay2 ## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. docker_container_storage_setup: false ## It must be define a disk path for docker_container_storage_setup_devs. ## Otherwise docker-storage-setup will be executed incorrectly. # docker_container_storage_setup_devs: /dev/vdb ## Uncomment this if you want to change the Docker Cgroup driver (native.cgroupdriver) ## Valid options are systemd or cgroupfs, default is systemd # docker_cgroup_driver: systemd ## Only set this if you have more than 3 nameservers: ## If true Kubespray will only use the first 3, otherwise it will fail docker_dns_servers_strict: false # Path used to store Docker data docker_daemon_graph: "/var/lib/docker" ## Used to set docker daemon iptables options to true docker_iptables_enabled: "false" # Docker log options # Rotate container stderr/stdout logs at 50m and keep last 5 docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" # define docker bin_dir docker_bin_dir: "/usr/bin" # keep docker packages after installation; speeds up repeated ansible provisioning runs when '1' # kubespray deletes the docker package on each run, so caching the package makes sense docker_rpm_keepcache: 1 ## An obvious use case is allowing insecure-registry access to self hosted registries. ## Can be ipaddress and domain_name. ## example define 172.19.16.11 or mirror.registry.io # docker_insecure_registries: # - mirror.registry.io # - 172.19.16.11 ## Add other registry,example China registry mirror. # docker_registry_mirrors: # - https://registry.docker-cn.com # - https://mirror.aliyuncs.com ## If non-empty will override default system MountFlags value. ## This option takes a mount propagation flag: shared, slave ## or private, which control whether mounts in the file system ## namespace set up for docker will receive or propagate mounts ## and unmounts. Leave empty for system default # docker_mount_flags: ## A string of extra options to pass to the docker daemon. ## This string should be exactly as you wish it to appear. # docker_options: "" ================================================ FILE: inventory/sample/group_vars/all/etcd.yml ================================================ --- ## Directory where etcd data stored etcd_data_dir: /var/lib/etcd ## Container runtime ## docker for docker, crio for cri-o and containerd for containerd. ## Additionally you can set this to kubeadm if you want to install etcd using kubeadm ## Kubeadm etcd deployment is experimental and only available for new deployments ## If this is not set, container manager will be inherited from the Kubespray defaults ## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want. ## Also this makes possible to use different container manager for etcd nodes. # container_manager: containerd ## Settings for etcd deployment type # Set this to docker if you are using container_manager: docker etcd_deployment_type: host ================================================ FILE: inventory/sample/group_vars/all/gcp.yml ================================================ ## GCP compute Persistent Disk CSI Driver credentials and parameters ## See docs/gcp-pd-csi.md for information about the implementation ## Specify the path to the file containing the service account credentials # gcp_pd_csi_sa_cred_file: "/my/safe/credentials/directory/cloud-sa.json" ## To enable GCP Persistent Disk CSI driver, uncomment below # gcp_pd_csi_enabled: true # gcp_pd_csi_controller_replicas: 1 # gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0" ================================================ FILE: inventory/sample/group_vars/all/hcloud.yml ================================================ ## Values for the external Hcloud Cloud Controller # external_hcloud_cloud: # hcloud_api_token: "" # token_secret_name: hcloud # with_networks: false # Use the hcloud controller-manager with networks support https://github.com/hetznercloud/hcloud-cloud-controller-manager#networks-support # network_name: # network name/ID: If you manage the network yourself it might still be required to let the CCM know about private networks # service_account_name: cloud-controller-manager # # controller_image_tag: "latest" # ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset # ## Format: # ## external_hcloud_cloud.controller_extra_args: # ## arg1: "value1" # ## arg2: "value2" # controller_extra_args: {} # # load_balancers_location: # mutually exclusive with load_balancers_network_zone # load_balancers_network_zone: # load_balancers_disable_private_ingress: # set to true if using IPVS based plugins https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/main/docs/load_balancers.md#sample-service-with-networks # load_balancers_use_private_ip: # set to true if using private networks # load_balancers_enabled: # network_routes_enabled: ================================================ FILE: inventory/sample/group_vars/all/huaweicloud.yml ================================================ ## Values for the external Huawei Cloud Controller # external_huaweicloud_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP" # external_huaweicloud_lbaas_network_id: "Neutron network ID to create LBaaS VIP" ## Credentials to authenticate against Keystone API ## All of them are required Per default these values will be ## read from the environment. # external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL') }}" # external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY') }}" # external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY') }}" # external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME') }}" # external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}" # external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}" ## The repo and tag of the external Huawei Cloud Controller image # external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" # external_huawei_cloud_controller_image_tag: "v0.26.8" ================================================ FILE: inventory/sample/group_vars/all/oci.yml ================================================ ## When External Oracle Cloud Infrastructure is used, set these variables ## External OCI Cloud Controller Manager ## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml # external_oracle_auth_region: "" # external_oracle_auth_tenancy: "" # external_oracle_auth_user: "" # external_oracle_auth_key: "" # external_oracle_auth_passphrase: "" # external_oracle_auth_fingerprint: "" # external_oracle_auth_use_instance_principals: false # external_oracle_compartment: "" # external_oracle_vcn: "" # external_oracle_load_balancer_subnet1: "" # external_oracle_load_balancer_subnet2: "" # external_oracle_load_balancer_security_list_management_mode: All # external_oracle_load_balancer_security_lists: {} # external_oracle_ratelimiter_qps_read: 20.0 # external_oracle_ratelimiter_bucket_read: 5 # external_oracle_ratelimiter_qps_write: 20.0 # external_oracle_ratelimiter_bucket_write: 5 # external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci # external_oracle_cloud_controller_image_tag: "v1.29.0" ## When Oracle Cloud Infrastructure is used, set these variables # oci_private_key: # oci_region_id: # oci_tenancy_id: # oci_user_id: # oci_user_fingerprint: # oci_compartment_id: # oci_vnc_id: # oci_subnet1_id: # oci_subnet2_id: ## Override these default/optional behaviors if you wish # oci_security_list_management: All ## If you would like the controller to manage specific lists per subnet. This is a mapping of subnet ocids to security list ocids. Below are examples. # oci_security_lists: # ocid1.subnet.oc1.phx.aaaaaaaasa53hlkzk6nzksqfccegk2qnkxmphkblst3riclzs4rhwg7rg57q: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q # ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q ## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint # oci_use_instance_principals: false ## If you would like to control OCI query rate limits for the controller # oci_rate_limit: # rate_limit_qps_read: # rate_limit_qps_write: # rate_limit_bucket_read: # rate_limit_bucket_write: ## Other optional variables # oci_cloud_controller_pull_source: (default iad.ocir.io/oracle/cloud-provider-oci) # oci_cloud_controller_pull_secret: (name of pull secret to use if you define your own mirror above) ================================================ FILE: inventory/sample/group_vars/all/offline.yml ================================================ --- ## Global Offline settings ### Private Container Image Registry # registry_host: "myprivateregisry.com" # files_repo: "http://myprivatehttpd" ### If using CentOS, RedHat, AlmaLinux or Fedora # yum_repo: "http://myinternalyumrepo" ### If using Debian # debian_repo: "http://myinternaldebianrepo" ### If using Ubuntu # ubuntu_repo: "http://myinternalubunturepo" ## Container Registry overrides # kube_image_repo: "{{ registry_host }}" # gcr_image_repo: "{{ registry_host }}" # github_image_repo: "{{ registry_host }}" # docker_image_repo: "{{ registry_host }}" # quay_image_repo: "{{ registry_host }}" ## Kubernetes components # kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm" # kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl" # kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet" ## Two options - Override entire repository or override only a single binary. ## [Optional] 1 - Override entire binary repository # github_url: "https://my_github_proxy" # dl_k8s_io_url: "https://my_dl_k8s_io_proxy" # storage_googleapis_url: "https://my_storage_googleapi_proxy" # get_helm_url: "https://my_helm_sh_proxy" ## [Optional] 2 - Override a specific binary ## CNI Plugins # cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz" ## cri-tools # crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" ## [Optional] etcd: only if you use etcd_deployment=host # etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" # [Optional] Calico: If using Calico network plugin # calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore # calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml" # [Optional] Cilium: If using Cilium network plugin # ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz" # [Optional] helm: only if you set helm_enabled: true # helm_download_url: "{{ files_repo }}/get.helm.sh/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz" # [Optional] crun: only if you set crun_enabled: true # crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}" # [Optional] kata: only if you set kata_containers_enabled: true # kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz" # [Optional] cri-dockerd: only if you set container_manager: docker # cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz" # [Optional] runc: if you set container_manager to containerd or crio # runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}" # [Optional] cri-o: only if you set container_manager: crio # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable" # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/" # crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz" # skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}" # [Optional] containerd: only if you set container_runtime: containerd # containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" # nerdctl_download_url: "{{ files_repo }}/github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" # [Optional] runsc,containerd-shim-runsc: only if you set gvisor_enabled: true # gvisor_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc" # gvisor_containerd_shim_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1" ## CentOS/Redhat/AlmaLinux ### For EL8, baseos and appstream must be available, ### By default we enable those repo automatically # rhel_enable_repos: false ### Docker / Containerd # docker_rh_repo_base_url: "{{ yum_repo }}/docker-ce/$releasever/$basearch" # docker_rh_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg" ## Fedora ### Docker # docker_fedora_repo_base_url: "{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}" # docker_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg" ### Containerd # containerd_fedora_repo_base_url: "{{ yum_repo }}/containerd" # containerd_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg" ## Debian ### Docker # docker_debian_repo_base_url: "{{ debian_repo }}/docker-ce" # docker_debian_repo_gpgkey: "{{ debian_repo }}/docker-ce/gpg" ### Containerd # containerd_debian_repo_base_url: "{{ debian_repo }}/containerd" # containerd_debian_repo_gpgkey: "{{ debian_repo }}/containerd/gpg" # containerd_debian_repo_repokey: 'YOURREPOKEY' ## Ubuntu ### Docker # docker_ubuntu_repo_base_url: "{{ ubuntu_repo }}/docker-ce" # docker_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/docker-ce/gpg" ### Containerd # containerd_ubuntu_repo_base_url: "{{ ubuntu_repo }}/containerd" # containerd_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/containerd/gpg" # containerd_ubuntu_repo_repokey: 'YOURREPOKEY' ================================================ FILE: inventory/sample/group_vars/all/openstack.yml ================================================ ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) # openstack_blockstorage_ignore_volume_az: yes ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. # openstack_lbaas_enabled: True # openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" ## To enable automatic floating ip provisioning, specify a subnet. # openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default" ## Override default LBaaS behavior # openstack_lbaas_use_octavia: False # openstack_lbaas_method: "ROUND_ROBIN" # openstack_lbaas_provider: "haproxy" # openstack_lbaas_create_monitor: "yes" # openstack_lbaas_monitor_delay: "1m" # openstack_lbaas_monitor_timeout: "30s" # openstack_lbaas_monitor_max_retries: "3" ## Values for the external OpenStack Cloud Controller # external_openstack_lbaas_enabled: true # external_openstack_lbaas_floating_network_id: "Neutron network ID to get floating IP from" # external_openstack_lbaas_floating_subnet_id: "Neutron subnet ID to get floating IP from" # external_openstack_lbaas_method: ROUND_ROBIN # external_openstack_lbaas_provider: amphora # external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP" # external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP" # external_openstack_lbaas_manage_security_groups: false # external_openstack_lbaas_create_monitor: false # external_openstack_lbaas_monitor_delay: 5s # external_openstack_lbaas_monitor_max_retries: 1 # external_openstack_lbaas_monitor_timeout: 3s # external_openstack_lbaas_internal_lb: false # external_openstack_network_ipv6_disabled: false # external_openstack_network_internal_networks: [] # external_openstack_network_public_networks: [] # external_openstack_metadata_search_order: "configDrive,metadataService" ## Application credentials to authenticate against Keystone API ## Those settings will take precedence over username and password that might be set your environment ## All of them are required # external_openstack_application_credential_name: # external_openstack_application_credential_id: # external_openstack_application_credential_secret: ## Tags for the Cinder CSI images ## registry.k8s.io/sig-storage/csi-attacher # cinder_csi_attacher_image_tag: "v4.4.2" ## registry.k8s.io/sig-storage/csi-provisioner # cinder_csi_provisioner_image_tag: "v3.6.2" ## registry.k8s.io/sig-storage/csi-snapshotter # cinder_csi_snapshotter_image_tag: "v6.3.2" ## registry.k8s.io/sig-storage/csi-resizer # cinder_csi_resizer_image_tag: "v1.9.2" ## registry.k8s.io/sig-storage/livenessprobe # cinder_csi_livenessprobe_image_tag: "v2.11.0" ## To use Cinder CSI plugin to provision volumes set this value to true ## Make sure to source in the openstack credentials # cinder_csi_enabled: true # cinder_csi_controller_replicas: 1 # storage_classes: # - name: "cinder-csi" # provisioner: "kubernetes.io/cinder" # mount_options: # - "discard" # parameters: # type: "thin" # availability: "nova" # reclaim_policy: "Delete" # volume_binding_mode: "WaitForFirstConsumer" ================================================ FILE: inventory/sample/group_vars/all/upcloud.yml ================================================ ## Repo for UpClouds csi-driver: https://github.com/UpCloudLtd/upcloud-csi ## To use UpClouds CSI plugin to provision volumes set this value to true ## Remember to set UPCLOUD_USERNAME and UPCLOUD_PASSWORD # upcloud_csi_enabled: true # upcloud_csi_controller_replicas: 1 ## Override used image tags # upcloud_csi_provisioner_image_tag: "v3.1.0" # upcloud_csi_attacher_image_tag: "v3.4.0" # upcloud_csi_resizer_image_tag: "v1.4.0" # upcloud_csi_plugin_image_tag: "v0.3.3" # upcloud_csi_node_image_tag: "v2.5.0" # upcloud_tolerations: [] ## Storage class options # storage_classes: # - name: standard # is_default: true # expand_persistent_volumes: true # parameters: # tier: maxiops # - name: hdd # is_default: false # expand_persistent_volumes: true # parameters: # tier: hdd ================================================ FILE: inventory/sample/group_vars/all/vsphere.yml ================================================ ## Values for the external vSphere Cloud Provider # external_vsphere_vcenter_ip: "myvcenter.domain.com" # external_vsphere_vcenter_port: "443" # external_vsphere_insecure: "true" # external_vsphere_user: "administrator@vsphere.local" # Can also be set via the `VSPHERE_USER` environment variable # external_vsphere_password: "K8s_admin" # Can also be set via the `VSPHERE_PASSWORD` environment variable # external_vsphere_datacenter: "DATACENTER_name" # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" ## To use vSphere CSI plugin to provision volumes set this value to true # vsphere_csi_enabled: true # vsphere_csi_controller_replicas: 1 ================================================ FILE: inventory/sample/group_vars/k8s_cluster/addons.yml ================================================ --- # Helm deployment helm_enabled: false # Registry deployment registry_enabled: false # registry_namespace: kube-system # registry_storage_class: "" # registry_disk_size: "10Gi" # Metrics Server deployment metrics_server_enabled: false # metrics_server_container_port: 10250 # metrics_server_kubelet_insecure_tls: true # metrics_server_metric_resolution: 15s # metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname" # metrics_server_host_network: false # metrics_server_replicas: 1 # Rancher Local Path Provisioner local_path_provisioner_enabled: false # local_path_provisioner_namespace: "local-path-storage" # local_path_provisioner_storage_class: "local-path" # local_path_provisioner_reclaim_policy: Delete # local_path_provisioner_claim_root: /opt/local-path-provisioner/ # local_path_provisioner_debug: false # local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner" # local_path_provisioner_helper_image_repo: "busybox" # local_path_provisioner_helper_image_tag: "latest" # Local volume provisioner deployment local_volume_provisioner_enabled: false # local_volume_provisioner_namespace: kube-system # local_volume_provisioner_nodelabels: # - kubernetes.io/hostname # - topology.kubernetes.io/region # - topology.kubernetes.io/zone # local_volume_provisioner_storage_classes: # local-storage: # host_dir: /mnt/disks # mount_dir: /mnt/disks # volume_mode: Filesystem # fs_type: ext4 # fast-disks: # host_dir: /mnt/fast-disks # mount_dir: /mnt/fast-disks # block_cleaner_command: # - "/scripts/shred.sh" # - "2" # volume_mode: Filesystem # fs_type: ext4 # local_volume_provisioner_tolerations: # - effect: NoSchedule # operator: Exists # CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots # currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller # Longhorn is an external CSI that would also require setting this to true but it is not included in kubespray # csi_snapshot_controller_enabled: false # csi snapshot namespace # snapshot_controller_namespace: kube-system # Gateway API CRDs gateway_api_enabled: false # ALB ingress controller deployment ingress_alb_enabled: false # alb_ingress_aws_region: "us-east-1" # alb_ingress_restrict_scheme: "false" # Enables logging on all outbound requests sent to the AWS API. # If logging is desired, set to true. # alb_ingress_aws_debug: "false" # Cert manager deployment cert_manager_enabled: false # cert_manager_namespace: "cert-manager" # cert_manager_tolerations: # - key: node-role.kubernetes.io/control-plane # effect: NoSchedule # cert_manager_affinity: # nodeAffinity: # preferredDuringSchedulingIgnoredDuringExecution: # - weight: 100 # preference: # matchExpressions: # - key: node-role.kubernetes.io/control-plane # operator: In # values: # - "" # cert_manager_nodeselector: # kubernetes.io/os: "linux" # cert_manager_trusted_internal_ca: | # -----BEGIN CERTIFICATE----- # [REPLACE with your CA certificate] # -----END CERTIFICATE----- # cert_manager_leader_election_namespace: kube-system # cert_manager_dns_policy: "ClusterFirst" # cert_manager_dns_config: # nameservers: # - "1.1.1.1" # - "8.8.8.8" # cert_manager_controller_extra_args: # - "--dns01-recursive-nameservers-only=true" # - "--dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53" # MetalLB deployment metallb_enabled: false metallb_speaker_enabled: "{{ metallb_enabled }}" metallb_namespace: "metallb-system" # metallb_protocol: "layer2" # metallb_port: "7472" # metallb_memberlist_port: "7946" # metallb_config: # speaker: # nodeselector: # kubernetes.io/os: "linux" # tolerations: # - key: "node-role.kubernetes.io/control-plane" # operator: "Equal" # value: "" # effect: "NoSchedule" # controller: # nodeselector: # kubernetes.io/os: "linux" # tolerations: # - key: "node-role.kubernetes.io/control-plane" # operator: "Equal" # value: "" # effect: "NoSchedule" # address_pools: # primary: # ip_range: # - 10.5.0.0/16 # auto_assign: true # pool1: # ip_range: # - 10.6.0.0/16 # auto_assign: true # pool2: # ip_range: # - 10.10.0.0/16 # auto_assign: true # layer2: # - primary # layer3: # defaults: # peer_port: 179 # hold_time: 120s # communities: # vpn-only: "1234:1" # NO_ADVERTISE: "65535:65282" # metallb_peers: # peer1: # peer_address: 10.6.0.1 # peer_asn: 64512 # my_asn: 4200000000 # communities: # - vpn-only # address_pool: # - pool1 # peer2: # peer_address: 10.10.0.1 # peer_asn: 64513 # my_asn: 4200000000 # communities: # - NO_ADVERTISE # address_pool: # - pool2 argocd_enabled: false # argocd_namespace: argocd # Default password: # - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli # --- # The initial password is autogenerated and stored in `argocd-initial-admin-secret` in the argocd namespace defined above. # Using the argocd CLI the generated password can be automatically be fetched from the current kubectl context with the command: # argocd admin initial-password -n argocd # --- # Use the following var to set admin password # argocd_admin_password: "password" # The plugin manager for kubectl # Kube VIP kube_vip_enabled: false # kube_vip_arp_enabled: true # kube_vip_controlplane_enabled: true # kube_vip_address: 192.168.56.120 # loadbalancer_apiserver: # address: "{{ kube_vip_address }}" # port: 6443 # kube_vip_interface: eth0 # kube_vip_services_enabled: false # kube_vip_dns_mode: first # kube_vip_cp_detect: false # kube_vip_leasename: plndr-cp-lock # kube_vip_enable_node_labeling: false # kube_vip_lb_fwdmethod: local # kube_vip_bgp_sourceip: # kube_vip_bgp_sourceif: # Node Feature Discovery node_feature_discovery_enabled: false # node_feature_discovery_gc_sa_name: node-feature-discovery # node_feature_discovery_gc_sa_create: false # node_feature_discovery_worker_sa_name: node-feature-discovery # node_feature_discovery_worker_sa_create: false # node_feature_discovery_master_config: # extraLabelNs: ["nvidia.com"] ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml ================================================ --- # Kubernetes configuration dirs and system namespace. # Those are where all the additional config stuff goes # the kubernetes normally puts in /srv/kubernetes. # This puts them in a sane location and namespace. # Editing those values will almost surely break something. kube_config_dir: /etc/kubernetes kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" kube_manifest_dir: "{{ kube_config_dir }}/manifests" # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/ssl" # This is where all of the bearer tokens will be stored kube_token_dir: "{{ kube_config_dir }}/tokens" kube_api_anonymous_auth: true # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) local_release_dir: "/tmp/releases" # Random shifts for retrying failed ops like pushing/downloading retry_stagger: 5 # This is the user that owns the cluster installation. # Note: cilium needs to set kube_owner to root https://kubespray.io/#/docs/CNI/cilium?id=unprivileged-agent-configuration kube_owner: kube # This is the group that the cert creation scripts chgrp the # cert files to. Not really changeable... kube_cert_group: kube-cert # Cluster Loglevel configuration kube_log_level: 2 # Directory where credentials will be stored credentials_dir: "{{ inventory_dir }}/credentials" ## It is possible to activate / deactivate selected authentication methods (oidc, static token auth) # kube_oidc_auth: false # kube_token_auth: false ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) # kube_oidc_url: https:// ... # kube_oidc_client_id: kubernetes ## Optional settings for OIDC # kube_oidc_ca_file: "{{ kube_cert_dir }}/ca.pem" # kube_oidc_username_claim: sub # kube_oidc_username_prefix: 'oidc:' # kube_oidc_groups_claim: groups # kube_oidc_groups_prefix: 'oidc:' ## Variables to control webhook authn/authz # kube_webhook_token_auth: false # kube_webhook_token_auth_url: https://... # kube_webhook_token_auth_url_skip_tls_verify: false ## For webhook authorization, authorization_modes must include Webhook or kube_apiserver_authorization_config_authorizers must configure a type: Webhook # kube_webhook_authorization: false # kube_webhook_authorization_url: https://... # kube_webhook_authorization_url_skip_tls_verify: false # Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico # Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni kube_network_plugin_multus: false # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 # internal network. When used, it will assign IP # addresses from this range to individual pods. # This network must be unused in your network infrastructure! kube_pods_subnet: 10.233.64.0/18 # internal network node size allocation (optional). This is the size allocated # to each node for pod IP address allocation. Note that the number of pods per node is # also limited by the kubelet_max_pods variable which defaults to 110. # # Example: # Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node: # - kube_pods_subnet: 10.233.64.0/18 # - kube_network_node_prefix: 24 # - kubelet_max_pods: 110 # # Example: # Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node: # - kube_pods_subnet: 10.233.64.0/18 # - kube_network_node_prefix: 25 # - kubelet_max_pods: 110 kube_network_node_prefix: 24 # Kubernetes internal network for IPv6 services, unused block of space. # This is only used if ipv6_stack is set to true # This provides 4096 IPv6 IPs kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 # Internal network. When used, it will assign IPv6 addresses from this range to individual pods. # This network must not already be in your network infrastructure! # This is only used if ipv6_stack is set to true. # This provides room for 256 nodes with 254 pods per node. kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # IPv6 subnet size allocated to each for pods. # This is only used if ipv6_stack is set to true # This provides room for 254 pods per node. kube_network_node_prefix_ipv6: 120 # The port the API Server will be listening on. kube_apiserver_ip: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" kube_apiserver_port: 6443 # (https) # Kube-proxy proxyMode configuration. # Can be ipvs, iptables, nftables # TODO: it needs to be changed to nftables when the upstream use nftables as default kube_proxy_mode: ipvs # configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface # must be set to true for MetalLB, kube-vip(ARP enabled) to work kube_proxy_strict_arp: false # A string slice of values which specify the addresses to use for NodePorts. # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). # The default empty string slice ([]) means to use all local addresses. # kube_proxy_nodeport_addresses_cidr is retained for legacy config kube_proxy_nodeport_addresses: >- {%- if kube_proxy_nodeport_addresses_cidr is defined -%} [{{ kube_proxy_nodeport_addresses_cidr }}] {%- else -%} [] {%- endif -%} # If non-empty, will use this string as identification instead of the actual hostname # kube_override_hostname: {{ inventory_hostname }} ## Encrypting Secret Data at Rest kube_encrypt_secret_data: false # Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/ # kubelet_shutdown_grace_period had to be greater than kubelet_shutdown_grace_period_critical_pods to allow # non-critical podsa to also terminate gracefully # kubelet_shutdown_grace_period: 60s # kubelet_shutdown_grace_period_critical_pods: 20s # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 # dns_timeout: 2 # dns_attempts: 2 # Custom search domains to be added in addition to the default cluster search domains # searchdomains: # - svc.{{ cluster_name }} # - default.svc.{{ cluster_name }} # Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). # remove_default_searchdomains: false # Can be coredns, coredns_dual, manual or none dns_mode: coredns # Set manual server if using a custom cluster DNS server # manual_dns_server: 10.x.x.x # Enable nodelocal dns cache enable_nodelocaldns: true enable_nodelocaldns_secondary: false nodelocaldns_ip: 169.254.25.10 nodelocaldns_health_port: 9254 nodelocaldns_second_health_port: 9256 nodelocaldns_bind_metrics_host_ip: false nodelocaldns_secondary_skew_seconds: 5 # nodelocaldns_external_zones: # - zones: # - example.com # - example.io:1053 # nameservers: # - 1.1.1.1 # - 2.2.2.2 # cache: 5 # - zones: # - https://mycompany.local:4453 # nameservers: # - 192.168.0.53 # cache: 0 # - zones: # - mydomain.tld # nameservers: # - 10.233.0.3 # cache: 5 # rewrite: # - name website.tld website.namespace.svc.cluster.local # Enable k8s_external plugin for CoreDNS enable_coredns_k8s_external: false coredns_k8s_external_zone: k8s_external.local # Enable endpoint_pod_names option for kubernetes plugin enable_coredns_k8s_endpoint_pod_names: false # Set forward options for upstream DNS servers in coredns (and nodelocaldns) config # dns_upstream_forward_extra_opts: # policy: sequential # Apply extra options to coredns kubernetes plugin # coredns_kubernetes_extra_opts: # - 'fallthrough example.local' # Forward extra domains to the coredns kubernetes plugin # coredns_kubernetes_extra_domains: '' # Can be docker_dns, host_resolvconf or none resolvconf_mode: host_resolvconf # Deploy netchecker app to verify DNS resolve as an HTTP service deploy_netchecker: false # Ip address of the kubernetes skydns service skydns_server: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" skydns_server_secondary: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" dns_domain: "{{ cluster_name }}" ## Container runtime ## docker for docker, crio for cri-o and containerd for containerd. ## Default: containerd container_manager: containerd # Additional container runtimes kata_containers_enabled: false kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}" # K8s image pull policy (imagePullPolicy) k8s_image_pull_policy: IfNotPresent # audit log for kubernetes kubernetes_audit: false # define kubelet config dir for dynamic kubelet # kubelet_config_dir: default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts # kubeconfig_localhost: false # Use ansible_host as external api ip when copying over kubeconfig. # kubeconfig_localhost_ansible_host: false # Download kubectl onto the host that runs Ansible in {{ bin_dir }} # kubectl_localhost: false # A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. # Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "". # kubelet_enforce_node_allocatable: pods ## Set runtime and kubelet cgroups when using systemd as cgroup driver (default) # kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service" # kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service" ## Set runtime and kubelet cgroups when using cgroupfs as cgroup driver # kubelet_runtime_cgroups_cgroupfs: "/system.slice/{{ container_manager }}.service" # kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service" # Whether to run kubelet and container-engine daemons in a dedicated cgroup. # kube_reserved: false ## Uncomment to override default values ## The following two items need to be set when kube_reserved is true # kube_reserved_cgroups_for_service_slice: kube.slice # kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" # kube_memory_reserved: 256Mi # kube_cpu_reserved: 100m # kube_ephemeral_storage_reserved: 2Gi # kube_pid_reserved: "1000" ## Optionally reserve resources for OS system daemons. # system_reserved: true ## Uncomment to override default values ## The following two items need to be set when system_reserved is true # system_reserved_cgroups_for_service_slice: system.slice # system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" # system_memory_reserved: 512Mi # system_cpu_reserved: 500m # system_ephemeral_storage_reserved: 2Gi ## Eviction Thresholds to avoid system OOMs # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds # eviction_hard: {} # eviction_hard_control_plane: {} # An alternative flexvolume plugin directory # kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec ## Supplementary addresses that can be added in kubernetes ssl keys. ## That can be useful for example to setup a keepalived virtual IP # supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3] ## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. ## See https://github.com/kubernetes-sigs/kubespray/issues/2141 ## Set this variable to true to get rid of this issue volume_cross_zone_attachment: false ## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, ## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI) persistent_volumes_enabled: false ## Container Engine Acceleration ## Enable container acceleration feature, for example use gpu acceleration in containers # nvidia_accelerator_enabled: true ## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset. ## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2' ## Array with nvida_gpu_nodes, leave empty or comment if you don't want to install drivers. ## Labels and taints won't be set to nodes if they are not in the array. # nvidia_gpu_nodes: # - kube-gpu-001 # nvidia_driver_version: "384.111" ## flavor can be tesla or gtx # nvidia_gpu_flavor: gtx ## NVIDIA driver installer images. Change them if you have trouble accessing gcr.io. # nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2 # nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63 ## NVIDIA GPU device plugin image. # nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" ## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. # tls_min_version: "" ## Support tls cipher suites. # tls_cipher_suites: {} # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA256 # - TLS_RSA_WITH_AES_128_GCM_SHA256 # - TLS_RSA_WITH_AES_256_CBC_SHA # - TLS_RSA_WITH_AES_256_GCM_SHA384 # - TLS_RSA_WITH_RC4_128_SHA ## Amount of time to retain events. (default 1h0m0s) event_ttl_duration: "1h0m0s" ## Automatically renew K8S control plane certificates on first Monday of each month auto_renew_certificates: false # First Monday of each month # auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00" kubeadm_patches_dir: "{{ kube_config_dir }}/patches" kubeadm_patches: [] # See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches # Correspondance with this link # patchtype = type # target = target # suffix -> managed automatically # extension -> always "yaml" # kubeadm_patches: # - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration # type: strategic(default)|json|merge # patch: # metadata: # annotations: # example.com/test: "true" # labels: # example.com/prod_level: "{{ prod_level }}" # - ... # Patches are applied in the order they are specified. # Set to true to remove the role binding to anonymous users created by kubeadm remove_anonymous_access: false ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml ================================================ --- # see roles/network_plugin/calico/defaults/main.yml # the default value of name calico_cni_name: k8s-pod-network ## With calico it is possible to distributed routes with border routers of the datacenter. ## Warning : enabling router peering will disable calico's default behavior ('node mesh'). ## The subnets of each nodes will be distributed by the datacenter router # peer_with_router: false # Enables Internet connectivity from containers # nat_outgoing: true # nat_outgoing_ipv6: true # Enables Calico CNI "host-local" IPAM plugin # calico_ipam_host_local: true # add default ippool name # calico_pool_name: "default-pool" # add default ippool blockSize calico_pool_blocksize: 26 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise) # calico_pool_cidr: 1.2.3.4/5 # Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set. # calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # Global as_num (/calico/bgp/v1/global/as_num) # global_as_num: "64512" # If doing peering with node-assigned asn where the globas does not match your nodes, you want this # to be true. All other cases, false. # calico_no_global_as_num: false # You can set MTU value here. If left undefined or empty, it will # not be specified in calico CNI config, so Calico will use built-in # defaults. The value should be a number, not a string. # calico_mtu: 1500 # Configure the MTU to use for workload interfaces and tunnels. # - If Wireguard is enabled, subtract 60 from your network MTU (i.e 1500-60=1440) # - Otherwise, if VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450) # - Otherwise, if IPIP is enabled, subtract 20 from your network MTU (i.e. 1500-20=1480) # - Otherwise, if not using any encapsulation, set to your network MTU (i.e. 1500) # calico_veth_mtu: 1440 # Advertise Cluster IPs # calico_advertise_cluster_ips: true # Advertise Service External IPs # calico_advertise_service_external_ips: # - x.x.x.x/24 # - y.y.y.y/32 # Advertise Service LoadBalancer IPs # calico_advertise_service_loadbalancer_ips: # - x.x.x.x/24 # - y.y.y.y/16 # Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) # calico_datastore: "kdd" # Choose Calico iptables backend: "Legacy", "Auto" or "NFT" # calico_iptables_backend: "Auto" # Use typha (only with kdd) # typha_enabled: false # Generate TLS certs for secure typha<->calico-node communication # typha_secure: false # Scaling typha: 1 replica per 100 nodes is adequate # Number of typha replicas # typha_replicas: 1 # Set max typha connections # typha_max_connections_lower_limit: 300 # Set calico network backend: "bird", "vxlan" or "none" # bird enable BGP routing, required for ipip and no encapsulation modes # calico_network_backend: vxlan # IP in IP and VXLAN is mutually exclusive modes. # set IP in IP encapsulation mode: "Always", "CrossSubnet", "Never" # calico_ipip_mode: 'Never' # set VXLAN encapsulation mode: "Always", "CrossSubnet", "Never" # calico_vxlan_mode: 'Always' # set VXLAN port and VNI # calico_vxlan_vni: 4096 # calico_vxlan_port: 4789 # Enable eBPF mode # calico_bpf_enabled: false # If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of: # * can-reach=DESTINATION # * interface=INTERFACE-REGEX # see https://docs.projectcalico.org/reference/node/configuration # calico_ip_auto_method: "interface=eth.*" # calico_ip6_auto_method: "interface=eth.*" # Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection. # see https://projectcalico.docs.tigera.io/reference/felix/configuration # calico_felix_mtu_iface_pattern: "^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)" # Choose the iptables insert mode for Calico: "Insert" or "Append". # calico_felix_chaininsertmode: Insert # If you want use the default route interface when you use multiple interface with dynamique route (iproute2) # see https://docs.projectcalico.org/reference/node/configuration : FELIX_DEVICEROUTESOURCEADDRESS # calico_use_default_route_src_ipaddr: false # Enable calico traffic encryption with wireguard # calico_wireguard_enabled: false # Under certain situations liveness and readiness probes may need tunning # calico_node_livenessprobe_timeout: 10 # calico_node_readinessprobe_timeout: 10 # Calico apiserver (only with kdd) # calico_apiserver_enabled: false ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml ================================================ --- # Log-level # cilium_debug: false # cilium_mtu: "" # cilium_enable_ipv4: true # cilium_enable_ipv6: false # Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/ cilium_l2announcements: false # Cilium agent health port # cilium_agent_health_port: "9879" # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). # These can be queried with: # `kubectl get ciliumid` # - "kvstore" stores identities in an etcd kvstore. # - In order to support External Workloads, "crd" is required # - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta # - KVStore operations are only required when cilium-operator is running with any of the below options: # - --synchronize-k8s-services # - --synchronize-k8s-nodes # - --identity-allocation-mode=kvstore # - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations # cilium_identity_allocation_mode: kvstore # Etcd SSL dirs # cilium_cert_dir: /etc/cilium/certs # kube_etcd_cacert_file: ca.pem # kube_etcd_cert_file: cert.pem # kube_etcd_key_file: cert-key.pem # Limits for apps # cilium_memory_limit: 500M # cilium_cpu_limit: 500m # cilium_memory_requests: 64M # cilium_cpu_requests: 100m # Overlay Network Mode # cilium_tunnel_mode: vxlan # LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode # cilium_loadbalancer_mode: snat # Optional features # cilium_enable_prometheus: false # Enable if you want to make use of hostPort mappings # cilium_enable_portmap: false # Monitor aggregation level (none/low/medium/maximum) # cilium_monitor_aggregation: medium # The monitor aggregation flags determine which TCP flags which, upon the # first observation, cause monitor notifications to be generated. # # Only effective when monitor aggregation is set to "medium" or higher. # cilium_monitor_aggregation_flags: "all" # Kube Proxy Replacement mode (true/false) # cilium_kube_proxy_replacement: false # If upgrading from Cilium < 1.5, you may want to override some of these options # to prevent service disruptions. See also: # http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action # cilium_preallocate_bpf_maps: false # `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9 # cilium_tofqdns_enable_poller: false # `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9 # cilium_enable_legacy_services: false # Unique ID of the cluster. Must be unique across all connected clusters and # in the range of 1 and 255. Only relevant when building a mesh of clusters. # This value is not defined by default # cilium_cluster_id: # Deploy cilium even if kube_network_plugin is not cilium. # This enables to deploy cilium alongside another CNI to replace kube-proxy. # cilium_deploy_additionally: false # Auto direct nodes routes can be used to advertise pods routes in your cluster # without any tunneling (with `cilium_tunnel_mode` sets to `disabled`). # This works only if you have a L2 connectivity between all your nodes. # You wil also have to specify the variable `cilium_native_routing_cidr` to # make this work. Please refer to the cilium documentation for more # information about this kind of setups. # cilium_auto_direct_node_routes: false # Allows to explicitly specify the IPv4 CIDR for native routing. # When specified, Cilium assumes networking for this CIDR is preconfigured and # hands traffic destined for that range to the Linux network stack without # applying any SNAT. # Generally speaking, specifying a native routing CIDR implies that Cilium can # depend on the underlying networking stack to route packets to their # destination. To offer a concrete example, if Cilium is configured to use # direct routing and the Kubernetes CIDR is included in the native routing CIDR, # the user must configure the routes to reach pods, either manually or by # setting the auto-direct-node-routes flag. # cilium_native_routing_cidr: "" # Allows to explicitly specify the IPv6 CIDR for native routing. # cilium_native_routing_cidr_ipv6: "" # Enable transparent network encryption. # cilium_encryption_enabled: false # Encryption method. Can be either ipsec or wireguard. # Only effective when `cilium_encryption_enabled` is set to true. # cilium_encryption_type: "ipsec" # Enable encryption for pure node to node traffic. # This option is only effective when `cilium_encryption_type` is set to `ipsec`. # cilium_ipsec_node_encryption: false # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. # When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, # it will fallback on the wireguard-go user-space implementation of WireGuard. # This option is only effective when `cilium_encryption_type` is set to `wireguard`. # cilium_wireguard_userspace_fallback: false # IP Masquerade Agent # https://docs.cilium.io/en/stable/concepts/networking/masquerading/ # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded # cilium_ip_masq_agent_enable: false ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded # cilium_non_masquerade_cidrs: # - 10.0.0.0/8 # - 172.16.0.0/12 # - 192.168.0.0/16 # - 100.64.0.0/10 # - 192.0.0.0/24 # - 192.0.2.0/24 # - 192.88.99.0/24 # - 198.18.0.0/15 # - 198.51.100.0/24 # - 203.0.113.0/24 # - 240.0.0.0/4 ### Indicates whether to masquerade traffic to the link local prefix. ### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list. # cilium_masq_link_local: false ### A time interval at which the agent attempts to reload config from disk # cilium_ip_masq_resync_interval: 60s ### Host Firewall and Policy Audit Mode # cilium_enable_host_firewall: false # cilium_policy_audit_mode: false # Hubble ### Enable Hubble without install # cilium_enable_hubble: false ### Enable Hubble-ui ### Installed by default when hubble is enabled. To disable set to false # cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}" ### Enable Hubble Metrics # cilium_enable_hubble_metrics: false ### if cilium_enable_hubble_metrics: true # cilium_hubble_metrics: {} # - dns # - drop # - tcp # - flow # - icmp # - http ### Enable Hubble install # cilium_hubble_install: false ### Enable auto generate certs if cilium_hubble_install: true # cilium_hubble_tls_generate: false ### Tune cilium_hubble_event_buffer_capacity & cilium_hubble_event_queue_size values to avoid dropping events when hubble is under heavy load ### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535 ### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095) # cilium_hubble_event_buffer_capacity: 4095 ### Buffer size of the channel to receive monitor events. # cilium_hubble_event_queue_size: 50 # Override the DNS suffix that Hubble-Relay uses to resolve its peer service. # It defaults to the inventory's `dns_domain`. # cilium_hubble_peer_service_cluster_domain: "{{ dns_domain }}" # IP address management mode for v1.9+. # https://docs.cilium.io/en/v1.9/concepts/networking/ipam/ # cilium_ipam_mode: kubernetes # Extra arguments for the Cilium agent # cilium_agent_custom_args: [] # For adding and mounting extra volumes to the cilium agent # cilium_agent_extra_volumes: [] # cilium_agent_extra_volume_mounts: [] # cilium_agent_extra_env_vars: [] # cilium_operator_replicas: 2 # The address at which the cillium operator bind health check api # cilium_operator_api_serve_addr: "127.0.0.1:9234" ## A dictionary of extra config variables to add to cilium-config, formatted like: ## cilium_config_extra_vars: ## var1: "value1" ## var2: "value2" # cilium_config_extra_vars: {} # For adding and mounting extra volumes to the cilium operator # cilium_operator_extra_volumes: [] # cilium_operator_extra_volume_mounts: [] # Extra arguments for the Cilium Operator # cilium_operator_custom_args: [] # Name of the cluster. Only relevant when building a mesh of clusters. # cilium_cluster_name: default # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. # This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. # Available for Cilium v1.10 and up. # cilium_cni_exclusive: true # Configure the log file for CNI logging with retention policy of 7 days. # Disable CNI file logging by setting this field to empty explicitly. # Available for Cilium v1.12 and up. # cilium_cni_log_file: "/var/run/cilium/cilium-cni.log" # -- Configure cgroup related configuration # -- Enable auto mount of cgroup2 filesystem. # When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at # `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod. # If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted # cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the # volume will be mounted inside the cilium agent pod at the same path. # Available for Cilium v1.11 and up # cilium_cgroup_auto_mount: true # -- Configure cgroup root where cgroup2 filesystem is mounted on the host # cilium_cgroup_host_root: "/run/cilium/cgroupv2" # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. # cilium_bpf_map_dynamic_size_ratio: "0.0" # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. # Available for Cilium v1.10 and up # cilium_enable_ipv4_masquerade: true # -- Enables masquerading of IPv6 traffic leaving the node from endpoints. # Available for Cilium v1.10 and up # cilium_enable_ipv6_masquerade: true # -- Enable native IP masquerade support in eBPF # cilium_enable_bpf_masquerade: false # -- Enable BGP Control Plane # cilium_enable_bgp_control_plane: false # -- Configure Loadbalancer IP Pools # cilium_loadbalancer_ip_pools: # - name: "blue-pool" # cidrs: # - "10.0.10.0/24" # ranges: # - start: "20.0.20.100" # stop: "20.0.20.200" # - start: "1.2.3.4" # -- Configure BGP Instances (New bgpv2 API v1.16+) # cilium_bgp_cluster_configs: # - name: "cilium-bgp" # spec: # bgpInstances: # - name: "instance-64512" # localASN: 64512 # peers: # - name: "peer-64512-tor1" # peerASN: 64512 # peerAddress: '10.47.1.1' # peerConfigRef: # name: "cilium-peer" # nodeSelector: # matchExpressions: # - {key: somekey, operator: NotIn, values: ['never-used-value']} # -- Configure BGP Peers (New bgpv2 API v1.16+) # cilium_bgp_peer_configs: # - name: cilium-peer # spec: # # authSecretRef: bgp-auth-secret # gracefulRestart: # enabled: true # restartTimeSeconds: 15 # families: # - afi: ipv4 # safi: unicast # advertisements: # matchLabels: # advertise: "bgp" # - afi: ipv6 # safi: unicast # advertisements: # matchLabels: # advertise: "bgp" # -- Configure BGP Advertisements (New bgpv2 API v1.16+) # cilium_bgp_advertisements: # - name: bgp-advertisements # labels: # advertise: bgp # spec: # advertisements: # # - advertisementType: "PodCIDR" # # attributes: # # communities: # # standard: [ "64512:99" ] # - advertisementType: "Service" # service: # addresses: # - ClusterIP # - ExternalIP # - LoadBalancerIP # selector: # matchExpressions: # - {key: somekey, operator: NotIn, values: ['never-used-value']} # -- Configure BGP Node Config Overrides (New bgpv2 API v1.16+) # cilium_bgp_node_config_overrides: # - name: bgp-node-config-override # spec: # bgpInstances: # - name: "instance-65000" # routerID: "192.168.10.1" # localPort: 1790 # peers: # - name: "peer-65000-tor1" # localAddress: fd00:10:0:2::2 # - name: "peer-65000-tor2" # localAddress: fd00:11:0:2::2 # -- Configure BGP Peers (Legacy v1.16+) # cilium_bgp_peering_policies: # - name: "01-bgp-peering-policy" # spec: # virtualRouters: # - localASN: 64512 # exportPodCIDR: false # neighbors: # - peerAddress: '10.47.1.1/24' # peerASN: 64512 # serviceSelector: # matchExpressions: # - {key: somekey, operator: NotIn, values: ['never-used-value']} # -- Configure whether direct routing mode should route traffic via # host stack (true) or directly and more efficiently out of BPF (false) if # the kernel supports it. The latter has the implication that it will also # bypass netfilter in the host namespace. # cilium_enable_host_legacy_routing: true # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity # cilium_enable_remote_node_identity: true # -- Enable the use of well-known identities. # cilium_enable_well_known_identities: false # -- Whether to enable CNP status updates. # cilium_disable_cnp_status_updates: true # A list of extra rules variables to add to clusterrole for cilium operator, formatted like: # cilium_clusterrole_rules_operator_extra_vars: # - apiGroups: # - '""' # resources: # - pods # verbs: # - delete # - apiGroups: # - '""' # resources: # - nodes # verbs: # - list # - watch # resourceNames: # - toto # cilium_clusterrole_rules_operator_extra_vars: [] # Cilium extra values, use any values from cilium Helm Chart # ref: https://docs.cilium.io/en/stable/helm-reference/ # cilium_extra_values: {} ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-custom-cni.yml ================================================ --- # custom_cni network plugin configuration # There are two deployment options to choose from, select one ## OPTION 1 - Static manifest files ## With this option, referred manifest file will be deployed ## as if the `kubectl apply -f` method was used with it. # ## List of Kubernetes resource manifest files ## See tests/files/custom_cni/README.md for example # custom_cni_manifests: [] ## OPTION 1 EXAMPLE - Cilium static manifests in Kubespray tree # custom_cni_manifests: # - "{{ playbook_dir }}/../tests/files/custom_cni/cilium.yaml" ## OPTION 2 - Helm chart application ## This allows the CNI backend to be deployed to Kubespray cluster ## as common Helm application. # ## Helm release name - how the local instance of deployed chart will be named # custom_cni_chart_release_name: "" # ## Kubernetes namespace to deploy into # custom_cni_chart_namespace: "kube-system" # ## Helm repository name - how the local record of Helm repository will be named # custom_cni_chart_repository_name: "" # ## Helm repository URL # custom_cni_chart_repository_url: "" # ## Helm chart reference - path to the chart in the repository # custom_cni_chart_ref: "" # ## Helm chart version # custom_cni_chart_version: "" # ## Custom Helm values to be used for deployment # custom_cni_chart_values: {} ## OPTION 2 EXAMPLE - Cilium deployed from official public Helm chart # custom_cni_chart_namespace: kube-system # custom_cni_chart_release_name: cilium # custom_cni_chart_repository_name: cilium # custom_cni_chart_repository_url: https://helm.cilium.io # custom_cni_chart_ref: cilium/cilium # custom_cni_chart_version: (e.g.: 1.14.3) # custom_cni_chart_values: # cluster: # name: "cilium-demo" ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-flannel.yml ================================================ # see roles/network_plugin/flannel/defaults/main.yml ## interface that should be used for flannel operations ## This is actually an inventory cluster-level item # flannel_interface: ## Select interface that should be used for flannel operations by regexp on Name or IP ## This is actually an inventory cluster-level item ## example: select interface with ip from net 10.0.0.0/23 ## single quote and escape backslashes # flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}' # You can choose what type of flannel backend to use: 'vxlan', 'host-gw' or 'wireguard' # please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md # flannel_backend_type: "vxlan" # flannel_vxlan_vni: 1 # flannel_vxlan_port: 8472 # flannel_vxlan_direct_routing: false ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-kube-ovn.yml ================================================ --- # geneve or vlan kube_ovn_network_type: geneve # geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module kube_ovn_tunnel_type: geneve ## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use. # kube_ovn_iface: eth1 ## The MTU used by pod iface in overlay networks (default iface MTU - 100) # kube_ovn_mtu: 1333 ## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port. kube_ovn_hw_offload: false # traffic mirror kube_ovn_traffic_mirror: false # kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # kube_ovn_default_interface_name: eth0 kube_ovn_external_address: 8.8.8.8 kube_ovn_external_address_ipv6: 2400:3200::1 kube_ovn_external_dns: alauda.cn # kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0 kube_ovn_default_gateway_check: true kube_ovn_default_logical_gateway: false # kube_ovn_default_exclude_ips: 10.16.0.1 kube_ovn_node_switch_cidr: 100.64.0.0/16 kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64 ## vlan config, set default interface name and vlan id # kube_ovn_default_interface_name: eth0 kube_ovn_default_vlan_id: 100 kube_ovn_vlan_name: product ## pod nic type, support: veth-pair or internal-port kube_ovn_pod_nic_type: veth_pair ## Enable load balancer kube_ovn_enable_lb: true ## Enable network policy support kube_ovn_enable_np: true ## Enable external vpc support kube_ovn_enable_external_vpc: true ## Enable checksum kube_ovn_encap_checksum: true ## enable ssl kube_ovn_enable_ssl: false ## dpdk kube_ovn_dpdk_enabled: false ## enable interconnection to an existing IC database server. kube_ovn_ic_enable: false kube_ovn_ic_autoroute: true kube_ovn_ic_dbhost: "127.0.0.1" kube_ovn_ic_zone: "kubernetes" ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml ================================================ # See roles/network_plugin/kube-router/defaults/main.yml # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP # kube_router_run_router: true # Enables Network Policy -- sets up iptables to provide ingress firewall for pods # kube_router_run_firewall: true # Enables Service Proxy -- sets up IPVS for Kubernetes Services # see docs/kube-router.md "Caveats" section # kube_router_run_service_proxy: false # Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers. # kube_router_advertise_cluster_ip: false # Add External IP of service to the RIB so that it gets advertised to the BGP peers. # kube_router_advertise_external_ip: false # Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers. # kube_router_advertise_loadbalancer_ip: false # Enables BGP graceful restarts # kube_router_bgp_graceful_restart: true # Adjust manifest of kube-router daemonset template with DSR needed changes # kube_router_enable_dsr: false # Array of arbitrary extra arguments to kube-router, see # https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md # kube_router_extra_args: [] # ASN number of the cluster, used when communicating with external BGP routers # kube_router_cluster_asn: ~ # ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. # kube_router_peer_router_asns: ~ # The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. # kube_router_peer_router_ips: ~ # The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used. # kube_router_peer_router_ports: ~ # Setups node CNI to allow hairpin mode, requires node reboots, see # https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode # kube_router_support_hairpin_mode: false # Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc. # kube_router_dns_policy: ClusterFirstWithHostNet # Array of annotations for master # kube_router_annotations_master: [] # Array of annotations for every node # kube_router_annotations_node: [] # Array of common annotations for every node # kube_router_annotations_all: [] # Enables scraping kube-router metrics with Prometheus # kube_router_enable_metrics: false # Path to serve Prometheus metrics on # kube_router_metrics_path: /metrics # Prometheus metrics port to use # kube_router_metrics_port: 9255 ================================================ FILE: inventory/sample/group_vars/k8s_cluster/k8s-net-macvlan.yml ================================================ --- # private interface, on a l2-network macvlan_interface: "eth1" # Enable nat in default gateway network interface enable_nat_default_gateway: true ================================================ FILE: inventory/sample/group_vars/k8s_cluster/kube_control_plane.yml ================================================ # Reservation for control plane kubernetes components # kube_memory_reserved: 512Mi # kube_cpu_reserved: 200m # kube_ephemeral_storage_reserved: 2Gi # kube_pid_reserved: "1000" # Reservation for control plane host system # system_memory_reserved: 256Mi # system_cpu_reserved: 250m # system_ephemeral_storage_reserved: 2Gi # system_pid_reserved: "1000" ================================================ FILE: inventory/sample/inventory.ini ================================================ # This inventory describe a HA typology with stacked etcd (== same nodes as control plane) # and 3 worker nodes # See https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html # for tips on building your # inventory # Configure 'ip' variable to bind kubernetes services on a different ip than the default iface # We should set etcd_member_name for etcd cluster. The node that are not etcd members do not need to set the value, # or can set the empty string value. [kube_control_plane] # node1 ansible_host=95.54.0.12 # ip=10.3.0.1 etcd_member_name=etcd1 # node2 ansible_host=95.54.0.13 # ip=10.3.0.2 etcd_member_name=etcd2 # node3 ansible_host=95.54.0.14 # ip=10.3.0.3 etcd_member_name=etcd3 [etcd:children] kube_control_plane [kube_node] # node4 ansible_host=95.54.0.15 # ip=10.3.0.4 # node5 ansible_host=95.54.0.16 # ip=10.3.0.5 # node6 ansible_host=95.54.0.17 # ip=10.3.0.6 ================================================ FILE: logo/LICENSE ================================================ # The Kubespray logo files are licensed under a choice of either Apache-2.0 or CC-BY-4.0 (Creative Commons Attribution 4.0 International). ================================================ FILE: logo/usage_guidelines.md ================================================ # Kubernetes Branding Guidelines These guidelines provide you with guidance for using the Kubespray logo. All artwork is made available under the Linux Foundation trademark usage [guidelines](https://www.linuxfoundation.org/trademark-usage/). This text from those guidelines, and the correct and incorrect usage examples, are particularly helpful: >Certain marks of The Linux Foundation have been created to enable you to >communicate compatibility or interoperability of software or products. In >addition to the requirement that any use of a mark to make an assertion of >compatibility must, of course, be accurate, the use of these marks must >avoid confusion regarding The Linux Foundation’s association with the >product. The use of the mark cannot imply that The Linux Foundation or >its projects are sponsoring or endorsing the product. Additionally, permission is granted to modify the Kubespray mark for non-commercial uses such as t-shirts and stickers. ================================================ FILE: meta/runtime.yml ================================================ --- requires_ansible: ">=2.18.0,<2.19.0" ================================================ FILE: pipeline.Dockerfile ================================================ # Use immutable image tags rather than mutable tags (like ubuntu:24.04) FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b # Some tools like yamllint need this # Pip needs this as well at the moment to install ansible # (and potentially other packages) # See: https://github.com/pypa/pip/issues/10219 ENV VAGRANT_VERSION=2.4.1 \ VAGRANT_DEFAULT_PROVIDER=libvirt \ VAGRANT_ANSIBLE_TAGS=facts \ LANG=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 RUN apt update -q \ && apt install -yq \ libssl-dev \ python3-dev \ python3-pip \ sshpass \ apt-transport-https \ jq \ moreutils \ libvirt-dev \ openssh-client \ rsync \ git \ ca-certificates \ curl \ gnupg2 \ unzip \ libvirt-clients \ qemu-utils \ qemu-kvm \ dnsmasq \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | tee /etc/apt/sources.list.d/docker.list \ && apt update -q \ && apt install --no-install-recommends -yq docker-ce \ && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* WORKDIR /kubespray ADD ./requirements.txt /kubespray/requirements.txt ADD ./tests/requirements.txt /kubespray/tests/requirements.txt RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \ && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \ && curl -L https://dl.k8s.io/release/v1.35.1/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/v1.35.1/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ # Install Vagrant && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ && vagrant plugin install vagrant-libvirt \ # Install Kubernetes collections && pip install --break-system-packages --no-compile --no-cache-dir kubernetes \ && ansible-galaxy collection install kubernetes.core ================================================ FILE: playbooks/ansible_version.yml ================================================ --- - name: Check Ansible version hosts: all gather_facts: false become: false run_once: true vars: minimal_ansible_version: 2.18.0 maximal_ansible_version: 2.19.0 tags: always tasks: - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}" assert: msg: "Ansible must be between {{ minimal_ansible_version }} and {{ maximal_ansible_version }} exclusive - you have {{ ansible_version.string }}" that: - ansible_version.string is version(minimal_ansible_version, ">=") - ansible_version.string is version(maximal_ansible_version, "<") tags: - check - name: "Check that python netaddr is installed" assert: msg: "Python netaddr is not present" that: "'127.0.0.1' | ansible.utils.ipaddr" tags: - check - name: "Check that jinja is not too old (install via pip)" assert: msg: "Your Jinja version is too old, install via pip" that: "{% set test %}It works{% endset %}{{ test == 'It works' }}" tags: - check ================================================ FILE: playbooks/boilerplate.yml ================================================ --- - name: Check ansible version import_playbook: ansible_version.yml # These are inventory compatibility tasks with two purposes: # - to ensure we keep compatibility with old style group names # - to reduce inventory boilerplate (defining parent groups / empty groups) - name: Inventory setup and validation hosts: all gather_facts: false tags: always roles: - dynamic_groups - validate_inventory - name: Install bastion ssh config hosts: bastion[0] gather_facts: false environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: bastion-ssh-config, tags: ["localhost", "bastion"] } ================================================ FILE: playbooks/cluster.yml ================================================ --- - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Gather facts import_playbook: internal_facts.yml - name: Prepare for etcd install hosts: k8s_cluster:etcd gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, tags: preinstall } - { role: "container-engine", tags: "container-engine", when: deploy_container_engine } - { role: download, tags: download, when: "not skip_downloads" } - name: Install etcd vars: etcd_cluster_setup: true etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" import_playbook: install_etcd.yml - name: Install Kubernetes nodes hosts: k8s_cluster gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/node, tags: node } - name: Install the control plane hosts: kube_control_plane gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/control-plane, tags: control-plane } - { role: kubernetes/client, tags: client } - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - name: Invoke kubeadm and install a CNI hosts: k8s_cluster gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/kubeadm, tags: kubeadm} - { role: kubernetes/node-label, tags: node-label } - { role: kubernetes/node-taint, tags: node-taint } - { role: kubernetes-apps/common_crds } - { role: network_plugin, tags: network } - name: Install Calico Route Reflector hosts: calico_rr gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] } - name: Patch Kubernetes for Windows hosts: kube_control_plane[0] gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] } - name: Install Kubernetes apps hosts: kube_control_plane gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller } - { role: kubernetes-apps/policy_controller, tags: policy-controller } - { role: kubernetes-apps/ingress_controller, tags: ingress-controller } - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } - { role: kubernetes-apps, tags: apps } - name: Apply resolv.conf changes now that cluster DNS is up hosts: k8s_cluster gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } ================================================ FILE: playbooks/facts.yml ================================================ --- - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Gather facts import_playbook: internal_facts.yml ================================================ FILE: playbooks/install_etcd.yml ================================================ --- - name: Add worker nodes to the etcd play if needed hosts: kube_node roles: - { role: kubespray_defaults } tasks: - name: Check if nodes needs etcd client certs (depends on network_plugin) group_by: key: "_kubespray_needs_etcd" when: - kube_network_plugin in ["flannel", "canal", "cilium"] or (cilium_deploy_additionally | default(false)) or (kube_network_plugin == "calico" and calico_datastore == "etcd") - etcd_deployment_type != "kubeadm" tags: etcd - name: Install etcd hosts: etcd:kube_control_plane:_kubespray_needs_etcd gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - role: etcd tags: etcd when: etcd_deployment_type != "kubeadm" ================================================ FILE: playbooks/internal_facts.yml ================================================ --- - name: Bootstrap hosts for Ansible hosts: k8s_cluster:etcd:calico_rr strategy: linear any_errors_fatal: "{{ any_errors_fatal | default(true) }}" gather_facts: false environment: "{{ proxy_disable_env }}" roles: - { role: bootstrap_os, tags: bootstrap_os} - name: Gather facts hosts: k8s_cluster:etcd:calico_rr gather_facts: false tags: always tasks: - name: Gather and compute network facts import_role: name: network_facts tags: - always - name: Gather minimal facts setup: gather_subset: '!all' # filter match the following variables: # ansible_default_ipv4 # ansible_default_ipv6 # ansible_all_ipv4_addresses # ansible_all_ipv6_addresses - name: Gather necessary facts (network) setup: gather_subset: '!all,!min,network' filter: "ansible_*_ipv[46]*" # filter match the following variables: # ansible_memtotal_mb # ansible_swaptotal_mb - name: Gather necessary facts (hardware) setup: gather_subset: '!all,!min,hardware' filter: "ansible_*total_mb" ================================================ FILE: playbooks/recover_control_plane.yml ================================================ --- - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Recover etcd hosts: etcd[0] environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults} - role: recover_control_plane/etcd when: etcd_deployment_type != "kubeadm" - name: Recover control plane hosts: kube_control_plane[0] environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults} - { role: recover_control_plane/control-plane } - name: Apply whole cluster install import_playbook: cluster.yml - name: Perform post recover tasks hosts: kube_control_plane environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults} - { role: recover_control_plane/post-recover } ================================================ FILE: playbooks/remove_node.yml ================================================ --- - name: Validate nodes for removal hosts: localhost gather_facts: false become: false tasks: - name: Assert that nodes are specified for removal assert: that: - node is defined - node | length > 0 msg: "No nodes specified for removal. The `node` variable must be set explicitly." - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Confirm node removal hosts: "{{ node | default('this_is_unreachable') }}" gather_facts: false tasks: - name: Confirm Execution pause: prompt: "Are you sure you want to delete nodes state? Type 'yes' to delete nodes." register: pause_result run_once: true when: - not (skip_confirmation | default(false) | bool) - name: Fail if user does not confirm deletion fail: msg: "Delete nodes confirmation failed" when: pause_result.user_input | default('yes') != 'yes' - name: Gather facts import_playbook: internal_facts.yml when: reset_nodes | default(True) | bool - name: Reset node hosts: "{{ node | default('this_is_unreachable') }}" gather_facts: false environment: "{{ proxy_disable_env }}" pre_tasks: - name: Gather information about installed services service_facts: when: reset_nodes | default(True) | bool roles: - { role: kubespray_defaults, when: reset_nodes | default(True) | bool } - { role: remove_node/pre_remove, tags: pre-remove } - role: remove-node/remove-etcd-node when: "'etcd' in group_names" - { role: reset, tags: reset, when: reset_nodes | default(True) | bool } # Currently cannot remove first control plane node or first etcd node - name: Post node removal hosts: "{{ node | default('this_is_unreachable') }}" gather_facts: false environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults, when: reset_nodes | default(True) | bool } - { role: remove-node/post-remove, tags: post-remove } ================================================ FILE: playbooks/reset.yml ================================================ --- - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Gather facts import_playbook: internal_facts.yml - name: Reset cluster hosts: etcd:k8s_cluster:calico_rr gather_facts: false pre_tasks: - name: Reset Confirmation pause: prompt: "Are you sure you want to reset cluster state? Type 'yes' to reset your cluster." register: reset_confirmation_prompt run_once: true when: - not (skip_confirmation | default(false) | bool) - reset_confirmation is not defined - name: Check confirmation fail: msg: "Reset confirmation failed" when: - not reset_confirmation | default(false) | bool - not reset_confirmation_prompt.user_input | default("") == "yes" - name: Gather information about installed services service_facts: environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults} - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true } - { role: reset, tags: reset } ================================================ FILE: playbooks/scale.yml ================================================ --- - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Gather facts import_playbook: internal_facts.yml - name: Install etcd vars: etcd_cluster_setup: false etcd_events_cluster_setup: false import_playbook: install_etcd.yml - name: Download images to ansible host cache via first kube_control_plane node hosts: kube_control_plane[0] gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" } - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" } - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" } - name: Target only workers to get kubelet installed and checking in on any new nodes(engine) hosts: kube_node gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, tags: preinstall } - { role: container-engine, tags: "container-engine", when: deploy_container_engine } - { role: download, tags: download, when: "not skip_downloads" } - role: etcd tags: etcd vars: etcd_cluster_setup: false when: - etcd_deployment_type != "kubeadm" - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool - kube_network_plugin != "calico" or calico_datastore == "etcd" - name: Target only workers to get kubelet installed and checking in on any new nodes(node) hosts: kube_node gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/node, tags: node } - name: Upload control plane certs and retrieve encryption key hosts: kube_control_plane | first environment: "{{ proxy_disable_env }}" gather_facts: false tags: kubeadm roles: - { role: kubespray_defaults } tasks: - name: Upload control plane certificates command: >- {{ bin_dir }}/kubeadm init phase --config {{ kube_config_dir }}/kubeadm-config.yaml upload-certs --upload-certs environment: "{{ proxy_disable_env }}" register: kubeadm_upload_cert changed_when: false - name: Set fact 'kubeadm_certificate_key' for later use set_fact: kubeadm_certificate_key: "{{ kubeadm_upload_cert.stdout_lines[-1] | trim }}" when: kubeadm_certificate_key is not defined - name: Target only workers to get kubelet installed and checking in on any new nodes(network) hosts: kube_node gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/kubeadm, tags: kubeadm } - { role: kubernetes/node-label, tags: node-label } - { role: kubernetes/node-taint, tags: node-taint } - { role: network_plugin, tags: network } - name: Apply resolv.conf changes now that cluster DNS is up hosts: k8s_cluster gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } ================================================ FILE: playbooks/upgrade_cluster.yml ================================================ --- - name: Common tasks for every playbooks import_playbook: boilerplate.yml - name: Gather facts import_playbook: internal_facts.yml - name: Download images to ansible host cache via first kube_control_plane node hosts: kube_control_plane[0] gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"} - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" } - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" } - name: Prepare nodes for upgrade hosts: k8s_cluster:etcd:calico_rr gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, tags: preinstall } - { role: download, tags: download, when: "not skip_downloads" } - name: Upgrade container engine on non-cluster nodes hosts: etcd:calico_rr:!k8s_cluster gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" serial: "{{ serial | default('20%') }}" roles: - { role: kubespray_defaults } - { role: container-engine, tags: "container-engine", when: deploy_container_engine } - name: Install etcd vars: etcd_cluster_setup: true etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" import_playbook: install_etcd.yml - name: Handle upgrades to control plane components first to maintain backwards compat. gather_facts: false hosts: kube_control_plane any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" serial: 1 roles: - { role: kubespray_defaults } - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: upgrade/system-upgrade, tags: system-upgrade } - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" } - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver } - { role: container-engine, tags: "container-engine", when: deploy_container_engine } - { role: kubernetes/node, tags: node } - { role: kubernetes/control-plane, tags: control-plane, upgrade_cluster_setup: true } - { role: kubernetes/client, tags: client } - { role: kubernetes/node-label, tags: node-label } - { role: kubernetes/node-taint, tags: node-taint } - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - { role: kubernetes-apps, tags: csi-driver } - { role: upgrade/post-upgrade, tags: post-upgrade } - name: Upgrade calico and external cloud provider on all control plane nodes, calico-rrs, and nodes hosts: kube_control_plane:calico_rr:kube_node gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" serial: "{{ serial | default('20%') }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller } - { role: network_plugin, tags: network } - { role: kubernetes-apps/policy_controller, tags: policy-controller } - name: Finally handle worker upgrades, based on given batch size hosts: kube_node:calico_rr:!kube_control_plane gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" serial: "{{ serial | default('20%') }}" roles: - { role: kubespray_defaults } - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: upgrade/system-upgrade, tags: system-upgrade } - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" } - { role: container-engine, tags: "container-engine", when: deploy_container_engine } - { role: kubernetes/node, tags: node } - { role: kubernetes/kubeadm, tags: kubeadm } - { role: kubernetes/node-label, tags: node-label } - { role: kubernetes/node-taint, tags: node-taint } - { role: upgrade/post-upgrade, tags: post-upgrade } - name: Patch Kubernetes for Windows hosts: kube_control_plane[0] gather_facts: false any_errors_fatal: true environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] } - name: Install Calico Route Reflector hosts: calico_rr gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: network_plugin/calico/rr, tags: network } - name: Install Kubernetes apps hosts: kube_control_plane gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes-apps/ingress_controller, tags: ingress-controller } - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } - { role: kubernetes-apps, tags: apps } - name: Apply resolv.conf changes now that cluster DNS is up hosts: k8s_cluster gather_facts: false any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } ================================================ FILE: plugins/modules/kube.py ================================================ #!/usr/bin/python # -*- coding: utf-8 -*- DOCUMENTATION = """ --- module: kube short_description: Manage Kubernetes Cluster description: - Create, replace, remove, and stop resources within a Kubernetes Cluster version_added: "2.0" options: name: required: false default: null description: - The name associated with resource filename: required: false default: null description: - The path and filename of the resource(s) definition file(s). - To operate on several files this can accept a comma separated list of files or a list of files. aliases: [ 'files', 'file', 'filenames' ] kubectl: required: false default: null description: - The path to the kubectl bin namespace: required: false default: null description: - The namespace associated with the resource(s) resource: required: false default: null description: - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc) label: required: false default: null description: - The labels used to filter specific resources. server: required: false default: null description: - The url for the API server that commands are executed against. kubeconfig: required: false default: null description: - The path to the kubeconfig. force: required: false default: false description: - A flag to indicate to force delete, replace, or stop. wait: required: false default: false description: - A flag to indicate to wait for resources to be created before continuing to the next step all: required: false default: false description: - A flag to indicate delete all, stop all, or all namespaces when checking exists. log_level: required: false default: 0 description: - Indicates the level of verbosity of logging by kubectl. state: required: false choices: ['present', 'absent', 'latest', 'reloaded', 'stopped'] default: present description: - present handles checking existence or creating if definition file provided, absent handles deleting resource(s) based on other options, latest handles creating or updating based on existence, reloaded handles updating resource(s) definition using definition file, stopped handles stopping resource(s) based on other options. recursive: required: false default: false description: - Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. requirements: - kubectl author: "Kenny Jones (@kenjones-cisco)" """ EXAMPLES = """ - name: test nginx is present kube: name=nginx resource=rc state=present - name: test nginx is stopped kube: name=nginx resource=rc state=stopped - name: test nginx is absent kube: name=nginx resource=rc state=absent - name: test nginx is present kube: filename=/tmp/nginx.yml - name: test nginx and postgresql are present kube: files=/tmp/nginx.yml,/tmp/postgresql.yml - name: test nginx and postgresql are present kube: files: - /tmp/nginx.yml - /tmp/postgresql.yml """ class KubeManager(object): def __init__(self, module): self.module = module self.kubectl = module.params.get('kubectl') if self.kubectl is None: self.kubectl = module.get_bin_path('kubectl', True) self.base_cmd = [self.kubectl] if module.params.get('server'): self.base_cmd.append('--server=' + module.params.get('server')) if module.params.get('kubeconfig'): self.base_cmd.append('--kubeconfig=' + module.params.get('kubeconfig')) if module.params.get('log_level'): self.base_cmd.append('--v=' + str(module.params.get('log_level'))) if module.params.get('namespace'): self.base_cmd.append('--namespace=' + module.params.get('namespace')) self.all = module.params.get('all') self.force = module.params.get('force') self.wait = module.params.get('wait') self.name = module.params.get('name') self.filename = [f.strip() for f in module.params.get('filename') or []] self.resource = module.params.get('resource') self.label = module.params.get('label') self.recursive = module.params.get('recursive') def _execute(self, cmd): args = self.base_cmd + cmd try: rc, out, err = self.module.run_command(args) if rc != 0: self.module.fail_json( msg='error running kubectl (%s) command (rc=%d), out=\'%s\', err=\'%s\'' % (' '.join(args), rc, out, err)) except Exception as exc: self.module.fail_json( msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc))) return out.splitlines() def _execute_nofail(self, cmd): args = self.base_cmd + cmd rc, out, err = self.module.run_command(args) if rc != 0: return None return out.splitlines() def create(self, check=True, force=True): if check and self.exists(): return [] cmd = ['apply'] if force: cmd.append('--force') if self.wait: cmd.append('--wait') if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) if not self.filename: self.module.fail_json(msg='filename required to create') cmd.append('--filename=' + ','.join(self.filename)) return self._execute(cmd) def replace(self, force=True): cmd = ['apply'] if force: cmd.append('--force') if self.wait: cmd.append('--wait') if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) if not self.filename: self.module.fail_json(msg='filename required to reload') cmd.append('--filename=' + ','.join(self.filename)) return self._execute(cmd) def delete(self): if not self.force and not self.exists(): return [] cmd = ['delete'] if self.filename: cmd.append('--filename=' + ','.join(self.filename)) if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) else: if not self.resource: self.module.fail_json(msg='resource required to delete without filename') cmd.append(self.resource) if self.name: cmd.append(self.name) if self.label: cmd.append('--selector=' + self.label) if self.all: cmd.append('--all') if self.force: cmd.append('--ignore-not-found') if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) return self._execute(cmd) def exists(self): cmd = ['get'] if self.filename: cmd.append('--filename=' + ','.join(self.filename)) if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) else: if not self.resource: self.module.fail_json(msg='resource required without filename') cmd.append(self.resource) if self.name: cmd.append(self.name) if self.label: cmd.append('--selector=' + self.label) if self.all: cmd.append('--all-namespaces') cmd.append('--no-headers') result = self._execute_nofail(cmd) if not result: return False return True # TODO: This is currently unused, perhaps convert to 'scale' with a replicas param? def stop(self): if not self.force and not self.exists(): return [] cmd = ['stop'] if self.filename: cmd.append('--filename=' + ','.join(self.filename)) if self.recursive: cmd.append('--recursive={}'.format(self.recursive)) else: if not self.resource: self.module.fail_json(msg='resource required to stop without filename') cmd.append(self.resource) if self.name: cmd.append(self.name) if self.label: cmd.append('--selector=' + self.label) if self.all: cmd.append('--all') if self.force: cmd.append('--ignore-not-found') return self._execute(cmd) def main(): module = AnsibleModule( argument_spec=dict( name=dict(), filename=dict(type='list', aliases=['files', 'file', 'filenames']), namespace=dict(), resource=dict(), label=dict(), server=dict(), kubeconfig=dict(), kubectl=dict(), force=dict(default=False, type='bool'), wait=dict(default=False, type='bool'), all=dict(default=False, type='bool'), log_level=dict(default=0, type='int'), state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped', 'exists']), recursive=dict(default=False, type='bool'), ), mutually_exclusive=[['filename', 'list']] ) changed = False manager = KubeManager(module) state = module.params.get('state') if state == 'present': result = manager.create(check=False) elif state == 'absent': result = manager.delete() elif state == 'reloaded': result = manager.replace() elif state == 'stopped': result = manager.stop() elif state == 'latest': result = manager.replace() elif state == 'exists': result = manager.exists() module.exit_json(changed=changed, msg='%s' % result) else: module.fail_json(msg='Unrecognized state %s.' % state) module.exit_json(changed=changed, msg='success: %s' % (' '.join(result)) ) from ansible.module_utils.basic import * # noqa if __name__ == '__main__': main() ================================================ FILE: recover-control-plane.yml ================================================ --- - name: Recover control plane ansible.builtin.import_playbook: playbooks/recover_control_plane.yml ================================================ FILE: remove-node.yml ================================================ --- - name: Remove node ansible.builtin.import_playbook: playbooks/remove_node.yml ================================================ FILE: remove_node.yml ================================================ --- - name: Remove node ansible.builtin.import_playbook: playbooks/remove_node.yml ================================================ FILE: requirements.txt ================================================ ansible==11.13.0 # Needed for community.crypto module cryptography==46.0.5 # Needed for jinja2 json_query templating jmespath==1.1.0 # Needed for ansible.utils.ipaddr netaddr==1.3.0 ================================================ FILE: reset.yml ================================================ --- - name: Reset the cluster ansible.builtin.import_playbook: playbooks/reset.yml ================================================ FILE: roles/adduser/defaults/main.yml ================================================ --- kube_owner: kube kube_cert_group: kube-cert etcd_data_dir: "/var/lib/etcd" addusers: etcd: name: etcd comment: "Etcd user" create_home: false system: true shell: /sbin/nologin kube: name: kube comment: "Kubernetes user" create_home: false system: true shell: /sbin/nologin group: "{{ kube_cert_group }}" adduser: name: "{{ user.name }}" group: "{{ user.name | default(None) }}" comment: "{{ user.comment | default(None) }}" shell: "{{ user.shell | default(None) }}" system: "{{ user.system | default(None) }}" create_home: "{{ user.create_home | default(None) }}" ================================================ FILE: roles/adduser/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true gather_facts: false roles: - role: adduser vars: user: name: foo ================================================ FILE: roles/adduser/molecule/default/molecule.yml ================================================ --- role_name_check: 1 dependency: name: galaxy platforms: - name: ubuntu22 cloud_image: ubuntu-2204 vm_cpu_cores: 1 vm_memory: 512 provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 playbooks: create: ../../../../tests/cloud_playbooks/create-kubevirt.yml verifier: name: testinfra ================================================ FILE: roles/adduser/tasks/main.yml ================================================ --- - name: User | Create User Group group: name: "{{ user.group | default(user.name) }}" system: "{{ user.system | default(omit) }}" - name: User | Create User user: comment: "{{ user.comment | default(omit) }}" create_home: "{{ user.create_home | default(omit) }}" group: "{{ user.group | default(user.name) }}" home: "{{ user.home | default(omit) }}" shell: "{{ user.shell | default(omit) }}" name: "{{ user.name }}" system: "{{ user.system | default(omit) }}" when: user.name != "root" ================================================ FILE: roles/adduser/vars/coreos.yml ================================================ --- addusers: - name: kube comment: "Kubernetes user" shell: /sbin/nologin system: true group: "{{ kube_cert_group }}" create_home: false ================================================ FILE: roles/adduser/vars/debian.yml ================================================ --- addusers: - name: etcd comment: "Etcd user" create_home: true home: "{{ etcd_data_dir }}" system: true shell: /sbin/nologin - name: kube comment: "Kubernetes user" create_home: false system: true shell: /sbin/nologin group: "{{ kube_cert_group }}" ================================================ FILE: roles/adduser/vars/redhat.yml ================================================ --- addusers: - name: etcd comment: "Etcd user" create_home: true home: "{{ etcd_data_dir }}" system: true shell: /sbin/nologin - name: kube comment: "Kubernetes user" create_home: false system: true shell: /sbin/nologin group: "{{ kube_cert_group }}" ================================================ FILE: roles/bastion-ssh-config/defaults/main.yml ================================================ --- ssh_bastion_config_name: ssh-bastion.conf ================================================ FILE: roles/bastion-ssh-config/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true gather_facts: false roles: - role: bastion-ssh-config tasks: - name: Copy config to remote host copy: src: "{{ playbook_dir }}/{{ ssh_bastion_config_name }}" dest: "{{ ssh_bastion_config_name }}" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" mode: "0644" ================================================ FILE: roles/bastion-ssh-config/molecule/default/molecule.yml ================================================ --- role_name_check: 1 dependency: name: galaxy platforms: - name: bastion-01 cloud_image: ubuntu-2204 vm_cpu_cores: 1 vm_memory: 512 provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 inventory: hosts: all: hosts: children: bastion: hosts: bastion-01: playbooks: create: ../../../../tests/cloud_playbooks/create-kubevirt.yml verifier: name: testinfra ================================================ FILE: roles/bastion-ssh-config/tasks/main.yml ================================================ --- - name: Set bastion host IP and port set_fact: bastion_ip: "{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}" bastion_port: "{{ hostvars[groups['bastion'][0]]['ansible_port'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_port']) | d(22) }}" delegate_to: localhost connection: local # As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly # To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user - name: Store the current ansible_user in the real_user fact set_fact: real_user: "{{ ansible_user }}" - name: Create ssh bastion conf become: false delegate_to: localhost connection: local template: src: "{{ ssh_bastion_config_name }}.j2" dest: "{{ playbook_dir }}/{{ ssh_bastion_config_name }}" mode: "0640" ================================================ FILE: roles/bastion-ssh-config/templates/ssh-bastion.conf.j2 ================================================ {% set vars={'hosts': ''} %} {% set user='' %} {% for h in groups['all'] %} {% if h not in groups['bastion'] %} {% if vars.update({'hosts': vars['hosts'] + ' ' + (hostvars[h].get('ansible_ssh_host') or hostvars[h]['ansible_host'])}) %}{% endif %} {% endif %} {% endfor %} Host {{ bastion_ip }} Hostname {{ bastion_ip }} StrictHostKeyChecking no ControlMaster auto ControlPath ~/.ssh/ansible-%r@%h:%p ControlPersist 5m Host {{ vars['hosts'] }} ProxyCommand ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -p {{ bastion_port }} {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} ================================================ FILE: roles/bootstrap-os/tasks/main.yml ================================================ --- - name: Warn for usage of deprecated role fail: msg: bootstrap-os is deprecated, switch to bootstrap_os ignore_errors: true # noqa ignore-errors run_once: true - name: Compat for direct role import import_role: name: bootstrap_os ================================================ FILE: roles/bootstrap_os/defaults/main.yml ================================================ --- ## CentOS/RHEL/AlmaLinux specific variables # Use the fastestmirror yum plugin centos_fastestmirror_enabled: false # Timeout (in seconds) for checking RHEL subscription status rh_subscription_check_timeout: 180 ## Flatcar Container Linux specific variables # Disable locksmithd or leave it in its current state coreos_locksmithd_disable: false # Install epel repo on Centos/RHEL epel_enabled: false ## openEuler specific variables # Enable metalink for openEuler repos (auto-selects fastest mirror by location) openeuler_metalink_enabled: false ## Oracle Linux specific variables # Install public repo on Oracle Linux use_oracle_public_repo: true ## Ubuntu specific variables # Disable unattended-upgrades for Linux kernel and all packages start with linux- on Ubuntu ubuntu_kernel_unattended_upgrades_disabled: false # Stop unattended-upgrades if it is currently running on Ubuntu ubuntu_stop_unattended_upgrades: false fedora_coreos_packages: - python - python3-libselinux - ethtool # required in kubeadm preflight phase for verifying the environment - ipset # required in kubeadm preflight phase for verifying the environment - conntrack-tools # required by kube-proxy - containernetworking-plugins # required by crio ## General # Set the hostname to inventory_hostname override_system_hostname: true is_fedora_coreos: false skip_http_proxy_on_os_packages: false ================================================ FILE: roles/bootstrap_os/files/bootstrap.sh ================================================ #!/bin/bash set -e BINDIR="/opt/bin" if [[ -e $BINDIR/.bootstrapped ]]; then exit 0 fi ARCH=$(uname -m) case $ARCH in "x86_64") PYPY_ARCH=linux64 PYPI_HASH=46818cb3d74b96b34787548343d266e2562b531ddbaf330383ba930ff1930ed5 ;; "aarch64") PYPY_ARCH=aarch64 PYPI_HASH=2e1ae193d98bc51439642a7618d521ea019f45b8fb226940f7e334c548d2b4b9 ;; *) echo "Unsupported Architecture: ${ARCH}" exit 1 esac PYTHON_VERSION=3.9 PYPY_VERSION=7.3.9 PYPY_FILENAME="pypy${PYTHON_VERSION}-v${PYPY_VERSION}-${PYPY_ARCH}" PYPI_URL="https://downloads.python.org/pypy/${PYPY_FILENAME}.tar.bz2" mkdir -p $BINDIR cd $BINDIR TAR_FILE=pyp.tar.bz2 wget -O "${TAR_FILE}" "${PYPI_URL}" echo "${PYPI_HASH} ${TAR_FILE}" | sha256sum -c - tar -xjf "${TAR_FILE}" && rm "${TAR_FILE}" mv -n "${PYPY_FILENAME}" pypy3 ln -s ./pypy3/bin/pypy3 python $BINDIR/python --version # install PyYAML ./python -m ensurepip ./python -m pip install pyyaml touch $BINDIR/.bootstrapped ================================================ FILE: roles/bootstrap_os/handlers/main.yml ================================================ --- - name: RHEL auto-attach subscription command: /sbin/subscription-manager attach --auto become: true ================================================ FILE: roles/bootstrap_os/meta/main.yml ================================================ --- dependencies: - role: kubespray_defaults ================================================ FILE: roles/bootstrap_os/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all gather_facts: false become: true roles: - role: bootstrap_os ================================================ FILE: roles/bootstrap_os/molecule/default/molecule.yml ================================================ --- role_name_check: 1 dependency: name: galaxy platforms: - name: ubuntu22 cloud_image: ubuntu-2204 vm_cpu_cores: 1 vm_memory: 512 - name: ubuntu24 cloud_image: ubuntu-2404 vm_cpu_cores: 1 vm_memory: 512 - name: almalinux9 cloud_image: almalinux-9 vm_cpu_cores: 1 vm_memory: 512 - name: debian12 cloud_image: debian-12 vm_cpu_cores: 1 vm_memory: 512 provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 inventory: group_vars: all: user: name: foo comment: My test comment playbooks: create: ../../../../tests/cloud_playbooks/create-kubevirt.yml verifier: name: testinfra ================================================ FILE: roles/bootstrap_os/molecule/default/tests/test_default.py ================================================ import os import testinfra.utils.ansible_runner testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE'] ).get_hosts('all') def test_python(host): assert host.exists('python3') or host.exists('python') ================================================ FILE: roles/bootstrap_os/tasks/almalinux.yml ================================================ --- - name: Import Centos boostrap for Alma Linux import_tasks: centos.yml ================================================ FILE: roles/bootstrap_os/tasks/amzn.yml ================================================ --- - name: Enable selinux-ng repo for Amazon Linux for container-selinux command: amazon-linux-extras enable selinux-ng - name: Enable EPEL repo for Amazon Linux yum_repository: name: epel file: epel description: Extra Packages for Enterprise Linux 7 - $basearch baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch gpgcheck: true gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 skip_if_unavailable: true enabled: true repo_gpgcheck: false when: epel_enabled ================================================ FILE: roles/bootstrap_os/tasks/centos.yml ================================================ --- - name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version setup: gather_subset: '!all' filter: ansible_distribution_*version - name: Add proxy to yum.conf or dnf.conf if http_proxy is defined community.general.ini_file: path: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}" section: main option: proxy value: "{{ http_proxy | default(omit) }}" state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}" no_extra_spaces: true mode: "0644" become: true when: not skip_http_proxy_on_os_packages # For Oracle Linux install public repo - name: Install EPEL for Oracle Linux repo package package: name: "oracle-epel-release-el{{ ansible_distribution_major_version }}" state: present when: - use_oracle_public_repo | default(true) - '''ID="ol"'' in os_release.stdout_lines' - (ansible_distribution_version | float) >= 7.6 - name: Enable Oracle Linux repo community.general.ini_file: dest: "/etc/yum.repos.d/oracle-linux-ol{{ ansible_distribution_major_version }}.repo" section: "ol{{ ansible_distribution_major_version }}_addons" option: "{{ item.option }}" value: "{{ item.value }}" mode: "0644" with_items: - { option: "name", value: "ol{{ ansible_distribution_major_version }}_addons" } - { option: "enabled", value: "1" } - { option: "baseurl", value: "http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/" } when: - use_oracle_public_repo | default(true) - '''ID="ol"'' in os_release.stdout_lines' - (ansible_distribution_version | float) >= 7.6 - name: Enable Centos extra repo for Oracle Linux community.general.ini_file: dest: "/etc/yum.repos.d/centos-extras.repo" section: "extras" option: "{{ item.option }}" value: "{{ item.value }}" mode: "0644" with_items: - { option: "name", value: "CentOS-{{ ansible_distribution_major_version }} - Extras" } - { option: "enabled", value: "1" } - { option: "gpgcheck", value: "0" } - { option: "baseurl", value: "http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/extras/$basearch/os/" } when: - use_oracle_public_repo | default(true) - '''ID="ol"'' in os_release.stdout_lines' - (ansible_distribution_version | float) >= 7.6 - (ansible_distribution_version | float) < 9 # CentOS ships with python installed - name: Check presence of fastestmirror.conf stat: path: /etc/yum/pluginconf.d/fastestmirror.conf get_attributes: false get_checksum: false get_mime: false register: fastestmirror # the fastestmirror plugin can actually slow down Ansible deployments - name: Disable fastestmirror plugin if requested lineinfile: dest: /etc/yum/pluginconf.d/fastestmirror.conf regexp: "^enabled=.*" line: "enabled=0" state: present become: true when: - fastestmirror.stat.exists - not centos_fastestmirror_enabled ================================================ FILE: roles/bootstrap_os/tasks/debian.yml ================================================ --- # Some Debian based distros ship without Python installed - name: Check if bootstrap is needed raw: which python3 register: need_bootstrap failed_when: false changed_when: false # This command should always run, even in check mode check_mode: false tags: - facts - name: Check http::proxy in apt configuration files raw: apt-config dump | grep -qsi 'Acquire::http::proxy' register: need_http_proxy failed_when: false changed_when: false # This command should always run, even in check mode check_mode: false - name: Add http_proxy to /etc/apt/apt.conf if http_proxy is defined raw: echo 'Acquire::http::proxy "{{ http_proxy }}";' >> /etc/apt/apt.conf become: true when: - http_proxy is defined - need_http_proxy.rc != 0 - not skip_http_proxy_on_os_packages - name: Check https::proxy in apt configuration files raw: apt-config dump | grep -qsi 'Acquire::https::proxy' register: need_https_proxy failed_when: false changed_when: false # This command should always run, even in check mode check_mode: false - name: Add https_proxy to /etc/apt/apt.conf if https_proxy is defined raw: echo 'Acquire::https::proxy "{{ https_proxy }}";' >> /etc/apt/apt.conf become: true when: - https_proxy is defined - need_https_proxy.rc != 0 - not skip_http_proxy_on_os_packages - name: Install python3 raw: apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal become: true when: - need_bootstrap.rc != 0 ================================================ FILE: roles/bootstrap_os/tasks/fedora-coreos.yml ================================================ --- - name: Check if bootstrap is needed raw: which python register: need_bootstrap failed_when: false changed_when: false tags: - facts - name: Remove podman network cni raw: "podman network rm podman" become: true ignore_errors: true # noqa ignore-errors when: need_bootstrap.rc != 0 - name: Clean up possible pending packages on fedora coreos raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree cleanup -p }}" become: true when: need_bootstrap.rc != 0 - name: Install required packages on fedora coreos raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install --allow-inactive {{ fedora_coreos_packages | join(' ') }}" become: true when: need_bootstrap.rc != 0 - name: Reboot immediately for updated ostree raw: "nohup bash -c 'sleep 5s && shutdown -r now'" become: true ignore_errors: true # noqa ignore-errors ignore_unreachable: true when: need_bootstrap.rc != 0 - name: Wait for the reboot to complete wait_for_connection: timeout: 240 connect_timeout: 20 delay: 5 sleep: 5 when: need_bootstrap.rc != 0 ================================================ FILE: roles/bootstrap_os/tasks/fedora.yml ================================================ --- # Some Fedora based distros ship without Python installed - name: Check if bootstrap is needed raw: which python register: need_bootstrap failed_when: false changed_when: false tags: - facts - name: Add proxy to dnf.conf if http_proxy is defined community.general.ini_file: path: "/etc/dnf/dnf.conf" section: main option: proxy value: "{{ http_proxy | default(omit) }}" state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}" no_extra_spaces: true mode: "0644" become: true when: not skip_http_proxy_on_os_packages # libselinux-python3 is required on SELinux enabled hosts # See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements - name: Install ansible requirements raw: "dnf install --assumeyes python3 python3-dnf libselinux-python3" become: true when: - need_bootstrap.rc != 0 ================================================ FILE: roles/bootstrap_os/tasks/flatcar.yml ================================================ --- # Flatcar Container Linux ships without Python installed - name: Check if bootstrap is needed raw: stat /opt/bin/.bootstrapped register: need_bootstrap failed_when: false changed_when: false tags: - facts - name: Run bootstrap.sh script: bootstrap.sh become: true environment: "{{ proxy_env }}" when: - need_bootstrap.rc != 0 # Workaround ansible https://github.com/ansible/ansible/pull/82821 # We set the interpreter rather than ansible_python_interpreter to allow # - using virtual env with task level ansible_python_interpreter later # - let users specify an ansible_python_interpreter in group_vars - name: Make interpreter discovery works on Flatcar set_fact: ansible_interpreter_python_fallback: "{{ (ansible_interpreter_python_fallback | default([])) + ['/opt/bin/python'] }}" - name: Disable auto-upgrade systemd_service: name: locksmithd.service masked: true state: stopped when: - coreos_locksmithd_disable ================================================ FILE: roles/bootstrap_os/tasks/main.yml ================================================ --- - name: Fetch /etc/os-release raw: cat /etc/os-release register: os_release changed_when: false # This command should always run, even in check mode check_mode: false - name: Include distro specifics vars and tasks vars: os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') | map('split', '=') | community.general.dict }}" block: - name: Include vars include_vars: "{{ item }}" tags: - facts with_first_found: - files: &search_files - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml" - "{{ os_release_dict['ID'] }}.yml" paths: - vars/ skip: true - name: Include tasks include_tasks: "{{ included_tasks_file }}" with_first_found: - files: *search_files skip: true loop_control: loop_var: included_tasks_file - name: Install system packages import_role: name: system_packages tags: - system-packages - name: Create remote_tmp for it is used by another module file: path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}" state: directory mode: "0700" - name: Gather facts setup: gather_subset: '!all' filter: ansible_* - name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora) hostname: name: "{{ inventory_hostname }}" when: override_system_hostname - name: Ensure bash_completion.d folder exists file: name: /etc/bash_completion.d/ state: directory owner: root group: root mode: "0755" ================================================ FILE: roles/bootstrap_os/tasks/openEuler.yml ================================================ --- - name: Import CentOS bootstrap for openEuler ansible.builtin.import_tasks: centos.yml - name: Get existing openEuler repo sections ansible.builtin.shell: cmd: "set -o pipefail && grep '^\\[' /etc/yum.repos.d/openEuler.repo | tr -d '[]'" executable: /bin/bash register: _openeuler_repo_sections changed_when: false failed_when: false check_mode: false become: true when: openeuler_metalink_enabled - name: Enable metalink for openEuler repos community.general.ini_file: path: /etc/yum.repos.d/openEuler.repo section: "{{ item.key }}" option: metalink value: "{{ item.value }}" no_extra_spaces: true mode: "0644" loop: "{{ _openeuler_metalink_repos | dict2items | selectattr('key', 'in', _openeuler_repo_sections.stdout_lines | default([])) }}" become: true when: openeuler_metalink_enabled register: _openeuler_metalink_result vars: _openeuler_metalink_repos: OS: "https://mirrors.openeuler.org/metalink?repo=$releasever/OS&arch=$basearch" everything: "https://mirrors.openeuler.org/metalink?repo=$releasever/everything&arch=$basearch" EPOL: "https://mirrors.openeuler.org/metalink?repo=$releasever/EPOL/main&arch=$basearch" debuginfo: "https://mirrors.openeuler.org/metalink?repo=$releasever/debuginfo&arch=$basearch" source: "https://mirrors.openeuler.org/metalink?repo=$releasever&arch=source" update: "https://mirrors.openeuler.org/metalink?repo=$releasever/update&arch=$basearch" update-source: "https://mirrors.openeuler.org/metalink?repo=$releasever/update&arch=source" - name: Clean dnf cache to apply metalink mirror selection ansible.builtin.command: dnf clean all become: true when: - openeuler_metalink_enabled - _openeuler_metalink_result.changed ================================================ FILE: roles/bootstrap_os/tasks/opensuse-leap.yml ================================================ --- - name: Import Opensuse bootstrap import_tasks: opensuse.yml ================================================ FILE: roles/bootstrap_os/tasks/opensuse-tumbleweed.yml ================================================ --- - name: Import Opensuse bootstrap import_tasks: opensuse.yml ================================================ FILE: roles/bootstrap_os/tasks/opensuse.yml ================================================ --- # OpenSUSE ships with Python installed - name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version setup: gather_subset: '!all' filter: ansible_distribution_*version - name: Check that /etc/sysconfig/proxy file exists stat: path: /etc/sysconfig/proxy get_attributes: false get_checksum: false get_mime: false register: stat_result - name: Create the /etc/sysconfig/proxy empty file file: # noqa risky-file-permissions path: /etc/sysconfig/proxy state: touch when: - http_proxy is defined or https_proxy is defined - not stat_result.stat.exists - name: Set the http_proxy in /etc/sysconfig/proxy lineinfile: path: /etc/sysconfig/proxy regexp: '^HTTP_PROXY=' line: 'HTTP_PROXY="{{ http_proxy }}"' become: true when: - http_proxy is defined - name: Set the https_proxy in /etc/sysconfig/proxy lineinfile: path: /etc/sysconfig/proxy regexp: '^HTTPS_PROXY=' line: 'HTTPS_PROXY="{{ https_proxy }}"' become: true when: - https_proxy is defined - name: Enable proxies lineinfile: path: /etc/sysconfig/proxy regexp: '^PROXY_ENABLED=' line: 'PROXY_ENABLED="yes"' become: true when: - http_proxy is defined or https_proxy is defined # Required for zypper module - name: Install python-xml shell: zypper refresh && zypper --non-interactive install python-xml changed_when: false become: true tags: - facts ================================================ FILE: roles/bootstrap_os/tasks/rhel.yml ================================================ --- - name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version setup: gather_subset: '!all' filter: ansible_distribution_*version - name: Add proxy to yum.conf or dnf.conf if http_proxy is defined community.general.ini_file: path: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}" section: main option: proxy value: "{{ http_proxy | default(omit) }}" state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}" no_extra_spaces: true mode: "0644" become: true when: not skip_http_proxy_on_os_packages - name: Add proxy to RHEL subscription-manager if http_proxy is defined command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\d+$') | regex_replace('^.*://') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }} become: true when: - not skip_http_proxy_on_os_packages - http_proxy is defined - name: Check RHEL subscription-manager status command: /sbin/subscription-manager status register: rh_subscription_status changed_when: "rh_subscription_status.rc != 0" ignore_errors: true # noqa ignore-errors timeout: "{{ rh_subscription_check_timeout }}" become: true - name: RHEL subscription Organization ID/Activation Key registration community.general.redhat_subscription: state: present org_id: "{{ rh_subscription_org_id }}" activationkey: "{{ rh_subscription_activation_key }}" force_register: true notify: RHEL auto-attach subscription become: true when: - rh_subscription_org_id is defined - rh_subscription_status.changed # this task has no_log set to prevent logging security sensitive information such as subscription passwords - name: RHEL subscription Username/Password registration community.general.redhat_subscription: state: present username: "{{ rh_subscription_username }}" password: "{{ rh_subscription_password }}" auto_attach: true force_register: true syspurpose: usage: "{{ rh_subscription_usage }}" role: "{{ rh_subscription_role }}" service_level_agreement: "{{ rh_subscription_sla }}" sync: true notify: RHEL auto-attach subscription become: true no_log: "{{ not (unsafe_show_logs | bool) }}" when: - rh_subscription_username is defined - rh_subscription_status.changed # container-selinux is in appstream repo - name: Enable RHEL 8 repos community.general.rhsm_repository: name: - "rhel-8-for-*-baseos-rpms" - "rhel-8-for-*-appstream-rpms" state: "{{ 'enabled' if (rhel_enable_repos | default(True) | bool) else 'disabled' }}" when: - ansible_distribution_major_version == "8" - (not rh_subscription_status.changed) or (rh_subscription_username is defined) or (rh_subscription_org_id is defined) - name: Check presence of fastestmirror.conf stat: path: /etc/yum/pluginconf.d/fastestmirror.conf get_attributes: false get_checksum: false get_mime: false register: fastestmirror # the fastestmirror plugin can actually slow down Ansible deployments - name: Disable fastestmirror plugin if requested lineinfile: dest: /etc/yum/pluginconf.d/fastestmirror.conf regexp: "^enabled=.*" line: "enabled=0" state: present become: true when: - fastestmirror.stat.exists - not centos_fastestmirror_enabled ================================================ FILE: roles/bootstrap_os/tasks/rocky.yml ================================================ --- - name: Import Centos boostrap for Rocky Linux import_tasks: centos.yml ================================================ FILE: roles/bootstrap_os/tasks/ubuntu.yml ================================================ --- - name: Import Debian bootstrap import_tasks: debian.yml - name: Check unattended-upgrades file exist stat: path: /etc/apt/apt.conf.d/50unattended-upgrades register: unattended_upgrades_file_stat when: - ubuntu_kernel_unattended_upgrades_disabled - name: Disable kernel unattended-upgrades lineinfile: path: "{{ unattended_upgrades_file_stat.stat.path }}" insertafter: "Unattended-Upgrade::Package-Blacklist" line: '"linux-";' state: present become: true when: - ubuntu_kernel_unattended_upgrades_disabled - unattended_upgrades_file_stat.stat.exists - name: Stop unattended-upgrades service service: name: unattended-upgrades state: stopped enabled: false become: true when: ubuntu_stop_unattended_upgrades ================================================ FILE: roles/bootstrap_os/vars/fedora-coreos.yml ================================================ --- is_fedora_coreos: true ================================================ FILE: roles/bootstrap_os/vars/flatcar.yml ================================================ --- bin_dir: "/opt/bin" ================================================ FILE: roles/container-engine/containerd/defaults/main.yml ================================================ --- containerd_storage_dir: "/var/lib/containerd" containerd_state_dir: "/run/containerd" containerd_systemd_dir: "/etc/systemd/system/containerd.service.d" # The default value is not -999 here because containerd's oom_score_adj has been # set to the -999 even if containerd_oom_score is 0. # Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242 containerd_oom_score: 0 containerd_default_runtime: "runc" containerd_snapshotter: "overlayfs" containerd_runc_runtime: name: runc type: "io.containerd.runc.v2" base_runtime_spec: cri-base.json options: Root: "" SystemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}" BinaryName: "{{ bin_dir }}/runc" containerd_additional_runtimes: [] # Example for Kata Containers as additional runtime: # - name: kata # type: "io.containerd.kata.v2" # options: # Root: "" containerd_base_runtime_spec_rlimit_nofile: 65535 containerd_default_base_runtime_spec_patch: process: rlimits: - type: RLIMIT_NOFILE hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}" soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}" # Only for containerd < 2.1; discard unpacked layers to save disk space # https://github.com/containerd/containerd/blob/release/2.1/docs/cri/config.md#image-pull-configuration-since-containerd-v21 containerd_discard_unpacked_layers: true containerd_base_runtime_specs: cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch, recursive=1) }}" containerd_grpc_max_recv_message_size: 16777216 containerd_grpc_max_send_message_size: 16777216 containerd_debug_address: "" containerd_debug_level: "info" containerd_debug_format: "" containerd_debug_uid: 0 containerd_debug_gid: 0 containerd_metrics_address: "" containerd_metrics_grpc_histogram: false containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://registry-1.docker.io capabilities: ["pull", "resolve"] skip_verify: false # ca: ["/etc/certs/mirror.pem"] # client: [["/etc/certs/client.pem", ""],["/etc/certs/client.cert", "/etc/certs/client.key"]] # header: # Authorization: "Basic XXX" containerd_max_container_log_line_size: 16384 # If enabled it will allow non root users to use port numbers <1024 containerd_enable_unprivileged_ports: false # If enabled it will allow non root users to use icmp sockets containerd_enable_unprivileged_icmp: false containerd_enable_selinux: false containerd_disable_apparmor: false containerd_tolerate_missing_hugetlb_controller: true containerd_disable_hugetlb_controller: true containerd_image_pull_progress_timeout: 5m containerd_cfg_dir: /etc/containerd # Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally containerd_extra_args: '' # Extra runtime configuration options to be injected into the containerd CRI runtime plugin section # [plugins."io.containerd.cri.v1.runtime"]. This is useful for adding containerd runtime # configuration options that aren't explicitly supported by Kubespray's default variables. # Example: # containerd_extra_runtime_args: # device_ownership_from_security_context: true # another_option: "value" containerd_extra_runtime_args: {} # Configure registry auth (if applicable to secure/insecure registries) containerd_registry_auth: [] # - registry: 10.0.0.2:5000 # username: user # password: pass # Configure containerd service containerd_limit_proc_num: "infinity" containerd_limit_core: "infinity" containerd_limit_open_file_num: 1048576 containerd_limit_mem_lock: "infinity" # OS distributions that already support containerd containerd_supported_distributions: - "CentOS" - "OracleLinux" - "RedHat" - "Ubuntu" - "Debian" - "Fedora" - "AlmaLinux" - "Rocky" - "Amazon" - "Flatcar" - "Flatcar Container Linux by Kinvolk" - "Suse" - "openSUSE Leap" - "openSUSE Tumbleweed" - "Kylin Linux Advanced Server" - "UnionTech" - "UniontechOS" - "openEuler" # Enable container device interface enable_cdi: false # For containerd tracing configuration please check out the official documentation: # https://github.com/containerd/containerd/blob/main/docs/tracing.md containerd_tracing_enabled: false containerd_tracing_endpoint: "[::]:4317" containerd_tracing_protocol: "grpc" containerd_tracing_sampling_ratio: 1.0 containerd_tracing_service_name: "containerd" ================================================ FILE: roles/container-engine/containerd/handlers/main.yml ================================================ --- - name: Containerd | restart containerd systemd_service: name: containerd state: restarted enabled: true daemon-reload: true masked: false listen: Restart containerd - name: Containerd | wait for containerd command: "{{ containerd_bin_dir }}/ctr images ls -q" register: containerd_ready retries: 8 delay: 4 until: containerd_ready.rc == 0 listen: Restart containerd ================================================ FILE: roles/container-engine/containerd/handlers/reset.yml ================================================ --- ================================================ FILE: roles/container-engine/containerd/meta/main.yml ================================================ --- dependencies: - role: container-engine/containerd-common - role: container-engine/runc - role: container-engine/crictl - role: container-engine/nerdctl ================================================ FILE: roles/container-engine/containerd/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true vars: container_manager: containerd roles: - role: kubespray_defaults - role: container-engine/containerd ================================================ FILE: roles/container-engine/containerd/molecule/default/molecule.yml ================================================ --- role_name_check: 1 platforms: - cloud_image: ubuntu-2404 name: ubuntu24 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster - cloud_image: debian-12 name: debian12 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster - cloud_image: almalinux-9 name: almalinux9 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: name: ansible ================================================ FILE: roles/container-engine/containerd/molecule/default/verify.yml ================================================ --- - name: Test containerd CRI import_playbook: ../../../molecule/test_cri.yml vars: container_manager: containerd cri_socket: unix:///var/run/containerd/containerd.sock cri_name: containerd - name: Test nerdctl hosts: all gather_facts: false become: true tasks: - name: Get kubespray defaults import_role: name: ../../../../../kubespray_defaults - name: Test nerdctl commands command: "{{ bin_dir }}/nerdctl {{ item | join(' ') }}" vars: image: quay.io/kubespray/hello-world:latest loop: - - pull - "{{ image }}" - - save - -o - /tmp/hello-world.tar - "{{ image }}" - - load - -i - /tmp/hello-world.tar - - -n - k8s.io - run - "{{ image }}" register: nerdctl - name: Check log from running a container assert: that: - ('Hello from Docker' in nerdctl.results[3].stdout) ================================================ FILE: roles/container-engine/containerd/tasks/main.yml ================================================ --- - name: Containerd | Download containerd include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.containerd) }}" - name: Containerd | Unpack containerd archive unarchive: src: "{{ downloads.containerd.dest }}" dest: "{{ containerd_bin_dir }}" mode: "0755" remote_src: true extra_opts: - --strip-components=1 notify: Restart containerd - name: Containerd | Generate systemd service for containerd template: src: containerd.service.j2 dest: /etc/systemd/system/containerd.service mode: "0644" validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:containerd.service'" # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) # Remove once we drop support for systemd < 250 notify: Restart containerd - name: Containerd | Ensure containerd directories exist file: dest: "{{ item }}" state: directory mode: "0755" owner: root group: root with_items: - "{{ containerd_systemd_dir }}" - "{{ containerd_cfg_dir }}" - name: Containerd | Write containerd proxy drop-in template: src: http-proxy.conf.j2 dest: "{{ containerd_systemd_dir }}/http-proxy.conf" mode: "0644" notify: Restart containerd when: http_proxy is defined or https_proxy is defined - name: Containerd | Generate default base_runtime_spec register: ctr_oci_spec command: "{{ containerd_bin_dir }}/ctr oci spec" check_mode: false changed_when: false - name: Containerd | Store generated default base_runtime_spec set_fact: containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}" - name: Containerd | Write base_runtime_specs copy: content: "{{ item.value }}" dest: "{{ containerd_cfg_dir }}/{{ item.key }}" owner: "root" mode: "0644" with_dict: "{{ containerd_base_runtime_specs | default({}) }}" notify: Restart containerd - name: Containerd | Copy containerd config file template: src: "{{ 'config.toml.j2' if containerd_version is version('2.0.0', '>=') else 'config-v1.toml.j2' }}" dest: "{{ containerd_cfg_dir }}/config.toml" owner: "root" mode: "0640" notify: Restart containerd - name: Containerd | Configure containerd registries # mirror configuration can contain sensitive information on headers configuration no_log: "{{ not (unsafe_show_logs | bool) }}" block: - name: Containerd | Create registry directories file: path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}" state: directory mode: "0755" loop: "{{ containerd_registries_mirrors }}" - name: Containerd | Write hosts.toml file template: src: hosts.toml.j2 dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml" mode: "0640" loop: "{{ containerd_registries_mirrors }}" # you can sometimes end up in a state where everything is installed # but containerd was not started / enabled - name: Containerd | Flush handlers meta: flush_handlers - name: Containerd | Ensure containerd is started and enabled systemd_service: name: containerd daemon_reload: true enabled: true state: started ================================================ FILE: roles/container-engine/containerd/tasks/reset.yml ================================================ --- - name: Containerd | Stop containerd service service: name: containerd daemon_reload: true enabled: false state: stopped tags: - reset_containerd - name: Containerd | Remove configuration files file: path: "{{ item }}" state: absent loop: - /etc/systemd/system/containerd.service - "{{ containerd_systemd_dir }}" - "{{ containerd_cfg_dir }}" - "{{ containerd_storage_dir }}" - "{{ containerd_state_dir }}" tags: - reset_containerd ================================================ FILE: roles/container-engine/containerd/templates/config-v1.toml.j2 ================================================ # This is for containerd v1 for compatibility version = 2 root = "{{ containerd_storage_dir }}" state = "{{ containerd_state_dir }}" oom_score = {{ containerd_oom_score }} {% if containerd_extra_args is defined %} {{ containerd_extra_args }} {% endif %} [grpc] max_recv_message_size = {{ containerd_grpc_max_recv_message_size }} max_send_message_size = {{ containerd_grpc_max_send_message_size }} [debug] address = "{{ containerd_debug_address }}" level = "{{ containerd_debug_level }}" format = "{{ containerd_debug_format }}" uid = {{ containerd_debug_uid }} gid = {{ containerd_debug_gid }} [metrics] address = "{{ containerd_metrics_address }}" grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }} [plugins] [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" max_container_log_line_size = {{ containerd_max_container_log_line_size }} enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }} enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }} enable_selinux = {{ containerd_enable_selinux | lower }} disable_apparmor = {{ containerd_disable_apparmor | lower }} tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }} disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }} image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}" {% if enable_cdi %} enable_cdi = true cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] {% endif %} [plugins."io.containerd.grpc.v1.cri".containerd] default_runtime_name = "{{ containerd_default_runtime }}" snapshotter = "{{ containerd_snapshotter }}" discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] {% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}] runtime_type = "{{ runtime.type }}" runtime_engine = "{{ runtime.engine }}" runtime_root = "{{ runtime.root }}" {% if runtime.base_runtime_spec is defined %} base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}" {% endif %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options] {% for key, value in runtime.options.items() %} {% if value | string != "true" and value | string != "false" %} {{ key }} = "{{ value }}" {% else %} {{ key }} = {{ value }} {% endif %} {% endfor %} {% endfor %} {% if kata_containers_enabled %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu] runtime_type = "io.containerd.kata-qemu.v2" {% endif %} {% if gvisor_enabled %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] runtime_type = "io.containerd.runsc.v1" {% endif %} [plugins."io.containerd.grpc.v1.cri".registry] config_path = "{{ containerd_cfg_dir }}/certs.d" {% for registry in containerd_registry_auth if registry['registry'] is defined %} {% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %} [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth] {% if registry['username'] is defined and registry['password'] is defined %} password = "{{ registry['password'] }}" username = "{{ registry['username'] }}" {% else %} auth = "{{ registry['auth'] }}" {% endif %} {% endif %} {% endfor %} {% if nri_enabled and containerd_version is version('1.7.0', '>=') %} [plugins."io.containerd.nri.v1.nri"] disable = false {% endif %} {% if containerd_tracing_enabled %} [plugins."io.containerd.tracing.processor.v1.otlp"] endpoint = "{{ containerd_tracing_endpoint }}" protocol = "{{ containerd_tracing_protocol }}" {% if containerd_tracing_protocol == "grpc" %} insecure = false {% endif %} [plugins."io.containerd.internal.v1.tracing"] sampling_ratio = {{ containerd_tracing_sampling_ratio }} service_name = "{{ containerd_tracing_service_name }}" {% endif %} ================================================ FILE: roles/container-engine/containerd/templates/config.toml.j2 ================================================ version = 3 root = "{{ containerd_storage_dir }}" state = "{{ containerd_state_dir }}" oom_score = {{ containerd_oom_score }} {% if containerd_extra_args is defined %} {{ containerd_extra_args }} {% endif %} [grpc] max_recv_message_size = {{ containerd_grpc_max_recv_message_size }} max_send_message_size = {{ containerd_grpc_max_send_message_size }} [debug] address = "{{ containerd_debug_address }}" level = "{{ containerd_debug_level }}" format = "{{ containerd_debug_format }}" uid = {{ containerd_debug_uid }} gid = {{ containerd_debug_gid }} [metrics] address = "{{ containerd_metrics_address }}" grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }} [plugins] [plugins."io.containerd.cri.v1.runtime"] max_container_log_line_size = {{ containerd_max_container_log_line_size }} enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }} enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }} enable_selinux = {{ containerd_enable_selinux | lower }} disable_apparmor = {{ containerd_disable_apparmor | lower }} tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }} disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }} {% if enable_cdi %} enable_cdi = true cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] {% endif %} {% for key, value in containerd_extra_runtime_args.items() %} {% if value is string %} {{ key }} = "{{ value }}" {% elif value is boolean %} {{ key }} = {{ value | lower }} {% else %} {{ key }} = {{ value }} {% endif %} {% endfor %} [plugins."io.containerd.cri.v1.runtime".containerd] default_runtime_name = "{{ containerd_default_runtime }}" [plugins."io.containerd.cri.v1.runtime".containerd.runtimes] {% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %} [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}] runtime_type = "{{ runtime.type }}" {% if runtime.base_runtime_spec is defined %} base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}" {% endif %} [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}.options] {% for key, value in runtime.options.items() %} {% if value | string != "true" and value | string != "false" %} {{ key }} = "{{ value }}" {% else %} {{ key }} = {{ value }} {% endif %} {% endfor %} {% endfor %} {% if kata_containers_enabled %} [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-qemu] runtime_type = "io.containerd.kata-qemu.v2" {% endif %} {% if gvisor_enabled %} [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runsc] runtime_type = "io.containerd.runsc.v1" {% endif %} [plugins."io.containerd.cri.v1.images"] snapshotter = "{{ containerd_snapshotter }}" {% if containerd_discard_unpacked_layers and containerd_version is version('2.1.0', '<') %} discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }} {% endif %} image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}" [plugins."io.containerd.cri.v1.images".pinned_images] sandbox = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" [plugins."io.containerd.cri.v1.images".registry] config_path = "{{ containerd_cfg_dir }}/certs.d" [plugins."io.containerd.nri.v1.nri"] disable = {{ 'false' if nri_enabled else 'true' }} {% if containerd_tracing_enabled %} [plugins."io.containerd.tracing.processor.v1.otlp"] endpoint = "{{ containerd_tracing_endpoint }}" protocol = "{{ containerd_tracing_protocol }}" {% if containerd_tracing_protocol == "grpc" %} insecure = false {% endif %} [plugins."io.containerd.internal.v1.tracing"] sampling_ratio = {{ containerd_tracing_sampling_ratio }} service_name = "{{ containerd_tracing_service_name }}" {% endif %} ================================================ FILE: roles/container-engine/containerd/templates/containerd.service.j2 ================================================ # Copyright The containerd Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. [Unit] Description=containerd container runtime Documentation=https://containerd.io After=network.target local-fs.target dbus.service [Service] ExecStartPre=-/sbin/modprobe overlay ExecStart={{ containerd_bin_dir }}/containerd Type=notify Delegate=yes KillMode=process Restart=always RestartSec=5 # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNPROC={{ containerd_limit_proc_num }} LimitCORE={{ containerd_limit_core }} LimitNOFILE={{ containerd_limit_open_file_num }} LimitMEMLOCK={{ containerd_limit_mem_lock }} # Comment TasksMax if your systemd version does not supports it. # Only systemd 226 and above support this version. TasksMax=infinity OOMScoreAdjust=-999 # Set the cgroup slice of the service so that kube reserved takes effect {% if kube_reserved is defined and kube_reserved|bool %} Slice={{ kube_reserved_cgroups_for_service_slice }} {% endif %} [Install] WantedBy=multi-user.target ================================================ FILE: roles/container-engine/containerd/templates/hosts.toml.j2 ================================================ server = "{{ item.server | default("https://" + item.prefix) }}" {% for mirror in item.mirrors %} [host."{{ mirror.host }}"] capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"] skip_verify = {{ mirror.skip_verify | default('false') | string | lower }} override_path = {{ mirror.override_path | default('false') | string | lower }} {% if mirror.ca is defined %} ca = ["{{ ([ mirror.ca ] | flatten ) | join('","') }}"] {% endif %} {% if mirror.client is defined %} client = [{% for pair in mirror.client %}["{{ pair[0] }}", "{{ pair[1] }}"]{% if not loop.last %},{% endif %}{% endfor %}] {% endif %} {% if mirror.header is defined %} [host."{{ mirror.host }}".header] {% for key, value in mirror.header.items() %} {{ key }} = ["{{ ([ value ] | flatten ) | join('","') }}"] {% endfor %} {% endif %} {% endfor %} ================================================ FILE: roles/container-engine/containerd/templates/http-proxy.conf.j2 ================================================ [Service] Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} ================================================ FILE: roles/container-engine/containerd-common/defaults/main.yml ================================================ --- # We keep these variables around to allow migration from package # manager controlled installs to direct download ones. containerd_package: 'containerd.io' yum_repo_dir: /etc/yum.repos.d ================================================ FILE: roles/container-engine/containerd-common/meta/main.yml ================================================ --- allow_duplicates: true ================================================ FILE: roles/container-engine/containerd-common/tasks/main.yml ================================================ --- - name: Containerd-common | check if fedora coreos stat: path: /run/ostree-booted get_attributes: false get_checksum: false get_mime: false register: ostree - name: Containerd-common | set is_ostree set_fact: is_ostree: "{{ ostree.stat.exists }}" - name: Containerd-common | gather os specific variables include_vars: "{{ item }}" with_first_found: - files: - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ host_architecture }}.yml" - "{{ ansible_distribution | lower }}.yml" - "{{ ansible_os_family | lower }}-{{ host_architecture }}.yml" - "{{ ansible_os_family | lower }}.yml" - defaults.yml paths: - ../vars skip: true tags: - facts ================================================ FILE: roles/container-engine/containerd-common/vars/amazon.yml ================================================ --- containerd_package: containerd ================================================ FILE: roles/container-engine/containerd-common/vars/suse.yml ================================================ --- containerd_package: containerd ================================================ FILE: roles/container-engine/cri-dockerd/defaults/main.yml ================================================ --- # Default is "info" (like if not provided). Possible values are any log level string parseable by logrus cri_dockerd_log_level: "info" ================================================ FILE: roles/container-engine/cri-dockerd/handlers/main.yml ================================================ --- - name: Cri-dockerd | reload systemd systemd_service: name: cri-dockerd daemon_reload: true masked: false listen: Restart and enable cri-dockerd - name: Cri-dockerd | reload cri-dockerd.socket service: name: cri-dockerd.socket state: restarted listen: Restart and enable cri-dockerd - name: Cri-dockerd | reload cri-dockerd.service service: name: cri-dockerd.service state: restarted listen: Restart and enable cri-dockerd - name: Cri-dockerd | enable cri-dockerd service service: name: cri-dockerd.service enabled: true listen: Restart and enable cri-dockerd ================================================ FILE: roles/container-engine/cri-dockerd/meta/main.yml ================================================ --- dependencies: - role: container-engine/docker - role: container-engine/crictl ================================================ FILE: roles/container-engine/cri-dockerd/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true vars: container_manager: docker roles: - role: kubespray_defaults - role: container-engine/cri-dockerd ================================================ FILE: roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf ================================================ { "cniVersion": "0.2.0", "name": "mynet", "type": "bridge", "bridge": "cni0", "isGateway": true, "ipMasq": true, "ipam": { "type": "host-local", "subnet": "172.19.0.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } } ================================================ FILE: roles/container-engine/cri-dockerd/molecule/default/files/container.json ================================================ { "metadata": { "name": "cri-dockerd1" }, "image": { "image": "quay.io/kubespray/hello-world:latest" }, "log_path": "cri-dockerd1.0.log", "linux": {} } ================================================ FILE: roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json ================================================ { "metadata": { "name": "cri-dockerd1", "namespace": "default", "attempt": 1, "uid": "hdishd83djaidwnduwk28bcsb" }, "linux": {}, "log_directory": "/tmp" } ================================================ FILE: roles/container-engine/cri-dockerd/molecule/default/molecule.yml ================================================ --- role_name_check: 1 platforms: - name: almalinux9 cloud_image: almalinux-9 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - name: ubuntu22 cloud_image: ubuntu-2204 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 inventory: group_vars: all: become: true k8s_cluster: container_manager: docker playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: name: ansible ================================================ FILE: roles/container-engine/cri-dockerd/molecule/default/verify.yml ================================================ --- - name: Test cri-dockerd import_playbook: ../../../molecule/test_cri.yml vars: container_manager: cri-dockerd cri_socket: unix:///var/run/cri-dockerd.sock cri_name: docker - name: Test running a container with docker import_playbook: ../../../molecule/test_runtime.yml vars: container_runtime: docker # cri-dockerd does not support multiple runtime handler before 0.4.0 # https://github.com/Mirantis/cri-dockerd/pull/350 # TODO: check this when we upgrade cri-dockerd ================================================ FILE: roles/container-engine/cri-dockerd/tasks/main.yml ================================================ --- - name: Runc | Download cri-dockerd binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.cri_dockerd) }}" - name: Copy cri-dockerd binary from download dir copy: src: "{{ local_release_dir }}/cri-dockerd" dest: "{{ bin_dir }}/cri-dockerd" mode: "0755" remote_src: true notify: - Restart and enable cri-dockerd - name: Generate cri-dockerd systemd unit files template: src: "{{ item }}.j2" dest: "/etc/systemd/system/{{ item }}" mode: "0644" validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{ item }}'" # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) # Remove once we drop support for systemd < 250 with_items: - cri-dockerd.service - cri-dockerd.socket notify: - Restart and enable cri-dockerd - name: Flush handlers meta: flush_handlers ================================================ FILE: roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 ================================================ [Unit] Description=CRI Interface for Docker Application Container Engine Documentation=https://docs.mirantis.com After=network-online.target firewalld.service docker.service Wants=network-online.target docker.service Requires=cri-dockerd.socket [Service] Type=notify ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnets }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if ipv6_stack %}--ipv6-dual-stack=True{% endif %} ExecReload=/bin/kill -s HUP $MAINPID TimeoutSec=0 RestartSec=2 Restart=always # Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. # Both the old, and new location are accepted by systemd 229 and up, so using the old location # to make them work for either version of systemd. StartLimitBurst=3 # Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. # Both the old, and new name are accepted by systemd 230 and up, so using the old name to make # this option work for either version of systemd. StartLimitInterval=60s # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNOFILE=infinity LimitNPROC=infinity LimitCORE=infinity # Comment TasksMax if your systemd version does not support it. # Only systemd 226 and above support this option. TasksMax=infinity Delegate=yes KillMode=process # Set the cgroup slice of the service so that kube reserved takes effect {% if kube_reserved is defined and kube_reserved|bool %} Slice={{ kube_reserved_cgroups_for_service_slice }} {% endif %} [Install] WantedBy=multi-user.target ================================================ FILE: roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 ================================================ [Unit] Description=CRI Docker Socket for the API PartOf=cri-dockerd.service [Socket] ListenStream=%t/cri-dockerd.sock SocketMode=0660 SocketUser=root SocketGroup=docker [Install] WantedBy=sockets.target ================================================ FILE: roles/container-engine/cri-o/defaults/main.yml ================================================ --- crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}" crio_conmon: "{{ bin_dir }}/conmon" crio_default_runtime: "crun" crio_libexec_dir: "/usr/libexec/crio" crio_runtime_switch: false crio_enable_metrics: false crio_log_level: "info" crio_metrics_port: "9090" crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}" # Registries defined within cri-o. # By default unqualified images are not allowed for security reasons crio_registries: [] # - prefix: docker.io # insecure: false # blocked: false # location: registry-1.docker.io ## REQUIRED # unqualified: false # mirrors: # - location: 172.20.100.52:5000 # insecure: true # - location: mirror.gcr.io # insecure: false crio_registry_auth: [] # - registry: 10.0.0.2:5000 # username: user # password: pass crio_seccomp_profile: "" crio_selinux: "{{ (preinstall_selinux_state == 'enforcing') | lower }}" crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}" # Set the pull progress timeout crio_pull_progress_timeout: "10s" # Override system default for storage driver # crio_storage_driver: "overlay" crio_stream_port: "10010" crio_required_version: "{{ kube_version | regex_replace('^(?P\\d+).(?P\\d+).(?P\\d+)$', '\\g.\\g') }}" crio_root: "/var/lib/containers/storage" # The crio_runtimes variable defines a list of OCI compatible runtimes. crio_runtimes: - name: crun path: "{{ crio_runtime_bin_dir }}/crun" # Use crun in cri-o distributions, don't use 'crun' role type: oci root: /run/crun # Kata Containers is an OCI runtime, where containers are run inside lightweight # VMs. Kata provides additional isolation towards the host, minimizing the host attack # surface and mitigating the consequences of containers breakout. kata_runtimes: # Kata Containers with the default configured VMM - name: kata-qemu path: /usr/local/bin/containerd-shim-kata-qemu-v2 type: vm root: /run/kata-containers privileged_without_host_devices: true runc_runtime: name: runc path: "{{ crio_runtime_bin_dir }}/runc" type: oci root: /run/runc # crun is a fast and low-memory footprint OCI Container Runtime fully written in C. crun_runtime: name: crun path: "{{ crio_runtime_bin_dir }}/crun" type: oci root: /run/crun # youki is an implementation of the OCI runtime-spec in Rust, similar to runc. youki_runtime: name: youki path: "{{ youki_bin_dir }}/youki" type: oci root: /run/youki # Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids) # at the end of the uid/gid space crio_remap_enable: false crio_remap_user: containers crio_subuid_start: 2130706432 crio_subuid_length: 16777216 crio_subgid_start: 2130706432 crio_subgid_length: 16777216 # cri-o manual files crio_man_files: 5: - crio.conf - crio.conf.d 8: - crio - crio-status # If set to true, it will enable the CRIU support in cri-o crio_criu_support_enabled: false # Configure default_capabilities in crio.conf crio_default_capabilities: - CHOWN - DAC_OVERRIDE - FSETID - FOWNER - SETGID - SETUID - SETPCAP - NET_BIND_SERVICE - KILL crio_additional_mounts: [] ================================================ FILE: roles/container-engine/cri-o/handlers/main.yml ================================================ --- - name: CRI-O | reload systemd systemd_service: daemon_reload: true listen: Restart crio - name: CRI-O | reload crio service: name: crio state: restarted enabled: true listen: Restart crio ================================================ FILE: roles/container-engine/cri-o/meta/main.yml ================================================ --- dependencies: - role: container-engine/crictl - role: container-engine/skopeo ================================================ FILE: roles/container-engine/cri-o/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true roles: - role: kubespray_defaults - role: container-engine/cri-o ================================================ FILE: roles/container-engine/cri-o/molecule/default/molecule.yml ================================================ --- role_name_check: 1 platforms: - name: ubuntu22 cloud_image: ubuntu-2204 vm_cpu_cores: 2 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster - name: almalinux9 cloud_image: almalinux-9 vm_cpu_cores: 2 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster - name: fedora cloud_image: fedora-39 vm_cpu_cores: 2 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster - name: debian12 cloud_image: debian-12 vm_cpu_cores: 2 vm_memory: 1024 node_groups: - kube_control_plane - kube_node - k8s_cluster provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 inventory: group_vars: k8s_cluster: container_manager: crio playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: name: ansible ================================================ FILE: roles/container-engine/cri-o/molecule/default/verify.yml ================================================ --- - name: Test CRI-O cri import_playbook: ../../../molecule/test_cri.yml vars: cri_socket: unix:///var/run/crio/crio.sock cri_name: cri-o - name: Test running a container with crun import_playbook: ../../../molecule/test_runtime.yml vars: container_runtime: crun ================================================ FILE: roles/container-engine/cri-o/tasks/load_vars.yml ================================================ --- - name: Cri-o | include vars/v1.29.yml include_vars: v1.29.yml when: crio_version is version("1.29.0", operator=">=") - name: Cri-o | include vars/v1.31.yml include_vars: v1.31.yml when: crio_version is version("1.31.0", operator=">=") ================================================ FILE: roles/container-engine/cri-o/tasks/main.yaml ================================================ --- - name: Cri-o | load vars import_tasks: load_vars.yml - name: Cri-o | check if fedora coreos stat: path: /run/ostree-booted get_attributes: false get_checksum: false get_mime: false register: ostree - name: Cri-o | set is_ostree set_fact: is_ostree: "{{ ostree.stat.exists }}" - name: Cri-o | get ostree version shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'" args: executable: /bin/bash register: ostree_version when: is_ostree - name: Cri-o | Download cri-o include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.crio) }}" - name: Cri-o | special handling for amazon linux import_tasks: "setup-amazon.yaml" when: ansible_distribution in ["Amazon"] - name: Cri-o | build a list of crio runtimes with Katacontainers runtimes set_fact: crio_runtimes: "{{ crio_runtimes + kata_runtimes }}" when: - kata_containers_enabled ## After CRI-O v1.31, crun is default runtime. # - name: Cri-o | build a list of crio runtimes with crun runtime # set_fact: # crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}" # when: # - crun_enabled - name: Cri-o | build a list of crio runtimes with runc runtime set_fact: crio_runtimes: "{{ crio_runtimes + [runc_runtime] }}" when: - runc_enabled - name: Cri-o | build a list of crio runtimes with youki runtime set_fact: crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}" when: - youki_enabled - name: Cri-o | Stop kubelet service if running service: name: kubelet state: stopped when: - crio_runtime_switch - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service'].state == 'running' - name: Cri-o | Get all pods ansible.builtin.command: "{{ bin_dir }}/crictl pods -o json" changed_when: false register: crio_pods when: - crio_runtime_switch - ansible_facts.services['crio.service'] is defined - name: Cri-o | Stop and remove pods not on host network ansible.builtin.command: "{{ bin_dir }}/crictl rmp -f {{ item.id }}" loop: "{{ (crio_pods.stdout | from_json).items | default([]) | selectattr('metadata.namespace', 'ne', 'NODE') }}" changed_when: true when: - crio_runtime_switch - ansible_facts.services['crio.service'] is defined - crio_pods.stdout is defined - name: Cri-o | Stop and remove all remaining pods ansible.builtin.command: "{{ bin_dir }}/crictl rmp -fa" changed_when: true when: - crio_runtime_switch - ansible_facts.services['crio.service'] is defined - name: Cri-o | stop crio service if running service: name: crio state: stopped when: - crio_runtime_switch - ansible_facts.services['crio.service'] is defined and ansible_facts.services['crio.service'].state == 'running' - name: Cri-o | make sure needed folders exist in the system with_items: - /etc/crio - /etc/containers - /etc/systemd/system/crio.service.d file: path: "{{ item }}" state: directory mode: "0755" - name: Cri-o | install cri-o config template: src: crio.conf.j2 dest: /etc/crio/crio.conf mode: "0644" register: config_install - name: Cri-o | install config.json template: src: config.json.j2 dest: /etc/crio/config.json mode: "0644" register: reg_auth_install - name: Cri-o | copy binaries copy: src: "{{ local_release_dir }}/cri-o/bin/{{ item }}" dest: "{{ bin_dir }}/{{ item }}" mode: "0755" remote_src: true with_items: - "{{ crio_bin_files }}" notify: Restart crio - name: Cri-o | create directory for libexec file: path: "{{ crio_libexec_dir }}" state: directory owner: root mode: "0755" - name: Cri-o | copy libexec copy: src: "{{ local_release_dir }}/cri-o/bin/{{ item }}" dest: "{{ crio_libexec_dir }}/{{ item }}" mode: "0755" remote_src: true with_items: - "{{ crio_libexec_files }}" notify: Restart crio - name: Cri-o | copy service file copy: src: "{{ local_release_dir }}/cri-o/contrib/crio.service" dest: /etc/systemd/system/crio.service mode: "0755" remote_src: true notify: Restart crio - name: Cri-o | configure crio to use kube reserved cgroups ansible.builtin.copy: dest: /etc/systemd/system/crio.service.d/00-slice.conf owner: root group: root mode: '0644' content: | [Service] Slice={{ kube_reserved_cgroups_for_service_slice }} notify: Restart crio when: - kube_reserved is defined and kube_reserved is true - kube_reserved_cgroups_for_service_slice is defined - name: Cri-o | update the bin dir for crio.service file replace: dest: /etc/systemd/system/crio.service regexp: "/usr/local/bin/crio" replace: "{{ bin_dir }}/crio" notify: Restart crio - name: Cri-o | copy default policy copy: src: "{{ local_release_dir }}/cri-o/contrib/policy.json" dest: /etc/containers/policy.json mode: "0755" remote_src: true notify: Restart crio - name: Cri-o | copy mounts.conf template: src: mounts.conf.j2 dest: /etc/containers/mounts.conf mode: "0644" when: - ansible_os_family == 'RedHat' notify: Restart crio - name: Cri-o | create directory for oci hooks file: path: /etc/containers/oci/hooks.d state: directory owner: root mode: "0755" - name: Cri-o | set overlay driver community.general.ini_file: dest: /etc/containers/storage.conf section: storage option: "{{ item.option }}" value: "{{ item.value }}" mode: "0644" with_items: - option: driver value: '"overlay"' - option: graphroot value: '"/var/lib/containers/storage"' - option: runroot value: '"/var/run/containers/storage"' # metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel - name: Cri-o | set metacopy mount options correctly community.general.ini_file: dest: /etc/containers/storage.conf section: storage.options.overlay option: mountopt value: '{{ ''"nodev"'' if ansible_kernel is version(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}' mode: "0644" - name: Cri-o | create directory registries configs file: path: /etc/containers/registries.conf.d state: directory owner: root mode: "0755" - name: Cri-o | write registries configs template: src: registry.conf.j2 dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':|/', '_') }}.conf" mode: "0644" loop: "{{ crio_registries }}" notify: Restart crio - name: Cri-o | configure unqualified registry settings template: src: unqualified.conf.j2 dest: "/etc/containers/registries.conf.d/01-unqualified.conf" mode: "0644" notify: Restart crio - name: Cri-o | write cri-o proxy drop-in template: src: http-proxy.conf.j2 dest: /etc/systemd/system/crio.service.d/http-proxy.conf mode: "0644" notify: Restart crio when: http_proxy is defined or https_proxy is defined - name: Cri-o | configure the uid/gid space for user namespaces lineinfile: path: '{{ item.path }}' line: '{{ item.entry }}' regex: '^\s*{{ crio_remap_user }}:' state: '{{ "present" if crio_remap_enable | bool else "absent" }}' loop: - path: /etc/subuid entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}' - path: /etc/subgid entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}' loop_control: label: '{{ item.path }}' - name: Cri-o | ensure crio service is started and enabled service: name: crio daemon_reload: true enabled: true state: started register: service_start - name: Cri-o | trigger service restart only when needed service: name: crio state: restarted when: - config_install.changed or reg_auth_install.changed - not service_start.changed - name: Cri-o | verify that crio is running command: "{{ bin_dir }}/{{ crio_status_command }} info" register: get_crio_info until: get_crio_info is succeeded changed_when: false retries: 5 delay: "{{ retry_stagger | random + 3 }}" # The kubelet service status can be 'not-found' if something depends on it. # This check prevents failures when the service is in this indeterminate state, # which can occur when adding new nodes to a cluster. # See: https://superuser.com/questions/1755211/cleaning-debugging-services/1755215#1755215 - name: Cri-o | ensure kubelet service is started if present and stopped service: name: kubelet state: started when: - crio_runtime_switch - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service']['status'] != 'not-found' ================================================ FILE: roles/container-engine/cri-o/tasks/reset.yml ================================================ --- - name: Cri-o | load vars import_tasks: load_vars.yml - name: CRI-O | Kubic repo name for debian os family set_fact: crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x', '')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}" when: ansible_os_family == "Debian" tags: - reset_crio - name: CRI-O | Remove kubic apt repo apt_repository: repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /" state: absent when: crio_kubic_debian_repo_name is defined tags: - reset_crio - name: CRI-O | Remove cri-o apt repo apt_repository: repo: "deb {{ crio_download_crio }}v{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /" state: absent filename: devel-kubic-libcontainers-stable-cri-o when: crio_kubic_debian_repo_name is defined tags: - reset_crio - name: CRI-O | Remove CRI-O kubic yum repo yum_repository: name: devel_kubic_libcontainers_stable state: absent when: ansible_distribution in ["Amazon"] tags: - reset_crio - name: CRI-O | Remove CRI-O kubic yum repo yum_repository: name: "devel_kubic_libcontainers_stable_cri-o_v{{ crio_version }}" state: absent when: - ansible_os_family == "RedHat" - ansible_distribution not in ["Amazon", "Fedora"] tags: - reset_crio - name: CRI-O | Run yum-clean-metadata command: yum clean metadata when: - ansible_os_family == "RedHat" tags: - reset_crio - name: CRI-O | Remove crictl file: name: "{{ item }}" state: absent loop: - /etc/crictl.yaml - "{{ bin_dir }}/crictl" tags: - reset_crio - name: CRI-O | Stop crio service service: name: crio daemon_reload: true enabled: false state: stopped tags: - reset_crio - name: CRI-O | Remove CRI-O configuration files file: name: "{{ item }}" state: absent loop: - /etc/crio - /etc/containers - /etc/systemd/system/crio.service.d tags: - reset_crio - name: CRI-O | Remove CRI-O binaries file: name: "{{ item }}" state: absent with_items: "{{ crio_bin_files }}" tags: - reset_crio - name: CRI-O | Remove CRI-O libexec file: name: "{{ item }}" state: absent with_items: "{{ crio_libexec_files }}" tags: - reset_crio ================================================ FILE: roles/container-engine/cri-o/tasks/setup-amazon.yaml ================================================ --- - name: Check that amzn2-extras.repo exists stat: path: /etc/yum.repos.d/amzn2-extras.repo register: amzn2_extras_file_stat - name: Find docker repo in amzn2-extras.repo file lineinfile: dest: /etc/yum.repos.d/amzn2-extras.repo line: "[amzn2extra-docker]" check_mode: true register: amzn2_extras_docker_repo when: - amzn2_extras_file_stat.stat.exists - name: Remove docker repository community.general.ini_file: dest: /etc/yum.repos.d/amzn2-extras.repo section: amzn2extra-docker option: enabled value: "0" backup: true mode: "0644" when: - amzn2_extras_file_stat.stat.exists - not amzn2_extras_docker_repo.changed ================================================ FILE: roles/container-engine/cri-o/templates/config.json.j2 ================================================ {% if crio_registry_auth is defined and crio_registry_auth|length %} { "auths": { {% for reg in crio_registry_auth %} "{{ reg.registry }}": { "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}" {% if not loop.last %} }, {% else %} } {% endif %} {% endfor %} } } {% else %} {} {% endif %} ================================================ FILE: roles/container-engine/cri-o/templates/crio.conf.j2 ================================================ # The CRI-O configuration file specifies all of the available configuration # options and command-line flags for the crio(8) OCI Kubernetes Container Runtime # daemon, but in a TOML format that can be more easily modified and versioned. # # Please refer to crio.conf(5) for details of all configuration options. # CRI-O supports partial configuration reload during runtime, which can be # done by sending SIGHUP to the running process. Currently supported options # are explicitly mentioned with: 'This option supports live configuration # reload'. # CRI-O reads its storage defaults from the containers-storage.conf(5) file # located at /etc/containers/storage.conf. Modify this storage configuration if # you want to change the system's defaults. If you want to modify storage just # for CRI-O, you can change the storage configuration options here. [crio] # Path to the "root directory". CRI-O stores all of its data, including # containers images, in this directory. root = "{{ crio_root }}" # Path to the "run directory". CRI-O stores all of its state in this directory. # Read from /etc/containers/storage.conf first so unnecessary here # runroot = "/var/run/containers/storage" # Storage driver used to manage the storage of images and containers. Please # refer to containers-storage.conf(5) to see all available storage drivers. {% if crio_storage_driver is defined %} storage_driver = "{{ crio_storage_driver }}" {% endif %} # List to pass options to the storage driver. Please refer to # containers-storage.conf(5) to see all available storage options. #storage_option = [ #] # The default log directory where all logs will go unless directly specified by # the kubelet. The log directory specified must be an absolute directory. log_dir = "/var/log/crio/pods" # Location for CRI-O to lay down the temporary version file. # It is used to check if crio wipe should wipe containers, which should # always happen on a node reboot version_file = "/var/run/crio/version" # Location for CRI-O to lay down the persistent version file. # It is used to check if crio wipe should wipe images, which should # only happen when CRI-O has been upgraded version_file_persist = "/var/lib/crio/version" # The crio.api table contains settings for the kubelet/gRPC interface. [crio.api] # Path to AF_LOCAL socket on which CRI-O will listen. listen = "/var/run/crio/crio.sock" # IP address on which the stream server will listen. stream_address = "127.0.0.1" # The port on which the stream server will listen. If the port is set to "0", then # CRI-O will allocate a random free port number. stream_port = "{{ crio_stream_port }}" # Enable encrypted TLS transport of the stream server. stream_enable_tls = false # Path to the x509 certificate file used to serve the encrypted stream. This # file can change, and CRI-O will automatically pick up the changes within 5 # minutes. stream_tls_cert = "" # Path to the key file used to serve the encrypted stream. This file can # change and CRI-O will automatically pick up the changes within 5 minutes. stream_tls_key = "" # Path to the x509 CA(s) file used to verify and authenticate client # communication with the encrypted stream. This file can change and CRI-O will # automatically pick up the changes within 5 minutes. stream_tls_ca = "" # Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. grpc_max_send_msg_size = 16777216 # Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. grpc_max_recv_msg_size = 16777216 # The crio.runtime table contains settings pertaining to the OCI runtime used # and options for how to set up and manage the OCI runtime. [crio.runtime] # A list of ulimits to be set in containers by default, specified as # "=:", for example: # "nofile=1024:2048" # If nothing is set here, settings will be inherited from the CRI-O daemon #default_ulimits = [ #] # default_runtime is the _name_ of the OCI runtime to be used as the default. # The name is matched against the runtimes map below. default_runtime = "{{ crio_default_runtime }}" # If true, the runtime will not use pivot_root, but instead use MS_MOVE. no_pivot = false # decryption_keys_path is the path where the keys required for # image decryption are stored. This option supports live configuration reload. decryption_keys_path = "/etc/crio/keys/" # Path to the conmon binary, used for monitoring the OCI runtime. # Will be searched for using $PATH if empty. conmon = "{{ crio_conmon }}" # Cgroup setting for conmon {% if crio_cgroup_manager == "cgroupfs" %} conmon_cgroup = "pod" {% else %} {% if kube_reserved is defined and kube_reserved|bool %} conmon_cgroup = "{{ kube_reserved_cgroups_for_service_slice }}" {% else %} conmon_cgroup = "system.slice" {% endif %} {% endif %} # Environment variable list for the conmon process, used for passing necessary # environment variables to conmon or the runtime. conmon_env = [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", ] # Additional environment variables to set for all the # containers. These are overridden if set in the # container image spec or in the container runtime configuration. default_env = [ ] # If true, SELinux will be used for pod separation on the host. selinux = {{ crio_selinux }} # Path to the seccomp.json profile which is used as the default seccomp profile # for the runtime. If not specified, then the internal default seccomp profile # will be used. This option supports live configuration reload. seccomp_profile = "{{ crio_seccomp_profile }}" # Used to change the name of the default AppArmor profile of CRI-O. The default # profile name is "crio-default". This profile only takes effect if the user # does not specify a profile via the Kubernetes Pod's metadata annotation. If # the profile is set to "unconfined", then this equals to disabling AppArmor. # This option supports live configuration reload. # apparmor_profile = "crio-default" # Cgroup management implementation used for the runtime. cgroup_manager = "{{ crio_cgroup_manager }}" # List of default capabilities for containers. If it is empty or commented out, # only the capabilities defined in the containers json file by the user/kube # will be added. default_capabilities = [ {%- for item in crio_default_capabilities %} "{{ item }}", {%- endfor %} ] # List of default sysctls. If it is empty or commented out, only the sysctls # defined in the container json file by the user/kube will be added. default_sysctls = [ ] # List of additional devices. specified as # "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". #If it is empty or commented out, only the devices # defined in the container json file by the user/kube will be added. additional_devices = [ ] # Path to OCI hooks directories for automatically executed hooks. If one of the # directories does not exist, then CRI-O will automatically skip them. hooks_dir = [ "/usr/share/containers/oci/hooks.d", ] # List of default mounts for each container. **Deprecated:** this option will # be removed in future versions in favor of default_mounts_file. default_mounts = [ ] # Path to the file specifying the defaults mounts for each container. The # format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads # its default mounts from the following two files: # # 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the # override file, where users can either add in their own default mounts, or # override the default mounts shipped with the package. # # 2) /usr/share/containers/mounts.conf: This is the default file read for # mounts. If you want CRI-O to read from a different, specific mounts file, # you can change the default_mounts_file. Note, if this is done, CRI-O will # only add mounts it finds in this file. # #default_mounts_file = "" # Maximum sized allowed for the container log file. Negative numbers indicate # that no size limit is imposed. If it is positive, it must be >= 8192 to # match/exceed conmon's read buffer. The file is truncated and re-opened so the # limit is never exceeded. log_size_max = -1 # Whether container output should be logged to journald in addition to the kubernetes log file log_to_journald = false # Path to directory in which container exit files are written to by conmon. container_exits_dir = "/var/run/crio/exits" # Path to directory for container attach sockets. container_attach_socket_dir = "/var/run/crio" # The prefix to use for the source of the bind mounts. bind_mount_prefix = "" # If set to true, all containers will run in read-only mode. read_only = false # Changes the verbosity of the logs based on the level it is set to. Options # are fatal, panic, error, warn, info, debug and trace. This option supports # live configuration reload. log_level = "{{ crio_log_level }}" # Filter the log messages by the provided regular expression. # This option supports live configuration reload. log_filter = "" # The UID mappings for the user namespace of each container. A range is # specified in the form containerUID:HostUID:Size. Multiple ranges must be # separated by comma. uid_mappings = "" # The GID mappings for the user namespace of each container. A range is # specified in the form containerGID:HostGID:Size. Multiple ranges must be # separated by comma. gid_mappings = "" # The minimal amount of time in seconds to wait before issuing a timeout # regarding the proper termination of the container. The lowest possible # value is 30s, whereas lower values are not considered by CRI-O. ctr_stop_timeout = 30 # **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below. # manage_network_ns_lifecycle = false # manage_ns_lifecycle determines whether we pin and remove namespaces # and manage their lifecycle {% if kata_containers_enabled %} manage_ns_lifecycle = true {% else %} manage_ns_lifecycle = false {% endif %} # The directory where the state of the managed namespaces gets tracked. # Only used when manage_ns_lifecycle is true. namespaces_dir = "/var/run" # pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle {% if bin_dir == "/usr/local/bin" %} pinns_path = "" {% else %} pinns_path = "{{ bin_dir }}/pinns" {% endif %} {% if crio_criu_support_enabled %} # Enable CRIU integration, requires that the criu binary is available in $PATH. enable_criu_support = true {% endif %} # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. # The runtime to use is picked based on the runtime_handler provided by the CRI. # If no runtime_handler is provided, the runtime will be picked based on the level # of trust of the workload. Each entry in the table should follow the format: # #[crio.runtime.runtimes.runtime-handler] # runtime_path = "/path/to/the/executable" # runtime_type = "oci" # runtime_root = "/path/to/the/root" # # Where: # - runtime-handler: name used to identify the runtime # - runtime_path (optional, string): absolute path to the runtime executable in # the host filesystem. If omitted, the runtime-handler identifier should match # the runtime executable name, and the runtime executable should be placed # in $PATH. # - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If # omitted, an "oci" runtime is assumed. # - runtime_root (optional, string): root directory for storage of containers # state. {% for runtime in crio_runtimes %} [crio.runtime.runtimes.{{ runtime.name }}] runtime_path = "{{ runtime.path }}" runtime_type = "{{ runtime.type }}" runtime_root = "{{ runtime.root }}" privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }} allowed_annotations = {{ runtime.allowed_annotations|default([])|to_json }} {% endfor %} # Kata Containers with the Firecracker VMM #[crio.runtime.runtimes.kata-fc] # The crio.image table contains settings pertaining to the management of OCI images. # # CRI-O reads its configured registries defaults from the system wide # containers-registries.conf(5) located in /etc/containers/registries.conf. If # you want to modify just CRI-O, you can change the registries configuration in # this file. Otherwise, leave insecure_registries and registries commented out to # use the system's defaults from /etc/containers/registries.conf. [crio.image] {% if crio_insecure_registries is defined and crio_insecure_registries|length>0 %} insecure_registries = {{ crio_insecure_registries }} {% endif %} # Default transport for pulling images from a remote container storage. default_transport = "docker://" # The path to a file containing credentials necessary for pulling images from # secure registries. The file is similar to that of /var/lib/kubelet/config.json global_auth_file = "/etc/crio/config.json" # The image used to instantiate infra containers. # This option supports live configuration reload. pause_image = "{{ crio_pause_image }}" # The path to a file containing credentials specific for pulling the pause_image from # above. The file is similar to that of /var/lib/kubelet/config.json # This option supports live configuration reload. pause_image_auth_file = "" # The command to run to have a container stay in the paused state. # When explicitly set to "", it will fallback to the entrypoint and command # specified in the pause image. When commented out, it will fallback to the # default: "/pause". This option supports live configuration reload. pause_command = "/pause" # Path to the file which decides what sort of policy we use when deciding # whether or not to trust an image that we've pulled. It is not recommended that # this option be used, as the default behavior of using the system-wide default # policy (i.e., /etc/containers/policy.json) is most often preferred. Please # refer to containers-policy.json(5) for more details. signature_policy = "{{ crio_signature_policy }}" # Controls how image volumes are handled. The valid values are mkdir, bind and # ignore; the latter will ignore volumes entirely. image_volumes = "mkdir" # The timeout for an image pull to make progress until the pull operation gets # canceled. This value will be also used for calculating the pull progress interval # to pull_progress_timeout / 10. Can be set to 0 to disable the timeout as well as # the progress output. pull_progress_timeout = "{{ crio_pull_progress_timeout }}" # The crio.network table containers settings pertaining to the management of # CNI plugins. [crio.network] # The default CNI network name to be selected. If not set or "", then # CRI-O will pick-up the first one found in network_dir. # cni_default_network = "" # Path to the directory where CNI configuration files are located. network_dir = "/etc/cni/net.d/" # Paths to directories where CNI plugin binaries are located. plugin_dirs = [ "/opt/cni/bin", "/usr/libexec/cni", ] # A necessary configuration for Prometheus based metrics retrieval [crio.metrics] # Globally enable or disable metrics support. enable_metrics = {{ crio_enable_metrics | bool | lower }} # The port on which the metrics server will listen. metrics_port = {{ crio_metrics_port }} {% if nri_enabled and crio_version is version('1.26.0', operator='>=') %} [crio.nri] enable_nri=true {% endif %} ================================================ FILE: roles/container-engine/cri-o/templates/http-proxy.conf.j2 ================================================ [Service] Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} ================================================ FILE: roles/container-engine/cri-o/templates/mounts.conf.j2 ================================================ /usr/share/rhel/secrets:/run/secrets {% for mount in crio_additional_mounts %} {{ mount }} {% endfor %} ================================================ FILE: roles/container-engine/cri-o/templates/registry.conf.j2 ================================================ [[registry]] prefix = "{{ item.prefix | default(item.location) }}" insecure = {{ item.insecure | default('false') | string | lower }} blocked = {{ item.blocked | default('false') | string | lower }} location = "{{ item.location }}" {% if item.mirrors is defined %} {% for mirror in item.mirrors %} [[registry.mirror]] location = "{{ mirror.location }}" insecure = {{ mirror.insecure | default('false') | string | lower }} {% endfor %} {% endif %} ================================================ FILE: roles/container-engine/cri-o/templates/unqualified.conf.j2 ================================================ {%- set _unqualified_registries = [] -%} {% for _registry in crio_registries if _registry.unqualified -%} {% if _registry.prefix is defined -%} {{ _unqualified_registries.append(_registry.prefix) }} {% else %} {{ _unqualified_registries.append(_registry.location) }} {%- endif %} {%- endfor %} unqualified-search-registries = {{ _unqualified_registries | string }} ================================================ FILE: roles/container-engine/cri-o/vars/v1.29.yml ================================================ --- crio_conmon: "{{ bin_dir }}/crio-conmon" crio_runtime_bin_dir: "{{ bin_dir }}" # cri-o binary files crio_bin_files: - crio-conmon - crio-conmonrs - crio-crun - crio-runc - crio - pinns crio_status_command: crio status ================================================ FILE: roles/container-engine/cri-o/vars/v1.31.yml ================================================ --- crio_conmon: "{{ crio_libexec_dir }}/conmon" crio_runtime_bin_dir: "{{ crio_libexec_dir }}" # cri-o binary files crio_bin_files: - crio - pinns crio_libexec_files: - conmon - conmonrs - crun - runc crio_status_command: crio status ================================================ FILE: roles/container-engine/crictl/handlers/main.yml ================================================ --- - name: Get crictl completion command: "{{ bin_dir }}/crictl completion" changed_when: false register: cri_completion check_mode: false - name: Install crictl completion copy: dest: /etc/bash_completion.d/crictl content: "{{ cri_completion.stdout }}" mode: "0644" ================================================ FILE: roles/container-engine/crictl/tasks/main.yml ================================================ --- - name: Crictl | Download crictl include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.crictl) }}" - name: Install crictl config template: src: crictl.yaml.j2 dest: /etc/crictl.yaml owner: root mode: "0644" - name: Copy crictl binary from download dir copy: src: "{{ local_release_dir }}/crictl" dest: "{{ bin_dir }}/crictl" mode: "0755" remote_src: true notify: - Get crictl completion - Install crictl completion ================================================ FILE: roles/container-engine/crictl/templates/crictl.yaml.j2 ================================================ runtime-endpoint: {{ cri_socket }} image-endpoint: {{ cri_socket }} timeout: 30 debug: false ================================================ FILE: roles/container-engine/crun/tasks/main.yml ================================================ --- - name: Crun | Download crun binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.crun) }}" - name: Copy crun binary from download dir copy: src: "{{ downloads.crun.dest }}" dest: "{{ bin_dir }}/crun" mode: "0755" remote_src: true ================================================ FILE: roles/container-engine/docker/defaults/main.yml ================================================ --- docker_version: '28.3' docker_cli_version: "{{ docker_version }}" docker_package_info: pkgs: # Path where to store repo key # docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg docker_repo_key_info: repo_keys: docker_repo_info: repos: docker_cgroup_driver: systemd docker_bin_dir: "/usr/bin" # flag to enable/disable docker cleanup docker_orphan_clean_up: false # old docker package names to be removed docker_remove_packages_yum: - docker - docker-common - docker-engine - docker-selinux.noarch - docker-client - docker-client-latest - docker-latest - docker-latest-logrotate - docker-logrotate - docker-engine-selinux.noarch # remove podman to avoid containerd.io confliction podman_remove_packages_yum: - podman docker_remove_packages_apt: - docker - docker-engine - docker.io # Docker specific repos should be part of the docker role not containerd-common anymore # Optional values for containerd apt repo containerd_package_info: pkgs: # Fedora docker-ce repo docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable' docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg' # CentOS/RedHat docker-ce repo docker_rh_repo_base_url: 'https://download.docker.com/linux/rhel/{{ ansible_distribution_major_version }}/$basearch/stable' docker_rh_repo_gpgkey: 'https://download.docker.com/linux/rhel/gpg' # Ubuntu docker-ce repo docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' # Debian docker-ce repo docker_debian_repo_base_url: "https://download.docker.com/linux/debian" docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' ================================================ FILE: roles/container-engine/docker/files/cleanup-docker-orphans.sh ================================================ #!/bin/bash list_descendants () { local children=$(ps -o pid= --ppid "$1") for pid in $children do list_descendants "$pid" done [[ -n "$children" ]] && echo "$children" } shim_search="^docker-containerd-shim|^containerd-shim" count_shim_processes=$(pgrep -f $shim_search | wc -l) if [ ${count_shim_processes} -gt 0 ]; then # Find all container pids from shims orphans=$(pgrep -P $(pgrep -d ',' -f $shim_search) |\ # Filter out valid docker pids, leaving the orphans egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf "%s%s",sep,$1; sep="|"}')) if [[ -n "$orphans" && -n "$(ps -o ppid= $orphans)" ]] then # Get shim pids of orphans orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans)) # Find all orphaned container PIDs orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done) # Recursively kill all child PIDs of orphan shims echo -e "Killing orphan container PIDs and descendants: \n$(ps -O ppid= $orphan_container_pids)" kill -9 $orphan_container_pids || true else echo "No orphaned containers found" fi else echo "The node doesn't have any shim processes." fi ================================================ FILE: roles/container-engine/docker/handlers/main.yml ================================================ --- - name: Docker | reload systemd systemd_service: name: docker daemon_reload: true masked: false listen: Restart docker - name: Docker | reload docker.socket service: name: docker.socket state: restarted when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos listen: Restart docker - name: Docker | reload docker service: name: docker state: restarted listen: Restart docker - name: Docker | wait for docker command: "{{ docker_bin_dir }}/docker images" register: docker_ready retries: 20 delay: 1 until: docker_ready.rc == 0 listen: Restart docker ================================================ FILE: roles/container-engine/docker/meta/main.yml ================================================ --- dependencies: - role: container-engine/containerd-common ================================================ FILE: roles/container-engine/docker/tasks/docker_plugin.yml ================================================ --- - name: Install Docker plugin command: docker plugin install --grant-all-permissions {{ docker_plugin | quote }} when: docker_plugin is defined register: docker_plugin_status failed_when: - docker_plugin_status.failed - '"already exists" not in docker_plugin_status.stderr' ================================================ FILE: roles/container-engine/docker/tasks/main.yml ================================================ --- - name: Check if fedora coreos stat: path: /run/ostree-booted get_attributes: false get_checksum: false get_mime: false register: ostree - name: Set is_ostree set_fact: is_ostree: "{{ ostree.stat.exists }}" - name: Gather os specific variables include_vars: "{{ item }}" with_first_found: - files: - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ host_architecture }}.yml" - "{{ ansible_distribution | lower }}.yml" - "{{ ansible_distribution.split(' ')[0] | lower }}.yml" - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_os_family | lower }}-{{ host_architecture }}.yml" - "{{ ansible_os_family | lower }}.yml" - defaults.yml paths: - ../vars skip: true tags: - facts - name: Warn about Docker version on SUSE debug: msg: "SUSE distributions always install Docker from the distro repos" when: ansible_pkg_mgr == 'zypper' - name: Gather DNS facts include_tasks: set_facts_dns.yml when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' tags: - facts - name: Pre-upgrade docker import_tasks: pre-upgrade.yml - name: Ensure docker-ce repository public key is installed apt_key: id: "{{ item }}" url: "{{ docker_repo_key_info.url }}" keyring: "{{ docker_repo_key_keyring | default(omit) }}" state: present register: keyserver_task_result until: keyserver_task_result is succeeded retries: 4 delay: "{{ retry_stagger }}" with_items: "{{ docker_repo_key_info.repo_keys }}" environment: "{{ proxy_env }}" when: ansible_pkg_mgr == 'apt' # ref to https://github.com/kubernetes-sigs/kubespray/issues/11086 & 12424 - name: Convert -backports sources to archive.debian.org for bullseye and older replace: path: "{{ item }}" regexp: '^(deb(?:-src)?\s+)(?:https?://)?(?:[^ ]+debian\.org)?([^ ]*/debian)(\s+{{ ansible_distribution_release }}-backports\b.*)' replace: '\1http://archive.debian.org/debian\3' backup: true loop: "{{ query('fileglob', '/etc/apt/sources.list') }}" when: - ansible_os_family == 'Debian' - ansible_distribution_release in ['bullseye', 'buster'] - name: Ensure docker-ce repository is enabled apt_repository: repo: "{{ item }}" state: present with_items: "{{ docker_repo_info.repos }}" when: ansible_pkg_mgr == 'apt' - name: Configure docker repository on Fedora template: src: "fedora_docker.repo.j2" dest: "{{ yum_repo_dir }}/docker.repo" mode: "0644" when: ansible_distribution == "Fedora" and not is_ostree - name: Configure docker repository on RedHat/CentOS/OracleLinux/AlmaLinux/KylinLinux template: src: "rh_docker.repo.j2" dest: "{{ yum_repo_dir }}/docker-ce.repo" mode: "0644" when: - ansible_os_family == "RedHat" - ansible_distribution != "Fedora" - not is_ostree - name: Remove dpkg hold dpkg_selections: name: "{{ item }}" selection: install when: ansible_pkg_mgr == 'apt' register: ret changed_when: false failed_when: - ret is failed - ret.msg != ( "Failed to find package '" + item + "' to perform selection 'install'." ) with_items: - "{{ containerd_package }}" - docker-ce - docker-ce-cli - name: Ensure docker packages are installed package: name: "{{ docker_package_info.pkgs }}" state: "{{ docker_package_info.state | default('present') }}" module_defaults: apt: update_cache: true dnf: enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}" disablerepo: "{{ docker_package_info.disablerepo | default(omit) }}" yum: enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}" zypper: update_cache: true register: docker_task_result until: docker_task_result is succeeded retries: 4 delay: "{{ retry_stagger }}" notify: Restart docker when: - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - not is_ostree - docker_package_info.pkgs | length > 0 # This is required to ensure any apt upgrade will not break kubernetes - name: Tell Debian hosts not to change the docker version with apt upgrade dpkg_selections: name: "{{ item }}" selection: hold when: ansible_pkg_mgr == 'apt' changed_when: false with_items: - "{{ containerd_package }}" - docker-ce - docker-ce-cli - name: Ensure docker started, remove our config if docker start failed and try again block: - name: Ensure service is started if docker packages are already present service: name: docker state: started when: docker_task_result is not changed rescue: - debug: # noqa name[missing] msg: "Docker start failed. Try to remove our config" - name: Remove kubespray generated config file: path: "{{ item }}" state: absent with_items: - /etc/systemd/system/docker.service.d/http-proxy.conf - /etc/systemd/system/docker.service.d/docker-options.conf - /etc/systemd/system/docker.service.d/docker-dns.conf - /etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf notify: Restart docker - name: Flush handlers so we can wait for docker to come up meta: flush_handlers # Install each plugin using a looped include to make error handling in the included task simpler. - name: Install docker plugin include_tasks: docker_plugin.yml loop: "{{ docker_plugins }}" loop_control: loop_var: docker_plugin - name: Set docker systemd config import_tasks: systemd.yml - name: Ensure docker service is started and enabled service: name: "{{ item }}" enabled: true state: started with_items: - docker ================================================ FILE: roles/container-engine/docker/tasks/pre-upgrade.yml ================================================ --- - name: Remove legacy docker repo file file: path: "{{ yum_repo_dir }}/docker.repo" state: absent when: - ansible_os_family == 'RedHat' - not is_ostree - name: Ensure old versions of Docker are not installed. | Debian apt: name: '{{ docker_remove_packages_apt }}' state: absent when: - ansible_os_family == 'Debian' - (docker_versioned_pkg[docker_version | string] is search('docker-ce')) - name: Ensure podman not installed. | RedHat package: name: '{{ podman_remove_packages_yum }}' state: absent when: - ansible_os_family == 'RedHat' - (docker_versioned_pkg[docker_version | string] is search('docker-ce')) - not is_ostree - name: Ensure old versions of Docker are not installed. | RedHat package: name: '{{ docker_remove_packages_yum }}' state: absent when: - ansible_os_family == 'RedHat' - (docker_versioned_pkg[docker_version | string] is search('docker-ce')) - not is_ostree ================================================ FILE: roles/container-engine/docker/tasks/reset.yml ================================================ --- - name: Docker | Get package facts package_facts: manager: auto - name: Docker | Find docker packages set_fact: docker_packages_list: "{{ ansible_facts.packages.keys() | select('search', '^docker+') }}" containerd_package: "{{ ansible_facts.packages.keys() | select('search', '^containerd+') }}" - name: Docker | Stop all running container shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -q | xargs -r {{ docker_bin_dir }}/docker kill" args: executable: /bin/bash register: stop_all_containers retries: 5 until: stop_all_containers.rc == 0 changed_when: true delay: 5 ignore_errors: true # noqa ignore-errors when: docker_packages_list | length>0 - name: Reset | remove all containers shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv" args: executable: /bin/bash register: remove_all_containers retries: 4 until: remove_all_containers.rc == 0 delay: 5 when: docker_packages_list | length>0 - name: Docker | Stop docker service service: name: "{{ item }}" enabled: false state: stopped loop: - docker - docker.socket - containerd when: docker_packages_list | length>0 - name: Docker | Remove dpkg hold dpkg_selections: name: "{{ item }}" selection: install when: ansible_pkg_mgr == 'apt' changed_when: false with_items: - "{{ docker_packages_list }}" - "{{ containerd_package }}" - name: Docker | Remove docker package package: name: "{{ item }}" state: absent changed_when: false with_items: - "{{ docker_packages_list }}" - "{{ containerd_package }}" when: - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - not is_ostree - docker_packages_list | length > 0 - name: Docker | ensure docker-ce repository is removed apt_repository: repo: "{{ item }}" state: absent with_items: "{{ docker_repo_info.repos }}" when: ansible_pkg_mgr == 'apt' - name: Docker | Remove docker repository on Fedora file: name: "{{ yum_repo_dir }}/docker.repo" state: absent when: ansible_distribution == "Fedora" and not is_ostree - name: Docker | Remove docker repository on RedHat/CentOS/Oracle/AlmaLinux Linux file: name: "{{ yum_repo_dir }}/docker-ce.repo" state: absent when: - ansible_os_family == "RedHat" - ansible_distribution != "Fedora" - not is_ostree - name: Docker | Remove docker configuration files file: name: "{{ item }}" state: absent loop: - /etc/systemd/system/docker.service.d/ - /etc/systemd/system/docker.socket - /etc/systemd/system/docker.service - /etc/systemd/system/containerd.service - /etc/systemd/system/containerd.service.d - /var/lib/docker - /etc/docker ignore_errors: true # noqa ignore-errors - name: Docker | systemctl daemon-reload # noqa no-handler systemd_service: daemon_reload: true ================================================ FILE: roles/container-engine/docker/tasks/set_facts_dns.yml ================================================ --- - name: Set dns server for docker set_fact: docker_dns_servers: "{{ dns_servers }}" - name: Show docker_dns_servers debug: msg: "{{ docker_dns_servers }}" - name: Add upstream dns servers set_fact: docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers }}" when: dns_mode in ['coredns', 'coredns_dual'] - name: Add global searchdomains set_fact: docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains }}" - name: Check system nameservers shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/' args: executable: /bin/bash changed_when: false register: system_nameservers check_mode: false - name: Check system search domains # noqa risky-shell-pipe - if resolf.conf has no search domain, grep will exit 1 which would force us to add failed_when: false # Therefore -o pipefail is not applicable in this specific instance shell: grep "^search" /etc/resolv.conf | sed -r 's/^search\s*([^#]+)\s*(#.*)?/\1/' args: executable: /bin/bash changed_when: false register: system_search_domains check_mode: false - name: Add system nameservers to docker options set_fact: docker_dns_servers: "{{ docker_dns_servers | union(system_nameservers.stdout_lines) | unique }}" when: system_nameservers.stdout - name: Add system search domains to docker options set_fact: docker_dns_search_domains: "{{ docker_dns_search_domains | union(system_search_domains.stdout.split() | default([])) | unique }}" when: system_search_domains.stdout - name: Check number of nameservers fail: msg: "Too many nameservers. You can relax this check by set docker_dns_servers_strict=false in docker.yml and we will only use the first 3." when: docker_dns_servers | length > 3 and docker_dns_servers_strict | bool - name: Rtrim number of nameservers to 3 set_fact: docker_dns_servers: "{{ docker_dns_servers[0:3] }}" when: docker_dns_servers | length > 3 and not docker_dns_servers_strict | bool - name: Check number of search domains fail: msg: "Too many search domains" when: docker_dns_search_domains | length > 6 - name: Check length of search domains fail: msg: "Search domains exceeded limit of 256 characters" when: docker_dns_search_domains | join(' ') | length > 256 ================================================ FILE: roles/container-engine/docker/tasks/systemd.yml ================================================ --- - name: Create docker service systemd directory if it doesn't exist file: path: /etc/systemd/system/docker.service.d state: directory mode: "0755" - name: Write docker proxy drop-in template: src: http-proxy.conf.j2 dest: /etc/systemd/system/docker.service.d/http-proxy.conf mode: "0644" notify: Restart docker when: http_proxy is defined or https_proxy is defined - name: Write docker.service systemd file template: src: docker.service.j2 dest: /etc/systemd/system/docker.service mode: "0644" register: docker_service_file notify: Restart docker when: - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - not is_fedora_coreos - name: Write docker options systemd drop-in template: src: docker-options.conf.j2 dest: "/etc/systemd/system/docker.service.d/docker-options.conf" mode: "0644" notify: Restart docker - name: Write docker dns systemd drop-in template: src: docker-dns.conf.j2 dest: "/etc/systemd/system/docker.service.d/docker-dns.conf" mode: "0644" notify: Restart docker when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' - name: Copy docker orphan clean up script to the node copy: src: cleanup-docker-orphans.sh dest: "{{ bin_dir }}/cleanup-docker-orphans.sh" mode: "0755" when: docker_orphan_clean_up | bool - name: Write docker orphan clean up systemd drop-in template: src: docker-orphan-cleanup.conf.j2 dest: "/etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf" mode: "0644" notify: Restart docker when: docker_orphan_clean_up | bool - name: Flush handlers meta: flush_handlers ================================================ FILE: roles/container-engine/docker/templates/docker-dns.conf.j2 ================================================ [Service] Environment="DOCKER_DNS_OPTIONS=\ {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \ {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \ {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \ " ================================================ FILE: roles/container-engine/docker/templates/docker-options.conf.j2 ================================================ [Service] Environment="DOCKER_OPTS={{ docker_options|default('') }} --iptables={{ docker_iptables_enabled | default('false') }} \ --exec-opt native.cgroupdriver={{ docker_cgroup_driver }} \ {% for i in docker_insecure_registries %}--insecure-registry={{ i }} {% endfor %} \ {% for i in docker_registry_mirrors %}--registry-mirror={{ i }} {% endfor %} \ --data-root={{ docker_daemon_graph }} \ {% if ansible_os_family not in ["openSUSE Leap", "openSUSE Tumbleweed", "Suse"] %}{{ docker_log_opts }}{% endif %}" {% if docker_mount_flags is defined and docker_mount_flags != "" %} MountFlags={{ docker_mount_flags }} {% endif %} ================================================ FILE: roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2 ================================================ [Service] ExecStartPost=-{{ bin_dir }}/cleanup-docker-orphans.sh ================================================ FILE: roles/container-engine/docker/templates/docker.service.j2 ================================================ [Unit] Description=Docker Application Container Engine Documentation=http://docs.docker.com After=network.target docker.socket containerd.service lvm2-monitor.service SuSEfirewall2.service {% if ansible_os_family != "Suse" %} BindsTo=containerd.service {% endif %} Wants=docker.socket [Service] Type=notify {% if docker_storage_options is defined %} Environment="DOCKER_STORAGE_OPTIONS={{ docker_storage_options }}" {% endif %} Environment=GOTRACEBACK=crash ExecReload=/bin/kill -s HUP $MAINPID Delegate=yes KillMode=process ExecStart={{ docker_bin_dir }}/dockerd \ {% if ansible_os_family == "Suse" %} --add-runtime oci=/usr/sbin/docker-runc \ {% endif %} $DOCKER_OPTS \ $DOCKER_STORAGE_OPTIONS \ $DOCKER_DNS_OPTIONS TasksMax=infinity LimitNOFILE=1048576 LimitNPROC=1048576 LimitCORE=infinity TimeoutStartSec=1min # restart the docker process if it exits prematurely Restart=on-failure StartLimitBurst=10 StartLimitInterval=60s # Set the cgroup slice of the service so that kube reserved takes effect {% if kube_reserved is defined and kube_reserved|bool %} Slice={{ kube_reserved_cgroups_for_service_slice }} {% endif %} [Install] WantedBy=multi-user.target ================================================ FILE: roles/container-engine/docker/templates/fedora_docker.repo.j2 ================================================ [docker-ce] name=Docker-CE Repository baseurl={{ docker_fedora_repo_base_url }} enabled=1 gpgcheck={{ '1' if docker_fedora_repo_gpgkey else '0' }} gpgkey={{ docker_fedora_repo_gpgkey }} {% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %} ================================================ FILE: roles/container-engine/docker/templates/http-proxy.conf.j2 ================================================ [Service] Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} ================================================ FILE: roles/container-engine/docker/templates/rh_docker.repo.j2 ================================================ [docker-ce] name=Docker-CE Repository baseurl={{ docker_rh_repo_base_url }} enabled=0 gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }} keepcache={{ docker_rpm_keepcache | default('1') }} gpgkey={{ docker_rh_repo_gpgkey }} {% if http_proxy is defined %} proxy={{ http_proxy }} {% endif %} ================================================ FILE: roles/container-engine/docker/vars/amazon.yml ================================================ --- # https://docs.aws.amazon.com/en_us/AmazonECS/latest/developerguide/docker-basics.html docker_versioned_pkg: 'latest': docker '18.09': docker-18.09.9ce-2.amzn2 '19.03': docker-19.03.13ce-1.amzn2 '20.10': docker-20.10.7-5.amzn2 '24.0': docker-24.0.5-1.amzn2 '25.0': docker-25.0.3-1.amzn2 docker_version: "latest" docker_package_info: pkgs: - "{{ docker_versioned_pkg[docker_version | string] }}" enablerepo: amzn2extra-docker ================================================ FILE: roles/container-engine/docker/vars/clearlinux.yml ================================================ --- docker_package_info: pkgs: - "containers-basic" ================================================ FILE: roles/container-engine/docker/vars/debian.yml ================================================ --- # containerd package info is only relevant for docker containerd_versioned_pkg: 'latest': "{{ containerd_package }}" '1.3.7': "{{ containerd_package }}=1.3.7-1" '1.3.9': "{{ containerd_package }}=1.3.9-1" '1.4.3': "{{ containerd_package }}=1.4.3-2" '1.4.4': "{{ containerd_package }}=1.4.4-1" '1.4.6': "{{ containerd_package }}=1.4.6-1" '1.4.9': "{{ containerd_package }}=1.4.9-1" '1.4.12': "{{ containerd_package }}=1.4.12-1" '1.6.4': "{{ containerd_package }}=1.6.4-1" '1.6.6': "{{ containerd_package }}=1.6.6-1" '1.6.7': "{{ containerd_package }}=1.6.7-1" '1.6.8': "{{ containerd_package }}=1.6.8-1" '1.6.9': "{{ containerd_package }}=1.6.9-1" '1.6.10': "{{ containerd_package }}=1.6.10-1" '1.6.11': "{{ containerd_package }}=1.6.11-1" '1.6.12': "{{ containerd_package }}=1.6.12-1" '1.6.13': "{{ containerd_package }}=1.6.13-1" '1.6.14': "{{ containerd_package }}=1.6.14-1" '1.6.15': "{{ containerd_package }}=1.6.15-1" '1.6.16': "{{ containerd_package }}=1.6.16-1" '1.6.18': "{{ containerd_package }}=1.6.18-1" '1.6.28': "{{ containerd_package }}=1.6.28-2" '1.6.31': "{{ containerd_package }}=1.6.31-1" '1.6.32': "{{ containerd_package }}=1.6.32-1" '1.6.33': "{{ containerd_package }}=1.6.33-1" '1.7.18': "{{ containerd_package }}=1.7.18-1" '1.7.19': "{{ containerd_package }}=1.7.19-1" '1.7.20': "{{ containerd_package }}=1.7.20-1" '1.7.21': "{{ containerd_package }}=1.7.21-1" '1.7.22': "{{ containerd_package }}=1.7.22-1" '1.7.23': "{{ containerd_package }}=1.7.23-1" '1.7.24': "{{ containerd_package }}=1.7.24-1" '1.7.25': "{{ containerd_package }}=1.7.25-1" '1.7.26': "{{ containerd_package }}=1.7.26-1" '1.7.27': "{{ containerd_package }}=1.7.27-1" 'stable': "{{ containerd_package }}=1.7.27-1" 'edge': "{{ containerd_package }}=1.7.27-1" # https://download.docker.com/linux/debian/ docker_versioned_pkg: 'latest': docker-ce '18.09': docker-ce=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }} '19.03': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }} '20.10': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} '23.0': docker-ce=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '24.0': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '25.0': docker-ce=5:25.0.5-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '26.0': docker-ce=5:26.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '26.1': docker-ce=5:26.1.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.0': docker-ce=5:27.0.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.1': docker-ce=5:27.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.2': docker-ce=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.3': docker-ce=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.4': docker-ce=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.5': docker-ce=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.0': docker-ce=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.1': docker-ce=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.2': docker-ce=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.3': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} 'stable': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} 'edge': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }} '19.03': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }} '20.10': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} '23.0': docker-ce-cli=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '24.0': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '25.0': docker-ce-cli=5:25.0.5-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '26.0': docker-ce-cli=5:26.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '26.1': docker-ce-cli=5:26.1.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.0': docker-ce-cli=5:27.0.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.1': docker-ce-cli=5:27.1.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.2': docker-ce-cli=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.3': docker-ce-cli=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.4': docker-ce-cli=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '27.5': docker-ce-cli=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.0': docker-ce-cli=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.1': docker-ce-cli=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.2': docker-ce-cli=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} '28.3': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} 'stable': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} 'edge': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} docker_package_info: pkgs: - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" docker_repo_key_info: url: '{{ docker_debian_repo_gpgkey }}' repo_keys: - '{{ docker_debian_repo_repokey }}' docker_repo_info: repos: - > deb {{ docker_debian_repo_base_url }} {{ ansible_distribution_release | lower }} stable ================================================ FILE: roles/container-engine/docker/vars/fedora.yml ================================================ --- # containerd versions are only relevant for docker containerd_versioned_pkg: 'latest': "{{ containerd_package }}" '1.3.7': "{{ containerd_package }}-1.3.7-3.1.fc{{ ansible_distribution_major_version }}" '1.3.9': "{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}" '1.4.3': "{{ containerd_package }}-1.4.3-3.2.fc{{ ansible_distribution_major_version }}" '1.4.4': "{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}" '1.4.6': "{{ containerd_package }}-1.4.6-3.1.fc{{ ansible_distribution_major_version }}" '1.4.9': "{{ containerd_package }}-1.4.9-3.1.fc{{ ansible_distribution_major_version }}" '1.4.12': "{{ containerd_package }}-1.4.12-3.1.fc{{ ansible_distribution_major_version }}" '1.6.4': "{{ containerd_package }}-1.6.4-3.1.fc{{ ansible_distribution_major_version }}" '1.6.6': "{{ containerd_package }}-1.6.6-3.1.fc{{ ansible_distribution_major_version }}" '1.6.7': "{{ containerd_package }}-1.6.7-3.1.fc{{ ansible_distribution_major_version }}" '1.6.8': "{{ containerd_package }}-1.6.8-3.1.fc{{ ansible_distribution_major_version }}" '1.6.9': "{{ containerd_package }}-1.6.9-3.1.fc{{ ansible_distribution_major_version }}" '1.6.10': "{{ containerd_package }}-1.6.10-3.1.fc{{ ansible_distribution_major_version }}" '1.6.11': "{{ containerd_package }}-1.6.11-3.1.fc{{ ansible_distribution_major_version }}" '1.6.12': "{{ containerd_package }}-1.6.12-3.1.fc{{ ansible_distribution_major_version }}" '1.6.13': "{{ containerd_package }}-1.6.13-3.1.fc{{ ansible_distribution_major_version }}" '1.6.14': "{{ containerd_package }}-1.6.14-3.1.fc{{ ansible_distribution_major_version }}" '1.6.15': "{{ containerd_package }}-1.6.15-3.1.fc{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.fc{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.fc{{ ansible_distribution_major_version }}" '1.6.28': "{{ containerd_package }}-1.6.28-3.2.fc{{ ansible_distribution_major_version }}" '1.6.31': "{{ containerd_package }}-1.6.31-3.1.fc{{ ansible_distribution_major_version }}" '1.6.32': "{{ containerd_package }}-1.6.32-3.1.fc{{ ansible_distribution_major_version }}" '1.6.33': "{{ containerd_package }}-1.6.33-3.1.fc{{ ansible_distribution_major_version }}" '1.7.18': "{{ containerd_package }}-1.7.18-3.1.fc{{ ansible_distribution_major_version }}" '1.7.19': "{{ containerd_package }}-1.7.19-3.1.fc{{ ansible_distribution_major_version }}" '1.7.20': "{{ containerd_package }}-1.7.20-3.1.fc{{ ansible_distribution_major_version }}" '1.7.21': "{{ containerd_package }}-1.7.21-3.1.fc{{ ansible_distribution_major_version }}" '1.7.22': "{{ containerd_package }}-1.7.22-3.1.fc{{ ansible_distribution_major_version }}" '1.7.23': "{{ containerd_package }}-1.7.23-3.1.fc{{ ansible_distribution_major_version }}" '1.7.24': "{{ containerd_package }}-1.7.24-3.1.fc{{ ansible_distribution_major_version }}" '1.7.25': "{{ containerd_package }}-1.7.25-3.1.fc{{ ansible_distribution_major_version }}" '1.7.26': "{{ containerd_package }}-1.7.26-3.1.fc{{ ansible_distribution_major_version }}" '1.7.27': "{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}" 'stable': "{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}" 'edge': "{{ containerd_package }}-1.7.27-3.1.fc{{ ansible_distribution_major_version }}" # https://docs.docker.com/install/linux/docker-ce/fedora/ # https://download.docker.com/linux/fedora//x86_64/stable/Packages/ docker_versioned_pkg: 'latest': docker-ce '19.03': docker-ce-19.03.15-3.fc{{ ansible_distribution_major_version }} '20.10': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }} '23.0': docker-ce-3:23.0.6-1.fc{{ ansible_distribution_major_version }} '24.0': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} '26.0': docker-ce-3:26.0.2-1.fc{{ ansible_distribution_major_version }} '26.1': docker-ce-3:26.1.4-1.fc{{ ansible_distribution_major_version }} '27.0': docker-ce-3:27.0.3-1.fc{{ ansible_distribution_major_version }} '27.1': docker-ce-3:27.1.2-1.fc{{ ansible_distribution_major_version }} '27.2': docker-ce-3:27.2.1-1.fc{{ ansible_distribution_major_version }} '27.3': docker-ce-3:27.3.1-1.fc{{ ansible_distribution_major_version }} '27.4': docker-ce-3:27.4.1-1.fc{{ ansible_distribution_major_version }} '27.5': docker-ce-3:27.5.1-1.fc{{ ansible_distribution_major_version }} '28.0': docker-ce-3:28.0.4-1.fc{{ ansible_distribution_major_version }} '28.1': docker-ce-3:28.1.1-1.fc{{ ansible_distribution_major_version }} '28.2': docker-ce-3:28.2.2-1.fc{{ ansible_distribution_major_version }} '28.3': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }} 'stable': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }} 'edge': docker-ce-3:28.3.3-1.fc{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '19.03': docker-ce-cli-19.03.15-3.fc{{ ansible_distribution_major_version }} '20.10': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }} '23.0': docker-ce-cli-1:23.0.6-1.fc{{ ansible_distribution_major_version }} '24.0': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} '26.0': docker-ce-cli-1:26.0.2-1.fc{{ ansible_distribution_major_version }} '26.1': docker-ce-cli-1:26.1.4-1.fc{{ ansible_distribution_major_version }} '27.0': docker-ce-cli-1:27.0.3-1.fc{{ ansible_distribution_major_version }} '27.1': docker-ce-cli-1:27.1.2-1.fc{{ ansible_distribution_major_version }} '27.2': docker-ce-cli-1:27.2.1-1.fc{{ ansible_distribution_major_version }} '27.3': docker-ce-cli-1:27.3.1-1.fc{{ ansible_distribution_major_version }} '27.4': docker-ce-cli-1:27.4.1-1.fc{{ ansible_distribution_major_version }} '27.5': docker-ce-cli-1:27.5.1-1.fc{{ ansible_distribution_major_version }} '28.0': docker-ce-cli-1:28.0.4-1.fc{{ ansible_distribution_major_version }} '28.1': docker-ce-cli-1:28.1.1-1.fc{{ ansible_distribution_major_version }} '28.2': docker-ce-cli-1:28.2.2-1.fc{{ ansible_distribution_major_version }} '28.3': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }} 'stable': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }} 'edge': docker-ce-cli-1:28.3.3-1.fc{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" pkgs: - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" ================================================ FILE: roles/container-engine/docker/vars/kylin.yml ================================================ --- docker_version: 26.1 docker_cli_version: "{{ docker_version }}" docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/8/$basearch/stable' # containerd versions are only relevant for docker containerd_versioned_pkg: 'latest': "{{ containerd_package }}" '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el8" '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el8" '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el8" '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el8" '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el8" '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el8" '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el8" '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el8" '1.6.6': "{{ containerd_package }}-1.6.6-3.1.el8" '1.6.7': "{{ containerd_package }}-1.6.7-3.1.el8" '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el8" '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el8" '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el8" '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el8" '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el8" '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el8" '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el8" '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el8" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el8" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el8" '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el8" '1.6.31': "{{ containerd_package }}-1.6.31-3.1.el8" '1.6.32': "{{ containerd_package }}-1.6.32-3.1.el8" 'stable': "{{ containerd_package }}-1.6.32-3.1.el8" 'edge': "{{ containerd_package }}-1.6.32-3.1.el8" # https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package # https://download.docker.com/linux/centos/8/x86_64/stable/Packages/ # or do 'yum --showduplicates list docker-engine' docker_versioned_pkg: 'latest': docker-ce '18.09': docker-ce-3:18.09.9-3.el8 '19.03': docker-ce-3:19.03.15-3.el8 '23.0': docker-ce-3:23.0.6-1.el8 '24.0': docker-ce-3:24.0.9-1.el8 '26.0': docker-ce-26.0.2-1.el8 '26.1': docker-ce-26.1.2-1.el8 'stable': docker-ce-26.1.2-1.el8 'edge': docker-ce-26.1.2-1.el8 docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli-1:18.09.9-3.el8 '19.03': docker-ce-cli-1:19.03.15-3.el8 '23.0': docker-ce-cli-1:23.0.6-1.el8 '24.0': docker-ce-cli-1:24.0.9-1.el8 '26.0': docker-ce-cli-26.0.2-1.el8 '26.1': docker-ce-cli-26.1.2-1.el8 'stable': docker-ce-cli-26.1.2-1.el8 'edge': docker-ce-cli-26.1.2-1.el8 docker_package_info: enablerepo: "docker-ce" pkgs: - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" ================================================ FILE: roles/container-engine/docker/vars/redhat.yml ================================================ --- # containerd versions are only relevant for docker containerd_versioned_pkg: 'latest': "{{ containerd_package }}" '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}" '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}" '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}" '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}" '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}" '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}" '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}" '1.6.6': "{{ containerd_package }}-1.6.6-3.1.el{{ ansible_distribution_major_version }}" '1.6.7': "{{ containerd_package }}-1.6.7-3.1.el{{ ansible_distribution_major_version }}" '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}" '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}" '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}" '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}" '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}" '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}" '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}" '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" '1.6.31': "{{ containerd_package }}-1.6.31-3.1.el{{ ansible_distribution_major_version }}" '1.6.32': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}" '1.6.33': "{{ containerd_package }}-1.6.33-3.1.el{{ ansible_distribution_major_version }}" '1.7.18': "{{ containerd_package }}-1.7.18-3.1.el{{ ansible_distribution_major_version }}" '1.7.19': "{{ containerd_package }}-1.7.19-3.1.el{{ ansible_distribution_major_version }}" '1.7.20': "{{ containerd_package }}-1.7.20-3.1.el{{ ansible_distribution_major_version }}" '1.7.21': "{{ containerd_package }}-1.7.21-3.1.el{{ ansible_distribution_major_version }}" '1.7.22': "{{ containerd_package }}-1.7.22-3.1.el{{ ansible_distribution_major_version }}" '1.7.23': "{{ containerd_package }}-1.7.23-3.1.el{{ ansible_distribution_major_version }}" '1.7.24': "{{ containerd_package }}-1.7.24-3.1.el{{ ansible_distribution_major_version }}" '1.7.25': "{{ containerd_package }}-1.7.25-3.1.el{{ ansible_distribution_major_version }}" '1.7.26': "{{ containerd_package }}-1.7.26-3.1.el{{ ansible_distribution_major_version }}" '1.7.27': "{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}" 'stable': "{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}" 'edge': "{{ containerd_package }}-1.7.27-3.1.el{{ ansible_distribution_major_version }}" # https://docs.docker.com/engine/installation/linux/rhel/#install-from-a-package # https://download.docker.com/linux/rhel/>/x86_64/stable/Packages/ # or do 'yum --showduplicates list docker-engine' docker_versioned_pkg: 'latest': docker-ce '18.09': docker-ce-3:18.09.9-3.el7 '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-3:20.10.24-3.el{{ ansible_distribution_major_version }} '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} '26.0': docker-ce-3:26.0.2-1.el{{ ansible_distribution_major_version }} '26.1': docker-ce-3:26.1.4-1.el{{ ansible_distribution_major_version }} '27.0': docker-ce-3:27.0.3-1.el{{ ansible_distribution_major_version }} '27.1': docker-ce-3:27.1.2-1.el{{ ansible_distribution_major_version }} '27.2': docker-ce-3:27.2.1-1.el{{ ansible_distribution_major_version }} '27.3': docker-ce-3:27.3.1-1.el{{ ansible_distribution_major_version }} '27.4': docker-ce-3:27.4.1-1.el{{ ansible_distribution_major_version }} '27.5': docker-ce-3:27.5.1-1.el{{ ansible_distribution_major_version }} '28.0': docker-ce-3:28.0.4-1.el{{ ansible_distribution_major_version }} '28.1': docker-ce-3:28.1.1-1.el{{ ansible_distribution_major_version }} '28.2': docker-ce-3:28.2.2-1.el{{ ansible_distribution_major_version }} '28.3': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }} 'stable': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }} 'edge': docker-ce-3:28.3.3-1.el{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli-1:18.09.9-3.el7 '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-cli-1:20.10.24-3.el{{ ansible_distribution_major_version }} '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} '26.0': docker-ce-cli-1:26.0.2-1.el{{ ansible_distribution_major_version }} '26.1': docker-ce-cli-1:26.1.4-1.el{{ ansible_distribution_major_version }} '27.0': docker-ce-cli-1:27.0.3-1.el{{ ansible_distribution_major_version }} '27.1': docker-ce-cli-1:27.1.2-1.el{{ ansible_distribution_major_version }} '27.2': docker-ce-cli-1:27.2.1-1.el{{ ansible_distribution_major_version }} '27.3': docker-ce-cli-1:27.3.1-1.el{{ ansible_distribution_major_version }} '27.4': docker-ce-cli-1:27.4.1-1.el{{ ansible_distribution_major_version }} '27.5': docker-ce-cli-1:27.5.1-1.el{{ ansible_distribution_major_version }} '28.0': docker-ce-cli-1:28.0.4-1.el{{ ansible_distribution_major_version }} '28.1': docker-ce-cli-1:28.1.1-1.el{{ ansible_distribution_major_version }} '28.2': docker-ce-cli-1:28.2.2-1.el{{ ansible_distribution_major_version }} '28.3': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }} 'stable': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }} 'edge': docker-ce-cli-1:28.3.3-1.el{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" pkgs: - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" ================================================ FILE: roles/container-engine/docker/vars/suse.yml ================================================ --- docker_package_info: state: latest pkgs: - docker - containerd ================================================ FILE: roles/container-engine/docker/vars/ubuntu.yml ================================================ --- # containerd versions are only relevant for docker containerd_versioned_pkg: 'latest': "{{ containerd_package }}" '1.6.4': "{{ containerd_package }}=1.6.4-1" '1.6.6': "{{ containerd_package }}=1.6.6-1" '1.6.7': "{{ containerd_package }}=1.6.7-1" '1.6.8': "{{ containerd_package }}=1.6.8-1" '1.6.9': "{{ containerd_package }}=1.6.9-1" '1.6.10': "{{ containerd_package }}=1.6.10-1" '1.6.11': "{{ containerd_package }}=1.6.11-1" '1.6.12': "{{ containerd_package }}=1.6.12-1" '1.6.13': "{{ containerd_package }}=1.6.13-1" '1.6.14': "{{ containerd_package }}=1.6.14-1" '1.6.15': "{{ containerd_package }}=1.6.15-1" '1.6.16': "{{ containerd_package }}=1.6.16-1" '1.6.18': "{{ containerd_package }}=1.6.18-1" '1.6.28': "{{ containerd_package }}=1.6.28-2" '1.6.31': "{{ containerd_package }}=1.6.31-1" '1.6.32': "{{ containerd_package }}=1.6.32-1" '1.6.33': "{{ containerd_package }}=1.6.33-1" '1.7.18': "{{ containerd_package }}=1.7.18-1" '1.7.19': "{{ containerd_package }}=1.7.19-1" '1.7.20': "{{ containerd_package }}=1.7.20-1" '1.7.21': "{{ containerd_package }}=1.7.21-1" '1.7.22': "{{ containerd_package }}=1.7.22-1" '1.7.23': "{{ containerd_package }}=1.7.23-1" '1.7.24': "{{ containerd_package }}=1.7.24-1" '1.7.25': "{{ containerd_package }}=1.7.25-1" '1.7.26': "{{ containerd_package }}=1.7.26-1" '1.7.27': "{{ containerd_package }}=1.7.27-1" 'stable': "{{ containerd_package }}=1.7.27-1" 'edge': "{{ containerd_package }}=1.7.27-1" # https://download.docker.com/linux/ubuntu/ docker_versioned_pkg: 'latest': docker-ce '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }} '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }} '20.10': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} '23.0': docker-ce=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '24.0': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '26.0': docker-ce=5:26.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '26.1': docker-ce=5:26.1.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.0': docker-ce=5:27.0.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.1': docker-ce=5:27.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.2': docker-ce=5:27.2.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.3': docker-ce=5:27.3.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.4': docker-ce=5:27.4.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.5': docker-ce=5:27.5.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.0': docker-ce=5:28.0.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.1': docker-ce=5:28.1.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.2': docker-ce=5:28.2.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.3': docker-ce=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }} '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }} '20.10': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} '23.0': docker-ce-cli=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '24.0': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '26.0': docker-ce-cli=5:26.0.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '26.1': docker-ce-cli=5:26.1.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.0': docker-ce-cli=5:27.0.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.1': docker-ce-cli=5:27.1.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.2': docker-ce-cli=5:27.2.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.3': docker-ce-cli=5:27.3.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.4': docker-ce-cli=5:27.4.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '27.5': docker-ce-cli=5:27.5.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.0': docker-ce-cli=5:28.0.4-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.1': docker-ce-cli=5:28.1.1-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.2': docker-ce-cli=5:28.2.2-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} '28.3': docker-ce-cli=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} 'stable': docker-ce-cli=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} 'edge': docker-ce-cli=5:28.3.3-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} docker_package_info: pkgs: - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" docker_repo_key_info: url: '{{ docker_ubuntu_repo_gpgkey }}' repo_keys: - '{{ docker_ubuntu_repo_repokey }}' docker_repo_info: repos: - > deb [arch={{ host_architecture }}] {{ docker_ubuntu_repo_base_url }} {{ ansible_distribution_release | lower }} stable ================================================ FILE: roles/container-engine/docker/vars/uniontech.yml ================================================ --- # containerd versions are only relevant for docker containerd_versioned_pkg: 'latest': "{{ containerd_package }}" '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}" '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}" '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}" '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}" '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}" '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}" '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}" '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}" '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}" '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}" '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}" '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}" '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}" '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}" '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" '1.6.32': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}" 'stable': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}" 'edge': "{{ containerd_package }}-1.6.32-3.1.el{{ ansible_distribution_major_version }}" docker_version: 19.03 docker_cli_version: 19.03 # https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package # https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ # or do 'yum --showduplicates list docker-engine' docker_versioned_pkg: 'latest': docker-ce '18.09': docker-ce-3:18.09.9-3.el7 '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }} '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli-1:18.09.9-3.el7 '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }} '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" disablerepo: "UniontechOS-20-AppStream" pkgs: - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" ================================================ FILE: roles/container-engine/gvisor/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true vars: gvisor_enabled: true container_manager: containerd roles: - role: kubespray_defaults - role: container-engine/containerd - role: container-engine/gvisor ================================================ FILE: roles/container-engine/gvisor/molecule/default/files/10-mynet.conf ================================================ { "cniVersion": "0.2.0", "name": "mynet", "type": "bridge", "bridge": "cni0", "isGateway": true, "ipMasq": true, "ipam": { "type": "host-local", "subnet": "172.19.0.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } } ================================================ FILE: roles/container-engine/gvisor/molecule/default/files/container.json ================================================ { "metadata": { "name": "gvisor1" }, "image": { "image": "quay.io/kubespray/hello-world:latest" }, "log_path": "gvisor1.0.log", "linux": {} } ================================================ FILE: roles/container-engine/gvisor/molecule/default/files/sandbox.json ================================================ { "metadata": { "name": "gvisor1", "namespace": "default", "attempt": 1, "uid": "hdishd83djaidwnduwk28bcsb" }, "linux": {}, "log_directory": "/tmp" } ================================================ FILE: roles/container-engine/gvisor/molecule/default/molecule.yml ================================================ --- role_name_check: 1 platforms: - cloud_image: ubuntu-2404 name: ubuntu24 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - name: almalinux9 cloud_image: almalinux-9 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 inventory: group_vars: k8s_cluster: gvisor_enabled: true container_manager: containerd playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: name: ansible ================================================ FILE: roles/container-engine/gvisor/molecule/default/verify.yml ================================================ --- - name: Test gvisor hosts: all gather_facts: false tasks: - name: Get kubespray defaults import_role: name: ../../../../../kubespray_defaults - name: Test version command: "{{ bin_dir }}/runsc --version" register: runsc_version failed_when: > runsc_version is failed or 'runsc version' not in runsc_version.stdout - name: Test run container import_playbook: ../../../molecule/test_runtime.yml vars: container_runtime: runsc ================================================ FILE: roles/container-engine/gvisor/tasks/main.yml ================================================ --- - name: GVisor | Download runsc binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}" - name: GVisor | Download containerd-shim-runsc-v1 binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}" - name: GVisor | Copy binaries copy: src: "{{ item.src }}" dest: "{{ bin_dir }}/{{ item.dest }}" mode: "0755" remote_src: true with_items: - { src: "{{ downloads.gvisor_runsc.dest }}", dest: "runsc" } - { src: "{{ downloads.gvisor_containerd_shim.dest }}", dest: "containerd-shim-runsc-v1" } ================================================ FILE: roles/container-engine/kata-containers/defaults/main.yml ================================================ --- kata_containers_dir: /opt/kata kata_containers_config_dir: /etc/kata-containers kata_containers_containerd_bin_dir: /usr/local/bin kata_containers_qemu_default_memory: "{{ ansible_memtotal_mb }}" kata_containers_qemu_debug: 'false' kata_containers_qemu_sandbox_cgroup_only: 'true' kata_containers_qemu_enable_mem_prealloc: 'false' kata_containers_virtio_fs_cache: 'always' ================================================ FILE: roles/container-engine/kata-containers/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true vars: kata_containers_enabled: true container_manager: containerd roles: - role: kubespray_defaults - role: container-engine/containerd - role: container-engine/kata-containers ================================================ FILE: roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf ================================================ { "cniVersion": "0.2.0", "name": "mynet", "type": "bridge", "bridge": "cni0", "isGateway": true, "ipMasq": true, "ipam": { "type": "host-local", "subnet": "172.19.0.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } } ================================================ FILE: roles/container-engine/kata-containers/molecule/default/files/container.json ================================================ { "metadata": { "name": "kata1" }, "image": { "image": "quay.io/kubespray/hello-world:latest" }, "log_path": "kata1.0.log", "linux": {} } ================================================ FILE: roles/container-engine/kata-containers/molecule/default/files/sandbox.json ================================================ { "metadata": { "name": "kata1", "namespace": "default", "attempt": 1, "uid": "hdishd83djaidwnduwk28bcsb" }, "linux": {}, "log_directory": "/tmp" } ================================================ FILE: roles/container-engine/kata-containers/molecule/default/molecule.yml ================================================ --- role_name_check: 1 platforms: - name: ubuntu22 cloud_image: ubuntu-2204 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - name: ubuntu24 cloud_image: ubuntu-2404 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: name: ansible ================================================ FILE: roles/container-engine/kata-containers/molecule/default/verify.yml ================================================ --- - name: Test kata-containers hosts: all gather_facts: false tasks: - name: Test version command: "/opt/kata/bin/kata-runtime version" register: version failed_when: > version is failed or 'kata-runtime' not in version.stdout - name: Test version command: "/opt/kata/bin/kata-runtime check" register: check failed_when: > check is failed or 'System is capable of running' not in check.stdout - name: Test run container import_playbook: ../../../molecule/test_runtime.yml vars: container_runtime: kata-qemu container_manager: containerd ================================================ FILE: roles/container-engine/kata-containers/tasks/main.yml ================================================ --- - name: Kata-containers | Download kata binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.kata_containers) }}" - name: Kata-containers | Copy kata-containers binary unarchive: src: "{{ downloads.kata_containers.dest }}" dest: "/" mode: "0755" owner: root group: root remote_src: true - name: Kata-containers | Create config directory file: path: "{{ kata_containers_config_dir }}" state: directory mode: "0755" - name: Kata-containers | Set configuration template: src: "{{ item }}.j2" dest: "{{ kata_containers_config_dir }}/{{ item }}" mode: "0644" with_items: - configuration-qemu.toml - name: Kata-containers | Set containerd bin vars: shim: "{{ item }}" template: dest: "{{ kata_containers_containerd_bin_dir }}/containerd-shim-kata-{{ item }}-v2" src: containerd-shim-kata-v2.j2 mode: "0755" with_items: - qemu - name: Kata-containers | Load vhost kernel modules community.general.modprobe: state: present name: "{{ item }}" with_items: - vhost_vsock - vhost_net - name: Kata-containers | Persist vhost kernel modules copy: dest: /etc/modules-load.d/kubespray-kata-containers.conf mode: "0644" content: | vhost_vsock vhost_net ================================================ FILE: roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 ================================================ # Copyright (c) 2017-2019 Intel Corporation # Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # # XXX: WARNING: this file is auto-generated. # XXX: # XXX: Source file: "config/configuration-qemu.toml.in" # XXX: Project: # XXX: Name: Kata Containers # XXX: Type: kata [hypervisor.qemu] path = "/opt/kata/bin/qemu-system-x86_64" {% if kata_containers_version is version('2.2.0', '>=') %} kernel = "/opt/kata/share/kata-containers/vmlinux.container" {% else %} kernel = "/opt/kata/share/kata-containers/vmlinuz.container" {% endif %} image = "/opt/kata/share/kata-containers/kata-containers.img" # initrd = "/opt/kata/share/kata-containers/kata-containers-initrd.img" machine_type = "q35" # rootfs filesystem type: # - ext4 (default) # - xfs # - erofs rootfs_type="ext4" # Enable confidential guest support. # Toggling that setting may trigger different hardware features, ranging # from memory encryption to both memory and CPU-state encryption and integrity. # The Kata Containers runtime dynamically detects the available feature set and # aims at enabling the largest possible one, returning an error if none is # available, or none is supported by the hypervisor. # # Known limitations: # * Does not work by design: # - CPU Hotplug # - Memory Hotplug # - NVDIMM devices # # Default false # confidential_guest = true # Choose AMD SEV-SNP confidential guests # In case of using confidential guests on AMD hardware that supports both SEV # and SEV-SNP, the following enables SEV-SNP guests. SEV guests are default. # Default false # sev_snp_guest = true # Enable running QEMU VMM as a non-root user. # By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as # a non-root random user. See documentation for the limitations of this mode. # rootless = true # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" enable_annotations = ["enable_iommu"] # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) # Your distribution recommends: ["/opt/kata/bin/qemu-system-x86_64"] valid_hypervisor_paths = ["/opt/kata/bin/qemu-system-x86_64"] # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. # # WARNING: - any parameter specified here will take priority over the default # parameter value of the same name used to start the virtual machine. # Do not set values here unless you understand the impact of doing so as you # may stop the virtual machine from booting. # To see the list of default parameters, enable hypervisor debug, create a # container and look for 'default-kernel-parameters' log entries. kernel_params = "" # Path to the firmware. # If you want that qemu uses the default firmware leave this option empty firmware = "" # Path to the firmware volume. # firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables # as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables # can be customized per each user while UEFI code is kept same. firmware_volume = "" # Machine accelerators # comma-separated list of machine accelerators to pass to the hypervisor. # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` machine_accelerators="" # Qemu seccomp sandbox feature # comma-separated list of seccomp sandbox features to control the syscall access. # For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"` # Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox # Another note: enabling this feature may reduce performance, you may enable # /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html #seccompsandbox="on,obsolete=deny,spawn=deny,resourcecontrol=deny" # CPU features # comma-separated list of cpu features to pass to the cpu # For example, `cpu_features = "pmu=off,vmx=off" cpu_features="pmu=off" # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to 1 # < 0 --> will be set to the actual number of physical cores # > 0 <= number of physical cores --> will be set to the specified number # > number of physical cores --> will be set to the actual number of physical cores default_vcpus = 1 # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded # > 0 <= number of physical cores --> will be set to the specified number # > number of physical cores --> will be set to the actual number of physical cores or to the maximum number # of vCPUs supported by KVM if that number is exceeded # WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when # the actual number of physical cores is greater than it. # WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU # the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs # can be added to a SB/VM, but the memory footprint will be big. Another example, with # `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of # vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, # unless you know what are you doing. # NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. default_maxvcpus = 0 # Bridges can be used to hot plug devices. # Limitations: # * Currently only pci bridges are supported # * Until 30 devices per bridge can be hot plugged. # * Until 5 PCI bridges can be cold plugged per VM. # This limitation could be a bug in qemu or in the kernel # Default number of bridges per SB/VM: # unspecified or 0 --> will be set to 1 # > 1 <= 5 --> will be set to the specified number # > 5 --> will be set to 5 default_bridges = 1 # Default memory size in MiB for SB/VM. # If unspecified then it will be set 2048 MiB. default_memory = {{ kata_containers_qemu_default_memory }} # # Default memory slots per SB/VM. # If unspecified then it will be set 10. # This is will determine the times that memory will be hotadded to sandbox/VM. #memory_slots = 10 # Default maximum memory in MiB per SB / VM # unspecified or == 0 --> will be set to the actual amount of physical RAM # > 0 <= amount of physical RAM --> will be set to the specified number # > amount of physical RAM --> will be set to the actual amount of physical RAM default_maxmemory = 0 # The size in MiB will be plused to max memory of hypervisor. # It is the memory address space for the NVDIMM devie. # If set block storage driver (block_device_driver) to "nvdimm", # should set memory_offset to the size of block device. # Default 0 #memory_offset = 0 # Specifies virtio-mem will be enabled or not. # Please note that this option should be used with the command # "echo 1 > /proc/sys/vm/overcommit_memory". # Default false #enable_virtio_mem = true # Disable block device from being used for a container's rootfs. # In case of a storage driver like devicemapper where a container's # root file system is backed by a block device, the block device is passed # directly to the hypervisor for performance reasons. # This flag prevents the block device from being passed to the hypervisor, # virtio-fs is used instead to pass the rootfs. disable_block_device_use = false # Shared file system type: # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus {% if kata_containers_version is version('2.2.0', '>=') %} shared_fs = "virtio-fs" {% else %} shared_fs = "virtio-9p" {% endif %} # Path to vhost-user-fs daemon. {% if kata_containers_version is version('2.5.0', '>=') %} virtio_fs_daemon = "/opt/kata/libexec/virtiofsd" {% else %} virtio_fs_daemon = "/opt/kata/libexec/kata-qemu/virtiofsd" {% endif %} # List of valid annotations values for the virtiofs daemon # The default if not set is empty (all annotations rejected.) # Your distribution recommends: ["/opt/kata/libexec/virtiofsd"] valid_virtio_fs_daemon_paths = [ "/opt/kata/libexec/virtiofsd", "/opt/kata/libexec/kata-qemu/virtiofsd", ] # Default size of DAX cache in MiB virtio_fs_cache_size = 0 # Default size of virtqueues virtio_fs_queue_size = 1024 # Extra args for virtiofsd daemon # # Format example: # ["--arg1=xxx", "--arg2=yyy"] # Examples: # Set virtiofsd log level to debug : ["--log-level=debug"] # # see `virtiofsd -h` for possible options. virtio_fs_extra_args = ["--thread-pool-size=1", "--announce-submounts"] # Cache mode: # # - never # Metadata, data, and pathname lookup are not cached in guest. They are # always fetched from host and any changes are immediately pushed to host. # # - auto # Metadata and pathname lookup cache expires after a configured amount of # time (default is 1 second). Data is cached while the file is open (close # to open consistency). # # - always # Metadata, data, and pathname lookup are cached in guest and never expire. virtio_fs_cache = "{{ kata_containers_virtio_fs_cache }}" # Block storage driver to be used for the hypervisor in case the container # rootfs is backed by a block device. This is virtio-scsi, virtio-blk # or nvdimm. block_device_driver = "virtio-scsi" # aio is the I/O mechanism used by qemu # Options: # # - threads # Pthread based disk I/O. # # - native # Native Linux I/O. # # - io_uring # Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and # qemu >=5.0. block_device_aio = "io_uring" # Specifies cache-related options will be set to block devices or not. # Default false #block_device_cache_set = true # Specifies cache-related options for block devices. # Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. # Default false #block_device_cache_direct = true # Specifies cache-related options for block devices. # Denotes whether flush requests for the device are ignored. # Default false #block_device_cache_noflush = true # Enable iothreads (data-plane) to be used. This causes IO to be # handled in a separate IO thread. This is currently only implemented # for SCSI. # enable_iothreads = false # Enable pre allocation of VM RAM, default false # Enabling this will result in lower container density # as all of the memory will be allocated and locked # This is useful when you want to reserve all the memory # upfront or in the cases where you want memory latencies # to be very predictable # Default false enable_mem_prealloc = {{ kata_containers_qemu_enable_mem_prealloc }} # Enable huge pages for VM RAM, default false # Enabling this will result in the VM memory # being allocated using huge pages. # This is useful when you want to use vhost-user network # stacks within the container. This will automatically # result in memory pre allocation #enable_hugepages = true # Enable vhost-user storage device, default false # Enabling this will result in some Linux reserved block type # major range 240-254 being chosen to represent vhost-user devices. enable_vhost_user_store = false # The base directory specifically used for vhost-user devices. # Its sub-path "block" is used for block devices; "block/sockets" is # where we expect vhost-user sockets to live; "block/devices" is where # simulated block device nodes for vhost-user devices to live. vhost_user_store_path = "/var/run/kata-containers/vhost-user" # Enable vIOMMU, default false # Enabling this will result in the VM having a vIOMMU device # This will also add the following options to the kernel's # command line: intel_iommu=on,iommu=pt #enable_iommu = true # Enable IOMMU_PLATFORM, default false # Enabling this will result in the VM device having iommu_platform=on set #enable_iommu_platform = true # List of valid annotations values for the vhost user store path # The default if not set is empty (all annotations rejected.) # Your distribution recommends: ["/var/run/kata-containers/vhost-user"] valid_vhost_user_store_paths = ["/var/run/kata-containers/vhost-user"] # The timeout for reconnecting on non-server spdk sockets when the remote end goes away. # qemu will delay this many seconds and then attempt to reconnect. # Zero disables reconnecting, and the default is zero. vhost_user_reconnect_timeout_sec = 0 # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. # This option will be ignored if VM templating is enabled. #file_mem_backend = "" # List of valid annotations values for the file_mem_backend annotation # The default if not set is empty (all annotations rejected.) # Your distribution recommends: [""] valid_file_mem_backends = [""] # -pflash can add image file to VM. The arguments of it should be in format # of ["/path/to/flash0.img", "/path/to/flash1.img"] pflashes = [] # This option changes the default hypervisor and kernel parameters # to enable debug output where available. And Debug also enables the hmp socket. # # Default false enable_debug = {{ kata_containers_qemu_debug }} # Disable the customizations done in the runtime when it detects # that it is running on top a VMM. This will result in the runtime # behaving as it would when running on bare metal. # #disable_nesting_checks = true # This is the msize used for 9p shares. It is the number of bytes # used for 9p packet payload. #msize_9p = 8192 # If false and nvdimm is supported, use nvdimm device to plug guest image. # Otherwise virtio-block device is used. # # nvdimm is not supported when `confidential_guest = true`. # # Default is false #disable_image_nvdimm = true # VFIO devices are hotplugged on a bridge by default. # Enable hotplugging on root bus. This may be required for devices with # a large PCI bar, as this is a current limitation with hotplugging on # a bridge. # Default false #hotplug_vfio_on_root_bus = true # Before hot plugging a PCIe device, you need to add a pcie_root_port device. # Use this parameter when using some large PCI bar devices, such as Nvidia GPU # The value means the number of pcie_root_port # This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35" # Default 0 #pcie_root_port = 2 # If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off # security (vhost-net runs ring0) for network I/O performance. #disable_vhost_net = true # # Default entropy source. # The path to a host source of entropy (including a real hardware RNG) # /dev/urandom and /dev/random are two main options. # Be aware that /dev/random is a blocking source of entropy. If the host # runs out of entropy, the VMs boot time will increase leading to get startup # timeouts. # The source of entropy /dev/urandom is non-blocking and provides a # generally acceptable source of entropy. It should work well for pretty much # all practical purposes. #entropy_source= "/dev/urandom" # List of valid annotations values for entropy_source # The default if not set is empty (all annotations rejected.) # Your distribution recommends: ["/dev/urandom","/dev/random",""] valid_entropy_sources = ["/dev/urandom","/dev/random",""] # Path to OCI hook binaries in the *guest rootfs*. # This does not affect host-side hooks which must instead be added to # the OCI spec passed to the runtime. # # You can create a rootfs with hooks by customizing the osbuilder scripts: # https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder # # Hooks must be stored in a subdirectory of guest_hook_path according to their # hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". # The agent will scan these directories for executable files and add them, in # lexicographical order, to the lifecycle of the guest container. # Hooks are executed in the runtime namespace of the guest. See the official documentation: # https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks # Warnings will be logged if any error is encountered while scanning for hooks, # but it will not abort container execution. #guest_hook_path = "/usr/share/oci/hooks" # # Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). # In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. # Default 0-sized value means unlimited rate. #rx_rate_limiter_max_rate = 0 # Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). # In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block) # to discipline traffic. # Default 0-sized value means unlimited rate. #tx_rate_limiter_max_rate = 0 # Set where to save the guest memory dump file. # If set, when GUEST_PANICKED event occurred, # guest memeory will be dumped to host filesystem under guest_memory_dump_path, # This directory will be created automatically if it does not exist. # # The dumped file(also called vmcore) can be processed with crash or gdb. # # WARNING: # Dump guest’s memory can take very long depending on the amount of guest memory # and use much disk space. #guest_memory_dump_path="/var/crash/kata" # If enable paging. # Basically, if you want to use "gdb" rather than "crash", # or need the guest-virtual addresses in the ELF vmcore, # then you should enable paging. # # See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details #guest_memory_dump_paging=false # Enable swap in the guest. Default false. # When enable_guest_swap is enabled, insert a raw file to the guest as the swap device # if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness") # is bigger than 0. # The size of the swap device should be # swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes. # If swap_in_bytes is not set, the size should be memory_limit_in_bytes. # If swap_in_bytes and memory_limit_in_bytes is not set, the size should # be default_memory. #enable_guest_swap = true # use legacy serial for guest console if available and implemented for architecture. Default false #use_legacy_serial = true # disable applying SELinux on the VMM process (default false) disable_selinux=false # disable applying SELinux on the container process # If set to false, the type `container_t` is applied to the container process by default. # Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built # with `SELINUX=yes`. # (default: true) disable_guest_selinux=true [factory] # VM templating support. Once enabled, new VMs are created from template # using vm cloning. They will share the same initial kernel, initramfs and # agent memory by mapping it readonly. It helps speeding up new container # creation and saves a lot of memory if there are many kata containers running # on the same host. # # When disabled, new VMs are created from scratch. # # Note: Requires "initrd=" to be set ("image=" is not supported). # # Default false #enable_template = true # Specifies the path of template. # # Default "/run/vc/vm/template" #template_path = "/run/vc/vm/template" # The number of caches of VMCache: # unspecified or == 0 --> VMCache is disabled # > 0 --> will be set to the specified number # # VMCache is a function that creates VMs as caches before using it. # It helps speed up new container creation. # The function consists of a server and some clients communicating # through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. # The VMCache server will create some VMs and cache them by factory cache. # It will convert the VM to gRPC format and transport it when gets # requestion from clients. # Factory grpccache is the VMCache client. It will request gRPC format # VM and convert it back to a VM. If VMCache function is enabled, # kata-runtime will request VM from factory grpccache when it creates # a new sandbox. # # Default 0 #vm_cache_number = 0 # Specify the address of the Unix socket that is used by VMCache. # # Default /var/run/kata-containers/cache.sock #vm_cache_endpoint = "/var/run/kata-containers/cache.sock" [agent.kata] # If enabled, make the agent display debug-level messages. # (default: disabled) enable_debug = {{ kata_containers_qemu_debug }} # Enable agent tracing. # # If enabled, the agent will generate OpenTelemetry trace spans. # # Notes: # # - If the runtime also has tracing enabled, the agent spans will be # associated with the appropriate runtime parent span. # - If enabled, the runtime will wait for the container to shutdown, # increasing the container shutdown time slightly. # # (default: disabled) #enable_tracing = true # Comma separated list of kernel modules and their parameters. # These modules will be loaded in the guest kernel using modprobe(8). # The following example can be used to load two kernel modules with parameters # - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] # The first word is considered as the module name and the rest as its parameters. # Container will not be started when: # * A kernel module is specified and the modprobe command is not installed in the guest # or it fails loading the module. # * The module is not available in the guest or it doesn't met the guest kernel # requirements, like architecture and version. # kernel_modules=[] # Enable debug console. # If enabled, user can connect guest OS running inside hypervisor # through "kata-runtime exec " command #debug_console_enabled = true # Agent connection dialing timeout value in seconds # (default: 30) #dial_timeout = 30 [runtime] # If enabled, the runtime will log additional debug messages to the # system log # (default: disabled) enable_debug = {{ kata_containers_qemu_debug }} # # Internetworking model # Determines how the VM should be connected to the # the container network interface # Options: # # - macvtap # Used when the Container network interface can be bridged using # macvtap. # # - none # Used when customize network. Only creates a tap device. No veth pair. # # - tcfilter # Uses tc filter rules to redirect traffic from the network interface # provided by plugin to a tap interface connected to the VM. # internetworking_model="tcfilter" # disable guest seccomp # Determines whether container seccomp profiles are passed to the virtual # machine and applied by the kata agent. If set to true, seccomp is not applied # within the guest # (default: true) disable_guest_seccomp=true # vCPUs pinning settings # if enabled, each vCPU thread will be scheduled to a fixed CPU # qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet) # enable_vcpus_pinning = false # Apply a custom SELinux security policy to the container process inside the VM. # This is used when you want to apply a type other than the default `container_t`, # so general users should not uncomment and apply it. # (format: "user:role:type") # Note: You cannot specify MCS policy with the label because the sensitivity levels and # categories are determined automatically by high-level container runtimes such as containerd. #guest_selinux_label="system_u:system_r:container_t" # If enabled, the runtime will create opentracing.io traces and spans. # (See https://www.jaegertracing.io/docs/getting-started). # (default: disabled) #enable_tracing = true # Set the full url to the Jaeger HTTP Thrift collector. # The default if not set will be "http://localhost:14268/api/traces" #jaeger_endpoint = "" # Sets the username to be used if basic auth is required for Jaeger. #jaeger_user = "" # Sets the password to be used if basic auth is required for Jaeger. #jaeger_password = "" # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. # `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge # (like OVS) directly. # (default: false) #disable_new_netns = true # if enabled, the runtime will add all the kata processes inside one dedicated cgroup. # The container cgroups in the host are not created, just one single cgroup per sandbox. # The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. # The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. # The sandbox cgroup is constrained if there is no container type annotation. # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType sandbox_cgroup_only={{ kata_containers_qemu_sandbox_cgroup_only }} # If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In # this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful # when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug. # Compatibility for determining appropriate sandbox (VM) size: # - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O # does not yet support sandbox sizing annotations. # - When running single containers using a tool like ctr, container sizing information will be available. static_sandbox_resource_mgmt=false # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. # If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` # These will not be exposed to the container workloads, and are only provided for potential guest services. sandbox_bind_mounts=[] # VFIO Mode # Determines how VFIO devices should be be presented to the container. # Options: # # - vfio # Matches behaviour of OCI runtimes (e.g. runc) as much as # possible. VFIO devices will appear in the container as VFIO # character devices under /dev/vfio. The exact names may differ # from the host (they need to match the VM's IOMMU group numbers # rather than the host's) # # - guest-kernel # This is a Kata-specific behaviour that's useful in certain cases. # The VFIO device is managed by whatever driver in the VM kernel # claims it. This means it will appear as one or more device nodes # or network interfaces depending on the nature of the device. # Using this mode requires specially built workloads that know how # to locate the relevant device interfaces within the VM. # vfio_mode="guest-kernel" # If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will # be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. disable_guest_empty_dir=false # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. # Supported experimental features: # (default: []) experimental=[] # If enabled, user can run pprof tools with shim v2 process through kata-monitor. # (default: false) # enable_pprof = true # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] # Container image service. # # Offload the CRI image management service to the Kata agent. # (default: false) #service_offload = true # Container image decryption keys provisioning. # Applies only if service_offload is true. # Keys can be provisioned locally (e.g. through a special command or # a local file) or remotely (usually after the guest is remotely attested). # The provision setting is a complete URL that lets the Kata agent decide # which method to use in order to fetch the keys. # # Keys can be stored in a local file, in a measured and attested initrd: #provision=data:///local/key/file # # Keys could be fetched through a special command or binary from the # initrd (guest) image, e.g. a firmware call: #provision=file:///path/to/bin/fetcher/in/guest # # Keys can be remotely provisioned. The Kata agent fetches them from e.g. # a HTTPS URL: #provision=https://my-key-broker.foo/tenant/ ================================================ FILE: roles/container-engine/kata-containers/templates/containerd-shim-kata-v2.j2 ================================================ #!/bin/bash KATA_CONF_FILE={{ kata_containers_config_dir }}/configuration-{{ shim }}.toml {{ kata_containers_dir }}/bin/containerd-shim-kata-v2 $@ ================================================ FILE: roles/container-engine/molecule/files/10-mynet.conf ================================================ { "cniVersion": "0.2.0", "name": "mynet", "type": "bridge", "bridge": "cni0", "isGateway": true, "ipMasq": true, "ipam": { "type": "host-local", "subnet": "172.19.0.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } } ================================================ FILE: roles/container-engine/molecule/prepare.yml ================================================ --- - name: Prepare hosts: all gather_facts: false become: true vars: ignore_assert_errors: true roles: - role: dynamic_groups - role: bootstrap_os - role: network_facts - role: kubernetes/preinstall - role: adduser user: "{{ addusers.kube }}" tasks: - name: Download CNI include_tasks: "../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.cni) }}" - name: Prepare CNI hosts: all gather_facts: false become: true vars: ignore_assert_errors: true kube_network_plugin: cni roles: - role: kubespray_defaults - role: network_plugin/cni tasks: - name: Create /etc/cni/net.d directory file: path: /etc/cni/net.d state: directory owner: root mode: "0755" - name: Config bridge host-local CNI copy: src: "10-mynet.conf" dest: "/etc/cni/net.d/" owner: root mode: "0644" ================================================ FILE: roles/container-engine/molecule/templates/container.json.j2 ================================================ { "metadata": { "name": "{{ container_runtime }}1" }, "image": { "image": "quay.io/kubespray/hello-world:latest" }, "log_path": "{{ container_runtime }}1.0.log", "linux": {} } ================================================ FILE: roles/container-engine/molecule/templates/sandbox.json.j2 ================================================ { "metadata": { "name": "{{ container_runtime }}1", "namespace": "default", "attempt": 1, "uid": "hdishd83djaidwnduwk28bcsb" }, "linux": {}, "log_directory": "/tmp" } ================================================ FILE: roles/container-engine/molecule/test_cri.yml ================================================ --- - name: Test container manager hosts: all gather_facts: false become: true tasks: - name: Get kubespray defaults import_role: name: ../../kubespray_defaults - name: Collect services facts ansible.builtin.service_facts: - name: Check container manager service is running assert: that: - ansible_facts.services[container_manager + '.service'].state == 'running' - ansible_facts.services[container_manager + '.service'].status == 'enabled' - name: Check runtime version command: "{{ bin_dir }}/crictl --runtime-endpoint {{ cri_socket }} version" register: cri_version failed_when: > cri_version is failed or ("RuntimeName: " + cri_name) not in cri_version.stdout ================================================ FILE: roles/container-engine/molecule/test_runtime.yml ================================================ --- - name: Test container runtime hosts: all gather_facts: false become: true roles: - role: ../../kubespray_defaults tasks: - name: Copy test container files template: src: "{{ item }}.j2" dest: "/tmp/{{ item }}" owner: root mode: "0644" loop: - container.json - sandbox.json - name: Check running a container with runtime {{ container_runtime }} block: - name: Run container command: argv: - "{{ bin_dir }}/crictl" - run - --with-pull - --runtime - "{{ container_runtime }}" - /tmp/container.json - /tmp/sandbox.json - name: Check log file slurp: src: "/tmp/{{ container_runtime }}1.0.log" register: log_file failed_when: > log_file is failed or 'Hello from Docker' not in (log_file.content | b64decode) rescue: - name: Display container manager config on error command: "{{ bin_dir }}/crictl info" - name: Check container manager logs command: journalctl -u {{ container_manager }} failed_when: true ================================================ FILE: roles/container-engine/nerdctl/handlers/main.yml ================================================ --- - name: Get nerdctl completion command: "{{ bin_dir }}/nerdctl completion bash" changed_when: false register: nerdctl_completion check_mode: false - name: Install nerdctl completion copy: dest: /etc/bash_completion.d/nerdctl content: "{{ nerdctl_completion.stdout }}" mode: "0644" ================================================ FILE: roles/container-engine/nerdctl/tasks/main.yml ================================================ --- - name: Nerdctl | Download nerdctl include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.nerdctl) }}" - name: Nerdctl | Copy nerdctl binary from download dir copy: src: "{{ local_release_dir }}/nerdctl" dest: "{{ bin_dir }}/nerdctl" mode: "0755" remote_src: true owner: root group: root become: true notify: - Get nerdctl completion - Install nerdctl completion - name: Nerdctl | Create configuration dir file: path: /etc/nerdctl state: directory mode: "0755" owner: root group: root become: true - name: Nerdctl | Install nerdctl configuration template: src: nerdctl.toml.j2 dest: /etc/nerdctl/nerdctl.toml mode: "0644" owner: root group: root become: true ================================================ FILE: roles/container-engine/nerdctl/templates/nerdctl.toml.j2 ================================================ debug = false debug_full = false address = "{{ cri_socket }}" namespace = "k8s.io" snapshotter = "{{ nerdctl_snapshotter | default('overlayfs') }}" cni_path = "/opt/cni/bin" cni_netconfpath = "/etc/cni/net.d" cgroup_manager = "{{ kubelet_cgroup_driver | default('systemd') }}" hosts_dir = ["{{ containerd_cfg_dir }}/certs.d"] ================================================ FILE: roles/container-engine/runc/defaults/main.yml ================================================ --- runc_bin_dir: "{{ bin_dir }}" runc_package_name: runc ================================================ FILE: roles/container-engine/runc/tasks/main.yml ================================================ --- - name: Runc | check if fedora coreos stat: path: /run/ostree-booted get_attributes: false get_checksum: false get_mime: false register: ostree - name: Runc | set is_ostree set_fact: is_ostree: "{{ ostree.stat.exists }}" - name: Runc | Uninstall runc package managed by package manager when: - not is_ostree - ansible_distribution != "Flatcar Container Linux by Kinvolk" - ansible_distribution != "Flatcar" block: - name: Runc | Remove package package: name: "{{ runc_package_name }}" state: absent - name: Runc | Remove orphaned binary file: path: /usr/bin/runc state: absent when: runc_bin_dir != "/usr/bin" - name: Runc | Download runc binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.runc) }}" - name: Copy runc binary from download dir copy: src: "{{ downloads.runc.dest }}" dest: "{{ runc_bin_dir }}/runc" mode: "0755" remote_src: true ================================================ FILE: roles/container-engine/skopeo/tasks/main.yml ================================================ --- - name: Skopeo | check if fedora coreos stat: path: /run/ostree-booted get_attributes: false get_checksum: false get_mime: false register: ostree - name: Skopeo | set is_ostree set_fact: is_ostree: "{{ ostree.stat.exists }}" - name: Skopeo | Uninstall skopeo package managed by package manager package: name: skopeo state: absent when: - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) ignore_errors: true # noqa ignore-errors - name: Skopeo | Download skopeo binary include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.skopeo) }}" - name: Copy skopeo binary from download dir copy: src: "{{ downloads.skopeo.dest }}" dest: "{{ bin_dir }}/skopeo" mode: "0755" remote_src: true ================================================ FILE: roles/container-engine/tasks/main.yml ================================================ --- - name: Validate container engine import_role: name: container-engine/validate-container-engine tags: - container-engine - validate-container-engine - name: Container runtimes include_role: name: "container-engine/{{ item.role }}" apply: tags: - container-engine - "{{ item.role }}" loop: - { role: 'kata-containers', enabled: "{{ kata_containers_enabled }}" } - { role: 'gvisor', enabled: "{{ gvisor_enabled and container_manager in ['docker', 'containerd'] }}" } - { role: 'crun', enabled: "{{ crun_enabled }}" } - { role: 'youki', enabled: "{{ youki_enabled and container_manager == 'crio' }}" } # TODO: Technically, this is more container-runtime than engine when: item.enabled tags: - container-engine - kata-containers - gvisor - crun - youki - name: Container Manager vars: container_manager_role: crio: cri-o docker: cri-dockerd containerd: containerd include_role: name: "container-engine/{{ container_manager_role[container_manager] }}" apply: tags: - container-engine - crio - docker - containerd tags: - container-engine - crio - docker - containerd ================================================ FILE: roles/container-engine/validate-container-engine/tasks/main.yml ================================================ --- - name: Validate-container-engine | check if fedora coreos stat: path: /run/ostree-booted get_attributes: false get_checksum: false get_mime: false register: ostree tags: - facts - name: Validate-container-engine | set is_ostree set_fact: is_ostree: "{{ ostree.stat.exists }}" tags: - facts - name: Ensure kubelet systemd unit exists stat: path: "/etc/systemd/system/kubelet.service" register: kubelet_systemd_unit_exists tags: - facts - name: Populate service facts service_facts: tags: - facts - name: Check if containerd is installed find: file_type: file recurse: true use_regex: true patterns: - containerd.service$ paths: - /lib/systemd - /etc/systemd - /run/systemd register: containerd_installed tags: - facts - name: Check if docker is installed find: file_type: file recurse: true use_regex: true patterns: - docker.service$ paths: - /lib/systemd - /etc/systemd - /run/systemd register: docker_installed tags: - facts - name: Check if crio is installed find: file_type: file recurse: true use_regex: true patterns: - crio.service$ paths: - /lib/systemd - /etc/systemd - /run/systemd register: crio_installed tags: - facts - name: Uninstall containerd vars: service_name: containerd.service when: - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) - container_manager != "containerd" - docker_installed.matched == 0 - containerd_installed.matched > 0 - ansible_facts.services[service_name]['state'] == 'running' block: - name: Drain node include_role: name: remove_node/pre_remove apply: tags: - pre-remove when: kubelet_systemd_unit_exists.stat.exists - name: Stop kubelet service: name: kubelet state: stopped when: kubelet_systemd_unit_exists.stat.exists - name: Remove Containerd import_role: name: container-engine/containerd tasks_from: reset handlers_from: reset - name: Uninstall docker vars: service_name: docker.service when: - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) - container_manager != "docker" - docker_installed.matched > 0 - ansible_facts.services[service_name]['state'] == 'running' block: - name: Drain node include_role: name: remove_node/pre_remove apply: tags: - pre-remove when: kubelet_systemd_unit_exists.stat.exists - name: Stop kubelet service: name: kubelet state: stopped when: kubelet_systemd_unit_exists.stat.exists - name: Remove Docker import_role: name: container-engine/docker tasks_from: reset - name: Uninstall crio vars: service_name: crio.service when: - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) - container_manager != "crio" - crio_installed.matched > 0 - ansible_facts.services[service_name]['state'] == 'running' block: - name: Drain node include_role: name: remove_node/pre_remove apply: tags: - pre-remove when: kubelet_systemd_unit_exists.stat.exists - name: Stop kubelet service: name: kubelet state: stopped when: kubelet_systemd_unit_exists.stat.exists - name: Remove CRI-O import_role: name: container-engine/cri-o tasks_from: reset ================================================ FILE: roles/container-engine/youki/defaults/main.yml ================================================ --- youki_bin_dir: "{{ bin_dir }}" ================================================ FILE: roles/container-engine/youki/molecule/default/converge.yml ================================================ --- - name: Converge hosts: all become: true vars: youki_enabled: true container_manager: crio roles: - role: kubespray_defaults - role: container-engine/cri-o - role: container-engine/youki ================================================ FILE: roles/container-engine/youki/molecule/default/files/10-mynet.conf ================================================ { "cniVersion": "0.4.0", "name": "mynet", "type": "bridge", "bridge": "cni0", "isGateway": true, "ipMasq": true, "ipam": { "type": "host-local", "subnet": "172.19.0.0/24", "routes": [ { "dst": "0.0.0.0/0" } ] } } ================================================ FILE: roles/container-engine/youki/molecule/default/files/container.json ================================================ { "metadata": { "name": "youki1" }, "image": { "image": "quay.io/kubespray/hello-world:latest" }, "log_path": "youki1.0.log", "linux": {} } ================================================ FILE: roles/container-engine/youki/molecule/default/files/sandbox.json ================================================ { "metadata": { "name": "youki1", "namespace": "default", "attempt": 1, "uid": "hdishd83djaidwnduwk28bcsb" }, "linux": {}, "log_directory": "/tmp" } ================================================ FILE: roles/container-engine/youki/molecule/default/molecule.yml ================================================ --- role_name_check: 1 platforms: - cloud_image: ubuntu-2404 name: ubuntu24 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane - name: almalinux9 cloud_image: almalinux-9 vm_cpu_cores: 1 vm_memory: 1024 node_groups: - kube_control_plane provisioner: name: ansible env: ANSIBLE_ROLES_PATH: ../../../../ config_options: defaults: callbacks_enabled: profile_tasks timeout: 120 inventory: group_vars: k8s_cluster: youki_enabled: true container_manager: crio playbooks: create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml prepare: ../../../molecule/prepare.yml verifier: name: ansible ================================================ FILE: roles/container-engine/youki/molecule/default/verify.yml ================================================ --- - name: Test youki hosts: all gather_facts: false tasks: - name: Get kubespray defaults import_role: name: ../../../../../kubespray_defaults - name: Test version command: "{{ bin_dir }}/youki --version" register: youki_version failed_when: > youki_version is failed or 'youki' not in youki_version.stdout - name: Test run container import_playbook: ../../../molecule/test_runtime.yml vars: container_runtime: youki ================================================ FILE: roles/container-engine/youki/tasks/main.yml ================================================ --- - name: Youki | Download youki include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.youki) }}" - name: Youki | Copy youki binary from download dir copy: src: "{{ local_release_dir }}/youki" dest: "{{ youki_bin_dir }}/youki" mode: "0755" remote_src: true ================================================ FILE: roles/download/meta/main.yml ================================================ --- allow_duplicates: true ================================================ FILE: roles/download/tasks/check_pull_required.yml ================================================ --- # The image_info_command depends on the Container Runtime and will output something like the following: # nginx:1.15,gcr.io/google-containers/kube-proxy:v1.14.1,gcr.io/google-containers/kube-proxy@sha256:44af2833c6cbd9a7fc2e9d2f5244a39dfd2e31ad91bf9d4b7d810678db738ee9,gcr.io/google-containers/kube-apiserver:v1.14.1,etc... - name: Check_pull_required | Generate a list of information about the images on a node # noqa command-instead-of-shell - image_info_command contains a pipe, therefore requiring shell shell: "{{ image_info_command }}" register: docker_images changed_when: false check_mode: false when: not download_always_pull - name: Check_pull_required | Set pull_required if the desired image is not yet loaded set_fact: pull_required: >- {%- if image_reponame | regex_replace('^docker\.io/(library/)?', '') in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%} when: not download_always_pull - name: Check_pull_required | Check that the local digest sha256 corresponds to the given image tag assert: that: "{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')" when: - not download_always_pull - not pull_required - pull_by_digest tags: - asserts ================================================ FILE: roles/download/tasks/download_container.yml ================================================ --- - tags: - download block: - name: Set default values for flag variables set_fact: image_is_cached: false image_changed: false pull_required: "{{ download_always_pull }}" tags: - facts - name: Download_container | Set a few facts import_tasks: set_container_facts.yml tags: - facts - name: Download_container | Prepare container download include_tasks: check_pull_required.yml when: - not download_always_pull - debug: # noqa name[missing] msg: "Pull {{ image_reponame }} required is: {{ pull_required }}" - name: Download_container | Determine if image is in cache stat: path: "{{ image_path_cached }}" get_attributes: false get_checksum: false get_mime: false delegate_to: localhost connection: local delegate_facts: false register: cache_image changed_when: false become: false when: - download_force_cache - name: Download_container | Set fact indicating if image is in cache set_fact: image_is_cached: "{{ cache_image.stat.exists }}" tags: - facts when: - download_force_cache - name: Stop if image not in cache on ansible host when download_force_cache=true assert: that: image_is_cached msg: "Image cache file {{ image_path_cached }} not found for {{ image_reponame }} on localhost" when: - download_force_cache - not download_run_once - name: Download_container | Download image if required command: "{{ image_pull_command_on_localhost if download_localhost else image_pull_command }} {{ image_reponame }}" delegate_to: "{{ download_delegate if download_run_once else inventory_hostname }}" delegate_facts: true run_once: "{{ download_run_once }}" register: pull_task_result until: pull_task_result is succeeded delay: "{{ retry_stagger | random + 3 }}" retries: "{{ download_retries }}" become: "{{ user_can_become_root | default(false) or not download_localhost }}" environment: "{{ proxy_env if container_manager == 'containerd' else omit }}" when: - pull_required or download_run_once - not image_is_cached - name: Download_container | Save and compress image shell: "{{ image_save_command_on_localhost if download_localhost else image_save_command }}" # noqa command-instead-of-shell - image_save_command_on_localhost contains a pipe, therefore requires shell delegate_to: "{{ download_delegate }}" delegate_facts: false register: container_save_status failed_when: container_save_status.stderr run_once: true become: "{{ user_can_become_root | default(false) or not download_localhost }}" when: - not image_is_cached - download_run_once - name: Download_container | Copy image to ansible host cache ansible.posix.synchronize: src: "{{ image_path_final }}" dest: "{{ image_path_cached }}" use_ssh_args: true mode: pull when: - not image_is_cached - download_run_once - not download_localhost - download_delegate == inventory_hostname - name: Download_container | Upload image to node if it is cached ansible.posix.synchronize: src: "{{ image_path_cached }}" dest: "{{ image_path_final }}" use_ssh_args: true mode: push delegate_facts: false register: upload_image failed_when: not upload_image until: upload_image is succeeded retries: "{{ download_retries }}" delay: "{{ retry_stagger | random + 3 }}" when: - pull_required - download_force_cache - name: Download_container | Load image into the local container registry shell: "{{ image_load_command }}" # noqa command-instead-of-shell - image_load_command uses pipes, therefore requires shell register: container_load_status failed_when: container_load_status is failed when: - pull_required - download_force_cache - name: Download_container | Remove container image from cache file: state: absent path: "{{ image_path_final }}" when: - not download_keep_remote_cache ================================================ FILE: roles/download/tasks/download_file.yml ================================================ --- - name: "Download_file | download {{ download.dest }}" tags: - download block: - name: Prep_download | Set a few facts set_fact: download_force_cache: "{{ true if download_run_once else download_force_cache }}" - name: Download_file | Show url of file to download when: unsafe_show_logs | bool debug: msg: "{{ download.url }}" run_once: "{{ download_run_once }}" - name: Download_file | Set pathname of cached file set_fact: file_path_cached: "{{ download_cache_dir }}/{{ download.dest | basename }}" tags: - facts - name: Download_file | Create dest directory on node file: path: "{{ download.dest | dirname }}" owner: "{{ download.owner | default(omit) }}" mode: "0755" state: directory recurse: true - name: Download_file | Create local cache directory file: path: "{{ file_path_cached | dirname }}" state: directory recurse: true delegate_to: localhost connection: local delegate_facts: false run_once: true become: false when: - download_force_cache tags: - localhost - name: Download_file | Create cache directory on download_delegate host file: path: "{{ file_path_cached | dirname }}" state: directory recurse: true delegate_to: "{{ download_delegate }}" delegate_facts: false run_once: true when: - download_force_cache - not download_localhost # This must always be called, to check if the checksum matches. On no-match the file is re-downloaded. # This task will avoid logging it's parameters to not leak environment passwords in the log - name: Download_file | Download item get_url: url: "{{ download.url }}" dest: "{{ file_path_cached if download_force_cache else download.dest }}" owner: "{{ omit if download_localhost else (download.owner | default(omit)) }}" mode: "{{ omit if download_localhost else (download.mode | default(omit)) }}" checksum: "{{ download.checksum }}" validate_certs: "{{ download_validate_certs }}" url_username: "{{ download.username | default(omit) }}" url_password: "{{ download.password | default(omit) }}" force_basic_auth: "{{ download.force_basic_auth | default(omit) }}" timeout: "{{ download.timeout | default(omit) }}" delegate_to: "{{ download_delegate if download_force_cache else inventory_hostname }}" run_once: "{{ download_force_cache }}" register: get_url_result become: "{{ not download_localhost }}" until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg or get_url_result.status_code | default() == 304" retries: "{{ download_retries }}" delay: "{{ retry_stagger | default(5) }}" environment: "{{ proxy_env }}" no_log: "{{ not (unsafe_show_logs | bool) }}" - name: Download_file | Copy file back to ansible host file cache ansible.posix.synchronize: src: "{{ file_path_cached }}" dest: "{{ file_path_cached }}" use_ssh_args: true mode: pull when: - download_force_cache - not download_localhost - download_delegate == inventory_hostname - name: Download_file | Copy file from cache to nodes, if it is available ansible.posix.synchronize: src: "{{ file_path_cached }}" dest: "{{ download.dest }}" use_ssh_args: true mode: push register: get_task until: get_task is succeeded delay: "{{ retry_stagger | random + 3 }}" retries: "{{ download_retries }}" when: - download_force_cache - name: Download_file | Set mode and owner file: path: "{{ download.dest }}" mode: "{{ download.mode | default(omit) }}" owner: "{{ download.owner | default(omit) }}" when: - download_force_cache - name: "Download_file | Extract file archives" include_tasks: "extract_file.yml" ================================================ FILE: roles/download/tasks/extract_file.yml ================================================ --- - name: Extract_file | Unpacking archive unarchive: src: "{{ download.dest }}" dest: "{{ download.dest | dirname }}" owner: "{{ download.owner | default(omit) }}" mode: "{{ download.mode | default(omit) }}" remote_src: true extra_opts: "{{ download.unarchive_extra_opts | default(omit) }}" when: - download.unarchive | default(false) ================================================ FILE: roles/download/tasks/main.yml ================================================ --- - name: Download | Prepare working directories and variables import_tasks: prep_download.yml when: - not skip_downloads | default(false) tags: - download - upload - name: Download | Get kubeadm binary and list of required images include_tasks: prep_kubeadm_images.yml when: - not skip_downloads | default(false) - ('kube_control_plane' in group_names) tags: - download - upload - name: Download | Download files / images include_tasks: "{{ include_file }}" loop: "{{ downloads | combine(kubeadm_images) | dict2items }}" vars: download: "{{ download_defaults | combine(item.value) }}" include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml" when: - not skip_downloads | default(false) - download.enabled - item.value.enabled - (not (item.value.container | default(false))) or (item.value.container and download_container) - (download_run_once and inventory_hostname == download_delegate) or (group_names | intersect(download.groups) | length) ================================================ FILE: roles/download/tasks/prep_download.yml ================================================ --- - name: Prep_download | Set a few facts set_fact: download_force_cache: "{{ true if download_run_once else download_force_cache }}" tags: - facts - name: Prep_download | On localhost, check if passwordless root is possible command: "true" delegate_to: localhost connection: local run_once: true register: test_become changed_when: false ignore_errors: true # noqa ignore-errors become: true when: - download_localhost tags: - localhost - asserts - name: Prep_download | On localhost, check if user has access to the container runtime without using sudo shell: "{{ image_info_command_on_localhost }}" # noqa command-instead-of-shell - image_info_command_on_localhost contains pipe, therefore requires shell delegate_to: localhost connection: local run_once: true register: test_docker changed_when: false ignore_errors: true # noqa ignore-errors become: false when: - download_localhost tags: - localhost - asserts - name: Prep_download | Parse the outputs of the previous commands set_fact: user_in_docker_group: "{{ not test_docker.failed }}" user_can_become_root: "{{ not test_become.failed }}" when: - download_localhost tags: - localhost - asserts - name: Prep_download | Check that local user is in group or can become root assert: that: "user_in_docker_group or user_can_become_root" msg: >- Error: User is not in docker group and cannot become root. When download_localhost is true, at least one of these two conditions must be met. when: - download_localhost tags: - localhost - asserts - name: Prep_download | Register docker images info shell: "{{ image_info_command }}" # noqa command-instead-of-shell - image_info_command contains pipe therefore requires shell no_log: "{{ not (unsafe_show_logs | bool) }}" register: docker_images failed_when: false changed_when: false check_mode: false when: download_container - name: Prep_download | Create staging directory on remote node file: path: "{{ local_release_dir }}/images" state: directory mode: "0755" owner: "{{ ansible_ssh_user | default(ansible_user_id) }}" when: - ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - name: Prep_download | Create local cache for files and images on control node file: path: "{{ download_cache_dir }}/images" state: directory mode: "0755" delegate_to: localhost connection: local delegate_facts: false run_once: true become: false when: download_force_cache or download_run_once tags: - localhost ================================================ FILE: roles/download/tasks/prep_kubeadm_images.yml ================================================ --- - name: Prep_kubeadm_images | Download kubeadm binary include_tasks: "download_file.yml" vars: download: "{{ download_defaults | combine(downloads.kubeadm) }}" when: - not skip_downloads | default(false) - downloads.kubeadm.enabled - name: Prep_kubeadm_images | Copy kubeadm binary from download dir to system path copy: src: "{{ downloads.kubeadm.dest }}" dest: "{{ bin_dir }}/kubeadm" mode: "0755" remote_src: true - name: Prep_kubeadm_images | Create kubeadm config template: src: "kubeadm-images.yaml.j2" dest: "{{ kube_config_dir }}/kubeadm-images.yaml" mode: "0644" validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}" when: - not skip_kubeadm_images | default(false) - name: Prep_kubeadm_images | Generate list of required images shell: "set -o pipefail && {{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml | grep -Ev 'coredns|pause'" args: executable: /bin/bash register: kubeadm_images_raw run_once: true changed_when: false when: - not skip_kubeadm_images | default(false) - name: Prep_kubeadm_images | Parse list of images vars: kubeadm_images_list: "{{ kubeadm_images_raw.stdout_lines }}" set_fact: kubeadm_image: key: "kubeadm_{{ (item | regex_replace('^(?:.*\\/)*', '')).split(':')[0] }}" value: enabled: true container: true repo: "{{ item | regex_replace('^(.*):.*$', '\\1') }}" tag: "{{ item | regex_replace('^.*:(.*)$', '\\1') }}" groups: - k8s_cluster loop: "{{ kubeadm_images_list | flatten(levels=1) }}" register: kubeadm_images_cooked run_once: true when: - not skip_kubeadm_images | default(false) - name: Prep_kubeadm_images | Convert list of images to dict for later use set_fact: kubeadm_images: "{{ kubeadm_images_cooked.results | map(attribute='ansible_facts.kubeadm_image') | list | items2dict }}" run_once: true when: - not skip_kubeadm_images | default(false) ================================================ FILE: roles/download/tasks/set_container_facts.yml ================================================ --- - name: Set_container_facts | Display the name of the image being processed debug: msg: "{{ download.repo }}" - name: Set_container_facts | Set if containers should be pulled by digest set_fact: pull_by_digest: "{{ download.sha256 is defined and download.sha256 }}" - name: Set_container_facts | Define by what name to pull the image set_fact: image_reponame: >- {%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%} - name: Set_container_facts | Define file name of image set_fact: image_filename: "{{ image_reponame | regex_replace('/|\0|:', '_') }}.tar" - name: Set_container_facts | Define path of image set_fact: image_path_cached: "{{ download_cache_dir }}/images/{{ image_filename }}" image_path_final: "{{ local_release_dir }}/images/{{ image_filename }}" - name: Set image save/load command for docker set_fact: image_save_command: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_final }}" image_load_command: "{{ docker_bin_dir }}/docker load < {{ image_path_final }}" when: container_manager == 'docker' - name: Set image save/load command for containerd set_fact: image_save_command: "{{ bin_dir }}/nerdctl -n k8s.io image save -o {{ image_path_final }} {{ image_reponame }}" image_load_command: "{{ bin_dir }}/nerdctl -n k8s.io image load < {{ image_path_final }}" when: container_manager == 'containerd' - name: Set image save/load command for crio set_fact: image_save_command: "{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null" image_load_command: "{{ bin_dir }}/skopeo copy docker-archive:{{ image_path_final }} containers-storage:{{ image_reponame }} 2>/dev/null" when: container_manager == 'crio' - name: Set image save/load command for docker on localhost set_fact: image_save_command_on_localhost: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached }}" when: container_manager_on_localhost == 'docker' - name: Set image save/load command for containerd on localhost set_fact: image_save_command_on_localhost: "{{ containerd_bin_dir }}/ctr -n k8s.io image export --platform linux/{{ image_arch }} {{ image_path_cached }} {{ image_reponame }}" when: container_manager_on_localhost == 'containerd' - name: Set image save/load command for crio on localhost set_fact: image_save_command_on_localhost: "{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null" when: container_manager_on_localhost == 'crio' ================================================ FILE: roles/download/templates/kubeadm-images.yaml.j2 ================================================ apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration nodeRegistration: criSocket: {{ cri_socket }} --- apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration imageRepository: {{ kubeadm_image_repo }} kubernetesVersion: v{{ kube_version }} etcd: {% if etcd_deployment_type == "kubeadm" %} local: imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" imageTag: "{{ etcd_image_tag }}" {% else %} external: endpoints: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} {% endif %} dns: imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }} imageTag: {{ coredns_image_tag }} ================================================ FILE: roles/dynamic_groups/tasks/main.yml ================================================ --- - name: Match needed groups by their old names or definition vars: group_mappings: kube_control_plane: - kube-master kube_node: - kube-node calico_rr: - calico-rr no_floating: - no-floating k8s_cluster: - kube_node - kube_control_plane - calico_rr group_by: key: "{{ item.key }}" when: group_names | intersect(item.value) | length > 0 loop: "{{ group_mappings | dict2items }}" ================================================ FILE: roles/etcd/handlers/backup.yml ================================================ --- - name: Refresh Time Fact setup: filter: ansible_date_time listen: Restart etcd when: etcd_cluster_is_healthy.rc == 0 - name: Set Backup Directory set_fact: etcd_backup_directory: "{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}" listen: Restart etcd - name: Create Backup Directory file: path: "{{ etcd_backup_directory }}" state: directory owner: root group: root mode: "0600" listen: Restart etcd when: etcd_cluster_is_healthy.rc == 0 - name: Stat etcd v2 data directory stat: path: "{{ etcd_data_dir }}/member" get_attributes: false get_checksum: false get_mime: false register: etcd_data_dir_member listen: Restart etcd when: etcd_cluster_is_healthy.rc == 0 - name: Backup etcd v2 data when: - etcd_data_dir_member.stat.exists - etcd_cluster_is_healthy.rc == 0 - etcd_version is version('3.6.0', '<') command: >- {{ bin_dir }}/etcdctl backup --data-dir {{ etcd_data_dir }} --backup-dir {{ etcd_backup_directory }} environment: ETCDCTL_API: "2" retries: 3 register: backup_v2_command until: backup_v2_command.rc == 0 delay: "{{ retry_stagger | random + 3 }}" listen: Restart etcd - name: Backup etcd v3 data command: >- {{ bin_dir }}/etcdctl snapshot save {{ etcd_backup_directory }}/snapshot.db environment: ETCDCTL_API: "3" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" retries: 3 register: etcd_backup_v3_command until: etcd_backup_v3_command.rc == 0 delay: "{{ retry_stagger | random + 3 }}" listen: Restart etcd when: etcd_cluster_is_healthy.rc == 0 ================================================ FILE: roles/etcd/handlers/backup_cleanup.yml ================================================ --- - name: Find old etcd backups ansible.builtin.find: file_type: directory recurse: false paths: "{{ etcd_backup_prefix }}" patterns: "etcd-*" register: _etcd_backups when: etcd_backup_retention_count >= 0 listen: Restart etcd - name: Remove old etcd backups ansible.builtin.file: state: absent path: "{{ item }}" loop: "{{ (_etcd_backups.files | sort(attribute='ctime', reverse=True))[etcd_backup_retention_count:] | map(attribute='path') }}" when: etcd_backup_retention_count >= 0 listen: Restart etcd ================================================ FILE: roles/etcd/handlers/main.yml ================================================ --- - name: Backup etcd import_tasks: backup.yml - name: Restart etcd systemd_service: name: etcd state: restarted daemon_reload: true when: ('etcd' in group_names) throttle: "{{ groups['etcd'] | length // 2 }}" # Etcd cluster MUST have an odd number of members # Truncated integer division by 2 will always return (majority - 1) which # means the cluster will keep quorum and stay available - name: Restart etcd-events systemd_service: name: etcd-events state: restarted daemon_reload: true # TODO: this seems odd. etcd-events should be a different group possibly ? when: ('etcd' in group_names) throttle: "{{ groups['etcd'] | length // 2 }}" - name: Wait for etcd up uri: url: "https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2379/health" validate_certs: false client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" register: result until: result.status is defined and result.status == 200 retries: 60 delay: 1 listen: Restart etcd - name: Cleanup etcd backups import_tasks: backup_cleanup.yml - name: Wait for etcd-events up uri: url: "https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2383/health" validate_certs: false client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" register: result until: result.status is defined and result.status == 200 retries: 60 delay: 1 listen: Restart etcd-events - name: Set etcd_secret_changed set_fact: etcd_secret_changed: true ================================================ FILE: roles/etcd/meta/main.yml ================================================ --- dependencies: - role: adduser user: "{{ addusers.etcd }}" when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) - role: adduser user: "{{ addusers.kube }}" when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) - role: etcd_defaults ================================================ FILE: roles/etcd/tasks/check_certs.yml ================================================ --- - name: "Check_certs | Register certs that have already been generated on first etcd node" find: paths: "{{ etcd_cert_dir }}" patterns: "ca.pem,node*.pem,member*.pem,admin*.pem" get_checksum: true delegate_to: "{{ groups['etcd'][0] }}" register: etcdcert_master run_once: true - name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false" set_fact: sync_certs: false gen_certs: false etcd_secret_changed: false - name: "Check certs | Register ca and etcd admin/member certs on etcd hosts" stat: path: "{{ etcd_cert_dir }}/{{ item }}" get_attributes: false get_checksum: true get_mime: false register: etcd_member_certs when: ('etcd' in group_names) with_items: - ca.pem - member-{{ inventory_hostname }}.pem - member-{{ inventory_hostname }}-key.pem - admin-{{ inventory_hostname }}.pem - admin-{{ inventory_hostname }}-key.pem - name: "Check certs | Register ca and etcd node certs on kubernetes hosts" stat: path: "{{ etcd_cert_dir }}/{{ item }}" register: etcd_node_certs when: ('k8s_cluster' in group_names) with_items: - ca.pem - node-{{ inventory_hostname }}.pem - node-{{ inventory_hostname }}-key.pem - name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(1/2)" set_fact: gen_certs: true when: force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list run_once: true with_items: "{{ expected_files }}" vars: expected_files: >- ['{{ etcd_cert_dir }}/ca.pem', {% set etcd_members = groups['etcd'] %} {% for host in etcd_members %} '{{ etcd_cert_dir }}/admin-{{ host }}.pem', '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', '{{ etcd_cert_dir }}/member-{{ host }}.pem', '{{ etcd_cert_dir }}/member-{{ host }}-key.pem', {% endfor %} {% set k8s_nodes = groups['kube_control_plane'] %} {% for host in k8s_nodes %} '{{ etcd_cert_dir }}/node-{{ host }}.pem', '{{ etcd_cert_dir }}/node-{{ host }}-key.pem' {% if not loop.last %}{{ ',' }}{% endif %} {% endfor %}] - name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(2/2)" set_fact: gen_certs: true run_once: true with_items: "{{ expected_files }}" vars: expected_files: >- ['{{ etcd_cert_dir }}/ca.pem', {% set etcd_members = groups['etcd'] %} {% for host in etcd_members %} '{{ etcd_cert_dir }}/admin-{{ host }}.pem', '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', '{{ etcd_cert_dir }}/member-{{ host }}.pem', '{{ etcd_cert_dir }}/member-{{ host }}-key.pem', {% endfor %} {% set k8s_nodes = groups['k8s_cluster'] | unique | sort %} {% for host in k8s_nodes %} '{{ etcd_cert_dir }}/node-{{ host }}.pem', '{{ etcd_cert_dir }}/node-{{ host }}-key.pem' {% if not loop.last %}{{ ',' }}{% endif %} {% endfor %}] when: - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list - name: "Check_certs | Set 'gen_*_certs' groups to track which nodes needs to have certs generated on first etcd node" vars: existing_certs: etcdcert_master.files | map(attribute='path') ansible.builtin.group_by: key: "gen_{{ item.node_type }}_certs_{{ force_etcd_cert_refresh or item.certs is not subset(existing_certs) }}" loop: "{{ cert_files | dict2items(key_name='node_type', value_name='certs') }}" - name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match" set_fact: etcd_member_requires_sync: true when: - ('etcd' in group_names) - (not etcd_member_certs.results[0].stat.exists | default(false)) or (not etcd_member_certs.results[1].stat.exists | default(false)) or (not etcd_member_certs.results[2].stat.exists | default(false)) or (not etcd_member_certs.results[3].stat.exists | default(false)) or (not etcd_member_certs.results[4].stat.exists | default(false)) or (etcd_member_certs.results[0].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[0].stat.path) | map(attribute="checksum") | first | default('')) or (etcd_member_certs.results[1].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[1].stat.path) | map(attribute="checksum") | first | default('')) or (etcd_member_certs.results[2].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[2].stat.path) | map(attribute="checksum") | first | default('')) or (etcd_member_certs.results[3].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[3].stat.path) | map(attribute="checksum") | first | default('')) or (etcd_member_certs.results[4].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[4].stat.path) | map(attribute="checksum") | first | default('')) - name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match" set_fact: kubernetes_host_requires_sync: true when: - ('k8s_cluster' in group_names) and inventory_hostname not in groups['etcd'] - (not etcd_node_certs.results[0].stat.exists | default(false)) or (not etcd_node_certs.results[1].stat.exists | default(false)) or (not etcd_node_certs.results[2].stat.exists | default(false)) or (etcd_node_certs.results[0].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_node_certs.results[0].stat.path) | map(attribute="checksum") | first | default('')) or (etcd_node_certs.results[1].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_node_certs.results[1].stat.path) | map(attribute="checksum") | first | default('')) or (etcd_node_certs.results[2].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_node_certs.results[2].stat.path) | map(attribute="checksum") | first | default('')) - name: "Check_certs | Set 'sync_certs' to true" set_fact: sync_certs: true when: - etcd_member_requires_sync | default(false) or kubernetes_host_requires_sync | default(false) or 'gen_master_certs_True' in group_names or 'gen_node_certs_True' in group_names ================================================ FILE: roles/etcd/tasks/clean_v2_store.yml ================================================ --- # When upgrading from etcd 3.5 to 3.6, need to clean up v2 store before upgrading. # Without this, etcd 3.6 will crash with following error: # "panic: detected disallowed v2 WAL for stage --v2-deprecation=write-only [recovered]" - name: Cleanup v2 store when upgrade etcd from <3.6 to >=3.6 when: - etcd_cluster_setup - etcd_current_version != '' - etcd_current_version is version('3.6.0', '<') - etcd_version is version('3.6.0', '>=') block: - name: Ensure etcd version is >=3.5.26 when: - etcd_current_version is version('3.5.26', '<') fail: msg: "You need to upgrade etcd to 3.5.26 or later before upgrade to 3.6. Current version is {{ etcd_current_version }}." # Workarounds: # Disable --enable-v2 (recommended in 20289) and do workaround of 20231 (MAX_WALS=1 and SNAPSHOT_COUNT=1) # - https://github.com/etcd-io/etcd/issues/20809 # - https://github.com/etcd-io/etcd/discussions/20231#discussioncomment-13958051 - name: Change etcd configuration temporally to limit number of WALs and snapshots to clean up v2 store ansible.builtin.lineinfile: path: /etc/etcd.env regexp: "{{ item.regexp }}" line: "{{ item.line }}" loop: - { regexp: '^ETCD_SNAPSHOT_COUNT=', line: 'ETCD_SNAPSHOT_COUNT=1' } - { regexp: '^ETCD_MAX_WALS=', line: 'ETCD_MAX_WALS=1' } - { regexp: '^ETCD_MAX_SNAPSHOTS=', line: 'ETCD_MAX_SNAPSHOTS=1' } - { regexp: '^ETCD_ENABLE_V2=', line: 'ETCD_ENABLE_V2=false' } # Restart etcd to apply temporal configuration and prevent some upgrade failures # See also: https://etcd.io/blog/2025/upgrade_from_3.5_to_3.6_issue_followup/ - name: Stop etcd service: name: etcd state: stopped - name: Start etcd service: name: etcd state: started ================================================ FILE: roles/etcd/tasks/configure.yml ================================================ --- - name: Configure | Check if etcd cluster is healthy shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" args: executable: /bin/bash register: etcd_cluster_is_healthy failed_when: false changed_when: false check_mode: false run_once: true when: - ('etcd' in group_names) - etcd_cluster_setup tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - name: Configure | Check if etcd-events cluster is healthy shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" args: executable: /bin/bash register: etcd_events_cluster_is_healthy failed_when: false changed_when: false check_mode: false run_once: true when: - ('etcd' in group_names) - etcd_events_cluster_setup tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - name: Configure | Refresh etcd config include_tasks: refresh_config.yml when: ('etcd' in group_names) - name: Configure | Copy etcd.service systemd file template: src: "etcd-{{ etcd_deployment_type }}.service.j2" dest: /etc/systemd/system/etcd.service backup: true mode: "0644" # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) # Remove once we drop support for systemd < 250 validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:etcd-{{ etcd_deployment_type }}.service'" when: - ('etcd' in group_names) - etcd_cluster_setup - name: Configure | Copy etcd-events.service systemd file template: src: "etcd-events-{{ etcd_deployment_type }}.service.j2" dest: /etc/systemd/system/etcd-events.service backup: true mode: "0644" validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:etcd-events-{{ etcd_deployment_type }}.service'" # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) # Remove once we drop support for systemd < 250 when: - ('etcd' in group_names) - etcd_events_cluster_setup - name: Configure | reload systemd systemd_service: daemon_reload: true when: ('etcd' in group_names) # when scaling new etcd will fail to start - name: Configure | Ensure etcd is running service: name: etcd state: started enabled: true ignore_errors: "{{ etcd_cluster_is_healthy.rc == 0 }}" # noqa ignore-errors when: - ('etcd' in group_names) - etcd_cluster_setup # when scaling new etcd will fail to start - name: Configure | Ensure etcd-events is running service: name: etcd-events state: started enabled: true ignore_errors: "{{ etcd_events_cluster_is_healthy.rc != 0 }}" # noqa ignore-errors when: - ('etcd' in group_names) - etcd_events_cluster_setup - name: Configure | Wait for etcd cluster to be healthy shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" args: executable: /bin/bash register: etcd_cluster_is_healthy until: etcd_cluster_is_healthy.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" changed_when: false check_mode: false run_once: true when: - ('etcd' in group_names) - etcd_cluster_setup tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - name: Configure | Wait for etcd-events cluster to be healthy shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" args: executable: /bin/bash register: etcd_events_cluster_is_healthy until: etcd_events_cluster_is_healthy.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" changed_when: false check_mode: false run_once: true when: - ('etcd' in group_names) - etcd_events_cluster_setup tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - name: Configure | Check if member is in etcd cluster shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address | replace('[', '') | replace(']', '') }}" register: etcd_member_in_cluster ignore_errors: true # noqa ignore-errors changed_when: false check_mode: false when: - ('etcd' in group_names) - etcd_cluster_setup tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - name: Configure | Check if member is in etcd-events cluster shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address | replace('[', '') | replace(']', '') }}" register: etcd_events_member_in_cluster ignore_errors: true # noqa ignore-errors changed_when: false check_mode: false when: - ('etcd' in group_names) - etcd_events_cluster_setup tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - name: Configure | Join member(s) to etcd cluster one at a time include_tasks: join_etcd_member.yml with_items: "{{ groups['etcd'] }}" when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 - name: Configure | Join member(s) to etcd-events cluster one at a time include_tasks: join_etcd-events_member.yml with_items: "{{ groups['etcd'] }}" when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0 ================================================ FILE: roles/etcd/tasks/gen_certs_script.yml ================================================ --- - name: Gen_certs | create etcd cert dir file: path: "{{ etcd_cert_dir }}" group: "{{ etcd_cert_group }}" state: directory owner: "{{ etcd_owner }}" mode: "0700" - name: "Gen_certs | create etcd script dir (on {{ groups['etcd'][0] }})" file: path: "{{ etcd_script_dir }}" state: directory owner: root mode: "0700" run_once: true when: inventory_hostname == groups['etcd'][0] - name: Gen_certs | write openssl config template: src: "openssl.conf.j2" dest: "{{ etcd_config_dir }}/openssl.conf" mode: "0640" run_once: true delegate_to: "{{ groups['etcd'][0] }}" when: - gen_certs | default(false) - inventory_hostname == groups['etcd'][0] - name: Gen_certs | copy certs generation script template: src: "make-ssl-etcd.sh.j2" dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh" mode: "0700" run_once: true when: - inventory_hostname == groups['etcd'][0] - name: Gen_certs | run cert generation script for etcd and kube control plane nodes command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" environment: MASTERS: "{{ groups['gen_master_certs_True'] | ansible.builtin.intersect(groups['etcd']) | join(' ') }}" HOSTS: "{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['kube_control_plane']) | join(' ') }}" run_once: true delegate_to: "{{ groups['etcd'][0] }}" when: gen_certs | default(false) notify: Set etcd_secret_changed - name: Gen_certs | run cert generation script for all clients command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" environment: HOSTS: "{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['k8s_cluster']) | join(' ') }}" run_once: true delegate_to: "{{ groups['etcd'][0] }}" when: - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - gen_certs | default(false) notify: Set etcd_secret_changed - name: Gen_certs | Gather etcd member/admin and kube_control_plane client certs from first etcd node slurp: src: "{{ item }}" register: etcd_master_certs with_items: - "{{ etcd_cert_dir }}/ca.pem" - "{{ etcd_cert_dir }}/ca-key.pem" - "[{% for node in groups['etcd'] %} '{{ etcd_cert_dir }}/admin-{{ node }}.pem', '{{ etcd_cert_dir }}/admin-{{ node }}-key.pem', '{{ etcd_cert_dir }}/member-{{ node }}.pem', '{{ etcd_cert_dir }}/member-{{ node }}-key.pem', {% endfor %}]" - "[{% for node in (groups['kube_control_plane']) %} '{{ etcd_cert_dir }}/node-{{ node }}.pem', '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', {% endfor %}]" delegate_to: "{{ groups['etcd'][0] }}" when: - ('etcd' in group_names) - sync_certs | default(false) - inventory_hostname != groups['etcd'][0] notify: Set etcd_secret_changed - name: Gen_certs | Write etcd member/admin and kube_control_plane client certs to other etcd nodes copy: dest: "{{ item.item }}" content: "{{ item.content | b64decode }}" group: "{{ etcd_cert_group }}" owner: "{{ etcd_owner }}" mode: "0640" with_items: "{{ etcd_master_certs.results }}" when: - ('etcd' in group_names) - sync_certs | default(false) - inventory_hostname != groups['etcd'][0] loop_control: label: "{{ item.item }}" - name: Gen_certs | Gather node certs from first etcd node slurp: src: "{{ item }}" register: etcd_master_node_certs with_items: - "[{% for node in groups['k8s_cluster'] %} '{{ etcd_cert_dir }}/node-{{ node }}.pem', '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', {% endfor %}]" delegate_to: "{{ groups['etcd'][0] }}" when: - ('etcd' in group_names) - inventory_hostname != groups['etcd'][0] - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" notify: Set etcd_secret_changed - name: Gen_certs | Write node certs to other etcd nodes copy: dest: "{{ item.item }}" content: "{{ item.content | b64decode }}" group: "{{ etcd_cert_group }}" owner: "{{ etcd_owner }}" mode: "0640" with_items: "{{ etcd_master_node_certs.results }}" when: - ('etcd' in group_names) - inventory_hostname != groups['etcd'][0] - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" loop_control: label: "{{ item.item }}" - name: Gen_certs | Generate etcd certs include_tasks: gen_nodes_certs_script.yml when: - ('kube_control_plane' in group_names) and sync_certs | default(false) and inventory_hostname not in groups['etcd'] - name: Gen_certs | Generate etcd certs on nodes if needed include_tasks: gen_nodes_certs_script.yml when: - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - ('k8s_cluster' in group_names) and sync_certs | default(false) and inventory_hostname not in groups['etcd'] # This is a hack around the fact kubeadm expect the same certs path on all kube_control_plane # TODO: fix certs generation to have the same file everywhere # OR work with kubeadm on node-specific config - name: Gen_certs | Pretend all control plane have all certs (with symlinks) file: state: link src: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}{{ item[0] }}.pem" dest: "{{ etcd_cert_dir }}/node-{{ item[1] }}{{ item[0] }}.pem" mode: "0640" loop: "{{ suffixes | product(groups['kube_control_plane']) }}" vars: suffixes: - '' - '-key' when: - ('kube_control_plane' in group_names) - item[1] != inventory_hostname register: symlink_created failed_when: - symlink_created is failed - ('refusing to convert from file to symlink' not in symlink_created.msg) ================================================ FILE: roles/etcd/tasks/gen_nodes_certs_script.yml ================================================ --- - name: Gen_certs | Set cert names per node set_fact: my_etcd_node_certs: [ 'ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem'] tags: - facts - name: "Check_certs | Set 'sync_certs' to true on nodes" set_fact: sync_certs: true with_items: - "{{ my_etcd_node_certs }}" - name: Gen_certs | Gather node certs vars: ansible_ssh_retries: 10 shell: "set -o pipefail && tar cfz - -C {{ etcd_cert_dir }} {{ my_etcd_node_certs | join(' ') }} | base64 --wrap=0" args: executable: /bin/bash no_log: "{{ not (unsafe_show_logs | bool) }}" register: etcd_node_certs check_mode: false delegate_to: "{{ groups['etcd'][0] }}" changed_when: false - name: Gen_certs | Copy certs on nodes shell: "set -o pipefail && base64 -d <<< '{{ etcd_node_certs.stdout | quote }}' | tar xz -C {{ etcd_cert_dir }}" args: executable: /bin/bash no_log: "{{ not (unsafe_show_logs | bool) }}" changed_when: false ================================================ FILE: roles/etcd/tasks/install_docker.yml ================================================ --- - name: Get currently-deployed etcd version shell: "{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}' --format='{{ '{{ .Image }}' }}'" register: etcd_current_docker_image when: etcd_cluster_setup - name: Get currently-deployed etcd-events version shell: "{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}-events' --format='{{ '{{ .Image }}' }}'" register: etcd_events_current_docker_image when: etcd_events_cluster_setup - name: Restart etcd if necessary command: /bin/true notify: Restart etcd when: - etcd_cluster_setup - etcd_image_tag not in etcd_current_docker_image.stdout | default('') - name: Restart etcd-events if necessary command: /bin/true notify: Restart etcd-events when: - etcd_events_cluster_setup - etcd_image_tag not in etcd_events_current_docker_image.stdout | default('') - name: Get currently-deployed etcd version as x.y.z format set_fact: etcd_current_version: "{{ (etcd_current_docker_image.stdout | regex_search('.*:v([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1'))[0] | default('') }}" when: etcd_cluster_setup - name: Cleanup v2 store data import_tasks: clean_v2_store.yml - name: Install etcd launch script template: src: etcd.j2 dest: "{{ bin_dir }}/etcd" owner: 'root' mode: "0750" backup: true when: etcd_cluster_setup - name: Install etcd-events launch script template: src: etcd-events.j2 dest: "{{ bin_dir }}/etcd-events" owner: 'root' mode: "0750" backup: true when: etcd_events_cluster_setup ================================================ FILE: roles/etcd/tasks/install_host.yml ================================================ --- - name: Get currently-deployed etcd version command: "{{ bin_dir }}/etcd --version" register: etcd_current_host_version # There's a chance this play could run before etcd is installed at all # TODO: figure out whether this happens. "A chance" is not enough information ignore_errors: true when: etcd_cluster_setup - name: Restart etcd if necessary command: /bin/true notify: Restart etcd when: - etcd_cluster_setup - etcd_version not in etcd_current_host_version.stdout | default('') - name: Restart etcd-events if necessary command: /bin/true notify: Restart etcd-events when: - etcd_events_cluster_setup - etcd_version not in etcd_current_host_version.stdout | default('') - name: Get currently-deployed etcd version as x.y.z format set_fact: etcd_current_version: "{{ (etcd_current_host_version.stdout | regex_search('etcd Version: ([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1'))[0] | default('') }}" when: etcd_cluster_setup - name: Cleanup v2 store data import_tasks: clean_v2_store.yml - name: Install | Copy etcd binary from download dir copy: src: "{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}" dest: "{{ bin_dir }}/{{ item }}" mode: "0755" remote_src: true with_items: - etcd when: etcd_cluster_setup ================================================ FILE: roles/etcd/tasks/join_etcd-events_member.yml ================================================ --- - name: Join Member | Add member to etcd-events cluster command: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}" register: member_add_result until: member_add_result.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - name: Join Member | Refresh etcd config include_tasks: refresh_config.yml vars: # noqa: jinja[spacing] etcd_events_peer_addresses: >- {% for host in groups['etcd'] -%} {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%} {{ "etcd" + loop.index | string }}=https://{{ hostvars[host].etcd_events_access_address | default(hostvars[host]['main_ip']) | ansible.utils.ipwrap }}:2382, {%- endif -%} {%- if loop.last -%} {{ etcd_member_name }}={{ etcd_events_peer_url }} {%- endif -%} {%- endfor -%} - name: Join Member | Ensure member is in etcd-events cluster shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep -w {{ etcd_events_access_address }} >/dev/null" args: executable: /bin/bash register: etcd_events_member_in_cluster changed_when: false check_mode: false tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - name: Configure | Ensure etcd-events is running service: name: etcd-events state: started enabled: true ================================================ FILE: roles/etcd/tasks/join_etcd_member.yml ================================================ --- - name: Join Member | Add member to etcd cluster command: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}" register: member_add_result until: member_add_result.rc == 0 or 'Peer URLs already exists' in member_add_result.stderr failed_when: member_add_result.rc != 0 and 'Peer URLs already exists' not in member_add_result.stderr retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - name: Join Member | Refresh etcd config include_tasks: refresh_config.yml vars: # noqa: jinja[spacing] etcd_peer_addresses: >- {% for host in groups['etcd'] -%} {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%} {{ "etcd" + loop.index | string }}=https://{{ hostvars[host].etcd_access_address | default(hostvars[host]['main_ip']) | ansible.utils.ipwrap }}:2380, {%- endif -%} {%- if loop.last -%} {{ etcd_member_name }}={{ etcd_peer_url }} {%- endif -%} {%- endfor -%} - name: Join Member | Ensure member is in etcd cluster shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep -w {{ etcd_access_address }} >/dev/null" args: executable: /bin/bash register: etcd_member_in_cluster changed_when: false check_mode: false retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" until: etcd_member_in_cluster.rc == 0 tags: - facts environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - name: Configure | Ensure etcd is running service: name: etcd state: started enabled: true ================================================ FILE: roles/etcd/tasks/main.yml ================================================ --- - name: Check etcd certs include_tasks: check_certs.yml when: cert_management == "script" tags: - etcd-secrets - facts - name: Generate etcd certs include_tasks: "gen_certs_script.yml" when: - cert_management == "script" tags: - etcd-secrets - name: Trust etcd CA include_tasks: upd_ca_trust.yml when: - ('etcd' in group_names) or ('kube_control_plane' in group_names) tags: - etcd-secrets - name: Trust etcd CA on nodes if needed include_tasks: upd_ca_trust.yml when: - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - ('k8s_cluster' in group_names) tags: - etcd-secrets - name: "Gen_certs | Get etcd certificate serials" command: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial" register: "etcd_client_cert_serial_result" changed_when: false check_mode: false when: - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - ('k8s_cluster' in group_names) tags: - control-plane - network - name: Set etcd_client_cert_serial set_fact: etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}" when: - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - ('k8s_cluster' in group_names) tags: - control-plane - network - name: Install etcd include_tasks: "install_{{ etcd_deployment_type }}.yml" when: ('etcd' in group_names) tags: - upgrade - name: Install etcdctl and etcdutl binary import_role: name: etcdctl_etcdutl tags: - etcdctl - etcdutl - upgrade when: - ('etcd' in group_names) - etcd_cluster_setup - name: Configure etcd include_tasks: configure.yml when: ('etcd' in group_names) - name: Refresh etcd config include_tasks: refresh_config.yml when: ('etcd' in group_names) - name: Restart etcd if certs changed command: /bin/true notify: Restart etcd when: - ('etcd' in group_names) - etcd_cluster_setup - etcd_secret_changed | default(false) - name: Restart etcd-events if certs changed command: /bin/true notify: Restart etcd when: - ('etcd' in group_names) - etcd_events_cluster_setup - etcd_secret_changed | default(false) # After etcd cluster is assembled, make sure that # initial state of the cluster is in `existing` # state instead of `new`. - name: Refresh etcd config again for idempotency include_tasks: refresh_config.yml when: ('etcd' in group_names) ================================================ FILE: roles/etcd/tasks/refresh_config.yml ================================================ --- - name: Refresh config | Create etcd config file template: src: etcd.env.j2 dest: /etc/etcd.env mode: "0640" notify: Restart etcd when: - ('etcd' in group_names) - etcd_cluster_setup - name: Refresh config | Create etcd-events config file template: src: etcd-events.env.j2 dest: /etc/etcd-events.env mode: "0640" notify: Restart etcd-events when: - ('etcd' in group_names) - etcd_events_cluster_setup ================================================ FILE: roles/etcd/tasks/upd_ca_trust.yml ================================================ --- - name: Gen_certs | target ca-certificate store file set_fact: ca_cert_path: |- {% if ansible_os_family == "Debian" -%} /usr/local/share/ca-certificates/etcd-ca.crt {%- elif ansible_os_family == "RedHat" -%} /etc/pki/ca-trust/source/anchors/etcd-ca.crt {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] -%} /etc/ssl/certs/etcd-ca.pem {%- elif ansible_os_family == "Suse" -%} /etc/pki/trust/anchors/etcd-ca.pem {%- elif ansible_os_family == "ClearLinux" -%} /usr/share/ca-certs/etcd-ca.pem {%- endif %} tags: - facts - name: Gen_certs | add CA to trusted CA dir copy: src: "{{ etcd_cert_dir }}/ca.pem" dest: "{{ ca_cert_path }}" remote_src: true mode: "0640" register: etcd_ca_cert - name: Gen_certs | update ca-certificates (Debian/Ubuntu/SUSE/Flatcar) # noqa no-handler command: update-ca-certificates when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "Flatcar", "Flatcar Container Linux by Kinvolk", "Suse"] - name: Gen_certs | update ca-certificates (RedHat) # noqa no-handler command: update-ca-trust extract when: etcd_ca_cert.changed and ansible_os_family == "RedHat" - name: Gen_certs | update ca-certificates (ClearLinux) # noqa no-handler command: clrtrust add "{{ ca_cert_path }}" when: etcd_ca_cert.changed and ansible_os_family == "ClearLinux" ================================================ FILE: roles/etcd/templates/etcd-docker.service.j2 ================================================ [Unit] Description=etcd docker wrapper Wants=docker.socket After=docker.service [Service] User=root PermissionsStartOnly=true EnvironmentFile=-/etc/etcd.env ExecStart={{ bin_dir }}/etcd ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name | default("etcd") }} ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name | default("etcd") }} Restart=always RestartSec=15s TimeoutStartSec=30s [Install] WantedBy=multi-user.target ================================================ FILE: roles/etcd/templates/etcd-events-docker.service.j2 ================================================ [Unit] Description=etcd docker wrapper Wants=docker.socket After=docker.service [Service] User=root PermissionsStartOnly=true EnvironmentFile=-/etc/etcd-events.env ExecStart={{ bin_dir }}/etcd-events ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events Restart=always RestartSec=15s TimeoutStartSec=30s [Install] WantedBy=multi-user.target ================================================ FILE: roles/etcd/templates/etcd-events-host.service.j2 ================================================ [Unit] Description=etcd After=network.target [Service] Type=notify User=root EnvironmentFile=/etc/etcd-events.env ExecStart={{ bin_dir }}/etcd NotifyAccess=all Restart=always RestartSec=10s LimitNOFILE=40000 [Install] WantedBy=multi-user.target ================================================ FILE: roles/etcd/templates/etcd-events.env.j2 ================================================ ETCD_DATA_DIR={{ etcd_events_data_dir }} ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }} ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }} ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %} ETCD_METRICS={{ etcd_metrics }} ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2383,https://127.0.0.1:2383 ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }} ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }} ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd ETCD_LISTEN_PEER_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2382 ETCD_NAME={{ etcd_member_name }}-events ETCD_PROXY=off ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }} ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}} ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }} {% if etcd_tls_cipher_suites is defined %} ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} {% endif %} {% for key, value in etcd_extra_vars.items() %} {{ key }}={{ value }} {% endfor %} ================================================ FILE: roles/etcd/templates/etcd-events.j2 ================================================ #!/bin/bash {{ docker_bin_dir }}/docker run \ --restart=on-failure:5 \ --env-file=/etc/etcd-events.env \ --net=host \ -v /etc/ssl/certs:/etc/ssl/certs:ro \ -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ -v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \ {% if etcd_memory_limit is defined %} --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \ {% endif %} {% if etcd_cpu_limit is defined %} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ {% endif %} {% if etcd_blkio_weight is defined %} --blkio-weight={{ etcd_blkio_weight }} \ {% endif %} --name={{ etcd_member_name }}-events \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ /usr/local/bin/etcd \ "$@" ================================================ FILE: roles/etcd/templates/etcd-host.service.j2 ================================================ [Unit] Description=etcd After=network.target [Service] Type=notify User=root EnvironmentFile=/etc/etcd.env ExecStart={{ bin_dir }}/etcd NotifyAccess=all Restart=always RestartSec=10s LimitNOFILE=40000 [Install] WantedBy=multi-user.target ================================================ FILE: roles/etcd/templates/etcd.env.j2 ================================================ # Environment file for etcd {{ etcd_version }} ETCD_DATA_DIR={{ etcd_data_dir }} ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }} ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }} ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %} ETCD_METRICS={{ etcd_metrics }} {% if etcd_listen_metrics_urls is defined %} ETCD_LISTEN_METRICS_URLS={{ etcd_listen_metrics_urls }} {% elif etcd_metrics_port is defined %} ETCD_LISTEN_METRICS_URLS=http://{{ etcd_address | ansible.utils.ipwrap }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }} {% endif %} ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2379,https://127.0.0.1:2379 ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }} ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }} ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd ETCD_LISTEN_PEER_URLS=https://{{ etcd_address | ansible.utils.ipwrap }}:2380 ETCD_NAME={{ etcd_member_name }} ETCD_PROXY=off ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} ETCD_LOG_LEVEL={{ etcd_log_level }} ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }} ETCD_MAX_WALS={{ etcd_max_wals }} # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}} ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }} {% if etcd_tls_cipher_suites is defined %} ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} {% endif %} {% for key, value in etcd_extra_vars.items() %} {{ key }}={{ value }} {% endfor %} # CLI settings ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem # ETCD 3.5.x issue # https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK={{ etcd_experimental_initial_corrupt_check }} {% if etcd_experimental_enable_distributed_tracing %} ETCD_EXPERIMENTAL_ENABLE_DISTRIBUTED_TRACING=true ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SAMPLING_RATE={{ etcd_experimental_distributed_tracing_sample_rate }} ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_ADDRESS={{ etcd_experimental_distributed_tracing_address }} ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SERVICE_NAME={{ etcd_experimental_distributed_tracing_service_name }} ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_INSTANCE_ID={{ etcd_member_name }} {% endif %} ETCD_EXPERIMENTAL_WATCH_PROGRESS_NOTIFY_INTERVAL={{ etcd_experimental_watch_progress_notify_interval }} ================================================ FILE: roles/etcd/templates/etcd.j2 ================================================ #!/bin/bash {{ docker_bin_dir }}/docker run \ --restart=on-failure:5 \ --env-file=/etc/etcd.env \ --net=host \ -v /etc/ssl/certs:/etc/ssl/certs:ro \ -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:rw \ {% if etcd_memory_limit is defined %} --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \ {% endif %} {% if etcd_cpu_limit is defined %} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ {% endif %} {% if etcd_blkio_weight is defined %} --blkio-weight={{ etcd_blkio_weight }} \ {% endif %} --name={{ etcd_member_name | default("etcd") }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ /usr/local/bin/etcd \ "$@" ================================================ FILE: roles/etcd/templates/make-ssl-etcd.sh.j2 ================================================ #!/bin/bash # Author: Smana smainklh@gmail.com # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o pipefail usage() { cat << EOF Create self signed certificates Usage : $(basename $0) -f [-d ] -h | --help : Show this message -f | --config : Openssl configuration file -d | --ssldir : Directory where the certificates will be installed ex : $(basename $0) -f openssl.conf -d /srv/ssl EOF } # Options parsing while (($#)); do case "$1" in -h | --help) usage; exit 0;; -f | --config) CONFIG=${2}; shift 2;; -d | --ssldir) SSLDIR="${2}"; shift 2;; *) usage echo "ERROR : Unknown option" exit 3 ;; esac done if [ -z ${CONFIG} ]; then echo "ERROR: the openssl configuration file is missing. option -f" exit 1 fi if [ -z ${SSLDIR} ]; then SSLDIR="/etc/ssl/etcd" fi tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX) trap 'rm -rf "${tmpdir}"' EXIT cd "${tmpdir}" mkdir -p "${SSLDIR}" # Root CA if [ -e "$SSLDIR/ca-key.pem" ]; then # Reuse existing CA cp $SSLDIR/{ca.pem,ca-key.pem} . else openssl genrsa -out ca-key.pem {{certificates_key_size}} openssl req -x509 -new -nodes -key ca-key.pem -days {{certificates_duration}} -out ca.pem -subj "/CN=etcd-ca" fi # ETCD member if [ -n "$MASTERS" ]; then for host in $MASTERS; do cn="${host%%.*}" # Member key openssl genrsa -out member-${host}-key.pem {{certificates_key_size}} openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} # Admin key openssl genrsa -out admin-${host}-key.pem {{certificates_key_size}} openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} done fi # Node keys if [ -n "$HOSTS" ]; then for host in $HOSTS; do cn="${host%%.*}" openssl genrsa -out node-${host}-key.pem {{certificates_key_size}} openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} done fi # Install certs if [ -e "$SSLDIR/ca-key.pem" ]; then # No pass existing CA rm -f ca.pem ca-key.pem fi if [ -n "$(ls -A *.pem)" ]; then mv *.pem ${SSLDIR}/ fi ================================================ FILE: roles/etcd/templates/openssl.conf.j2 ================================================ {% set counter = {'dns': 2,'ip': 1,} %}{% macro increment(dct, key, inc=1)%}{% if dct.update({key: dct[key] + inc}) %} {% endif %}{% endmacro %}[req] req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [ ssl_client ] extendedKeyUsage = clientAuth, serverAuth basicConstraints = CA:FALSE subjectKeyIdentifier=hash authorityKeyIdentifier=keyid,issuer subjectAltName = @alt_names [ v3_ca ] basicConstraints = CA:TRUE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names authorityKeyIdentifier=keyid:always,issuer [alt_names] DNS.1 = localhost {% for host in groups['etcd'] %} {% if hostvars[host]['etcd_access_address'] is defined and not (hostvars[host]['etcd_access_address'] | ansible.utils.ipaddr) %} {# If defined, the address which etcd uses to access its members must be included in the SAN, otherwise etcd will fail with a TLS error upon startup. #} DNS.{{ counter["dns"] }} = {{ hostvars[host]['etcd_access_address'] }}{{ increment(counter, 'dns') }} {% endif %} {# This will always expand to inventory_hostname, which can be a completely arbitrary name, that etcd will not know or care about, hence this line is (probably) redundant. #} DNS.{{ counter["dns"] }} = {{ host }}{{ increment(counter, 'dns') }} {% endfor %} {% for etcd_alt_name in etcd_cert_alt_names %} DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }} {% endfor %} {% for host in groups['etcd'] %} {% for address in hostvars[host]['main_access_ips'] %} IP.{{ counter["ip"] }} = {{ address }}{{ increment(counter, 'ip') }} {% endfor %} {% for address in hostvars[host]['main_ips'] %} IP.{{ counter["ip"] }} = {{ address }}{{ increment(counter, 'ip') }} {% endfor %} {% endfor %} {% for cert_alt_ip in etcd_cert_alt_ips %} IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }} {% endfor %} IP.{{ counter["ip"] }} = 127.0.0.1{{ increment(counter, 'ip') }} IP.{{ counter["ip"] }} = ::1 ================================================ FILE: roles/etcd_defaults/defaults/main.yml ================================================ --- # Set etcd user etcd_owner: etcd # Set to false to only do certificate management etcd_cluster_setup: true etcd_events_cluster_setup: false # Set to true to separate k8s events to a different etcd cluster etcd_events_cluster_enabled: false etcd_backup_prefix: "/var/backups" etcd_data_dir: "/var/lib/etcd" # Number of etcd backups to retain. Set to a value < 0 to retain all backups etcd_backup_retention_count: -1 force_etcd_cert_refresh: true etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" etcd_cert_group: root # Note: This does not set up DNS entries. It simply adds the following DNS # entries to the certificate etcd_cert_alt_names: - "etcd.kube-system.svc.{{ dns_domain }}" - "etcd.kube-system.svc" - "etcd.kube-system" - "etcd" etcd_cert_alt_ips: [] etcd_script_dir: "{{ bin_dir }}/etcd-scripts" etcd_heartbeat_interval: "250" etcd_election_timeout: "5000" ## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. etcd_metrics: "basic" # Define in inventory to set a separate port for etcd to expose metrics on # etcd_metrics_port: 2381 ## A dictionary of extra environment variables to add to etcd.env, formatted like: ## etcd_extra_vars: ## ETCD_VAR1: "value1" ## ETCD_VAR2: "value2" etcd_extra_vars: {} # Limits ## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing. ## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM. ## This value is only relevant when deploying etcd with `etcd_deployment_type: docker` etcd_memory_limit: "{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}" ## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than ## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check ## etcd documentation for more information. # 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. etcd_quota_backend_bytes: "2147483648" # Maximum client request size in bytes the server will accept. # etcd is designed to handle small key value pairs typical for metadata. # Larger requests will work, but may increase the latency of other requests etcd_max_request_bytes: "1572864" # Uncomment to set CPU share for etcd # etcd_cpu_limit: 300m etcd_blkio_weight: 1000 etcd_node_cert_hosts: "{{ groups['k8s_cluster'] }}" ## Etcd auto compaction retention for mvcc key value store in hour etcd_compaction_retention: "8" # Force clients like etcdctl to use TLS certs (different than peer security) etcd_secure_client: true # Enable peer client cert authentication etcd_peer_client_auth: true # Maximum number of snapshot files to retain (0 is unlimited) etcd_max_snapshots: 5 # Maximum number of wal files to retain (0 is unlimited) etcd_max_wals: 5 # Number of loop retries etcd_retries: 4 ## Support tls cipher suites. # etcd_tls_cipher_suites: {} # - TLS_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA # - TLS_RSA_WITH_AES_256_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA256 # - TLS_RSA_WITH_AES_128_GCM_SHA256 # - TLS_RSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # ETCD 3.5.x issue # https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer etcd_experimental_initial_corrupt_check: true # Enable distributed tracing # https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing etcd_experimental_enable_distributed_tracing: false etcd_experimental_distributed_tracing_sample_rate: 100 # Per million spans etcd_experimental_distributed_tracing_address: "localhost:4317" etcd_experimental_distributed_tracing_service_name: etcd # The interval for etcd watch progress notify events etcd_experimental_watch_progress_notify_interval: 5s etcd_log_level: info ================================================ FILE: roles/etcd_defaults/vars/main.yml ================================================ --- cert_files: master: - "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" - "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" node: - "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" ================================================ FILE: roles/etcdctl_etcdutl/tasks/main.yml ================================================ --- - name: Copy etcdctl and etcdutl binary from docker container command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdxtl-binarycopy; {{ docker_bin_dir }}/docker create --name etcdxtl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && {{ docker_bin_dir }}/docker cp etcdxtl-binarycopy:/usr/local/bin/{{ item }} {{ bin_dir }}/{{ item }} && {{ docker_bin_dir }}/docker rm -f etcdxtl-binarycopy" with_items: - etcdctl - etcdutl register: etcdxtl_install_result until: etcdxtl_install_result.rc == 0 retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false when: container_manager == "docker" - name: Download etcd binary include_tasks: "../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.etcd) }}" when: container_manager in ['crio', 'containerd'] - name: Copy etcd binary unarchive: src: "{{ downloads.etcd.dest }}" dest: "{{ local_release_dir }}/" remote_src: true when: container_manager in ['crio', 'containerd'] - name: Copy etcdctl and etcdutl binary from download dir copy: src: "{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}" dest: "{{ bin_dir }}/{{ item }}" mode: "0755" remote_src: true with_items: - etcdctl - etcdutl when: container_manager in ['crio', 'containerd'] - name: Create etcdctl wrapper script template: src: etcdctl.sh.j2 dest: "{{ bin_dir }}/etcdctl.sh" mode: "0755" ================================================ FILE: roles/etcdctl_etcdutl/templates/etcdctl.sh.j2 ================================================ #!/bin/bash # {{ ansible_managed }} # example invocation: etcdctl.sh get --keys-only --from-key "" etcdctl \ {% if etcd_deployment_type == "kubeadm" %} --cacert {{ kube_cert_dir }}/etcd/ca.crt \ --cert {{ kube_cert_dir }}/etcd/server.crt \ --key {{ kube_cert_dir }}/etcd/server.key "$@" {% else %} --cacert {{ etcd_cert_dir }}/ca.pem \ --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem \ --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem "$@" {% endif %} ================================================ FILE: roles/helm-apps/README.md ================================================ Role Name ========= This role is intended to be used to fetch and deploy Helm Charts as part of cluster installation or upgrading with kubespray. Requirements ------------ The role needs to be executed on a host with access to the Kubernetes API, and with the helm binary in place. Role Variables -------------- See meta/argument_specs.yml Playbook example: ```yaml --- - hosts: kube_control_plane[0] gather_facts: no roles: - name: helm-apps releases: - name: app namespace: app chart_ref: simple-app/simple-app - name: app2 namespace: app chart_ref: simple-app/simple-app wait_timeout: "10m" # override the same option in `release_common_opts` repositories: "{{ repos }}" - name: simple-app url: "https://blog.leiwang.info/simple-app" release_common_opts: "{{ helm_params }}" wait_timeout: "5m" ``` ================================================ FILE: roles/helm-apps/meta/argument_specs.yml ================================================ --- argument_specs: main: short_description: Install a list of Helm charts. options: releases: type: list elements: dict required: true description: | List of dictionaries passed as arguments to kubernetes.core.helm. Arguments passed here will override those in `helm_settings`. For structure of the dictionary, see the documentation for kubernetes.core.helm ansible module. options: chart_ref: type: path required: true chart_version: type: str name: type: str required: true namespace: type: str required: true values: type: dict # Possibly general options create_namespace: type: bool chart_repo_url: type: str disable_hook: type: bool history_max: type: int purge: type: bool replace: type: bool skip_crds: type: bool wait: type: bool default: true wait_timeout: type: str atomic: type: bool default: true repositories: type: list elements: dict description: | List of dictionaries passed as arguments to kubernetes.core.helm_repository. default: [] options: name: type: str required: true password: type: str username: type: str url: type: str release_common_opts: type: dict description: | Common arguments for every helm invocation. default: {} options: create_namespace: type: bool default: true chart_repo_url: type: str disable_hook: type: bool history_max: type: int purge: type: bool replace: type: bool skip_crds: type: bool wait: type: bool default: true wait_timeout: type: str default: "5m" atomic: type: bool default: true ================================================ FILE: roles/helm-apps/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/helm ================================================ FILE: roles/helm-apps/tasks/main.yml ================================================ --- - name: Add Helm repositories kubernetes.core.helm_repository: "{{ helm_repository_defaults | combine(item) }}" # noqa args[module] loop: "{{ repositories }}" - name: Update Helm repositories kubernetes.core.helm: state: absent binary_path: "{{ bin_dir }}/helm" release_name: dummy # trick needed to refresh in separate step release_namespace: kube-system update_repo_cache: true when: - repositories != [] - helm_update - name: Install Helm Applications kubernetes.core.helm: "{{ helm_defaults | combine(release_common_opts, item) }}" # noqa args[module] loop: "{{ releases }}" ================================================ FILE: roles/helm-apps/vars/main.yml ================================================ --- helm_update: true helm_defaults: atomic: true binary_path: "{{ bin_dir }}/helm" helm_repository_defaults: binary_path: "{{ bin_dir }}/helm" force_update: true ================================================ FILE: roles/kubernetes/client/defaults/main.yml ================================================ --- kubeconfig_localhost: false kubeconfig_localhost_ansible_host: false kubectl_localhost: false artifacts_dir: "{{ inventory_dir }}/artifacts" kube_config_dir: "/etc/kubernetes" kube_apiserver_port: "6443" ================================================ FILE: roles/kubernetes/client/tasks/main.yml ================================================ --- - name: Set external kube-apiserver endpoint set_fact: # noqa: jinja[spacing] external_apiserver_address: >- {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%} {{ loadbalancer_apiserver.address }} {%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%} {{ hostvars[groups['kube_control_plane'][0]].ansible_host }} {%- else -%} {{ kube_apiserver_access_address }} {%- endif -%} # noqa: jinja[spacing] external_apiserver_port: >- {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%} {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} {%- else -%} {{ kube_apiserver_port }} {%- endif -%} tags: - facts - name: Create kube config dir for current/ansible become user file: path: "{{ ansible_env.HOME | default('/root') }}/.kube" mode: "0700" state: directory - name: Write admin kubeconfig to current/ansible become user home copy: src: "{{ kube_config_dir }}/admin.conf" dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config" remote_src: true mode: "0600" backup: true - name: Create kube artifacts dir file: path: "{{ artifacts_dir }}" mode: "0750" state: directory delegate_to: localhost connection: local become: false run_once: true when: kubeconfig_localhost - name: Wait for k8s apiserver wait_for: host: "{{ kube_apiserver_access_address }}" port: "{{ kube_apiserver_port }}" timeout: 180 - name: Create kubeconfig localhost artifacts when: kubeconfig_localhost block: - name: Generate admin kubeconfig using kubeadm command: >- {{ bin_dir }}/kubeadm kubeconfig user --client-name=kubernetes-admin-{{ cluster_name }} --org=kubeadm:cluster-admins --config {{ kube_config_dir }}/kubeadm-config.yaml register: kubeadm_admin_kubeconfig changed_when: false run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Write admin kubeconfig on ansible host copy: content: "{{ kubeadm_admin_kubeconfig.stdout | from_yaml | combine(override, recursive=true) | to_nice_yaml(indent=2) }}" dest: "{{ artifacts_dir }}/admin.conf" mode: "0600" vars: admin_kubeconfig: "{{ kubeadm_admin_kubeconfig.stdout | from_yaml }}" context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}" override: clusters: - "{{ admin_kubeconfig['clusters'][0] | combine({'name': cluster_name, 'cluster': admin_kubeconfig['clusters'][0]['cluster'] | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})}, recursive=true) }}" contexts: - "{{ admin_kubeconfig['contexts'][0] | combine({'name': context, 'context': admin_kubeconfig['contexts'][0]['context'] | combine({'cluster': cluster_name})}, recursive=true) }}" current-context: "{{ context }}" delegate_to: localhost connection: local become: false run_once: true - name: Copy kubectl binary to ansible host fetch: src: "{{ bin_dir }}/kubectl" dest: "{{ artifacts_dir }}/kubectl" flat: true validate_checksum: false register: copy_binary_result until: copy_binary_result is not failed retries: 20 become: false run_once: true when: kubectl_localhost - name: Create helper script kubectl.sh on ansible host copy: content: | #!/bin/bash ${BASH_SOURCE%/*}/kubectl --kubeconfig=${BASH_SOURCE%/*}/admin.conf "$@" dest: "{{ artifacts_dir }}/kubectl.sh" mode: "0755" become: false run_once: true delegate_to: localhost connection: local when: kubectl_localhost and kubeconfig_localhost ================================================ FILE: roles/kubernetes/control-plane/defaults/main/etcd.yml ================================================ --- # Set etcd user/group etcd_owner: etcd # Note: This does not set up DNS entries. It simply adds the following DNS # entries to the certificate etcd_cert_alt_names: - "etcd.kube-system.svc.{{ dns_domain }}" - "etcd.kube-system.svc" - "etcd.kube-system" - "etcd" etcd_cert_alt_ips: [] etcd_heartbeat_interval: "250" etcd_election_timeout: "5000" etcd_metrics: "basic" ## A dictionary of extra environment variables to add to etcd.env, formatted like: ## etcd_extra_vars: ## var1: "value1" ## var2: "value2" ## Note this is different from the etcd role with ETCD_ prfexi, caps, and underscores etcd_extra_vars: {} # etcd_quota_backend_bytes: "2147483648" # etcd_max_request_bytes: "1572864" etcd_compaction_retention: "8" ================================================ FILE: roles/kubernetes/control-plane/defaults/main/kube-proxy.yml ================================================ --- # bind address for kube-proxy kube_proxy_bind_address: '0.0.0.0' # acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the # default value of 'application/json'. This field will control all connections to the server used by a particular # client. kube_proxy_client_accept_content_types: '' # burst allows extra queries to accumulate when a client is exceeding its rate. kube_proxy_client_burst: 10 # contentType is the content type used when sending data to the server from this client. kube_proxy_client_content_type: application/vnd.kubernetes.protobuf # kubeconfig is the path to a KubeConfig file. # Leave as empty string to generate from other fields kube_proxy_client_kubeconfig: '' # qps controls the number of queries per second allowed for this connection. kube_proxy_client_qps: 5 # How often configuration from the apiserver is refreshed. Must be greater than 0. kube_proxy_config_sync_period: 15m0s ### Conntrack # maxPerCore is the maximum number of NAT connections to track # per CPU core (0 to leave the limit as-is and ignore min). kube_proxy_conntrack_max_per_core: 32768 # min is the minimum value of connect-tracking records to allocate, # regardless of conntrackMaxPerCore (set maxPerCore=0 to leave the limit as-is). kube_proxy_conntrack_min: 131072 # tcpCloseWaitTimeout is how long an idle conntrack entry # in CLOSE_WAIT state will remain in the conntrack # table. (e.g. '60s'). Must be greater than 0 to set. kube_proxy_conntrack_tcp_close_wait_timeout: 1h0m0s # tcpEstablishedTimeout is how long an idle TCP connection will be kept open # (e.g. '2s'). Must be greater than 0 to set. kube_proxy_conntrack_tcp_established_timeout: 24h0m0s # Enables profiling via web interface on /debug/pprof handler. # Profiling handlers will be handled by metrics server. kube_proxy_enable_profiling: false # bind address for kube-proxy health check kube_proxy_healthz_bind_address: 0.0.0.0:10256 # If using the pure iptables proxy, SNAT everything. Note that it breaks any # policy engine. kube_proxy_masquerade_all: false # If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. # Must be within the range [0, 31]. kube_proxy_masquerade_bit: 14 # The minimum interval of how often the iptables or ipvs rules can be refreshed as # endpoints and services change (e.g. '5s', '1m', '2h22m'). kube_proxy_min_sync_period: 0s # The maximum interval of how often iptables or ipvs rules are refreshed (e.g. '5s', '1m', '2h22m'). # Must be greater than 0. kube_proxy_sync_period: 30s # A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules. kube_proxy_exclude_cidrs: [] # The ipvs scheduler type when proxy mode is ipvs # rr: round-robin # lc: least connection # dh: destination hashing # sh: source hashing # sed: shortest expected delay # nq: never queue kube_proxy_scheduler: rr # configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface # must be set to true for MetalLB, kube-vip(ARP enabled) to work kube_proxy_strict_arp: false # kube_proxy_tcp_timeout is the timeout value used for idle IPVS TCP sessions. # The default value is 0, which preserves the current timeout value on the system. kube_proxy_tcp_timeout: 0s # kube_proxy_tcp_fin_timeout is the timeout value used for IPVS TCP sessions after receiving a FIN. # The default value is 0, which preserves the current timeout value on the system. kube_proxy_tcp_fin_timeout: 0s # kube_proxy_udp_timeout is the timeout value used for IPVS UDP packets. # The default value is 0, which preserves the current timeout value on the system. kube_proxy_udp_timeout: 0s # The IP address and port for the metrics server to serve on # (set to 0.0.0.0 for all IPv4 interfaces and `::` for all IPv6 interfaces) kube_proxy_metrics_bind_address: 127.0.0.1:10249 # A string slice of values which specify the addresses to use for NodePorts. # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). # The default empty string slice ([]) means to use all local addresses. kube_proxy_nodeport_addresses: >- {%- if kube_proxy_nodeport_addresses_cidr is defined -%} [{{ kube_proxy_nodeport_addresses_cidr }}] {%- else -%} [] {%- endif -%} # oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000] kube_proxy_oom_score_adj: -999 # portRange is the range of host ports (beginPort-endPort, inclusive) that may be consumed # in order to proxy service traffic. If unspecified, 0, or (0-0) then ports will be randomly chosen. kube_proxy_port_range: '' ================================================ FILE: roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml ================================================ --- # Extra args passed by kubeadm kube_kubeadm_scheduler_extra_args: {} # Associated interface must be reachable by the rest of the cluster, and by # CLI/web clients. kube_scheduler_bind_address: "::" # ClientConnection options (e.g. Burst, QPS) except from kubeconfig. kube_scheduler_client_conn_extra_opts: {} # Additional KubeSchedulerConfiguration settings (e.g. metricsBindAddress). kube_scheduler_config_extra_opts: {} # List of scheduler extenders (dicts), each holding the values of how to # communicate with the extender. kube_scheduler_extenders: [] # Leader Election options (e.g. ResourceName, RetryPerion) except from # LeaseDuration and Renew deadline which are defined in following vars. kube_scheduler_leader_elect_extra_opts: {} # Leader election lease duration kube_scheduler_leader_elect_lease_duration: 15s # Leader election lease timeout kube_scheduler_leader_elect_renew_deadline: 10s # Lisf of scheduling profiles (ditcs) supported by kube-scheduler kube_scheduler_profiles: [] # Extra volume mounts scheduler_extra_volumes: {} ================================================ FILE: roles/kubernetes/control-plane/defaults/main/main.yml ================================================ --- # disable upgrade cluster upgrade_cluster_setup: false # Number of retries (with 5 seconds interval) to check that new control plane nodes # are in Ready condition after joining control_plane_node_become_ready_tries: 24 # By default the external API listens on all interfaces, this can be changed to # listen on a specific address/interface. # NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost # loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too. kube_apiserver_bind_address: "::" # A port range to reserve for services with NodePort visibility. # Inclusive at both ends of the range. kube_apiserver_node_port_range: "30000-32767" # ETCD backend for k8s data kube_apiserver_storage_backend: etcd3 # The interval of compaction requests. If 0, the compaction request from apiserver is disabled. kube_apiserver_etcd_compaction_interval: "5m0s" # CIS 1.2.26 # Validate that the service account token # in the request is actually present in etcd. kube_apiserver_service_account_lookup: true kube_etcd_cacert_file: ca.pem kube_etcd_cert_file: node-{{ inventory_hostname }}.pem kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem # Associated interfaces must be reachable by the rest of the cluster, and by # CLI/web clients. kube_controller_manager_bind_address: "::" ## Control plane health check settings control_plane_health_retries: 60 # Default retries for apiserver, scheduler, controller-manager health checks # Leader election lease durations and timeouts for controller-manager kube_controller_manager_leader_elect_lease_duration: 15s kube_controller_manager_leader_elect_renew_deadline: 10s # discovery_timeout modifies the discovery timeout discovery_timeout: 5m0s # Instruct first control plane node to refresh kubeadm token kubeadm_refresh_token: true # Scale down coredns replicas to 0 if not using coredns dns_mode kubeadm_scale_down_coredns_enabled: true # audit support kubernetes_audit: false # path to audit log file audit_log_path: /var/log/audit/kube-apiserver-audit.log # num days audit_log_maxage: 30 # the num of audit logs to retain audit_log_maxbackups: 10 # the max size in MB to retain audit_log_maxsize: 100 # policy file audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" # custom audit policy rules (to replace the default ones) # audit_policy_custom_rules: | # - level: None # users: [] # verbs: [] # resources: [] # audit log hostpath audit_log_name: audit-logs audit_log_hostpath: /var/log/kubernetes/audit audit_log_mountpath: "{{ audit_log_path | dirname }}" # audit policy hostpath audit_policy_name: audit-policy audit_policy_hostpath: "{{ audit_policy_file | dirname }}" audit_policy_mountpath: "{{ audit_policy_hostpath }}" # audit webhook support kubernetes_audit_webhook: false # path to audit webhook config file audit_webhook_config_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml" audit_webhook_server_url: "https://audit.app" audit_webhook_server_extra_args: {} audit_webhook_mode: batch audit_webhook_batch_max_size: 100 audit_webhook_batch_max_wait: 1s kube_controller_node_monitor_grace_period: 40s kube_controller_node_monitor_period: 5s kube_controller_terminated_pod_gc_threshold: 12500 kube_apiserver_request_timeout: "1m0s" kube_apiserver_pod_eviction_not_ready_timeout_seconds: "300" kube_apiserver_pod_eviction_unreachable_timeout_seconds: "300" # 1.10+ admission plugins kube_apiserver_enable_admission_plugins: [] # enable admission plugins configuration kube_apiserver_admission_control_config_file: false # data structure to configure EventRateLimit admission plugin # this should have the following structure: # kube_apiserver_admission_event_rate_limits: # : # type: # qps: # burst: # cache_size: kube_apiserver_admission_event_rate_limits: {} ## PodSecurityAdmission plugin configuration kube_pod_security_use_default: false kube_pod_security_default_enforce: baseline kube_pod_security_default_enforce_version: "v{{ kube_major_version }}" kube_pod_security_default_audit: restricted kube_pod_security_default_audit_version: "v{{ kube_major_version }}" kube_pod_security_default_warn: restricted kube_pod_security_default_warn_version: "v{{ kube_major_version }}" kube_pod_security_exemptions_usernames: [] kube_pod_security_exemptions_runtime_class_names: [] kube_pod_security_exemptions_namespaces: - kube-system ## ResourceQuota plugin configuration ## Resources that ResourceQuota should limit by default if no quota exists ## Example below enforces quota on all storage classes # kube_resource_quota_limited_resources: # - apiGroup: "" # resource: persistentvolumeclaims # matchContains: # - .storageclass.storage.k8s.io/requests.storage kube_resource_quota_limited_resources: [] # 1.10+ list of disabled admission plugins kube_apiserver_disable_admission_plugins: [] # extra runtime config kube_api_runtime_config: [] ## Enable/Disable Kube API Server Authentication Methods kube_token_auth: false kube_oidc_auth: false ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication kube_webhook_token_auth: false kube_webhook_token_auth_url_skip_tls_verify: false # kube_webhook_token_auth_url: https://... ## base64-encoded string of the webhook's CA certificate # kube_webhook_token_auth_ca_data: "LS0t..." ## Variables for webhook token authz https://kubernetes.io/docs/reference/access-authn-authz/webhook/ # kube_webhook_authorization_url: https://... kube_webhook_authorization: false kube_webhook_authorization_url_skip_tls_verify: false # Default podnodeselector kube_apiserver_admission_plugins_podnodeselector_default_node_selector: "" ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) # kube_oidc_url: https:// ... # kube_oidc_client_id: kubernetes ## Optional settings for OIDC # kube_oidc_username_claim: sub # kube_oidc_username_prefix: 'oidc:' # kube_oidc_groups_claim: groups # kube_oidc_groups_prefix: 'oidc:' # Copy oidc CA file to the following path if needed # kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem # Optionally include a base64-encoded oidc CA cert # kube_oidc_ca_cert: c3RhY2thYnVzZS5jb20... # List of the preferred NodeAddressTypes to use for kubelet connections. kubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP' ## Extra args for k8s components passing by kubeadm kube_kubeadm_apiserver_extra_args: {} kube_kubeadm_controller_extra_args: {} ## Extra control plane host volume mounts ## Example: # apiserver_extra_volumes: # - name: name # hostPath: /host/path # mountPath: /mount/path # readOnly: true apiserver_extra_volumes: {} controller_manager_extra_volumes: {} ## Encrypting Secret Data at Rest kube_encrypt_secret_data: false kube_encrypt_token: "{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}" # Must be either: aescbc, secretbox or aesgcm kube_encryption_algorithm: "secretbox" # Which kubernetes resources to encrypt kube_encryption_resources: [secrets] secrets_encryption_query: "resources[*].providers[0].{{ kube_encryption_algorithm }}.keys[0].secret" ## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. # tls_min_version: "" ## Support tls cipher suites. # tls_cipher_suites: # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA256 # - TLS_RSA_WITH_AES_128_GCM_SHA256 # - TLS_RSA_WITH_AES_256_CBC_SHA # - TLS_RSA_WITH_AES_256_GCM_SHA384 # - TLS_RSA_WITH_RC4_128_SHA ## Amount of time to retain events. (default 1h0m0s) event_ttl_duration: "1h0m0s" ## Automatically renew K8S control plane certificates on first Monday of each month auto_renew_certificates: false # First Monday of each month auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00" # kubeadm renews all the certificates during control plane upgrade. # If we have requirement like without renewing certs upgrade the cluster, # we can opt out from the default behavior by setting kubeadm_upgrade_auto_cert_renewal to false kubeadm_upgrade_auto_cert_renewal: true # Add Subject Alternative Names to the Kubernetes apiserver certificates. # Useful if you access the API from multiples load balancers, for instance. supplementary_addresses_in_ssl_keys: [] # Bash alias of kubectl to interact with Kubernetes cluster much easier # kubectl_alias: k ## Enable distributed tracing for kube-apiserver kube_apiserver_tracing: false kube_apiserver_tracing_endpoint: "[::]:4317" kube_apiserver_tracing_sampling_rate_per_million: 100 # Enable kubeadm file discovery if anonymous access has been removed kubeadm_use_file_discovery: "{{ remove_anonymous_access }}" # imagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel. Default: true kubeadm_image_pull_serial: true # Supported asymmetric encryption algorithm types for the cluster's keys and certificates. # can be one of RSA-2048(default), RSA-3072, RSA-4096, ECDSA-P256 # ref: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/#kubeadm-k8s-io-v1beta4-ClusterConfiguration kube_asymmetric_encryption_algorithm: "RSA-2048" # certificates validity period configuration # non-CA certificate validity period, default 1 year (365d × 24h = 8760h) kube_cert_validity_period: 8760h # CA certificate validity period, default 10 years (365d × 24h × 10 = 87600h) kube_ca_cert_validity_period: 87600h ================================================ FILE: roles/kubernetes/control-plane/handlers/main.yml ================================================ --- - name: Control plane | reload systemd systemd_service: daemon_reload: true listen: Control plane | restart kubelet - name: Control plane | reload kubelet service: name: kubelet state: restarted listen: Control plane | restart kubelet - name: Control plane | Remove apiserver container docker shell: "set -o pipefail && docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f" args: executable: /bin/bash register: remove_apiserver_container retries: 10 until: remove_apiserver_container.rc == 0 delay: 1 when: container_manager == "docker" listen: Control plane | Restart apiserver - name: Control plane | Remove apiserver container containerd/crio shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-apiserver*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" args: executable: /bin/bash register: remove_apiserver_container retries: 10 until: remove_apiserver_container.rc == 0 delay: 1 when: container_manager in ['containerd', 'crio'] listen: Control plane | Restart apiserver - name: Control plane | Remove scheduler container docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" args: executable: /bin/bash register: remove_scheduler_container retries: 10 until: remove_scheduler_container.rc == 0 delay: 1 when: container_manager == "docker" listen: Control plane | Restart kube-scheduler - name: Control plane | Remove scheduler container containerd/crio shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-scheduler*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" args: executable: /bin/bash register: remove_scheduler_container retries: 10 until: remove_scheduler_container.rc == 0 delay: 1 when: container_manager in ['containerd', 'crio'] listen: Control plane | Restart kube-scheduler - name: Control plane | Remove controller manager container docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" args: executable: /bin/bash register: remove_cm_container retries: 10 until: remove_cm_container.rc == 0 delay: 1 when: container_manager == "docker" listen: Control plane | Restart kube-controller-manager - name: Control plane | Remove controller manager container containerd/crio shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-controller-manager*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" args: executable: /bin/bash register: remove_cm_container retries: 10 until: remove_cm_container.rc == 0 delay: 1 when: container_manager in ['containerd', 'crio'] listen: Control plane | Restart kube-controller-manager - name: Control plane | wait for kube-scheduler vars: endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '::' else 'localhost' }}" uri: url: https://{{ endpoint }}:10259/healthz validate_certs: false register: scheduler_result until: scheduler_result.status == 200 retries: "{{ control_plane_health_retries }}" delay: 1 listen: - Control plane | restart kubelet - Control plane | Restart kube-scheduler - name: Control plane | wait for kube-controller-manager vars: endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '::' else 'localhost' }}" uri: url: https://{{ endpoint }}:10257/healthz validate_certs: false register: controller_manager_result until: controller_manager_result.status == 200 retries: "{{ control_plane_health_retries }}" delay: 1 listen: - Control plane | restart kubelet - Control plane | Restart kube-controller-manager - name: Control plane | wait for the apiserver to be running uri: url: "{{ kube_apiserver_endpoint }}/healthz" validate_certs: false register: result until: result.status == 200 retries: "{{ control_plane_health_retries }}" delay: 1 listen: - Control plane | restart kubelet - Control plane | Restart apiserver ================================================ FILE: roles/kubernetes/control-plane/meta/main.yml ================================================ --- dependencies: - role: kubernetes/kubeadm_common - role: adduser user: "{{ addusers.etcd }}" when: - etcd_deployment_type == "kubeadm" - not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) - role: network_plugin/calico_defaults - role: etcd_defaults ================================================ FILE: roles/kubernetes/control-plane/tasks/check-api.yml ================================================ --- - name: Kubeadm | Check api is up uri: url: "https://{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}/healthz" validate_certs: false when: ('kube_control_plane' in group_names) register: _result retries: 60 delay: 5 until: _result.status == 200 ================================================ FILE: roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml ================================================ --- - name: Check if secret for encrypting data at rest already exist stat: path: "{{ kube_cert_dir }}/secrets_encryption.yaml" get_attributes: false get_checksum: false get_mime: false register: secrets_encryption_file - name: Slurp secrets_encryption file if it exists slurp: src: "{{ kube_cert_dir }}/secrets_encryption.yaml" register: secret_file_encoded when: secrets_encryption_file.stat.exists - name: Base 64 Decode slurped secrets_encryption.yaml file set_fact: secret_file_decoded: "{{ secret_file_encoded['content'] | b64decode | from_yaml }}" when: secrets_encryption_file.stat.exists - name: Extract secret value from secrets_encryption.yaml set_fact: kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}" when: secrets_encryption_file.stat.exists - name: Set kube_encrypt_token across control plane nodes set_fact: kube_encrypt_token: "{{ kube_encrypt_token_extracted }}" delegate_to: "{{ item }}" delegate_facts: true with_inventory_hostnames: kube_control_plane when: kube_encrypt_token_extracted is defined - name: Write secrets for encrypting secret data at rest template: src: secrets_encryption.yaml.j2 dest: "{{ kube_cert_dir }}/secrets_encryption.yaml" owner: root group: "{{ kube_cert_group }}" mode: "0640" ================================================ FILE: roles/kubernetes/control-plane/tasks/kubeadm-backup.yml ================================================ --- - name: Backup old certs and keys copy: src: "{{ kube_cert_dir }}/{{ item }}" dest: "{{ kube_cert_dir }}/{{ item }}.old" mode: preserve remote_src: true with_items: - apiserver.crt - apiserver.key - apiserver-kubelet-client.crt - apiserver-kubelet-client.key - front-proxy-client.crt - front-proxy-client.key ignore_errors: true # noqa ignore-errors - name: Backup old confs copy: src: "{{ kube_config_dir }}/{{ item }}" dest: "{{ kube_config_dir }}/{{ item }}.old" mode: preserve remote_src: true with_items: - admin.conf - controller-manager.conf - kubelet.conf - scheduler.conf ignore_errors: true # noqa ignore-errors ================================================ FILE: roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml ================================================ --- - name: Calculate etcd cert serial command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial" register: "etcd_client_cert_serial_result" changed_when: false tags: - network - name: Set etcd_client_cert_serial set_fact: etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}" tags: - network - name: Ensure etcdctl and etcdutl script is installed import_role: name: etcdctl_etcdutl when: etcd_deployment_type == "kubeadm" tags: - etcdctl - etcdutl - name: Set ownership for etcd data directory file: path: "{{ etcd_data_dir }}" owner: "{{ etcd_owner }}" group: "{{ etcd_owner }}" mode: "0700" when: etcd_deployment_type == "kubeadm" ================================================ FILE: roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml ================================================ --- - name: Set kubeadm_discovery_address set_fact: # noqa: jinja[spacing] kubeadm_discovery_address: >- {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} {%- else -%} {{ kube_apiserver_endpoint | regex_replace('https://', '') }} {%- endif %} tags: - facts - name: Obtain kubeadm certificate key for joining control planes nodes when: - not kube_external_ca_mode run_once: true block: - name: Upload certificates so they are fresh and not expired command: >- {{ bin_dir }}/kubeadm init phase --config {{ kube_config_dir }}/kubeadm-config.yaml upload-certs --upload-certs register: kubeadm_upload_cert delegate_to: "{{ first_kube_control_plane }}" - name: Parse certificate key if not set set_fact: kubeadm_certificate_key: "{{ kubeadm_upload_cert.stdout_lines[-1] | trim }}" - name: Wait for k8s apiserver wait_for: host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}" port: "{{ kubeadm_discovery_address.split(':')[-1] }}" timeout: 180 - name: Check already run debug: msg: "{{ kubeadm_already_run.stat.exists }}" - name: Reset cert directory shell: >- if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then {{ bin_dir }}/kubeadm reset -f --cert-dir {{ kube_cert_dir }}; fi environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" when: - inventory_hostname != first_kube_control_plane - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - not kube_external_ca_mode - name: Get kubeconfig for join discovery process command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'" register: kubeconfig_file_discovery run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" when: - kubeadm_use_file_discovery - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - name: Copy discovery kubeconfig copy: dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" content: "{{ kubeconfig_file_discovery.stdout }}" owner: "root" mode: "0644" when: - inventory_hostname != first_kube_control_plane - kubeadm_use_file_discovery - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - name: Create kubeadm ControlPlane config template: src: "kubeadm-controlplane.yaml.j2" dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml" mode: "0640" backup: true validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}" when: - inventory_hostname != first_kube_control_plane - not kubeadm_already_run.stat.exists - name: Joining control plane node to the cluster. command: >- {{ bin_dir }}/kubeadm join --config {{ kube_config_dir }}/kubeadm-controlplane.yaml --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }} --skip-phases={{ kubeadm_join_phases_skip | join(',') }} environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" register: kubeadm_join_control_plane retries: 3 throttle: 1 until: kubeadm_join_control_plane is succeeded when: - inventory_hostname != first_kube_control_plane - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - name: Wait for new control plane nodes to be Ready when: kubeadm_already_run.stat.exists run_once: true command: > {{ kubectl }} get nodes --selector node-role.kubernetes.io/control-plane -o jsonpath-as-json="{.items[*].status.conditions[?(@.type == 'Ready')]}" register: control_plane_node_ready_conditions retries: "{{ control_plane_node_become_ready_tries }}" delay: 5 delegate_to: "{{ groups['kube_control_plane'][0] }}" until: > control_plane_node_ready_conditions.stdout | from_json | selectattr('status', '==', 'True') | length == (groups['kube_control_plane'] | length) ================================================ FILE: roles/kubernetes/control-plane/tasks/kubeadm-setup.yml ================================================ --- - name: Install OIDC certificate copy: content: "{{ kube_oidc_ca_cert | b64decode }}" dest: "{{ kube_oidc_ca_file }}" owner: root group: root mode: "0644" when: - kube_oidc_auth - kube_oidc_ca_cert is defined - name: Kubeadm | Check if kubeadm has already run stat: path: "/var/lib/kubelet/config.yaml" get_attributes: false get_checksum: false get_mime: false register: kubeadm_already_run - name: Kubeadm | Backup kubeadm certs / kubeconfig import_tasks: kubeadm-backup.yml when: - kubeadm_already_run.stat.exists - name: Kubeadm | aggregate all SANs set_fact: apiserver_sans: "{{ _apiserver_sans | flatten | select | unique }}" vars: _apiserver_sans: - "kubernetes" - "kubernetes.default" - "kubernetes.default.svc" - "kubernetes.default.svc.{{ dns_domain }}" - "{{ kube_apiserver_ip }}" - "localhost" - "127.0.0.1" - "::1" - "{{ apiserver_loadbalancer_domain_name | d('') }}" - "{{ loadbalancer_apiserver.address | d('') }}" - "{{ supplementary_addresses_in_ssl_keys }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | select('defined') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'address']) | select('defined') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'ansible_hostname') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'ansible_fqdn') }}" - "{{ kube_override_hostname }}" - "{{ kube_vip_address }}" tags: facts - name: Create audit-policy directory file: path: "{{ audit_policy_file | dirname }}" state: directory mode: "0640" when: kubernetes_audit or kubernetes_audit_webhook - name: Write api audit policy yaml template: src: apiserver-audit-policy.yaml.j2 dest: "{{ audit_policy_file }}" mode: "0640" when: kubernetes_audit or kubernetes_audit_webhook notify: Control plane | Restart apiserver - name: Write api audit webhook config yaml template: src: apiserver-audit-webhook-config.yaml.j2 dest: "{{ audit_webhook_config_file }}" mode: "0640" when: kubernetes_audit_webhook notify: Control plane | Restart apiserver - name: Create apiserver tracing config directory file: path: "{{ kube_config_dir }}/tracing" state: directory mode: "0640" when: kube_apiserver_tracing - name: Write apiserver tracing config yaml template: src: apiserver-tracing.yaml.j2 dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml" mode: "0640" when: kube_apiserver_tracing notify: Control plane | Restart apiserver # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. - name: Set kubeadm_config_api_fqdn define set_fact: kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name }}" when: loadbalancer_apiserver is defined - name: Kubeadm | Create kubeadm config template: src: "kubeadm-config.v1beta4.yaml.j2" dest: "{{ kube_config_dir }}/kubeadm-config.yaml" mode: "0640" validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}" - name: Kubeadm | Create directory to store admission control configurations file: path: "{{ kube_config_dir }}/admission-controls" state: directory mode: "0640" when: kube_apiserver_admission_control_config_file - name: Kubeadm | Push admission control config file template: src: "admission-controls.yaml.j2" dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml" mode: "0640" when: kube_apiserver_admission_control_config_file notify: Control plane | Restart apiserver - name: Kubeadm | Push admission control config files template: src: "{{ item | lower }}.yaml.j2" dest: "{{ kube_config_dir }}/admission-controls/{{ item | lower }}.yaml" mode: "0640" when: - kube_apiserver_admission_control_config_file - item in kube_apiserver_admission_plugins_needs_configuration loop: "{{ kube_apiserver_enable_admission_plugins }}" notify: Control plane | Restart apiserver - name: Kubeadm | Check apiserver.crt SANs vars: apiserver_ips: "{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}" apiserver_hosts: "{{ apiserver_sans | difference(apiserver_ips) }}" when: - kubeadm_already_run.stat.exists - not kube_external_ca_mode block: - name: Kubeadm | Check apiserver.crt SAN IPs command: cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkip {{ item }}" loop: "{{ apiserver_ips }}" register: apiserver_sans_ip_check changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate') failed_when: apiserver_sans_ip_check.rc != 0 and apiserver_sans_ip_check.stdout is not search('does NOT match certificate') - name: Kubeadm | Check apiserver.crt SAN hosts command: cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}" loop: "{{ apiserver_hosts }}" register: apiserver_sans_host_check changed_when: apiserver_sans_host_check.stdout is not search('does match certificate') failed_when: apiserver_sans_host_check.rc != 0 and apiserver_sans_host_check.stdout is not search('does NOT match certificate') - name: Kubeadm | regenerate apiserver cert 1/2 file: state: absent path: "{{ kube_cert_dir }}/{{ item }}" with_items: - apiserver.crt - apiserver.key when: - kubeadm_already_run.stat.exists - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed - not kube_external_ca_mode - name: Kubeadm | regenerate apiserver cert 2/2 command: >- {{ bin_dir }}/kubeadm init phase certs apiserver --config={{ kube_config_dir }}/kubeadm-config.yaml when: - kubeadm_already_run.stat.exists - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed - not kube_external_ca_mode # TODO: Remove --skip-phases from command when v1beta4 UpgradeConfiguration supports skipPhases - name: Kubeadm | Initialize first control plane node when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists vars: kubeadm_init_first_control_plane_cmd: >- timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }} {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors={{ _ignore_errors | flatten | join(',') }} --skip-phases={{ kubeadm_init_phases_skip | join(',') }} {{ kube_external_ca_mode | ternary('', '--upload-certs') }} _ignore_errors: "{{ kubeadm_ignore_preflight_errors }}" environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" notify: Control plane | restart kubelet block: - name: Kubeadm | Initialize first control plane node (1st try) command: "{{ kubeadm_init_first_control_plane_cmd }}" register: kubeadm_init failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr rescue: # Retry is because upload config sometimes fails # This retry task is separated from 1st task to show log of failure of 1st task. - name: Kubeadm | Initialize first control plane node (retry) command: "{{ kubeadm_init_first_control_plane_cmd }}" vars: _errors_from_first_try: - 'FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml' - 'FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml' - 'FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml' - 'Port-10250' _ignore_errors: - "{{ kubeadm_ignore_preflight_errors }}" - "{{ _errors_from_first_try if 'all' not in kubeadm_ignore_preflight_errors else [] }}" register: kubeadm_init retries: 2 until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr - name: Set kubeadm certificate key set_fact: kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)', '\\1') | first }}" with_items: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_init'].stdout_lines | default([]) }}" when: - kubeadm_certificate_key is not defined - (item | trim) is match('.*--certificate-key.*') - name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined) shell: >- {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token delete {{ kubeadm_token }} || :; {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }} changed_when: false when: - inventory_hostname == first_kube_control_plane - kubeadm_token is defined - kubeadm_refresh_token tags: - kubeadm_token - name: Remove binding to anonymous user command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found" when: inventory_hostname == first_kube_control_plane and remove_anonymous_access - name: Create kubeadm token for joining nodes with 24h expiration (default) command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create" changed_when: false register: temp_token retries: 5 delay: 5 until: temp_token is succeeded delegate_to: "{{ first_kube_control_plane }}" when: kubeadm_token is not defined tags: - kubeadm_token - name: Set kubeadm_token set_fact: kubeadm_token: "{{ temp_token.stdout }}" when: temp_token.stdout is defined tags: - kubeadm_token - name: Kubeadm | Join other control plane nodes include_tasks: kubeadm-secondary.yml - name: Kubeadm | upgrade kubernetes cluster to {{ kube_version }} include_tasks: kubeadm-upgrade.yml when: - upgrade_cluster_setup - kubeadm_already_run.stat.exists # FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file. - name: Kubeadm | Remove taint for control plane node with node role command: "{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}" delegate_to: "{{ first_kube_control_plane }}" with_items: - "node-role.kubernetes.io/control-plane:NoSchedule-" when: ('kube_node' in group_names) failed_when: false ================================================ FILE: roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml ================================================ --- - name: Ensure kube-apiserver is up before upgrade import_tasks: check-api.yml - name: Kubeadm | Upgrade first control plane node to {{ kube_version }} command: >- timeout -k 600s 600s {{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.yaml register: kubeadm_upgrade when: inventory_hostname == first_kube_control_plane failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" - name: Kubeadm | Upgrade other control plane nodes to {{ kube_version }} command: >- {{ bin_dir }}/kubeadm upgrade node --config={{ kube_config_dir }}/kubeadm-config.yaml register: kubeadm_upgrade when: inventory_hostname != first_kube_control_plane failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" # kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied - name: Update kubeadm and kubelet configmaps after upgrade command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml" register: kubeadm_upload_config # Retry is because upload config sometimes fails retries: 3 until: kubeadm_upload_config.rc == 0 when: - inventory_hostname == first_kube_control_plane - name: Update kube-proxy configmap after upgrade command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml" register: kube_proxy_upload_config # Retry is because upload config sometimes fails retries: 3 until: kube_proxy_upload_config.rc == 0 when: - inventory_hostname == first_kube_control_plane - ('addon/kube-proxy' not in kubeadm_init_phases_skip) - name: Rewrite kubeadm managed etcd static pod manifests with updated configmap command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml" when: - etcd_deployment_type == "kubeadm" notify: Control plane | restart kubelet - name: Rewrite kubernetes control plane static pod manifests with updated configmap command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml" notify: Control plane | restart kubelet - name: Flush kubelet handlers meta: flush_handlers - name: Ensure kube-apiserver is up after upgrade and control plane configuration updates import_tasks: check-api.yml - name: Kubeadm | Remove binding to anonymous user command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found" when: remove_anonymous_access - name: Kubeadm | clean kubectl cache to refresh api types file: path: "{{ item }}" state: absent with_items: - /root/.kube/cache - /root/.kube/http-cache # FIXME: https://github.com/kubernetes/kubeadm/issues/1318 - name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode command: >- {{ kubectl }} -n kube-system scale deployment/coredns --replicas 0 register: scale_down_coredns retries: 6 delay: 5 until: scale_down_coredns is succeeded run_once: true when: - kubeadm_scale_down_coredns_enabled - dns_mode not in ['coredns', 'coredns_dual'] changed_when: false ================================================ FILE: roles/kubernetes/control-plane/tasks/main.yml ================================================ --- - name: Pre-upgrade control plane import_tasks: pre-upgrade.yml tags: - k8s-pre-upgrade - name: Create webhook token auth config template: src: webhook-token-auth-config.yaml.j2 dest: "{{ kube_config_dir }}/webhook-token-auth-config.yaml" mode: "0640" when: kube_webhook_token_auth | default(false) - name: Create webhook authorization config template: src: webhook-authorization-config.yaml.j2 dest: "{{ kube_config_dir }}/webhook-authorization-config.yaml" mode: "0640" when: kube_webhook_authorization | default(false) - name: Create structured AuthorizationConfiguration file copy: content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}" dest: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml" mode: "0640" vars: authz_config: apiVersion: apiserver.config.k8s.io/{{ kube_apiserver_authorization_config_api_version }} kind: AuthorizationConfiguration authorizers: "{{ kube_apiserver_authorization_config_authorizers }}" when: kube_apiserver_use_authorization_config_file - name: Create kube-scheduler config template: src: kubescheduler-config.yaml.j2 dest: "{{ kube_config_dir }}/kubescheduler-config.yaml" mode: "0644" - name: Apply Kubernetes encrypt at rest config import_tasks: encrypt-at-rest.yml when: - kube_encrypt_secret_data tags: - kube-apiserver - name: Install | Copy kubectl binary from download dir copy: src: "{{ downloads.kubectl.dest }}" dest: "{{ bin_dir }}/kubectl" mode: "0755" remote_src: true tags: - kubectl - upgrade - name: Install kubectl bash completion shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh" when: ansible_os_family in ["Debian","RedHat", "Suse"] tags: - kubectl ignore_errors: true # noqa ignore-errors - name: Set kubectl bash completion file permissions file: path: /etc/bash_completion.d/kubectl.sh owner: root group: root mode: "0755" when: ansible_os_family in ["Debian","RedHat", "Suse"] tags: - kubectl - upgrade ignore_errors: true # noqa ignore-errors - name: Set bash alias for kubectl blockinfile: path: /etc/bash_completion.d/kubectl.sh block: |- alias {{ kubectl_alias }}=kubectl if [[ $(type -t compopt) = "builtin" ]]; then complete -o default -F __start_kubectl {{ kubectl_alias }} else complete -o default -o nospace -F __start_kubectl {{ kubectl_alias }} fi state: present marker: "# Ansible entries {mark}" when: - ansible_os_family in ["Debian","RedHat", "Suse"] - kubectl_alias is defined and kubectl_alias != "" tags: - kubectl - upgrade ignore_errors: true # noqa ignore-errors - name: Include kubeadm setup import_tasks: kubeadm-setup.yml - name: Include kubeadm etcd extra tasks include_tasks: kubeadm-etcd.yml when: etcd_deployment_type == "kubeadm" - name: Cleanup unused AuthorizationConfiguration file versions file: path: "{{ kube_config_dir }}/apiserver-authorization-config-{{ item }}.yaml" state: absent loop: "{{ ['v1alpha1', 'v1beta1', 'v1'] | reject('equalto', kube_apiserver_authorization_config_api_version) | list }}" when: kube_apiserver_use_authorization_config_file - name: Install script to renew K8S control plane certificates template: src: k8s-certs-renew.sh.j2 dest: "{{ bin_dir }}/k8s-certs-renew.sh" mode: "0755" - name: Renew K8S control plane certificates monthly 1/2 template: src: "{{ item }}.j2" dest: "/etc/systemd/system/{{ item }}" mode: "0644" validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{item}}'" # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) # Remove once we drop support for systemd < 250 with_items: - k8s-certs-renew.service - k8s-certs-renew.timer register: k8s_certs_units when: auto_renew_certificates - name: Renew K8S control plane certificates monthly 2/2 systemd_service: name: k8s-certs-renew.timer enabled: true state: started daemon_reload: "{{ k8s_certs_units is changed }}" when: auto_renew_certificates ================================================ FILE: roles/kubernetes/control-plane/tasks/pre-upgrade.yml ================================================ --- - name: "Pre-upgrade | Delete control plane manifests if etcd secrets changed" file: path: "/etc/kubernetes/manifests/{{ item }}.manifest" state: absent with_items: - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] register: kube_apiserver_manifest_replaced when: etcd_secret_changed | default(false) - name: "Pre-upgrade | Delete control plane containers forcefully" # noqa no-handler shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f" args: executable: /bin/bash with_items: - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] when: kube_apiserver_manifest_replaced.changed register: remove_control_plane_container retries: 10 until: remove_control_plane_container.rc == 0 delay: 1 ================================================ FILE: roles/kubernetes/control-plane/templates/admission-controls.yaml.j2 ================================================ apiVersion: apiserver.config.k8s.io/v1 kind: AdmissionConfiguration plugins: {% for plugin in kube_apiserver_enable_admission_plugins %} {% if plugin in kube_apiserver_admission_plugins_needs_configuration %} - name: {{ plugin }} path: {{ kube_config_dir }}/{{ plugin | lower }}.yaml {% endif %} {% endfor %} ================================================ FILE: roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2 ================================================ apiVersion: audit.k8s.io/v1 kind: Policy rules: {% if audit_policy_custom_rules is defined and audit_policy_custom_rules != "" %} {{ audit_policy_custom_rules | indent(2, true) }} {% else %} # The following requests were manually identified as high-volume and low-risk, # so drop them. - level: None users: ["system:kube-proxy"] verbs: ["watch"] resources: - group: "" # core resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads `configmaps/ingress-uid` through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. users: ["system:unsecured"] namespaces: ["kube-system"] verbs: ["get"] resources: - group: "" # core resources: ["configmaps"] - level: None users: ["kubelet"] # legacy kubelet identity verbs: ["get"] resources: - group: "" # core resources: ["nodes", "nodes/status"] - level: None userGroups: ["system:nodes"] verbs: ["get"] resources: - group: "" # core resources: ["nodes", "nodes/status"] - level: None users: - system:kube-controller-manager - system:kube-scheduler - system:serviceaccount:kube-system:endpoint-controller verbs: ["get", "update"] namespaces: ["kube-system"] resources: - group: "" # core resources: ["endpoints"] - level: None users: ["system:apiserver"] verbs: ["get"] resources: - group: "" # core resources: ["namespaces", "namespaces/status", "namespaces/finalize"] # Don't log HPA fetching metrics. - level: None users: - system:kube-controller-manager verbs: ["get", "list"] resources: - group: "metrics.k8s.io" # Don't log these read-only URLs. - level: None nonResourceURLs: - /healthz* - /version - /swagger* # Don't log events requests. - level: None resources: - group: "" # core resources: ["events"] # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data, # so only log at the Metadata level. - level: Metadata resources: - group: "" # core resources: ["secrets", "configmaps", "serviceaccounts/token"] - group: authentication.k8s.io resources: ["tokenreviews"] omitStages: - "RequestReceived" # Get responses can be large; skip them. - level: Request verbs: ["get", "list", "watch"] resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" - group: "apiregistration.k8s.io" - group: "apps" - group: "authentication.k8s.io" - group: "authorization.k8s.io" - group: "autoscaling" - group: "batch" - group: "certificates.k8s.io" - group: "extensions" - group: "metrics.k8s.io" - group: "networking.k8s.io" - group: "policy" - group: "rbac.authorization.k8s.io" - group: "settings.k8s.io" - group: "storage.k8s.io" omitStages: - "RequestReceived" # Default level for known APIs - level: RequestResponse resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" - group: "apiregistration.k8s.io" - group: "apps" - group: "authentication.k8s.io" - group: "authorization.k8s.io" - group: "autoscaling" - group: "batch" - group: "certificates.k8s.io" - group: "extensions" - group: "metrics.k8s.io" - group: "networking.k8s.io" - group: "policy" - group: "rbac.authorization.k8s.io" - group: "settings.k8s.io" - group: "storage.k8s.io" omitStages: - "RequestReceived" # Default level for all other requests. - level: Metadata omitStages: - "RequestReceived" {% endif %} ================================================ FILE: roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2 ================================================ apiVersion: v1 kind: Config clusters: - cluster: server: {{ audit_webhook_server_url }} {% for key in audit_webhook_server_extra_args %} {{ key }}: "{{ audit_webhook_server_extra_args[key] }}" {% endfor %} name: auditsink contexts: - context: cluster: auditsink user: "" name: default-context current-context: default-context preferences: {} users: [] ================================================ FILE: roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 ================================================ apiVersion: apiserver.config.k8s.io/v1beta1 kind: TracingConfiguration endpoint: "{{ kube_apiserver_tracing_endpoint }}" samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }} ================================================ FILE: roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2 ================================================ apiVersion: eventratelimit.admission.k8s.io/v1alpha1 kind: Configuration limits: {% for limit in kube_apiserver_admission_event_rate_limits.values() %} - type: {{ limit.type }} qps: {{ limit.qps }} burst: {{ limit.burst }} {% if limit.cache_size is defined %} cacheSize: {{ limit.cache_size }} {% endif %} {% endfor %} ================================================ FILE: roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 ================================================ [Unit] Description=Renew K8S control plane certificates [Service] Type=oneshot ExecStart={{ bin_dir }}/k8s-certs-renew.sh ================================================ FILE: roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 ================================================ #!/bin/bash echo "## Check Expiration before renewal ##" {{ bin_dir }}/kubeadm certs check-expiration days_buffer=7 # set a time margin, because we should not renew at the last moment next_time=$(systemctl show k8s-certs-renew.timer -p NextElapseUSecRealtime --value) if [ "${next_time}" == "" ]; then echo "## Skip expiry comparison due to fail to parse next elapse from systemd calendar,do renewal directly ##" else current_time=$(date +%s) target_time=$(date -d "${next_time} + ${days_buffer} days" +%s) # $next_time - $days_buffer days expiry_threshold=$(( ${target_time} - ${current_time} )) expired_certs=$({{ bin_dir }}/kubeadm certs check-expiration -o jsonpath="{.certificates[?(@.residualTime<${expiry_threshold}.0)]}") if [ "${expired_certs}" == "" ];then echo "## Skip cert renew and K8S container restart, since all residualTimes are beyond threshold ##" exit 0 fi fi echo "## Renewing certificates managed by kubeadm ##" {{ bin_dir }}/kubeadm certs renew all echo "## Restarting control plane pods managed by kubeadm ##" {% if container_manager == "docker" %} {{ docker_bin_dir }}/docker ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs {{ docker_bin_dir }}/docker rm -f {% else %} {{ bin_dir }}/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs {{ bin_dir }}/crictl rmp -f {% endif %} echo "## Updating /root/.kube/config ##" cp {{ kube_config_dir }}/admin.conf /root/.kube/config echo "## Waiting for apiserver to be up again ##" until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/{{ kube_apiserver_port | default(6443) }}; do sleep 1; done echo "## Expiration after renewal ##" {{ bin_dir }}/kubeadm certs check-expiration ================================================ FILE: roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 ================================================ [Unit] Description=Timer to renew K8S control plane certificates [Timer] OnCalendar={{ auto_renew_certificates_systemd_calendar }} RandomizedDelaySec={{ 10 * (groups['kube_control_plane'] | length) }}min FixedRandomDelay=yes Persistent=yes [Install] WantedBy=multi-user.target ================================================ FILE: roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 ================================================ apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration {% if kubeadm_token is defined %} bootstrapTokens: - token: "{{ kubeadm_token }}" description: "kubespray kubeadm bootstrap token" ttl: "24h" {% endif %} localAPIEndpoint: advertiseAddress: "{{ kube_apiserver_address }}" bindPort: {{ kube_apiserver_port }} {% if kubeadm_certificate_key is defined %} certificateKey: {{ kubeadm_certificate_key }} {% endif %} nodeRegistration: {% if kube_override_hostname | default('') %} name: "{{ kube_override_hostname }}" {% endif %} {% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %} taints: - effect: NoSchedule key: node-role.kubernetes.io/control-plane {% else %} taints: [] {% endif %} criSocket: {{ cri_socket }} {% if cloud_provider == "external" %} kubeletExtraArgs: - name: cloud-provider value: external {% endif %} imagePullPolicy: {{ k8s_image_pull_policy }} imagePullSerial: {{ kubeadm_image_pull_serial | lower }} {% if kubeadm_patches | length > 0 %} patches: directory: {{ kubeadm_patches_dir }} {% endif %} --- apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration clusterName: {{ cluster_name }} encryptionAlgorithm: {{ kube_asymmetric_encryption_algorithm }} certificateValidityPeriod: {{ kube_cert_validity_period }} caCertificateValidityPeriod: {{ kube_ca_cert_validity_period }} etcd: {% if etcd_deployment_type != "kubeadm" %} external: endpoints: {% for endpoint in etcd_access_addresses.split(',') %} - "{{ endpoint }}" {% endfor %} caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} {% elif etcd_deployment_type == "kubeadm" %} local: imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" imageTag: "{{ etcd_image_tag }}" dataDir: "{{ etcd_data_dir }}" extraArgs: - name: metrics value: {{ etcd_metrics }} - name: election-timeout value: "{{ etcd_election_timeout }}" - name: heartbeat-interval value: "{{ etcd_heartbeat_interval }}" - name: auto-compaction-retention value: "{{ etcd_compaction_retention }}" {% if etcd_listen_metrics_urls is defined %} - name: listen-metrics-urls value: "{{ etcd_listen_metrics_urls }}" {% endif %} - name: snapshot-count value: "{{ etcd_snapshot_count }}" - name: quota-backend-bytes value: "{{ etcd_quota_backend_bytes }}" - name: max-request-bytes value: "{{ etcd_max_request_bytes }}" - name: log-level value: "{{ etcd_log_level }}" {% for key, value in etcd_extra_vars.items() %} - name: {{ key }} value: "{{ value }}" {% endfor %} serverCertSANs: {% for san in etcd_cert_alt_names %} - "{{ san }}" {% endfor %} {% for san in etcd_cert_alt_ips %} - "{{ san }}" {% endfor %} peerCertSANs: {% for san in etcd_cert_alt_names %} - "{{ san }}" {% endfor %} {% for san in etcd_cert_alt_ips %} - "{{ san }}" {% endfor %} {% endif %} dns: {% if 'addon/coredns' in kubeadm_init_phases_skip %} disabled: true {% endif %} imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }} imageTag: {{ coredns_image_tag }} networking: dnsDomain: {{ dns_domain }} serviceSubnet: "{{ kube_service_subnets }}" {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} podSubnet: "{{ kube_pods_subnets }}" {% endif %} {% if kubeadm_feature_gates %} featureGates: {% for feature in kubeadm_feature_gates %} {{ feature | replace("=", ": ") }} {% endfor %} {% endif %} kubernetesVersion: v{{ kube_version }} {% if kubeadm_config_api_fqdn is defined %} controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" {% else %} controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}" {% endif %} certificatesDir: {{ kube_cert_dir }} imageRepository: {{ kubeadm_image_repo }} apiServer: extraArgs: - name: etcd-compaction-interval value: "{{ kube_apiserver_etcd_compaction_interval }}" - name: default-not-ready-toleration-seconds value: "{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}" - name: default-unreachable-toleration-seconds value: "{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}" {% if kube_api_anonymous_auth is defined %} {# TODO: rework once suppport for structured auth lands #} - name: anonymous-auth value: "{{ kube_api_anonymous_auth }}" {% endif %} {% if kube_apiserver_use_authorization_config_file %} - name: authorization-config value: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml" {% else %} - name: authorization-mode value: "{{ authorization_modes | join(',') }}" {% endif %} - name: bind-address value: "{{ kube_apiserver_bind_address }}" {% if kube_apiserver_enable_admission_plugins | length > 0 %} - name: enable-admission-plugins value: "{{ kube_apiserver_enable_admission_plugins | join(',') }}" {% endif %} {% if kube_apiserver_admission_control_config_file %} - name: admission-control-config-file value: "{{ kube_config_dir }}/admission-controls.yaml" {% endif %} {% if kube_apiserver_disable_admission_plugins | length > 0 %} - name: disable-admission-plugins value: "{{ kube_apiserver_disable_admission_plugins | join(',') }}" {% endif %} - name: apiserver-count value: "{{ kube_apiserver_count }}" - name: endpoint-reconciler-type value: lease {% if etcd_events_cluster_enabled %} - name: etcd-servers-overrides value: "/events#{{ etcd_events_access_addresses_semicolon }}" {% endif %} - name: service-node-port-range value: "{{ kube_apiserver_node_port_range }}" - name: service-cluster-ip-range value: "{{ kube_service_subnets }}" - name: kubelet-preferred-address-types value: "{{ kubelet_preferred_address_types }}" - name: profiling value: "{{ kube_profiling }}" - name: request-timeout value: "{{ kube_apiserver_request_timeout }}" - name: enable-aggregator-routing value: "{{ kube_api_aggregator_routing }}" {% if kube_apiserver_service_account_lookup %} - name: service-account-lookup value: "{{ kube_apiserver_service_account_lookup }}" {% endif %} {% if kube_oidc_auth and kube_oidc_url is defined and kube_oidc_client_id is defined %} - name: oidc-issuer-url value: "{{ kube_oidc_url }}" - name: oidc-client-id value: "{{ kube_oidc_client_id }}" {% if kube_oidc_ca_file is defined %} - name: oidc-ca-file value: "{{ kube_oidc_ca_file }}" {% endif %} {% if kube_oidc_username_claim is defined %} - name: oidc-username-claim value: "{{ kube_oidc_username_claim }}" {% endif %} {% if kube_oidc_groups_claim is defined %} - name: oidc-groups-claim value: "{{ kube_oidc_groups_claim }}" {% endif %} {% if kube_oidc_username_prefix is defined %} - name: oidc-username-prefix value: "{{ kube_oidc_username_prefix }}" {% endif %} {% if kube_oidc_groups_prefix is defined %} - name: oidc-groups-prefix value: "{{ kube_oidc_groups_prefix }}" {% endif %} {% endif %} {% if kube_webhook_token_auth %} - name: authentication-token-webhook-config-file value: "{{ kube_config_dir }}/webhook-token-auth-config.yaml" {% endif %} {% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %} - name: authorization-webhook-config-file value: "{{ kube_config_dir }}/webhook-authorization-config.yaml" {% endif %} {% if kube_encrypt_secret_data %} - name: encryption-provider-config value: "{{ kube_cert_dir }}/secrets_encryption.yaml" {% endif %} - name: storage-backend value: "{{ kube_apiserver_storage_backend }}" {% if kube_api_runtime_config | length > 0 %} - name: runtime-config value: "{{ kube_api_runtime_config | join(',') }}" {% endif %} - name: allow-privileged value: "true" {% if kubernetes_audit or kubernetes_audit_webhook %} - name: audit-policy-file value: "{{ audit_policy_file }}" {% endif %} {% if kubernetes_audit %} - name: audit-log-path value: "{{ audit_log_path }}" - name: audit-log-maxage value: "{{ audit_log_maxage }}" - name: audit-log-maxbackup value: "{{ audit_log_maxbackups }}" - name: audit-log-maxsize value: "{{ audit_log_maxsize }}" {% endif %} {% if kubernetes_audit_webhook %} - name: audit-webhook-config-file value: "{{ audit_webhook_config_file }}" - name: audit-webhook-mode value: "{{ audit_webhook_mode }}" {% if audit_webhook_mode == "batch" %} - name: audit-webhook-batch-max-size value: "{{ audit_webhook_batch_max_size }}" - name: audit-webhook-batch-max-wait value: "{{ audit_webhook_batch_max_wait }}" {% endif %} {% endif %} {% for key in kube_kubeadm_apiserver_extra_args %} - name: "{{ key }}" value: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} {% if kube_apiserver_feature_gates or kube_feature_gates %} - name: feature-gates value: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}" {% endif %} {% if tls_min_version is defined %} - name: tls-min-version value: "{{ tls_min_version }}" {% endif %} {% if tls_cipher_suites is defined %} - name: tls-cipher-suites value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}" {% endif %} - name: event-ttl value: "{{ event_ttl_duration }}" {% if kubelet_rotate_server_certificates %} - name: kubelet-certificate-authority value: "{{ kube_cert_dir }}/ca.crt" {% endif %} {% if kube_apiserver_tracing %} - name: tracing-config-file value: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml" {% endif %} {% if kubernetes_audit or kube_token_auth or kube_webhook_token_auth or apiserver_extra_volumes or ssl_ca_dirs | length %} extraVolumes: {% if kube_token_auth %} - name: token-auth-config hostPath: {{ kube_token_dir }} mountPath: {{ kube_token_dir }} {% endif %} {% if kube_webhook_token_auth %} - name: webhook-token-auth-config hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml {% endif %} {% if kube_webhook_authorization %} - name: webhook-authorization-config hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml {% endif %} {% if kube_apiserver_use_authorization_config_file %} - name: authorization-config hostPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml mountPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml {% endif %} {% if kubernetes_audit or kubernetes_audit_webhook %} - name: {{ audit_policy_name }} hostPath: {{ audit_policy_hostpath }} mountPath: {{ audit_policy_mountpath }} {% if audit_log_path != "-" %} - name: {{ audit_log_name }} hostPath: {{ audit_log_hostpath }} mountPath: {{ audit_log_mountpath }} readOnly: false {% endif %} {% endif %} {% if kube_apiserver_admission_control_config_file %} - name: admission-control-configs hostPath: {{ kube_config_dir }}/admission-controls mountPath: {{ kube_config_dir }} readOnly: false pathType: DirectoryOrCreate {% endif %} {% if kube_apiserver_tracing %} - name: tracing hostPath: {{ kube_config_dir }}/tracing mountPath: {{ kube_config_dir }}/tracing readOnly: true pathType: DirectoryOrCreate {% endif %} {% for volume in apiserver_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} mountPath: {{ volume.mountPath }} readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} {% endfor %} {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} hostPath: {{ dir }} mountPath: {{ dir }} readOnly: true {% endfor %} {% endif %} {% endif %} certSANs: {% for san in apiserver_sans %} - "{{ san }}" {% endfor %} controllerManager: extraArgs: - name: node-monitor-grace-period value: "{{ kube_controller_node_monitor_grace_period }}" - name: node-monitor-period value: "{{ kube_controller_node_monitor_period }}" {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} - name: cluster-cidr value: "{{ kube_pods_subnets }}" {% endif %} - name: service-cluster-ip-range value: "{{ kube_service_subnets }}" {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %} - name: allocate-node-cidrs value: "false" {% else %} {% if ipv4_stack %} - name: node-cidr-mask-size-ipv4 value: "{{ kube_network_node_prefix }}" {% endif %} {% if ipv6_stack %} - name: node-cidr-mask-size-ipv6 value: "{{ kube_network_node_prefix_ipv6 }}" {% endif %} {% endif %} - name: profiling value: "{{ kube_profiling }}" - name: terminated-pod-gc-threshold value: "{{ kube_controller_terminated_pod_gc_threshold }}" - name: bind-address value: "{{ kube_controller_manager_bind_address }}" - name: leader-elect-lease-duration value: "{{ kube_controller_manager_leader_elect_lease_duration }}" - name: leader-elect-renew-deadline value: "{{ kube_controller_manager_leader_elect_renew_deadline }}" {% if kube_controller_feature_gates or kube_feature_gates %} - name: feature-gates value: "{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}" {% endif %} {% for key in kube_kubeadm_controller_extra_args %} - name: "{{ key }}" value: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} {% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %} - name: configure-cloud-routes value: "false" {% endif %} {% if kubelet_flexvolumes_plugins_dir is defined %} - name: flex-volume-plugin-dir value: "{{ kubelet_flexvolumes_plugins_dir }}" {% endif %} {% if tls_min_version is defined %} - name: tls-min-version value: "{{ tls_min_version }}" {% endif %} {% if tls_cipher_suites is defined %} - name: tls-cipher-suites value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}" {% endif %} {% if controller_manager_extra_volumes %} extraVolumes: {% for volume in controller_manager_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} mountPath: {{ volume.mountPath }} readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} {% endfor %} {% endif %} scheduler: extraArgs: - name: bind-address value: "{{ kube_scheduler_bind_address }}" - name: config value: "{{ kube_config_dir }}/kubescheduler-config.yaml" {% if kube_scheduler_feature_gates or kube_feature_gates %} - name: feature-gates value: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}" {% endif %} - name: profiling value: "{{ kube_profiling }}" {% if kube_kubeadm_scheduler_extra_args | length > 0 %} {% for key in kube_kubeadm_scheduler_extra_args %} - name: "{{ key }}" value: "{{ kube_kubeadm_scheduler_extra_args[key] }}" {% endfor %} {% endif %} {% if tls_min_version is defined %} - name: tls-min-version value: "{{ tls_min_version }}" {% endif %} {% if tls_cipher_suites is defined %} - name: tls-cipher-suites value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}" {% endif %} extraVolumes: - name: kubescheduler-config hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml readOnly: true {% if scheduler_extra_volumes %} {% for volume in scheduler_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} mountPath: {{ volume.mountPath }} readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} {% endfor %} {% endif %} --- apiVersion: kubeadm.k8s.io/v1beta4 kind: UpgradeConfiguration apply: kubernetesVersion: v{{ kube_version }} allowExperimentalUpgrades: true certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }} etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }} forceUpgrade: true {% if kubeadm_ignore_preflight_errors | length > 0 %} ignorePreflightErrors: {% for ignore_error in kubeadm_ignore_preflight_errors %} - "{{ ignore_error }}" {% endfor %} {% endif %} {% if kubeadm_patches | length > 0 %} patches: directory: {{ kubeadm_patches_dir }} {% endif %} imagePullPolicy: {{ k8s_image_pull_policy }} imagePullSerial: {{ kubeadm_image_pull_serial | lower }} {% for skip_phase in kubeadm_init_phases_skip %} {% if loop.first %} skipPhases: {% endif %} - "{{ skip_phase }}" {% endfor %} node: certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }} etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }} {% if kubeadm_ignore_preflight_errors | length > 0 %} ignorePreflightErrors: {% for ignore_error in kubeadm_ignore_preflight_errors %} - "{{ ignore_error }}" {% endfor %} {% endif %} {% if kubeadm_patches | length > 0 %} patches: directory: {{ kubeadm_patches_dir }} {% endif %} imagePullPolicy: {{ k8s_image_pull_policy }} imagePullSerial: {{ kubeadm_image_pull_serial | lower }} {% for skip_phase in kubeadm_upgrade_node_phases_skip %} {% if loop.first %} skipPhases: {% endif %} - "{{ skip_phase }}" {% endfor %} --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration bindAddress: "{{ kube_proxy_bind_address }}" clientConnection: acceptContentTypes: {{ kube_proxy_client_accept_content_types }} burst: {{ kube_proxy_client_burst }} contentType: {{ kube_proxy_client_content_type }} kubeconfig: {{ kube_proxy_client_kubeconfig }} qps: {{ kube_proxy_client_qps }} {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} clusterCIDR: "{{ kube_pods_subnets }}" {% endif %} configSyncPeriod: {{ kube_proxy_config_sync_period }} conntrack: maxPerCore: {{ kube_proxy_conntrack_max_per_core }} min: {{ kube_proxy_conntrack_min }} tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }} tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }} enableProfiling: {{ kube_proxy_enable_profiling }} healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}" hostnameOverride: "{{ kube_override_hostname }}" iptables: masqueradeAll: {{ kube_proxy_masquerade_all }} masqueradeBit: {{ kube_proxy_masquerade_bit }} minSyncPeriod: {{ kube_proxy_min_sync_period }} syncPeriod: {{ kube_proxy_sync_period }} ipvs: excludeCIDRs: {{ kube_proxy_exclude_cidrs }} minSyncPeriod: {{ kube_proxy_min_sync_period }} scheduler: {{ kube_proxy_scheduler }} syncPeriod: {{ kube_proxy_sync_period }} strictARP: {{ kube_proxy_strict_arp }} tcpTimeout: {{ kube_proxy_tcp_timeout }} tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }} udpTimeout: {{ kube_proxy_udp_timeout }} metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}" mode: {{ kube_proxy_mode }} nodePortAddresses: {{ kube_proxy_nodeport_addresses }} oomScoreAdj: {{ kube_proxy_oom_score_adj }} portRange: {{ kube_proxy_port_range }} {% if kube_proxy_feature_gates or kube_feature_gates %} {% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %} featureGates: {% for feature in feature_gates %} {{ feature | replace("=", ": ") }} {% endfor %} {% endif %} {# DNS settings for kubelet #} {% if enable_nodelocaldns %} {% set kubelet_cluster_dns = [nodelocaldns_ip] %} {% elif dns_mode in ['coredns'] %} {% set kubelet_cluster_dns = [skydns_server] %} {% elif dns_mode == 'coredns_dual' %} {% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %} {% elif dns_mode == 'manual' %} {% set kubelet_cluster_dns = [manual_dns_server] %} {% else %} {% set kubelet_cluster_dns = [] %} {% endif %} --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration {% if kube_version is version('1.35.0', '>=') %} failCgroupV1: {{ kubelet_fail_cgroup_v1 }} {% endif %} clusterDNS: {% for dns_address in kubelet_cluster_dns %} - {{ dns_address }} {% endfor %} {% if kubelet_feature_gates or kube_feature_gates %} {% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %} featureGates: {% for feature in feature_gates %} {{ feature | replace("=", ": ") }} {% endfor %} {% endif %} ================================================ FILE: roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2 ================================================ apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration discovery: {% if kubeadm_use_file_discovery %} file: kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml {% else %} bootstrapToken: {% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} {% else %} apiServerEndpoint: "{{ kubeadm_discovery_address }}" {% endif %} token: {{ kubeadm_token }} unsafeSkipCAVerification: true {% endif %} tlsBootstrapToken: {{ kubeadm_token }} timeouts: discovery: {{ discovery_timeout }} controlPlane: localAPIEndpoint: advertiseAddress: "{{ kube_apiserver_address }}" bindPort: {{ kube_apiserver_port }} certificateKey: {{ kubeadm_certificate_key }} nodeRegistration: name: {{ kube_override_hostname | default(inventory_hostname) }} criSocket: {{ cri_socket }} {% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %} taints: - effect: NoSchedule key: node-role.kubernetes.io/control-plane {% else %} taints: [] {% endif %} {% if kubeadm_patches | length > 0 %} patches: directory: {{ kubeadm_patches_dir }} {% endif %} ================================================ FILE: roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2 ================================================ apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: kubeconfig: "{{ kube_config_dir }}/scheduler.conf" {% for key in kube_scheduler_client_conn_extra_opts %} {{ key }}: {{ kube_scheduler_client_conn_extra_opts[key] }} {% endfor %} {% if kube_scheduler_extenders %} extenders: {{ kube_scheduler_extenders | to_nice_yaml(indent=2, width=256) }} {% endif %} leaderElection: leaseDuration: {{ kube_scheduler_leader_elect_lease_duration }} renewDeadline: {{ kube_scheduler_leader_elect_renew_deadline }} {% for key in kube_scheduler_leader_elect_extra_opts %} {{ key }}: {{ kube_scheduler_leader_elect_extra_opts[key] }} {% endfor %} {% if kube_scheduler_profiles %} profiles: {{ kube_scheduler_profiles | to_nice_yaml(indent=2, width=256) }} {% endif %} {% for key in kube_scheduler_config_extra_opts %} {{ key }}: {{ kube_scheduler_config_extra_opts[key] }} {% endfor %} ================================================ FILE: roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 ================================================ podNodeSelectorPluginConfig: clusterDefaultNodeSelector: {{ kube_apiserver_admission_plugins_podnodeselector_default_node_selector }} ================================================ FILE: roles/kubernetes/control-plane/templates/podsecurity.yaml.j2 ================================================ apiVersion: pod-security.admission.config.k8s.io/v1 kind: PodSecurityConfiguration {% if kube_pod_security_use_default %} defaults: enforce: "{{ kube_pod_security_default_enforce }}" enforce-version: "{{ kube_pod_security_default_enforce_version }}" audit: "{{ kube_pod_security_default_audit }}" audit-version: "{{ kube_pod_security_default_audit_version }}" warn: "{{ kube_pod_security_default_warn }}" warn-version: "{{ kube_pod_security_default_warn_version }}" exemptions: usernames: {{ kube_pod_security_exemptions_usernames | to_json }} runtimeClasses: {{ kube_pod_security_exemptions_runtime_class_names | to_json }} namespaces: {{ kube_pod_security_exemptions_namespaces | to_json }} {% else %} # This file is intentinally left empty as kube_pod_security_use_default={{ kube_pod_security_use_default }} {% endif %} ================================================ FILE: roles/kubernetes/control-plane/templates/resourcequota.yaml.j2 ================================================ apiVersion: apiserver.config.k8s.io/v1 kind: ResourceQuotaConfiguration {% if kube_resource_quota_limited_resources | d(false) -%} limitedResources: {{ kube_resource_quota_limited_resources | to_nice_yaml(indent=2, sort_keys=false) }} {% else %} # No limitedResources configured. If limitedResources are required, please set kube_resource_quota_limited_resources. {%- endif %} ================================================ FILE: roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2 ================================================ apiVersion: apiserver.config.k8s.io/v1 kind: EncryptionConfiguration resources: - resources: {{ kube_encryption_resources | to_nice_yaml | indent(4, True) }} providers: - {{ kube_encryption_algorithm }}: keys: - name: key secret: {{ kube_encrypt_token | b64encode }} - identity: {} ================================================ FILE: roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2 ================================================ # clusters refers to the remote service. clusters: - name: webhook-token-authz-cluster cluster: server: {{ kube_webhook_authorization_url }} insecure-skip-tls-verify: {{ kube_webhook_authorization_url_skip_tls_verify }} # users refers to the API server's webhook configuration. users: - name: webhook-token-authz-user # kubeconfig files require a context. Provide one for the API server. current-context: webhook-token-authz contexts: - context: cluster: webhook-token-authz-cluster user: webhook-token-authz-user name: webhook-token-authz ================================================ FILE: roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2 ================================================ # clusters refers to the remote service. clusters: - name: webhook-token-auth-cluster cluster: server: {{ kube_webhook_token_auth_url }} insecure-skip-tls-verify: {{ kube_webhook_token_auth_url_skip_tls_verify }} {% if kube_webhook_token_auth_ca_data is defined %} certificate-authority-data: {{ kube_webhook_token_auth_ca_data }} {% endif %} # users refers to the API server's webhook configuration. users: - name: webhook-token-auth-user # kubeconfig files require a context. Provide one for the API server. current-context: webhook-token-auth contexts: - context: cluster: webhook-token-auth-cluster user: webhook-token-auth-user name: webhook-token-auth ================================================ FILE: roles/kubernetes/control-plane/vars/main.yaml ================================================ --- # list of admission plugins that needs to be configured # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/ kube_apiserver_admission_plugins_needs_configuration: - EventRateLimit - ImagePolicyWebhook - PodSecurity - PodNodeSelector - ResourceQuota ================================================ FILE: roles/kubernetes/kubeadm/defaults/main.yml ================================================ --- # discovery_timeout modifies the discovery timeout # This value must be smaller than kubeadm_join_timeout discovery_timeout: 60s kubeadm_join_timeout: 120s # Enable kubeadm file discovery if anonymous access has been removed kubeadm_use_file_discovery: "{{ remove_anonymous_access }}" ================================================ FILE: roles/kubernetes/kubeadm/handlers/main.yml ================================================ --- - name: Kubeadm | reload systemd systemd_service: daemon_reload: true listen: Kubeadm | restart kubelet - name: Kubeadm | reload kubelet service: name: kubelet state: restarted listen: Kubeadm | restart kubelet ================================================ FILE: roles/kubernetes/kubeadm/meta/main.yml ================================================ --- dependencies: - role: kubernetes/kubeadm_common ================================================ FILE: roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml ================================================ --- - name: Parse certificate key if not set set_fact: kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_certificate_key'] }}" when: kubeadm_certificate_key is undefined - name: Create kubeadm cert controlplane config template: src: "kubeadm-client.conf.j2" dest: "{{ kube_config_dir }}/kubeadm-cert-controlplane.conf" mode: "0640" validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}" vars: kubeadm_cert_controlplane: true - name: Pull control plane certs down shell: >- {{ bin_dir }}/kubeadm join phase control-plane-prepare download-certs --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf && {{ bin_dir }}/kubeadm join phase control-plane-prepare certs --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf args: creates: "{{ kube_cert_dir }}/apiserver-etcd-client.key" - name: Delete unneeded certificates file: path: "{{ item }}" state: absent with_items: - "{{ kube_cert_dir }}/apiserver.crt" - "{{ kube_cert_dir }}/apiserver.key" - "{{ kube_cert_dir }}/ca.key" - "{{ kube_cert_dir }}/etcd/ca.key" - "{{ kube_cert_dir }}/etcd/healthcheck-client.crt" - "{{ kube_cert_dir }}/etcd/healthcheck-client.key" - "{{ kube_cert_dir }}/etcd/peer.crt" - "{{ kube_cert_dir }}/etcd/peer.key" - "{{ kube_cert_dir }}/etcd/server.crt" - "{{ kube_cert_dir }}/etcd/server.key" - "{{ kube_cert_dir }}/front-proxy-ca.crt" - "{{ kube_cert_dir }}/front-proxy-ca.key" - "{{ kube_cert_dir }}/front-proxy-client.crt" - "{{ kube_cert_dir }}/front-proxy-client.key" - "{{ kube_cert_dir }}/sa.key" - "{{ kube_cert_dir }}/sa.pub" - name: Calculate etcd cert serial command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial" register: "etcd_client_cert_serial_result" changed_when: false when: - group_names | intersect(['k8s_cluster', 'calico_rr']) | length > 0 tags: - network - name: Set etcd_client_cert_serial set_fact: etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}" tags: - network ================================================ FILE: roles/kubernetes/kubeadm/tasks/main.yml ================================================ --- - name: Set kubeadm_discovery_address set_fact: # noqa: jinja[spacing] kubeadm_discovery_address: >- {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} {%- else -%} {{ kube_apiserver_endpoint | replace("https://", "") }} {%- endif %} tags: - facts - name: Check if kubelet.conf exists stat: path: "{{ kube_config_dir }}/kubelet.conf" get_attributes: false get_checksum: false get_mime: false register: kubelet_conf - name: Check if kubeadm CA cert is accessible stat: path: "{{ kube_cert_dir }}/ca.crt" get_attributes: false get_checksum: false get_mime: false register: kubeadm_ca_stat delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true - name: Fetch CA certificate from control plane node slurp: src: "{{ kube_cert_dir }}/ca.crt" register: ca_cert_content when: - kubeadm_ca_stat.stat is defined - kubeadm_ca_stat.stat.exists delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true - name: Create kubeadm token for joining nodes with 24h expiration (default) command: "{{ bin_dir }}/kubeadm token create" register: temp_token delegate_to: "{{ groups['kube_control_plane'][0] }}" when: kubeadm_token is not defined changed_when: false - name: Set kubeadm_token to generated token set_fact: kubeadm_token: "{{ temp_token.stdout }}" when: kubeadm_token is not defined - name: Get kubeconfig for join discovery process command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'" register: kubeconfig_file_discovery run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" when: kubeadm_use_file_discovery - name: Check if discovery kubeconfig exists stat: path: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" register: cluster_info_discovery_kubeconfig - name: Copy discovery kubeconfig copy: dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" content: "{{ kubeconfig_file_discovery.stdout }}" owner: "root" mode: "0644" when: - ('kube_control_plane' not in group_names) - not kubelet_conf.stat.exists or not cluster_info_discovery_kubeconfig.stat.exists - kubeadm_use_file_discovery - name: Create kubeadm client config template: src: "kubeadm-client.conf.j2" dest: "{{ kube_config_dir }}/kubeadm-client.conf" backup: true mode: "0640" validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}" when: ('kube_control_plane' not in group_names) - name: Join to cluster if needed environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin" when: - ('kube_control_plane' not in group_names) - not kubelet_conf.stat.exists vars: ignored: - "{{ 'DirAvailable--etc-kubernetes-manifests' if 'all' not in kubeadm_ignore_preflight_errors }}" - "{{ kubeadm_ignore_preflight_errors }}" command: >- timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }} {{ bin_dir }}/kubeadm join --config {{ kube_config_dir }}/kubeadm-client.conf --ignore-preflight-errors={{ ignored | select | flatten | join(',') }} --skip-phases={{ kubeadm_join_phases_skip | join(',') }} - name: Update server field in kubelet kubeconfig lineinfile: dest: "{{ kube_config_dir }}/kubelet.conf" regexp: 'server:' line: ' server: {{ kube_apiserver_endpoint }}' backup: true when: - kubeadm_config_api_fqdn is not defined - ('kube_control_plane' not in group_names) - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") notify: Kubeadm | restart kubelet - name: Update server field in kubelet kubeconfig - external lb lineinfile: dest: "{{ kube_config_dir }}/kubelet.conf" regexp: '^ server: https' line: ' server: {{ kube_apiserver_endpoint }}' backup: true when: - ('kube_control_plane' not in group_names) - loadbalancer_apiserver is defined notify: Kubeadm | restart kubelet - name: Get current resourceVersion of kube-proxy configmap command: "{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'" register: original_configmap_resource_version run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_facts: false when: - kube_proxy_deployed tags: - kube-proxy # FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point # incorrectly to first control plane node, creating SPoF. - name: Update server field in kube-proxy kubeconfig shell: >- set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g' | {{ kubectl }} replace -f - args: executable: /bin/bash run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_facts: false when: - kubeadm_config_api_fqdn is not defined - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") - kube_proxy_deployed - loadbalancer_apiserver_localhost tags: - kube-proxy - name: Update server field in kube-proxy kubeconfig - external lb shell: >- set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml | sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g' | {{ kubectl }} replace -f - args: executable: /bin/bash run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_facts: false when: - kube_proxy_deployed - loadbalancer_apiserver is defined tags: - kube-proxy - name: Get new resourceVersion of kube-proxy configmap command: "{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'" register: new_configmap_resource_version run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_facts: false when: - kube_proxy_deployed tags: - kube-proxy - name: Set ca.crt file permission file: path: "{{ kube_cert_dir }}/ca.crt" owner: root group: root mode: "0644" - name: Restart all kube-proxy pods to ensure that they load the new configmap command: "{{ kubectl }} delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0" run_once: true delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_facts: false when: - kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") or loadbalancer_apiserver is defined - kube_proxy_deployed - original_configmap_resource_version.stdout != new_configmap_resource_version.stdout tags: - kube-proxy - name: Extract etcd certs from control plane if using etcd kubeadm mode include_tasks: kubeadm_etcd_node.yml when: - etcd_deployment_type == "kubeadm" - inventory_hostname not in groups['kube_control_plane'] - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally - kube_network_plugin != "calico" or calico_datastore == "etcd" - kube_network_plugin != "cilium" or cilium_identity_allocation_mode != 'crd' ================================================ FILE: roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 ================================================ --- apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration discovery: {% if kubeadm_use_file_discovery %} file: kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml {% else %} bootstrapToken: {% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" {% else %} apiServerEndpoint: "{{ kubeadm_discovery_address }}" {% endif %} token: {{ kubeadm_token }} {% if ca_cert_content is defined %} caCertHashes: - sha256:{{ (ca_cert_content.content | b64decode | community.crypto.x509_certificate_info).public_key_fingerprints.sha256.replace(':', '') }} {% else %} unsafeSkipCAVerification: true {% endif %} {% endif %} tlsBootstrapToken: {{ kubeadm_token }} timeouts: discovery: {{ discovery_timeout }} caCertPath: {{ kube_cert_dir }}/ca.crt {% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %} controlPlane: localAPIEndpoint: advertiseAddress: "{{ kube_apiserver_address }}" bindPort: {{ kube_apiserver_port }} certificateKey: {{ kubeadm_certificate_key }} {% endif %} nodeRegistration: name: '{{ kube_override_hostname }}' criSocket: {{ cri_socket }} {% if 'calico_rr' in group_names and 'kube_node' not in group_names %} taints: - effect: NoSchedule key: node-role.kubernetes.io/calico-rr {% endif %} {% if kubeadm_patches | length > 0 %} patches: directory: {{ kubeadm_patches_dir }} {% endif %} ================================================ FILE: roles/kubernetes/kubeadm_common/defaults/main.yml ================================================ --- kubeadm_patches_dir: "{{ kube_config_dir }}/patches" kubeadm_patches: [] # See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches # Correspondance with this link # patchtype = type # target = target # suffix -> managed automatically # extension -> always "yaml" # kubeadm_patches: # - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration # type: strategic(default)|json|merge # patch: # metadata: # annotations: # example.com/test: "true" # labels: # example.com/prod_level: "{{ prod_level }}" # - ... # Patches are applied in the order they are specified. # List of errors to ignore during kubeadm preflight checks kubeadm_ignore_preflight_errors: [] ================================================ FILE: roles/kubernetes/kubeadm_common/tasks/main.yml ================================================ --- - name: Kubeadm | Create directory to store kubeadm patches file: path: "{{ kubeadm_patches_dir }}" state: directory mode: "0750" when: kubeadm_patches | length > 0 - name: Kubeadm | List existing kubeadm patches find: paths: - "{{ kubeadm_patches_dir }}" file_type: file use_regex: true patterns: - '^(kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration)[0-9]+\+(strategic|json|merge).yaml$' register: existing_kubeadm_patches - name: Kubeadm | Copy kubeadm patches from inventory files copy: content: "{{ item.patch | to_yaml }}" dest: "{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml" owner: "root" mode: "0644" loop: "{{ kubeadm_patches }}" loop_control: index_var: suffix register: current_kubeadm_patches - name: Kubeadm | Delete old patches loop: "{{ existing_kubeadm_patches.files | map(attribute='path') | difference( current_kubeadm_patches.results | map(attribute='dest') ) }}" file: state: absent path: "{{ item }}" ================================================ FILE: roles/kubernetes/node/defaults/main.yml ================================================ --- # advertised host IP for kubelet. This affects network plugin config. Take caution # add ipv6 manual for dualstack mode because ipv4 priority in main_ip for dualstack kubelet_address: "{{ main_ips | join(',') }}" # bind address for kubelet. Set to :: to listen on all interfaces kubelet_bind_address: "{{ main_ip | default('::') }}" # resolv.conf to base dns config kube_resolv_conf: "/etc/resolv.conf" # Set to empty to avoid cgroup creation kubelet_enforce_node_allocatable: "\"\"" # Set runtime and kubelet cgroups when using systemd as cgroup driver (default) kube_service_cgroups: "{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}" kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service" kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service" # Set runtime and kubelet cgroups when using cgroupfs as cgroup driver kubelet_runtime_cgroups_cgroupfs: "/system.slice/{{ container_manager }}.service" kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service" # Set systemd service hardening features kubelet_systemd_hardening: false # Kubelet service dependencies other than container runtime kubelet_systemd_wants_dependencies: [] # List of secure IPs for kubelet # don't forget ipv6 addresses for dualstack(because "main_ip" prioritizes ipv4) kube_node_addresses: >- {%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%} {{ hostvars[host]['main_ips'] | join(' ') }}{{ ' ' if not loop.last else '' }} {%- endfor -%} kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnets | regex_replace(',', ' ') }} {{ kube_node_addresses }}" # Reserve this space for kube resources # Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations). kube_reserved: false kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" kube_memory_reserved: "256Mi" kube_cpu_reserved: "100m" kube_ephemeral_storage_reserved: "500Mi" kube_pid_reserved: "1000" # Set to true to reserve resources for system daemons system_reserved: false system_reserved_cgroups_for_service_slice: system.slice system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" system_memory_reserved: "512Mi" system_cpu_reserved: "500m" system_ephemeral_storage_reserved: "500Mi" system_pid_reserved: 1000 ## Eviction Thresholds to avoid system OOMs # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds eviction_hard: {} eviction_hard_control_plane: {} kubelet_status_update_frequency: 10s # kube-vip kube_vip_arp_enabled: false kube_vip_interface: kube_vip_services_interface: kube_vip_cidr: 32 kube_vip_dns_mode: first kube_vip_controlplane_enabled: false kube_vip_ddns_enabled: false kube_vip_cp_detect: false kube_vip_services_enabled: false kube_vip_leader_election_enabled: "{{ kube_vip_arp_enabled }}" kube_vip_bgp_enabled: false kube_vip_bgp_routerid: kube_vip_local_as: 65000 kube_vip_bgp_peeraddress: kube_vip_bgp_peerpass: kube_vip_bgp_peeras: 65000 kube_vip_bgppeers: kube_vip_enableServicesElection: false kube_vip_lb_enable: false kube_vip_leasename: plndr-cp-lock kube_vip_svc_leasename: plndr-svcs-lock kube_vip_leaseduration: 5 kube_vip_renewdeadline: 3 kube_vip_retryperiod: 1 kube_vip_enable_node_labeling: false kube_vip_bgp_sourceip: kube_vip_bgp_sourceif: # Requests for load balancer app loadbalancer_apiserver_memory_requests: 32M loadbalancer_apiserver_cpu_requests: 25m loadbalancer_apiserver_keepalive_timeout: 5m loadbalancer_apiserver_pod_name: "{% if loadbalancer_apiserver_type == 'nginx' %}nginx-proxy{% else %}haproxy{% endif %}" # Uncomment if you need to enable deprecated runtimes # kube_api_runtime_config: # - apps/v1beta1=true # - apps/v1beta2=true # - extensions/v1beta1/daemonsets=true # - extensions/v1beta1/deployments=true # - extensions/v1beta1/replicasets=true # - extensions/v1beta1/networkpolicies=true # A port range to reserve for services with NodePort visibility. # Inclusive at both ends of the range. kube_apiserver_node_port_range: "30000-32767" # Configure the amount of pods able to run on single node # default is equal to application default kubelet_max_pods: 110 # Sets the maximum number of processes running per Pod # Default value -1 = unlimited kubelet_pod_pids_limit: -1 ## Support parameters to be passed to kubelet via kubelet-config.yaml kubelet_config_extra_args: {} ## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver kubelet_config_extra_args_cgroupfs: systemCgroups: /system.slice cgroupRoot: / # Maximum number of container log files that can be present for a container. kubelet_logfiles_max_nr: 5 # Maximum size of the container log file before it is rotated kubelet_logfiles_max_size: 10Mi ## Support custom flags to be passed to kubelet kubelet_custom_flags: [] # The read-only port for the Kubelet to serve on with no authentication/authorization. kube_read_only_port: 0 # Port for healthz for Kubelet kubelet_healthz_port: 10248 # Bind address for healthz for Kubelet kubelet_healthz_bind_address: 127.0.0.1 # sysctl_file_path to add sysctl conf to sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" ## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. # tls_min_version: "" ## Support tls cipher suites. # tls_cipher_suites: # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA256 # - TLS_RSA_WITH_AES_128_GCM_SHA256 # - TLS_RSA_WITH_AES_256_CBC_SHA # - TLS_RSA_WITH_AES_256_GCM_SHA384 # - TLS_RSA_WITH_RC4_128_SHA kube_proxy_ipvs_modules: - ip_vs - ip_vs_rr - ip_vs_wrr - ip_vs_sh - ip_vs_wlc - ip_vs_lc ## Enable distributed tracing for kubelet kubelet_tracing: false kubelet_tracing_endpoint: "[::]:4317" kubelet_tracing_sampling_rate_per_million: 100 # The maximum number of image pulls in parallel. Set it to a integer great than 1 to enable image pulling in parallel. kubelet_max_parallel_image_pulls: 1 ================================================ FILE: roles/kubernetes/node/handlers/main.yml ================================================ --- - name: Kubelet | reload systemd systemd_service: daemon_reload: true listen: Node | restart kubelet - name: Kubelet | restart kubelet service: name: kubelet state: restarted listen: Node | restart kubelet ================================================ FILE: roles/kubernetes/node/tasks/facts.yml ================================================ --- - name: Gather cgroups facts for docker when: container_manager == 'docker' block: - name: Look up docker cgroup driver shell: "set -o pipefail && docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'" args: executable: /bin/bash register: docker_cgroup_driver_result changed_when: false check_mode: false - name: Set kubelet_cgroup_driver_detected fact for docker set_fact: kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}" - name: Gather cgroups facts for crio when: container_manager == 'crio' block: - name: Look up crio cgroup driver shell: "set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'" args: executable: /bin/bash register: crio_cgroup_driver_result changed_when: false - name: Set kubelet_cgroup_driver_detected fact for crio set_fact: kubelet_cgroup_driver_detected: "{{ crio_cgroup_driver_result.stdout }}" - name: Set kubelet_cgroup_driver_detected fact for containerd when: container_manager == 'containerd' set_fact: kubelet_cgroup_driver_detected: >- {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%} - name: Set kubelet_cgroup_driver set_fact: kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}" when: kubelet_cgroup_driver is undefined - name: Set kubelet_cgroups options when cgroupfs is used set_fact: kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}" kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}" when: kubelet_cgroup_driver == 'cgroupfs' - name: Set kubelet_config_extra_args options when cgroupfs is used set_fact: kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}" when: kubelet_cgroup_driver == 'cgroupfs' - name: Os specific vars include_vars: "{{ item }}" with_first_found: - files: - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}.yml" - "{{ ansible_os_family | lower }}.yml" skip: true ================================================ FILE: roles/kubernetes/node/tasks/install.yml ================================================ --- - name: Install | Copy kubeadm binary from download dir copy: src: "{{ downloads.kubeadm.dest }}" dest: "{{ bin_dir }}/kubeadm" mode: "0755" remote_src: true tags: - kubeadm when: - not ('kube_control_plane' in group_names) - name: Install | Copy kubelet binary from download dir copy: src: "{{ downloads.kubelet.dest }}" dest: "{{ bin_dir }}/kubelet" mode: "0755" remote_src: true tags: - kubelet - upgrade notify: Node | restart kubelet ================================================ FILE: roles/kubernetes/node/tasks/kubelet.yml ================================================ --- - name: Set kubelet api version to v1beta1 set_fact: kubeletConfig_api_version: v1beta1 tags: - kubelet - kubeadm - name: Write kubelet environment config file (kubeadm) template: src: "kubelet.env.{{ kubeletConfig_api_version }}.j2" dest: "{{ kube_config_dir }}/kubelet.env" setype: "{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}" backup: true mode: "0600" notify: Node | restart kubelet tags: - kubelet - kubeadm - name: Write kubelet config file template: src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2" dest: "{{ kube_config_dir }}/kubelet-config.yaml" mode: "0600" notify: Kubelet | restart kubelet tags: - kubelet - kubeadm - name: Write kubelet systemd init file template: src: "kubelet.service.j2" dest: "/etc/systemd/system/kubelet.service" backup: true mode: "0600" validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:kubelet.service'" # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) # Remove once we drop support for systemd < 250 notify: Node | restart kubelet tags: - kubelet - kubeadm - name: Flush_handlers and reload-systemd meta: flush_handlers - name: Enable kubelet service: name: kubelet enabled: true state: started tags: - kubelet notify: Kubelet | restart kubelet ================================================ FILE: roles/kubernetes/node/tasks/loadbalancer/haproxy.yml ================================================ --- - name: Haproxy | Cleanup potentially deployed nginx-proxy file: path: "{{ kube_manifest_dir }}/nginx-proxy.yml" state: absent - name: Haproxy | Make haproxy directory file: path: "{{ haproxy_config_dir }}" state: directory mode: "0755" owner: root - name: Haproxy | Write haproxy configuration template: src: "loadbalancer/haproxy.cfg.j2" dest: "{{ haproxy_config_dir }}/haproxy.cfg" owner: root mode: "0755" backup: true register: haproxy_conf - name: Haproxy | Write static pod template: src: manifests/haproxy.manifest.j2 dest: "{{ kube_manifest_dir }}/haproxy.yml" mode: "0640" ================================================ FILE: roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml ================================================ --- - name: Kube-vip | Check cluster settings for kube-vip fail: msg: "kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md" when: - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp - kube_vip_arp_enabled - name: Kube-vip | Check mutually exclusive BGP source settings vars: kube_vip_bgp_sourceip_normalized: "{{ kube_vip_bgp_sourceip | default('', true) | string | trim }}" kube_vip_bgp_sourceif_normalized: "{{ kube_vip_bgp_sourceif | default('', true) | string | trim }}" assert: that: - kube_vip_bgp_sourceip_normalized == '' or kube_vip_bgp_sourceif_normalized == '' fail_msg: "kube-vip allows only one of kube_vip_bgp_sourceip or kube_vip_bgp_sourceif." when: - kube_vip_bgp_enabled | default(false) - name: Kube-vip | Check if super-admin.conf exists stat: path: "{{ kube_config_dir }}/super-admin.conf" failed_when: false changed_when: false register: stat_kube_vip_super_admin - name: Kube-vip | Check if kubeadm has already run stat: path: "/var/lib/kubelet/config.yaml" get_attributes: false get_checksum: false get_mime: false register: kubeadm_already_run - name: Kube-vip | Set admin.conf set_fact: kube_vip_admin_conf: admin.conf - name: Kube-vip | Set admin.conf for first Control Plane set_fact: kube_vip_admin_conf: super-admin.conf when: - inventory_hostname == groups['kube_control_plane'] | first - (stat_kube_vip_super_admin.stat.exists and stat_kube_vip_super_admin.stat.isreg) or (not kubeadm_already_run.stat.exists ) - name: Kube-vip | Write static pod template: src: manifests/kube-vip.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-vip.yml" mode: "0640" ================================================ FILE: roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml ================================================ --- - name: Haproxy | Cleanup potentially deployed haproxy file: path: "{{ kube_manifest_dir }}/haproxy.yml" state: absent - name: Nginx-proxy | Make nginx directory file: path: "{{ nginx_config_dir }}" state: directory mode: "0700" owner: root - name: Nginx-proxy | Write nginx-proxy configuration template: src: "loadbalancer/nginx.conf.j2" dest: "{{ nginx_config_dir }}/nginx.conf" owner: root mode: "0755" backup: true register: nginx_conf - name: Nginx-proxy | Write static pod template: src: manifests/nginx-proxy.manifest.j2 dest: "{{ kube_manifest_dir }}/nginx-proxy.yml" mode: "0640" ================================================ FILE: roles/kubernetes/node/tasks/main.yml ================================================ --- - name: Fetch facts import_tasks: facts.yml tags: - facts - kubelet - name: Ensure /var/lib/cni exists file: path: /var/lib/cni state: directory mode: "0755" - name: Install kubelet binary import_tasks: install.yml tags: - kubelet - name: Install kube-vip import_tasks: loadbalancer/kube-vip.yml when: - ('kube_control_plane' in group_names) - kube_vip_enabled tags: - kube-vip - name: Install nginx-proxy import_tasks: loadbalancer/nginx-proxy.yml when: - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::') - loadbalancer_apiserver_localhost - loadbalancer_apiserver_type == 'nginx' tags: - nginx - name: Install haproxy import_tasks: loadbalancer/haproxy.yml when: - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::') - loadbalancer_apiserver_localhost - loadbalancer_apiserver_type == 'haproxy' tags: - haproxy - name: Ensure nodePort range is reserved ansible.posix.sysctl: name: net.ipv4.ip_local_reserved_ports value: "{{ kube_apiserver_node_port_range }}" sysctl_set: true sysctl_file: "{{ sysctl_file_path }}" state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" when: kube_apiserver_node_port_range is defined tags: - kube-proxy - name: Verify if br_netfilter module exists command: "modinfo br_netfilter" environment: PATH: "{{ ansible_env.PATH }}:/sbin" # Make sure we can workaround RH's conservative path management register: modinfo_br_netfilter failed_when: modinfo_br_netfilter.rc not in [0, 1] changed_when: false check_mode: false - name: Verify br_netfilter module path exists file: path: "{{ item }}" state: directory mode: "0755" loop: - /etc/modules-load.d - /etc/modprobe.d - name: Enable br_netfilter module community.general.modprobe: name: br_netfilter state: present when: modinfo_br_netfilter.rc == 0 - name: Persist br_netfilter module copy: dest: /etc/modules-load.d/kubespray-br_netfilter.conf content: br_netfilter mode: "0644" when: modinfo_br_netfilter.rc == 0 # kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module - name: Check if bridge-nf-call-iptables key exists command: "sysctl net.bridge.bridge-nf-call-iptables" failed_when: false changed_when: false check_mode: false register: sysctl_bridge_nf_call_iptables - name: Enable bridge-nf-call tables ansible.posix.sysctl: name: "{{ item }}" state: present sysctl_file: "{{ sysctl_file_path }}" value: "1" reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" when: sysctl_bridge_nf_call_iptables.rc == 0 with_items: - net.bridge.bridge-nf-call-iptables - net.bridge.bridge-nf-call-arptables - net.bridge.bridge-nf-call-ip6tables - name: Modprobe Kernel Module for IPVS community.general.modprobe: name: "{{ item }}" state: present persistent: present loop: "{{ kube_proxy_ipvs_modules }}" when: kube_proxy_mode == 'ipvs' tags: - kube-proxy - name: Modprobe conntrack module community.general.modprobe: name: "{{ item }}" state: present persistent: present register: modprobe_conntrack_module ignore_errors: true # noqa ignore-errors loop: - nf_conntrack - nf_conntrack_ipv4 when: - kube_proxy_mode == 'ipvs' - modprobe_conntrack_module is not defined or modprobe_conntrack_module is ansible.builtin.failed # loop until first success tags: - kube-proxy - name: Modprobe Kernel Module for nftables community.general.modprobe: name: "nf_tables" state: present persistent: present when: kube_proxy_mode == 'nftables' tags: - kube-proxy - name: Install kubelet import_tasks: kubelet.yml tags: - kubelet - kubeadm ================================================ FILE: roles/kubernetes/node/templates/http-proxy.conf.j2 ================================================ [Service] Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %} ================================================ FILE: roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 ================================================ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}" failSwapOn: {{ kubelet_fail_swap_on }} authentication: anonymous: enabled: false webhook: enabled: {{ kubelet_authentication_token_webhook }} x509: clientCAFile: {{ kube_cert_dir }}/ca.crt authorization: {% if kubelet_authorization_mode_webhook %} mode: Webhook {% else %} mode: AlwaysAllow {% endif %} {% if kube_version is version('1.35.0', '>=') %} failCgroupV1: {{ kubelet_fail_cgroup_v1 }} {% endif %} {% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %} {% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split(",") %} enforceNodeAllocatable: {% for item in kubelet_enforce_node_allocatable_list %} - {{ item }} {% endfor %} {% endif %} staticPodPath: {{ kube_manifest_dir }} cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }} containerLogMaxFiles: {{ kubelet_logfiles_max_nr }} containerLogMaxSize: {{ kubelet_logfiles_max_size }} containerRuntimeEndpoint : {{ cri_socket }} maxPods: {{ kubelet_max_pods }} podPidsLimit: {{ kubelet_pod_pids_limit }} address: "{{ kubelet_bind_address }}" readOnlyPort: {{ kube_read_only_port }} healthzPort: {{ kubelet_healthz_port }} healthzBindAddress: "{{ kubelet_healthz_bind_address }}" kubeletCgroups: {{ kubelet_kubelet_cgroups }} clusterDomain: {{ dns_domain }} {% if kubelet_protect_kernel_defaults | bool %} protectKernelDefaults: true {% endif %} {% if kubelet_rotate_certificates | bool %} rotateCertificates: true {% endif %} {% if kubelet_rotate_server_certificates | bool %} serverTLSBootstrap: true {% endif %} {# DNS settings for kubelet #} {% if enable_nodelocaldns %} {% set kubelet_cluster_dns = [nodelocaldns_ip] %} {% elif dns_mode in ['coredns'] %} {% set kubelet_cluster_dns = [skydns_server] %} {% elif dns_mode == 'coredns_dual' %} {% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %} {% elif dns_mode == 'manual' %} {% set kubelet_cluster_dns = [manual_dns_server] %} {% else %} {% set kubelet_cluster_dns = [] %} {% endif %} clusterDNS: {% for dns_address in kubelet_cluster_dns %} - {{ dns_address }} {% endfor %} {# Node reserved CPU/memory #} {% for scope in "kube", "system" %} {% if lookup('ansible.builtin.vars', scope + "_reserved") | bool %} {{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }} {% endif %} {{ scope }}Reserved: {% for resource in "cpu", "memory", "ephemeral-storage", "pid" %} {{ resource }}: "{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}" {% endfor %} {% endfor %} {% if eviction_hard is defined and eviction_hard %} evictionHard: {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }} {% endif %} resolvConf: "{{ kube_resolv_conf }}" {% if kubelet_config_extra_args %} {{ kubelet_config_extra_args | to_nice_yaml(indent=2) }} {% endif %} {% if kubelet_feature_gates or kube_feature_gates %} featureGates: {% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %} {{ feature | replace("=", ": ") }} {% endfor %} {% endif %} {% if tls_min_version is defined %} tlsMinVersion: {{ tls_min_version }} {% endif %} {% if tls_cipher_suites is defined %} tlsCipherSuites: {% for tls in tls_cipher_suites %} - {{ tls }} {% endfor %} {% endif %} eventRecordQPS: {{ kubelet_event_record_qps }} shutdownGracePeriod: {{ kubelet_shutdown_grace_period }} shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }} {% if not kubelet_fail_swap_on %} memorySwap: swapBehavior: {{ kubelet_swap_behavior }} {% endif %} {% if kubelet_streaming_connection_idle_timeout is defined %} streamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }} {% endif %} {% if kubelet_image_gc_high_threshold is defined %} imageGCHighThresholdPercent: {{ kubelet_image_gc_high_threshold }} {% endif %} {% if kubelet_image_gc_low_threshold is defined %} imageGCLowThresholdPercent: {{ kubelet_image_gc_low_threshold }} {% endif %} {% if kubelet_make_iptables_util_chains is defined %} makeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }} {% endif %} {% if kubelet_seccomp_default is defined %} seccompDefault: {{ kubelet_seccomp_default | bool }} {% endif %} {% if kubelet_cpu_manager_policy is defined %} cpuManagerPolicy: {{ kubelet_cpu_manager_policy }} {% endif %} {% if kubelet_cpu_manager_policy_options is defined %} cpuManagerPolicyOptions: {{ kubelet_cpu_manager_policy_options | to_nice_yaml(indent=2) | indent(width=2) }} {% endif %} {% if kubelet_topology_manager_policy is defined %} topologyManagerPolicy: {{ kubelet_topology_manager_policy }} {% endif %} {% if kubelet_topology_manager_scope is defined %} topologyManagerScope: {{ kubelet_topology_manager_scope }} {% endif %} {% if kubelet_tracing %} tracing: endpoint: "{{ kubelet_tracing_endpoint }}" samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }} {% endif %} maxParallelImagePulls: {{ kubelet_max_parallel_image_pulls }} ================================================ FILE: roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 ================================================ KUBE_LOG_LEVEL="--v={{ kube_log_level }}" KUBELET_ADDRESS="--node-ip={{ kubelet_address }}" {% if kube_override_hostname|default('') %} KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {# Base kubelet args #} {% set kubelet_args_base -%} {# start kubeadm specific settings #} --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --config={{ kube_config_dir }}/kubelet-config.yaml \ --kubeconfig={{ kube_config_dir }}/kubelet.conf \ {# end kubeadm specific settings #} --runtime-cgroups={{ kubelet_runtime_cgroups }} \ {% endset %} KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}" {% if kubelet_flexvolumes_plugins_dir is defined %} KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" {% endif %} {% if kube_network_plugin is defined and kube_network_plugin == "cloud" %} KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" {% endif %} {% if cloud_provider == "external" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}" {% else %} KUBELET_CLOUDPROVIDER="" {% endif %} PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ================================================ FILE: roles/kubernetes/node/templates/kubelet.service.j2 ================================================ [Unit] Description=Kubernetes Kubelet Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After={{ container_manager }}.service {% if container_manager == 'docker' %} Wants=docker.socket {% else %} Wants={{ container_manager }}.service {% endif %} {% for kubelet_dependency in kubelet_systemd_wants_dependencies|default([]) %} {% if kubelet_dependency|length > 0 %} Wants={{ kubelet_dependency }} {% endif %} {% endfor %} [Service] EnvironmentFile=-{{ kube_config_dir }}/kubelet.env {% if system_reserved|bool %} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ system_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ system_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ system_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ system_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ system_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ system_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ system_reserved_cgroups_for_service_slice }} {% endif %} {% if kube_reserved|bool %} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ kube_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ kube_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ kube_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ kube_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ kube_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ kube_reserved_cgroups_for_service_slice }} ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ kube_reserved_cgroups_for_service_slice }} {% endif %} ExecStart={{ bin_dir }}/kubelet \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBELET_API_SERVER \ $KUBELET_ADDRESS \ $KUBELET_PORT \ $KUBELET_HOSTNAME \ $KUBELET_ARGS \ $DOCKER_SOCKET \ $KUBELET_NETWORK_PLUGIN \ $KUBELET_VOLUME_PLUGIN \ $KUBELET_CLOUDPROVIDER Restart=always RestartSec=10s {% if kubelet_systemd_hardening %} # Hardening setup IPAddressDeny=any IPAddressAllow={{ kubelet_secure_addresses }} {% endif %} [Install] WantedBy=multi-user.target ================================================ FILE: roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 ================================================ global maxconn 4000 log 127.0.0.1 local0 defaults mode http log global option httplog option dontlognull option http-server-close option redispatch retries 5 timeout http-request 5m timeout queue 5m timeout connect 30s timeout client {{ loadbalancer_apiserver_keepalive_timeout }} timeout server 15m timeout http-keep-alive 30s timeout check 30s maxconn 4000 {% if loadbalancer_apiserver_healthcheck_port is defined -%} frontend healthz bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }} {% if ipv6_stack -%} bind :::{{ loadbalancer_apiserver_healthcheck_port }} {% endif -%} mode http monitor-uri /healthz {% endif %} frontend kube_api_frontend bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }} {% if ipv6_stack -%} bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }} {% endif -%} mode tcp option tcplog default_backend kube_api_backend backend kube_api_backend mode tcp balance leastconn default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100 option httpchk GET /healthz http-check expect status 200 {% for host in groups['kube_control_plane'] -%} server {{ host }} {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} check check-ssl verify none {% endfor -%} ================================================ FILE: roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 ================================================ error_log stderr notice; worker_processes 2; worker_rlimit_nofile 130048; worker_shutdown_timeout 10s; events { multi_accept on; use epoll; worker_connections 16384; } stream { upstream kube_apiserver { least_conn; {% for host in groups['kube_control_plane'] -%} server {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}; {% endfor -%} } server { listen 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}; {% if ipv6_stack -%} listen [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}; {% endif -%} proxy_pass kube_apiserver; proxy_timeout 10m; proxy_connect_timeout 1s; } } http { aio threads; aio_write on; tcp_nopush on; tcp_nodelay on; keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }}; keepalive_requests 100; reset_timedout_connection on; server_tokens off; autoindex off; {% if loadbalancer_apiserver_healthcheck_port is defined -%} server { listen {{ loadbalancer_apiserver_healthcheck_port }}; {% if ipv6_stack -%} listen [::]:{{ loadbalancer_apiserver_healthcheck_port }}; {% endif -%} location /healthz { access_log off; return 200; } location /stub_status { stub_status on; access_log off; } } {% endif %} } ================================================ FILE: roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 ================================================ apiVersion: v1 kind: Pod metadata: name: {{ loadbalancer_apiserver_pod_name }} namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile k8s-app: kube-haproxy annotations: haproxy-cfg-checksum: "{{ haproxy_conf.checksum }}" spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical containers: - name: haproxy image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} resources: requests: cpu: {{ loadbalancer_apiserver_cpu_requests }} memory: {{ loadbalancer_apiserver_memory_requests }} {% if loadbalancer_apiserver_healthcheck_port is defined -%} livenessProbe: httpGet: path: /healthz port: {{ loadbalancer_apiserver_healthcheck_port }} readinessProbe: httpGet: path: /healthz port: {{ loadbalancer_apiserver_healthcheck_port }} {% endif -%} volumeMounts: - mountPath: /usr/local/etc/haproxy/ name: etc-haproxy readOnly: true volumes: - name: etc-haproxy hostPath: path: {{ haproxy_config_dir }} ================================================ FILE: roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 ================================================ # Inspired by https://github.com/kube-vip/kube-vip/blob/v1.0.3/pkg/kubevip/config_generator.go#L103 apiVersion: v1 kind: Pod metadata: name: kube-vip namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile k8s-app: kube-vip spec: containers: - args: - manager env: - name: vip_arp value: {{ kube_vip_arp_enabled | string | to_json }} - name: port value: {{ kube_apiserver_port | string | to_json }} - name: vip_nodename value: {{ inventory_hostname }} {% if kube_vip_interface %} - name: vip_interface value: {{ kube_vip_interface | string | to_json }} {% endif %} {% if kube_vip_services_interface %} - name: vip_servicesinterface value: {{ kube_vip_services_interface | string | to_json }} {% endif %} {% if kube_vip_cidr %} - name: vip_{{ "subnet" if kube_vip_version is version('0.9.0', '>=') else "cidr" }} value: {{ kube_vip_cidr | string | to_json }} {% endif %} {% if kube_vip_dns_mode %} - name: dns_mode value: {{ kube_vip_dns_mode | string | to_json }} {% endif %} {% if kube_vip_controlplane_enabled %} - name: cp_enable value: "true" - name: cp_namespace value: kube-system - name: vip_ddns value: {{ kube_vip_ddns_enabled | string | to_json }} - name: cp_detect value: {{ kube_vip_cp_detect | string | to_json }} {% endif %} {% if kube_vip_services_enabled %} - name: svc_enable value: "true" {% endif %} {% if kube_vip_svc_leasename %} - name: svc_leasename value: {{ kube_vip_svc_leasename | string | to_json }} {% endif %} {% if kube_vip_enableServicesElection %} - name: svc_election value: "true" {% endif %} {% if kube_vip_leader_election_enabled %} - name: vip_leaderelection value: "true" - name: vip_leasename value: {{ kube_vip_leasename | string | to_json }} - name: vip_leaseduration value: {{ kube_vip_leaseduration | string | to_json }} - name: vip_renewdeadline value: {{ kube_vip_renewdeadline | string | to_json }} - name: vip_retryperiod value: {{ kube_vip_retryperiod | string | to_json }} {% endif %} {% if kube_vip_enable_node_labeling %} - name: enable_node_labeling value: {{ kube_vip_enable_node_labeling | string | to_json }} {% endif %} {% if kube_vip_bgp_enabled %} - name: bgp_enable value: "true" - name: bgp_routerid value: {{ kube_vip_bgp_routerid | string | to_json }} - name: bgp_as value: {{ kube_vip_local_as | string | to_json }} - name: bgp_peeraddress value: {{ kube_vip_bgp_peeraddress | to_json }} - name: bgp_peerpass value: {{ kube_vip_bgp_peerpass | to_json }} - name: bgp_peeras value: {{ kube_vip_bgp_peeras | string | to_json }} {% set kube_vip_bgp_sourceip_normalized = kube_vip_bgp_sourceip | default('', true) | string | trim %} {% if kube_vip_bgp_sourceip_normalized %} - name: bgp_sourceip value: {{ kube_vip_bgp_sourceip_normalized | to_json }} {% endif %} {% set kube_vip_bgp_sourceif_normalized = kube_vip_bgp_sourceif | default('', true) | string | trim %} {% if kube_vip_bgp_sourceif_normalized %} - name: bgp_sourceif value: {{ kube_vip_bgp_sourceif_normalized | to_json }} {% endif %} {% if kube_vip_bgppeers %} - name: bgp_peers value: {{ kube_vip_bgppeers | join(',') | to_json }} {% endif %} {% endif %} - name: address value: {{ kube_vip_address | to_json }} {% if kube_vip_lb_enable %} - name: lb_enable value: "true" {% endif %} {% if kube_vip_lb_fwdmethod %} - name: lb_fwdmethod value: {{ kube_vip_lb_fwdmethod | string | to_json }} {% endif %} image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} name: kube-vip resources: {} {% if kube_vip_lb_fwdmethod == "masquerade" %} securityContext: privileged: true {% else %} securityContext: capabilities: add: - NET_ADMIN - NET_RAW drop: - ALL {% endif %} volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig hostAliases: - hostnames: - kubernetes ip: 127.0.0.1 hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/{{kube_vip_admin_conf}} name: kubeconfig status: {} ================================================ FILE: roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 ================================================ apiVersion: v1 kind: Pod metadata: name: {{ loadbalancer_apiserver_pod_name }} namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile k8s-app: kube-nginx annotations: nginx-cfg-checksum: "{{ nginx_conf.checksum }}" spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical containers: - name: nginx-proxy image: {{ nginx_image_repo }}:{{ nginx_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} resources: requests: cpu: {{ loadbalancer_apiserver_cpu_requests }} memory: {{ loadbalancer_apiserver_memory_requests }} {% if loadbalancer_apiserver_healthcheck_port is defined -%} livenessProbe: httpGet: path: /healthz port: {{ loadbalancer_apiserver_healthcheck_port }} readinessProbe: httpGet: path: /healthz port: {{ loadbalancer_apiserver_healthcheck_port }} {% endif -%} volumeMounts: - mountPath: /etc/nginx name: etc-nginx readOnly: true volumes: - name: etc-nginx hostPath: path: {{ nginx_config_dir }} ================================================ FILE: roles/kubernetes/node/vars/fedora.yml ================================================ --- kube_resolv_conf: "/run/systemd/resolve/resolv.conf" ================================================ FILE: roles/kubernetes/node/vars/ubuntu-18.yml ================================================ --- kube_resolv_conf: "/run/systemd/resolve/resolv.conf" ================================================ FILE: roles/kubernetes/node/vars/ubuntu-20.yml ================================================ --- kube_resolv_conf: "/run/systemd/resolve/resolv.conf" ================================================ FILE: roles/kubernetes/node/vars/ubuntu-22.yml ================================================ --- kube_resolv_conf: "/run/systemd/resolve/resolv.conf" ================================================ FILE: roles/kubernetes/node/vars/ubuntu-24.yml ================================================ --- kube_resolv_conf: "/run/systemd/resolve/resolv.conf" ================================================ FILE: roles/kubernetes/node-label/tasks/main.yml ================================================ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: url: "{{ kube_apiserver_endpoint }}/healthz" validate_certs: false client_cert: "{{ kube_apiserver_client_cert }}" client_key: "{{ kube_apiserver_client_key }}" register: result until: result.status == 200 retries: 10 delay: 6 when: inventory_hostname == groups['kube_control_plane'][0] - name: Set role node label to empty list set_fact: role_node_labels: [] - name: Node label for nvidia GPU nodes set_fact: role_node_labels: "{{ role_node_labels + ['nvidia.com/gpu=true'] }}" when: - nvidia_gpu_nodes is defined - nvidia_accelerator_enabled | bool - inventory_hostname in nvidia_gpu_nodes - name: Set inventory node label to empty list set_fact: inventory_node_labels: [] - name: Populate inventory node label set_fact: inventory_node_labels: "{{ inventory_node_labels + ['%s=%s' | format(item.key, item.value)] }}" loop: "{{ node_labels | d({}) | dict2items }}" when: - node_labels is defined - node_labels is mapping - debug: # noqa name[missing] var: role_node_labels - debug: # noqa name[missing] var: inventory_node_labels - name: Set label to node command: >- {{ kubectl }} label node {% if kube_override_hostname %}{{ kube_override_hostname }}{% else %}{{ inventory_hostname }}{% endif %} {{ item }} --overwrite=true loop: "{{ role_node_labels + inventory_node_labels }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" changed_when: false ... ================================================ FILE: roles/kubernetes/node-taint/defaults/main.yml ================================================ --- node_taints: [] ================================================ FILE: roles/kubernetes/node-taint/tasks/main.yml ================================================ --- - name: Set role and inventory node taint to empty list set_fact: role_node_taints: [] inventory_node_taints: [] - name: Node taint for nvidia GPU nodes set_fact: role_node_taints: "{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}" when: - nvidia_gpu_nodes is defined - nvidia_accelerator_enabled | bool - inventory_hostname in nvidia_gpu_nodes - name: Populate inventory node taint set_fact: inventory_node_taints: "{{ inventory_node_taints + node_taints }}" when: - node_taints is defined - node_taints is not string - node_taints is not mapping - node_taints is iterable - debug: # noqa name[missing] var: role_node_taints - debug: # noqa name[missing] var: inventory_node_taints - name: Set taint to node command: >- {{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true delegate_to: "{{ groups['kube_control_plane'][0] }}" changed_when: false when: - (role_node_taints + inventory_node_taints) | length > 0 ================================================ FILE: roles/kubernetes/preinstall/defaults/main.yml ================================================ --- # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false # Set to false to disable the backup parameter, set to true to accumulate backups of config files. leave_etc_backup_files: true nameservers: [] cloud_resolver: [] disable_host_nameservers: false # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf dns_late: false # Set to true if your network does not support IPv6 # This may be necessary for pulling Docker images from # GCE docker repository disable_ipv6_dns: false # Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). remove_default_searchdomains: false kube_owner: kube kube_cert_group: kube-cert kube_config_dir: /etc/kubernetes kube_cert_dir: "{{ kube_config_dir }}/ssl" kube_cert_compat_dir: /etc/kubernetes/pki kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec # Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content # for hostnet pods and infra needs resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf # sysctl_file_path to add sysctl conf to sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" # Minimal memory requirement in MB for safety checks minimal_node_memory_mb: 1024 minimal_master_memory_mb: 1500 ## NTP Settings # Manage the NTP configuration file. ntp_manage_config: false # Specify the NTP servers # Only takes effect when ntp_manage_config is true. ntp_servers: - "0.pool.ntp.org iburst" - "1.pool.ntp.org iburst" - "2.pool.ntp.org iburst" - "3.pool.ntp.org iburst" # Restrict NTP access to these hosts. # Only takes effect when ntp_manage_config is true. ntp_restrict: - "127.0.0.1" - "::1" # Specify whether to filter interfaces ntp_filter_interface: false # Specify the interfaces # Only takes effect when ntp_filter_interface is true # ntp_interfaces: # - ignore wildcard # - listen xxx # The NTP driftfile path # Only takes effect when ntp_manage_config is true. # Default value is `/var/lib/ntp/ntp.drift`, for ntpsec use '/var/lib/ntpsec/ntp.drift' ntp_driftfile: >- {% if ntp_package == "ntpsec" -%} /var/lib/ntpsec/ntp.drift {%- else -%} /var/lib/ntp/ntp.drift {%- endif -%} # Only takes effect when ntp_manage_config is true. ntp_tinker_panic: false # Force sync time immediately after the ntp installed, which is useful in a newly installed system. ntp_force_sync_immediately: false # Set the timezone for your server. eg: "Etc/UTC","Etc/GMT-8". If not set, the timezone will not change. ntp_timezone: "" # Currently known os distributions supported_os_distributions: - 'RedHat' - 'CentOS' - 'Fedora' - 'Ubuntu' - 'Debian' - 'Flatcar' - 'Flatcar Container Linux by Kinvolk' - 'Suse' - 'openSUSE Leap' - 'openSUSE Tumbleweed' - 'ClearLinux' - 'OracleLinux' - 'AlmaLinux' - 'Rocky' - 'Amazon' - 'Kylin Linux Advanced Server' - 'UnionTech' - 'UniontechOS' - 'openEuler' # Extending some distributions into the redhat os family redhat_os_family_extensions: - "UnionTech" - "UniontechOS" # Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use" systemd_resolved_disable_stub_listener: "{{ ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] }}" # Used to disable File Access Policy Daemon service. # If service is enabled, the CNI plugin installation will fail disable_fapolicyd: true ================================================ FILE: roles/kubernetes/preinstall/files/dhclient_nodnsupdate ================================================ #!/bin/sh make_resolv_conf() { : } ================================================ FILE: roles/kubernetes/preinstall/handlers/main.yml ================================================ --- - name: Preinstall | apply resolvconf cloud-init command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] listen: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk - name: Preinstall | reload NetworkManager service: name: NetworkManager.service state: restarted listen: Preinstall | update resolvconf for networkmanager - name: Preinstall | reload kubelet service: name: kubelet state: restarted notify: - Preinstall | kube-controller configured - Preinstall | kube-apiserver configured - Preinstall | restart kube-controller-manager docker - Preinstall | restart kube-controller-manager crio/containerd - Preinstall | restart kube-apiserver docker - Preinstall | restart kube-apiserver crio/containerd when: not dns_early | bool listen: - Preinstall | propagate resolvconf to k8s components - Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk - Preinstall | update resolvconf for networkmanager # FIXME(mattymo): Also restart for kubeadm mode - name: Preinstall | kube-apiserver configured stat: path: "{{ kube_manifest_dir }}/kube-apiserver.yaml" get_attributes: false get_checksum: false get_mime: false register: kube_apiserver_set when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' listen: Preinstall | propagate resolvconf to k8s components # FIXME(mattymo): Also restart for kubeadm mode - name: Preinstall | kube-controller configured stat: path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml" get_attributes: false get_checksum: false get_mime: false register: kube_controller_set when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-controller-manager docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" args: executable: /bin/bash when: - container_manager == "docker" - ('kube_control_plane' in group_names) - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_controller_set.stat.exists listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-controller-manager crio/containerd shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" args: executable: /bin/bash register: preinstall_restart_controller_manager retries: 10 delay: 1 until: preinstall_restart_controller_manager.rc == 0 when: - container_manager in ['crio', 'containerd'] - ('kube_control_plane' in group_names) - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_controller_set.stat.exists listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-apiserver docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" args: executable: /bin/bash when: - container_manager == "docker" - ('kube_control_plane' in group_names) - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_apiserver_set.stat.exists listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-apiserver crio/containerd shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" args: executable: /bin/bash register: preinstall_restart_apiserver retries: 10 until: preinstall_restart_apiserver.rc == 0 delay: 1 when: - container_manager in ['crio', 'containerd'] - ('kube_control_plane' in group_names) - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_apiserver_set.stat.exists listen: Preinstall | propagate resolvconf to k8s components # When running this as the last phase ensure we wait for kube-apiserver to come up - name: Preinstall | wait for the apiserver to be running uri: url: "{{ kube_apiserver_endpoint }}/healthz" validate_certs: false register: result until: result.status == 200 retries: 60 delay: 1 when: - dns_late - ('kube_control_plane' in group_names) - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | Restart systemd-resolved service: name: systemd-resolved state: restarted - name: Preinstall | restart ntp service: name: "{{ ntp_service_name }}" state: restarted when: ntp_enabled ================================================ FILE: roles/kubernetes/preinstall/meta/main.yml ================================================ --- dependencies: - role: adduser user: "{{ addusers.kube }}" when: - not is_fedora_coreos tags: - kubelet ================================================ FILE: roles/kubernetes/preinstall/tasks/0010-swapoff.yml ================================================ --- - name: Check if /etc/fstab exists stat: path: "/etc/fstab" get_attributes: false get_checksum: false get_mime: false register: fstab_file - name: Remove swapfile from /etc/fstab ansible.posix.mount: name: "{{ item }}" fstype: swap state: absent loop: - swap - none when: fstab_file.stat.exists - name: Mask swap.target (persist swapoff) ansible.builtin.systemd_service: name: swap.target masked: true - name: Disable swap command: /sbin/swapoff -a ================================================ FILE: roles/kubernetes/preinstall/tasks/0020-set_facts.yml ================================================ --- - name: Set os_family fact for other redhat-based operating systems set_fact: ansible_os_family: "RedHat" ansible_distribution_major_version: "8" when: ansible_distribution in redhat_os_family_extensions tags: - facts - name: Check resolvconf command: which resolvconf register: resolvconf failed_when: false changed_when: false check_mode: false - name: Check existence of /etc/resolvconf/resolv.conf.d stat: path: /etc/resolvconf/resolv.conf.d get_attributes: false get_checksum: false get_mime: false failed_when: false register: resolvconfd_path - name: Check status of /etc/resolv.conf stat: path: /etc/resolv.conf follow: false get_attributes: false get_checksum: false get_mime: false failed_when: false register: resolvconf_stat # Used in vars/ - name: Fetch resolv.conf when: resolvconf_stat.stat.exists slurp: src: /etc/resolv.conf register: resolvconf_slurp - name: NetworkManager | Check if host has NetworkManager # noqa command-instead-of-module - Should we use service_facts for this? command: systemctl is-active --quiet NetworkManager.service register: networkmanager_enabled failed_when: false changed_when: false check_mode: false - name: Check systemd-resolved # noqa command-instead-of-module - Should we use service_facts for this? command: systemctl is-active systemd-resolved register: systemd_resolved_enabled failed_when: false changed_when: false check_mode: false - name: Set default dns if remove_default_searchdomains is false set_fact: default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0) - name: Set dns facts set_fact: resolvconf: >- {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} - name: Check if kubelet is configured stat: path: "{{ kube_config_dir }}/kubelet.env" get_attributes: false get_checksum: false get_mime: false register: kubelet_configured changed_when: false - name: Check if early DNS configuration stage set_fact: dns_early: "{{ not kubelet_configured.stat.exists }}" - name: Target resolv.conf files set_fact: resolvconffile: /etc/resolv.conf base: >- {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%} head: >- {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%} when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos - name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS) set_fact: resolvconffile: /tmp/resolveconf_cloud_init_conf when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos - name: Check if /etc/dhclient.conf exists stat: path: /etc/dhclient.conf get_attributes: false get_checksum: false get_mime: false register: dhclient_stat - name: Target dhclient conf file for /etc/dhclient.conf set_fact: dhclientconffile: /etc/dhclient.conf when: dhclient_stat.stat.exists - name: Check if /etc/dhcp/dhclient.conf exists stat: path: /etc/dhcp/dhclient.conf get_attributes: false get_checksum: false get_mime: false register: dhcp_dhclient_stat - name: Target dhclient conf file for /etc/dhcp/dhclient.conf set_fact: dhclientconffile: /etc/dhcp/dhclient.conf when: dhcp_dhclient_stat.stat.exists - name: Target dhclient hook file for Red Hat family set_fact: dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh when: ansible_os_family == "RedHat" - name: Target dhclient hook file for Debian family set_fact: dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate when: ansible_os_family == "Debian" - name: Set etcd vars if using kubeadm mode set_fact: etcd_cert_dir: "{{ kube_cert_dir }}" kube_etcd_cacert_file: "etcd/ca.crt" kube_etcd_cert_file: "apiserver-etcd-client.crt" kube_etcd_key_file: "apiserver-etcd-client.key" when: - etcd_deployment_type == "kubeadm" - name: Check /usr readonly stat: path: "/usr" get_attributes: false get_checksum: false get_mime: false register: usr - name: Set alternate flexvolume path set_fact: kubelet_flexvolumes_plugins_dir: /var/lib/kubelet/volumeplugins when: not usr.stat.writeable ================================================ FILE: roles/kubernetes/preinstall/tasks/0040-verify-settings.yml ================================================ --- - name: Stop if any host not in '--limit' does not have a fact cache vars: uncached_hosts: "{{ hostvars | dict2items | selectattr('value.ansible_default_ipv6', 'undefined') | selectattr('value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}" excluded_hosts: "{{ groups['k8s_cluster'] | difference(query('inventory_hostnames', ansible_limit)) }}" assert: that: uncached_hosts | intersect(excluded_hosts) == [] fail_msg: | Kubespray does not support '--limit' without a populated facts cache for the excluded hosts. Please run the facts.yml playbook first without '--limit'. The following excluded hosts are not cached: {{ uncached_hosts | intersect(excluded_hosts) }} run_once: true when: - ansible_limit is defined - not ignore_assert_errors - name: Stop if non systemd OS type assert: that: ansible_service_mgr == "systemd" when: not ignore_assert_errors - name: Stop if the os does not support assert: that: (allow_unsupported_distribution_setup | default(false)) or ansible_distribution in supported_os_distributions msg: "{{ ansible_distribution }} is not a known OS" when: not ignore_assert_errors - name: Stop if memory is too small for control plane nodes assert: that: ansible_memtotal_mb >= minimal_master_memory_mb when: - not ignore_assert_errors - ('kube_control_plane' in group_names) - name: Stop if memory is too small for nodes assert: that: ansible_memtotal_mb >= minimal_node_memory_mb when: - not ignore_assert_errors - ('kube_node' in group_names) # This command will fail if cgroups are not enabled on the node. # For reference: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version - name: Stop if cgroups are not enabled on nodes command: stat -fc %T /sys/fs/cgroup/ changed_when: false when: not ignore_assert_errors - name: Stop if ip var does not match local ips assert: that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses) msg: "IPv4: '{{ ansible_all_ipv4_addresses }}' and IPv6: '{{ ansible_all_ipv6_addresses }}' do not contain '{{ ip }}'" when: - not ignore_assert_errors - ip is defined - name: Stop if access_ip is not pingable command: ping -c1 {{ main_access_ip }} when: - main_access_ip is defined - not ignore_assert_errors - ping_access_ip changed_when: false - name: Stop if kernel version is too low for cilium assert: that: ansible_kernel.split('-')[0] is version('4.9.17', '>=') when: - kube_network_plugin == 'cilium' or cilium_deploy_additionally - not ignore_assert_errors - name: Stop if kernel version is too low for nftables assert: that: ansible_kernel.split('-')[0] is version('5.13', '>=') when: - kube_proxy_mode == 'nftables' - not ignore_assert_errors - name: Stop if bad hostname assert: that: inventory_hostname is match("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$") msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character" when: not ignore_assert_errors - name: Stop if /etc/resolv.conf has no configured nameservers assert: that: configured_nameservers | length>0 fail_msg: "nameserver should not be empty in /etc/resolv.conf" when: - upstream_dns_servers | length == 0 - not disable_host_nameservers - dns_mode in ['coredns', 'coredns_dual'] - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] msg: "download_run_once not supported for Flatcar Container Linux" when: download_run_once or download_force_cache ================================================ FILE: roles/kubernetes/preinstall/tasks/0050-create_directories.yml ================================================ --- - name: Create kubernetes directories file: path: "{{ item }}" state: directory owner: "{{ kube_owner }}" mode: "0755" when: ('k8s_cluster' in group_names) become: true tags: - kubelet - kube-controller-manager - kube-apiserver - bootstrap_os - apps - network - control-plane - node with_items: - "{{ kube_config_dir }}" - "{{ kube_manifest_dir }}" - "{{ kube_script_dir }}" - "{{ kubelet_flexvolumes_plugins_dir }}" - name: Create other directories of root owner file: path: "{{ item }}" state: directory owner: root mode: "0755" when: ('k8s_cluster' in group_names) become: true tags: - kubelet - kube-controller-manager - kube-apiserver - bootstrap_os - apps - network - control-plane - node with_items: - "{{ kube_cert_dir }}" - "{{ bin_dir }}" - name: Check if kubernetes kubeadm compat cert dir exists stat: path: "{{ kube_cert_compat_dir }}" get_attributes: false get_checksum: false get_mime: false register: kube_cert_compat_dir_check when: - ('k8s_cluster' in group_names) - kube_cert_dir != kube_cert_compat_dir - name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498) file: src: "{{ kube_cert_dir }}" dest: "{{ kube_cert_compat_dir }}" state: link mode: "0755" when: - ('k8s_cluster' in group_names) - kube_cert_dir != kube_cert_compat_dir - not kube_cert_compat_dir_check.stat.exists - name: Create cni directories file: path: "{{ item }}" state: directory owner: "{{ kube_owner }}" mode: "0755" with_items: - "/etc/cni/net.d" - "/opt/cni/bin" when: - kube_network_plugin in ["calico", "flannel", "cilium", "kube-ovn", "kube-router", "macvlan"] - ('k8s_cluster' in group_names) tags: - network - cilium - calico - kube-ovn - kube-router - bootstrap_os - name: Create calico cni directories file: path: "{{ item }}" state: directory owner: "{{ kube_owner }}" mode: "0755" with_items: - "/var/lib/calico" when: - kube_network_plugin == "calico" - ('k8s_cluster' in group_names) tags: - network - calico - bootstrap_os - name: Create local volume provisioner directories file: path: "{{ local_volume_provisioner_storage_classes[item].host_dir }}" state: directory owner: root group: root mode: "{{ local_volume_provisioner_directory_mode }}" with_items: "{{ local_volume_provisioner_storage_classes.keys() | list }}" when: - ('k8s_cluster' in group_names) - local_volume_provisioner_enabled tags: - persistent_volumes ================================================ FILE: roles/kubernetes/preinstall/tasks/0060-resolvconf.yml ================================================ --- - name: Create temporary resolveconf cloud init file command: cp -f /etc/resolv.conf "{{ resolvconffile }}" when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - name: Add domain/search/nameservers/options to resolv.conf blockinfile: path: "{{ resolvconffile }}" block: |- domain {{ dns_domain }} search {{ (default_searchdomains + searchdomains) | join(' ') }} {% for item in nameserverentries %} nameserver {{ item }} {% endfor %} options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }} state: present insertbefore: BOF create: true backup: "{{ not resolvconf_stat.stat.islnk }}" marker: "# Ansible entries {mark}" mode: "0644" notify: Preinstall | propagate resolvconf to k8s components - name: Remove search/domain/nameserver options before block replace: path: "{{ item[0] }}" regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)' backup: "{{ not resolvconf_stat.stat.islnk }}" with_nested: - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}" - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ] notify: Preinstall | propagate resolvconf to k8s components - name: Remove search/domain/nameserver options after block replace: path: "{{ item[0] }}" regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+' replace: '\1' backup: "{{ not resolvconf_stat.stat.islnk }}" with_nested: - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}" - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ] notify: Preinstall | propagate resolvconf to k8s components - name: Get temporary resolveconf cloud init file content command: cat {{ resolvconffile }} register: cloud_config when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - name: Persist resolvconf cloud init file template: dest: "{{ resolveconf_cloud_init_conf }}" src: resolvconf.j2 owner: root mode: "0644" notify: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] ================================================ FILE: roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml ================================================ --- - name: Create systemd-resolved drop-in directory file: state: directory name: /etc/systemd/resolved.conf.d/ mode: "0755" - name: Write Kubespray DNS settings to systemd-resolved template: src: resolved.conf.j2 dest: /etc/systemd/resolved.conf.d/kubespray.conf owner: root group: root mode: "0644" notify: Preinstall | Restart systemd-resolved ================================================ FILE: roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml ================================================ --- - name: NetworkManager | Ensure NetworkManager conf.d dir file: path: "/etc/NetworkManager/conf.d" state: directory recurse: true - name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico) copy: content: | [keyfile] unmanaged-devices+=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico dest: /etc/NetworkManager/conf.d/calico.conf mode: "0644" when: - kube_network_plugin == "calico" notify: Preinstall | reload NetworkManager # TODO: add other network_plugin interfaces - name: NetworkManager | Prevent NetworkManager from managing K8S interfaces (kube-ipvs0/nodelocaldns) copy: content: | [keyfile] unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns dest: /etc/NetworkManager/conf.d/k8s.conf mode: "0644" notify: Preinstall | reload NetworkManager ================================================ FILE: roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml ================================================ --- - name: NetworkManager | Add nameservers to NM configuration community.general.ini_file: path: /etc/NetworkManager/conf.d/dns.conf section: global-dns-domain-* option: servers value: "{{ nameserverentries | join(',') }}" mode: '0600' backup: "{{ leave_etc_backup_files }}" when: - ('127.0.0.53' not in nameserverentries or systemd_resolved_enabled.rc != 0) notify: Preinstall | update resolvconf for networkmanager - name: Set default dns if remove_default_searchdomains is false set_fact: default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0) - name: NetworkManager | Add DNS search to NM configuration community.general.ini_file: path: /etc/NetworkManager/conf.d/dns.conf section: global-dns option: searches value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}" mode: '0600' backup: "{{ leave_etc_backup_files }}" notify: Preinstall | update resolvconf for networkmanager - name: NetworkManager | Add DNS options to NM configuration community.general.ini_file: path: /etc/NetworkManager/conf.d/dns.conf section: global-dns option: options value: "ndots:{{ ndots }},timeout:{{ dns_timeout | default('2') }},attempts:{{ dns_attempts | default('2') }}" mode: '0600' backup: "{{ leave_etc_backup_files }}" notify: Preinstall | update resolvconf for networkmanager ================================================ FILE: roles/kubernetes/preinstall/tasks/0080-system-configurations.yml ================================================ --- # Todo : selinux configuration - name: Confirm selinux deployed stat: path: /etc/selinux/config get_attributes: false get_checksum: false get_mime: false when: - ansible_os_family == "RedHat" - "'Amazon' not in ansible_distribution" register: slc - name: Set selinux policy ansible.posix.selinux: policy: targeted state: "{{ preinstall_selinux_state }}" when: - ansible_os_family == "RedHat" - "'Amazon' not in ansible_distribution" - slc.stat.exists tags: - bootstrap_os - name: Disable IPv6 DNS lookup lineinfile: dest: /etc/gai.conf line: "precedence ::ffff:0:0/96 100" state: present create: true backup: "{{ leave_etc_backup_files }}" mode: "0644" when: - disable_ipv6_dns - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] tags: - bootstrap_os - name: Clean previously used sysctl file locations file: path: "/etc/sysctl.d/{{ item }}" state: absent with_items: - ipv4-ip_forward.conf - bridge-nf-call.conf - name: Stat sysctl file configuration stat: path: "{{ sysctl_file_path }}" get_attributes: false get_checksum: false get_mime: false register: sysctl_file_stat tags: - bootstrap_os - name: Change sysctl file path to link source if linked set_fact: sysctl_file_path: "{{ sysctl_file_stat.stat.lnk_source }}" when: - sysctl_file_stat.stat.islnk is defined - sysctl_file_stat.stat.islnk tags: - bootstrap_os - name: Make sure sysctl file path folder exists file: name: "{{ sysctl_file_path | dirname }}" state: directory mode: "0755" - name: Enable ip forwarding ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: net.ipv4.ip_forward value: "1" state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" when: ipv4_stack | bool - name: Enable ipv6 forwarding ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: net.ipv6.conf.all.forwarding value: "1" state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" when: ipv6_stack | bool - name: Check if we need to set fs.may_detach_mounts stat: path: /proc/sys/fs/may_detach_mounts get_attributes: false get_checksum: false get_mime: false register: fs_may_detach_mounts ignore_errors: true # noqa ignore-errors - name: Set fs.may_detach_mounts if needed ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: fs.may_detach_mounts value: 1 state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" when: fs_may_detach_mounts.stat.exists | d(false) - name: Ensure kubelet expected parameters are set ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: "{{ item.name }}" value: "{{ item.value }}" state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" with_items: - { name: kernel.keys.root_maxbytes, value: 25000000 } - { name: kernel.keys.root_maxkeys, value: 1000000 } - { name: kernel.panic, value: 10 } - { name: kernel.panic_on_oops, value: 1 } - { name: vm.overcommit_memory, value: 1 } - { name: vm.panic_on_oom, value: 0 } when: kubelet_protect_kernel_defaults | bool - name: Check dummy module community.general.modprobe: name: dummy state: present params: 'numdummies=0' when: enable_nodelocaldns - name: Set additional sysctl variables ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: "{{ item.name }}" value: "{{ item.value }}" state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" with_items: "{{ additional_sysctl }}" - name: Disable fapolicyd service failed_when: false systemd_service: name: fapolicyd state: stopped enabled: false when: disable_fapolicyd ================================================ FILE: roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml ================================================ --- - name: Disable systemd-timesyncd service: name: systemd-timesyncd.service enabled: false state: stopped failed_when: false - name: Set fact NTP settings set_fact: # noqa: jinja[spacing] ntp_config_file: >- {% if ntp_package == "ntp" -%} /etc/ntp.conf {%- elif ntp_package == "ntpsec" -%} /etc/ntpsec/ntp.conf {%- elif ansible_os_family in ['RedHat', 'Suse'] -%} /etc/chrony.conf {%- else -%} /etc/chrony/chrony.conf {%- endif -%} # noqa: jinja[spacing] ntp_service_name: >- {% if ntp_package == "chrony" -%} chronyd {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse"] -%} ntpd {%- else -%} ntp {%- endif %} - name: Generate NTP configuration file. template: src: "{{ ntp_config_file | basename }}.j2" dest: "{{ ntp_config_file }}" mode: "0644" notify: Preinstall | restart ntp when: - ntp_manage_config - name: Stop the NTP Deamon For Sync Immediately # `ntpd -gq`,`chronyd -q` requires the ntp daemon stop service: name: "{{ ntp_service_name }}" state: stopped when: - ntp_force_sync_immediately - name: Force Sync NTP Immediately # noqa: jinja[spacing] command: >- timeout -k 60s 60s {% if ntp_package == "chrony" -%} chronyd -q {%- else -%} ntpd -gq {%- endif -%} when: - ntp_force_sync_immediately - name: Ensure NTP service is started and enabled service: name: "{{ ntp_service_name }}" state: started enabled: true - name: Ensure tzdata package package: name: - tzdata state: present when: - ntp_timezone - not is_fedora_coreos - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - name: Gather selinux facts ansible.builtin.setup: gather_subset: selinux when: - ntp_timezone - ansible_os_family == "RedHat" - name: Put SELinux in permissive mode, logging actions that would be blocked. ansible.posix.selinux: policy: targeted state: permissive when: - ntp_timezone - ansible_os_family == "RedHat" - ansible_facts.selinux.status == 'enabled' - ansible_facts.selinux.mode == 'enforcing' - name: Set ntp_timezone community.general.timezone: name: "{{ ntp_timezone }}" when: - ntp_timezone - name: Re-enable SELinux ansible.posix.selinux: policy: targeted state: "{{ preinstall_selinux_state }}" when: - ntp_timezone - ansible_os_family == "RedHat" - ansible_facts.selinux.status == 'enabled' ================================================ FILE: roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml ================================================ --- - name: Configure dhclient to supersede search/domain/nameservers blockinfile: # 1 is the 2nd item of a tuple in items() block: |- {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%} {% if key == "domain-name-servers" -%} supersede {{ key }} {{ val | join(',') }}; {% else -%} supersede {{ key }} "{{ val | join('","') }}"; {% endif -%} {% endfor %} path: "{{ dhclientconffile }}" create: true state: present insertbefore: BOF backup: "{{ leave_etc_backup_files }}" marker: "# Ansible entries {mark}" mode: "0644" notify: Preinstall | propagate resolvconf to k8s components - name: Configure dhclient hooks for resolv.conf (non-RH) template: src: dhclient_dnsupdate.sh.j2 dest: "{{ dhclienthookfile }}" owner: root mode: "0755" notify: Preinstall | propagate resolvconf to k8s components when: ansible_os_family not in [ "RedHat", "Suse" ] - name: Configure dhclient hooks for resolv.conf (RH-only) template: src: dhclient_dnsupdate_rh.sh.j2 dest: "{{ dhclienthookfile }}" owner: root mode: "0755" notify: Preinstall | propagate resolvconf to k8s components when: ansible_os_family == "RedHat" ================================================ FILE: roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml ================================================ --- # These tasks will undo changes done by kubespray in the past if needed (e.g. when upgrading from kubespray 2.0.x # or when changing resolvconf_mode) - name: Remove kubespray specific config from dhclient config blockinfile: path: "{{ dhclientconffile }}" state: absent backup: "{{ leave_etc_backup_files }}" marker: "# Ansible entries {mark}" notify: Preinstall | propagate resolvconf to k8s components - name: Remove kubespray specific dhclient hook file: path: "{{ dhclienthookfile }}" state: absent notify: Preinstall | propagate resolvconf to k8s components ================================================ FILE: roles/kubernetes/preinstall/tasks/main.yml ================================================ --- # Disable swap - name: Disable swap import_tasks: 0010-swapoff.yml when: - not dns_late - kubelet_fail_swap_on - name: Set facts import_tasks: 0020-set_facts.yml tags: - resolvconf - facts - name: Check settings import_tasks: 0040-verify-settings.yml when: - not dns_late tags: - asserts - name: Create directories import_tasks: 0050-create_directories.yml when: - not dns_late - name: Apply resolvconf settings import_tasks: 0060-resolvconf.yml when: - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - systemd_resolved_enabled.rc != 0 - networkmanager_enabled.rc != 0 tags: - bootstrap_os - resolvconf - name: Apply systemd-resolved settings import_tasks: 0061-systemd-resolved.yml when: - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - systemd_resolved_enabled.rc == 0 tags: - bootstrap_os - resolvconf - name: Apply networkmanager unmanaged devices settings import_tasks: 0062-networkmanager-unmanaged-devices.yml when: - networkmanager_enabled.rc == 0 tags: - bootstrap_os - name: Apply networkmanager DNS settings import_tasks: 0063-networkmanager-dns.yml when: - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - networkmanager_enabled.rc == 0 tags: - bootstrap_os - resolvconf - name: Apply system configurations import_tasks: 0080-system-configurations.yml when: - not dns_late tags: - bootstrap_os - name: Configure NTP import_tasks: 0081-ntp-configurations.yml when: - not dns_late - ntp_enabled tags: - bootstrap_os - name: Configure dhclient import_tasks: 0100-dhclient-hooks.yml when: - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - dhclientconffile is defined - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] tags: - bootstrap_os - resolvconf - name: Configure dhclient dhclient hooks import_tasks: 0110-dhclient-hooks-undo.yml when: - dns_mode != 'none' - resolvconf_mode != 'host_resolvconf' - dhclientconffile is defined - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] tags: - bootstrap_os - resolvconf # We need to make sure the network is restarted early enough so that docker can later pick up the correct system # nameservers and search domains - name: Flush handlers meta: flush_handlers - name: Check if we are running inside a Azure VM stat: path: /var/lib/waagent/ get_attributes: false get_checksum: false get_mime: false register: azure_check when: - not dns_late tags: - bootstrap_os - name: Run calico checks include_role: name: network_plugin/calico tasks_from: check when: - kube_network_plugin == 'calico' - not ignore_assert_errors ================================================ FILE: roles/kubernetes/preinstall/templates/ansible_git.j2 ================================================ ; This file contains the information which identifies the deployment state relative to the git repo [default] {{ gitinfo.stdout }} ================================================ FILE: roles/kubernetes/preinstall/templates/chrony.conf.j2 ================================================ # {{ ansible_managed }} # Specify one or more NTP servers. # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). {% for server in ntp_servers %} server {{ server }} {% endfor %} # Record the rate at which the system clock gains/losses time. driftfile /var/lib/chrony/drift {% if ntp_tinker_panic is sameas true %} # Force time sync if the drift exceeds the threshold specified # Useful for VMs that can be paused and much later resumed. makestep 1.0 -1 {% else %} # Allow the system clock to be stepped in the first three updates # if its offset is larger than 1 second. makestep 1.0 3 {% endif %} # Enable kernel synchronization of the real-time clock (RTC). rtcsync # Specify directory for log files. logdir /var/log/chrony ================================================ FILE: roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 ================================================ #!/bin/sh # # Prepend resolver options to /etc/resolv.conf after dhclient` # regenerates the file. See man (5) resolver for more details. # if [ $reason = "BOUND" ]; then if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d') OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}" printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf fi fi ================================================ FILE: roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 ================================================ #!/bin/sh # # Prepend resolver options to /etc/resolv.conf after dhclient` # regenerates the file. See man (5) resolver for more details. # zdnsupdate_config() { if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d') OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}" echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf fi } zdnsupdate_restore() { : } ================================================ FILE: roles/kubernetes/preinstall/templates/ntp.conf.j2 ================================================ # {{ ansible_managed }} # /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help driftfile {{ ntp_driftfile }} {% if ntp_tinker_panic is sameas true %} # Always reset the clock, even if the new time is more than 1000s away # from the current system time. Useful for VMs that can be paused # and much later resumed. tinker panic 0 {% endif %} # Specify one or more NTP servers. # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). {% for item in ntp_servers %} pool {{ item }} {% endfor %} # Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for # details. The web page # might also be helpful. # # Note that "restrict" applies to both servers and clients, so a configuration # that might be intended to block requests from certain clients could also end # up blocking replies from your own upstream servers. # By default, exchange time with everybody, but don't allow configuration. restrict -4 default kod notrap nomodify nopeer noquery limited restrict -6 default kod notrap nomodify nopeer noquery limited # Local users may interrogate the ntp server more closely. {% for item in ntp_restrict %} restrict {{ item }} {% endfor %} # Needed for filtering interfaces {% if ntp_filter_interface %} {% for item in ntp_interfaces %} interface {{ item }} {% endfor %} {% endif %} # Needed for adding pool entries restrict source notrap nomodify noquery # Disable the monitoring facility to prevent amplification attacks using ntpdc # monlist command when default restrict does not include the noquery flag. See # CVE-2013-5211 for more details. # Note: Monitoring will not be disabled with the limited restriction flag. disable monitor ================================================ FILE: roles/kubernetes/preinstall/templates/resolvconf.j2 ================================================ #cloud-config write_files: - path: "/etc/resolv.conf" permissions: "0644" owner: "root" content: | {% for l in cloud_config.stdout_lines %} {{ l }} {% endfor %} # ================================================ FILE: roles/kubernetes/preinstall/templates/resolved.conf.j2 ================================================ [Resolve] {% if not dns_early and dns_late %} DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }} {% endif %} FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }} {% if remove_default_searchdomains and searchdomains | length != 0 %} Domains={{ searchdomains | join(' ') }} {% else %} Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }} {% endif %} DNSSEC=no Cache=no-negative {% if systemd_resolved_disable_stub_listener | bool %} DNSStubListener=no {% endif %} ================================================ FILE: roles/kubernetes/preinstall/vars/main.yml ================================================ --- coredns_server_by_mode: coredns: "{{ [skydns_server] }}" coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}" manual: "{{ manual_dns_server.split(',') }}" none: [] coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}" _nameserverentries: late: - "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}" early: - "{{ nameservers }}" - "{{ cloud_resolver }}" - "{{ configured_nameservers if not disable_host_nameservers else [] }}" nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}" dhclient_supersede: domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}" domain-name: "{{ [dns_domain] }}" domain-search: "{{ default_searchdomains + searchdomains }}" configured_nameservers: "{{ (resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr) if resolvconf_stat.stat.exists else [] }}" ================================================ FILE: roles/kubernetes-apps/ansible/defaults/main.yml ================================================ --- # Limits for coredns # uncomment the line below to customize the DNS cpu limit value # dns_cpu_limit: 300m dns_memory_limit: 300Mi dns_cpu_requests: 100m dns_memory_requests: 70Mi dns_min_replicas: "{{ [2, groups['k8s_cluster'] | length] | min }}" dns_nodes_per_replica: 16 dns_cores_per_replica: 256 dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas | int > 1 else 'false' }}" enable_coredns_reverse_dns_lookups: true coredns_svc_name: "coredns" coredns_ordinal_suffix: "" # dns_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}] coredns_affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: k8s-app operator: In values: ["kube-dns"] topologyKey: kubernetes.io/hostname coredns_deployment_nodeselector: "kubernetes.io/os: linux" coredns_default_zone_cache_block: | cache 30 coredns_pod_disruption_budget: false # when enable_dns_autoscaler is false, coredns_replicas is used to set the number of replicas coredns_replicas: 2 # value for coredns pdb coredns_pod_disruption_budget_max_unavailable: "30%" deploy_coredns: true # coredns_additional_configs adds any extra configuration to coredns # coredns_additional_configs: | # whoami # local # coredns_rewrite_block: | # rewrite stop { # name regex (.*)\.my\.domain {1}.svc.cluster.local # answer name (.*)\.svc\.cluster\.local {1}.my.domain # } # coredns_additional_error_config: | # consolidate 5m ".* i/o timeout$" warning # Configure coredns and nodelocaldns to correctly answer DNS queries when you changed # your 'dns_domain' and some workloads used it directly. old_dns_domains: [] # dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section # dns_upstream_forward_extra_opts: # policy: sequential # Apply extra options to coredns kubernetes plugin # coredns_kubernetes_extra_opts: # - 'fallthrough example.local' # nodelocaldns nodelocaldns_cpu_requests: 100m nodelocaldns_memory_limit: 200Mi nodelocaldns_memory_requests: 70Mi nodelocaldns_ds_nodeselector: "kubernetes.io/os: linux" nodelocaldns_prometheus_port: 9253 nodelocaldns_secondary_prometheus_port: 9255 # nodelocaldns_additional_configs adds any extra configuration to coredns # nodelocaldns_additional_configs: | # whoami # local # Limits for dns-autoscaler dns_autoscaler_cpu_requests: 20m dns_autoscaler_memory_requests: 10Mi dns_autoscaler_deployment_nodeselector: "kubernetes.io/os: linux" # dns_autoscaler_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}] dns_autoscaler_affinity: {} # etcd metrics # etcd_metrics_service_labels: # k8s-app: etcd # app.kubernetes.io/managed-by: Kubespray # app: kube-prometheus-stack-kube-etcd # release: prometheus-stack # Policy Controllers # policy_controller_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}] ================================================ FILE: roles/kubernetes-apps/ansible/tasks/main.yml ================================================ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: url: "{{ kube_apiserver_endpoint }}/healthz" validate_certs: false client_cert: "{{ kube_apiserver_client_cert }}" client_key: "{{ kube_apiserver_client_key }}" register: result until: result.status == 200 retries: 20 delay: 1 when: inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Apps | CoreDNS command: cmd: "{{ kubectl_apply_stdin }}" stdin: "{{ lookup('template', item) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true loop: "{{ coredns_manifests | flatten }}" tags: - coredns vars: clusterIP: "{{ skydns_server }}" when: - dns_mode in ['coredns', 'coredns_dual'] - deploy_coredns - name: Kubernetes Apps | CoreDNS Secondary command: cmd: "{{ kubectl_apply_stdin }}" stdin: "{{ lookup('template', item) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true loop: "{{ coredns_manifests | flatten }}" tags: - coredns vars: clusterIP: "{{ skydns_server_secondary }}" coredns_ordinal_suffix: "-secondary" when: - dns_mode == 'coredns_dual' - deploy_coredns - name: Kubernetes Apps | nodelocalDNS command: cmd: "{{ kubectl_apply_stdin }}" stdin: "{{ lookup('template', item) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true loop: "{{ nodelocaldns_manifests | flatten }}" when: - enable_nodelocaldns tags: - nodelocaldns - coredns vars: primaryClusterIP: >- {%- if dns_mode in ['coredns', 'coredns_dual'] -%} {{ skydns_server }} {%- elif dns_mode == 'manual' -%} {{ manual_dns_server }} {%- endif -%} secondaryclusterIP: "{{ skydns_server_secondary }}" forwardTarget: >- {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%} {{ primaryClusterIP }} {{ secondaryclusterIP }} {%- else -%} {{ primaryClusterIP }} {%- endif -%} upstreamForwardTarget: >- {%- if upstream_dns_servers | length > 0 -%} {{ upstream_dns_servers | join(' ') }} {%- else -%} /etc/resolv.conf {%- endif -%} - name: Kubernetes Apps | Etcd metrics endpoints command: cmd: "{{ kubectl_apply_stdin }}" stdin: "{{ lookup('template', item) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true loop: - etcd_metrics-endpoints.yml.j2 - etcd_metrics-service.yml.j2 when: etcd_metrics_port is defined and etcd_metrics_service_labels is defined tags: - etcd_metrics ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: kubernetes.io/bootstrapping: rbac-defaults addonmanager.kubernetes.io/mode: Reconcile name: system:coredns rules: - apiGroups: - "" resources: - endpoints - services - pods - namespaces verbs: - list - watch - apiGroups: - "" resources: - nodes verbs: - get - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - list - watch ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: kubernetes.io/bootstrapping: rbac-defaults addonmanager.kubernetes.io/mode: EnsureExists name: system:coredns roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:coredns subjects: - kind: ServiceAccount name: coredns namespace: kube-system ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 ================================================ --- apiVersion: v1 kind: ConfigMap metadata: name: coredns namespace: kube-system labels: addonmanager.kubernetes.io/mode: EnsureExists data: Corefile: | {% if coredns_external_zones is defined and coredns_external_zones | length > 0 %} {% for block in coredns_external_zones %} {{ block['zones'] | join(' ') }} { log errors { {% if coredns_additional_error_config is defined %} {{ coredns_additional_error_config | indent(width=10, first=False) }} {% endif %} } {% if block['rewrite'] is defined and block['rewrite'] | length > 0 %} {% for rewrite_match in block['rewrite'] %} rewrite {{ rewrite_match }} {% endfor %} {% endif %} forward . {{ block['nameservers'] | join(' ') }} loadbalance cache {{ block['cache'] | default(5) }} reload {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } {% endfor %} {% endif %} .:53 { {% if coredns_additional_configs is defined %} {{ coredns_additional_configs | indent(width=8, first=False) }} {% endif %} errors { {% if coredns_additional_error_config is defined %} {{ coredns_additional_error_config | indent(width=10, first=False) }} {% endif %} } health { lameduck 5s } {% if coredns_rewrite_block is defined %} {{ coredns_rewrite_block | indent(width=8, first=False) }} {% endif %} {% for old_dns_domain in old_dns_domains %} rewrite name suffix {{ old_dns_domain }} {{ dns_domain }} answer auto {% endfor %} ready kubernetes {{ dns_domain }} {% if coredns_kubernetes_extra_domains is defined %}{{ coredns_kubernetes_extra_domains }} {% endif %}{% if enable_coredns_reverse_dns_lookups %}in-addr.arpa ip6.arpa {% endif %}{ pods insecure {% if enable_coredns_k8s_endpoint_pod_names %} endpoint_pod_names {% endif %} {% if enable_coredns_reverse_dns_lookups %} fallthrough in-addr.arpa ip6.arpa {% endif %} {% if coredns_kubernetes_extra_opts is defined %} {% for opt in coredns_kubernetes_extra_opts %} {{ opt }} {% endfor %} {% endif %} } prometheus :9153 forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} { prefer_udp max_concurrent 1000 {% if dns_upstream_forward_extra_opts is defined %} {% for optname, optvalue in dns_upstream_forward_extra_opts.items() %} {{ (optname ~ ' ' ~ optvalue) | trim }} {# do not add a trailing space when optvalue == '' workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #} {% endfor %} {% endif %} } {% if enable_coredns_k8s_external %} k8s_external {{ coredns_k8s_external_zone }} {% endif %} {{ coredns_default_zone_cache_block | indent(width=8, first=False) }} loop reload loadbalance {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } {% if dns_etchosts | default(None) %} hosts: | {{ dns_etchosts | indent(width=4, first=False) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: "coredns{{ coredns_ordinal_suffix }}" namespace: kube-system labels: k8s-app: "kube-dns{{ coredns_ordinal_suffix }}" addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "coredns{{ coredns_ordinal_suffix }}" spec: {% if not enable_dns_autoscaler %} replicas: {{ coredns_replicas }} {% endif %} strategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 maxSurge: 10% selector: matchLabels: k8s-app: kube-dns{{ coredns_ordinal_suffix }} template: metadata: labels: k8s-app: kube-dns{{ coredns_ordinal_suffix }} annotations: createdby: 'kubespray' checksum/config: "{{ lookup('template', 'coredns-config.yml.j2') | checksum }}" spec: securityContext: seccompProfile: type: RuntimeDefault nodeSelector: {{ coredns_deployment_nodeselector }} priorityClassName: system-cluster-critical serviceAccountName: coredns tolerations: - key: node-role.kubernetes.io/control-plane effect: NoSchedule {% if dns_extra_tolerations is defined %} {{ dns_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} {% endif %} affinity: {{ coredns_affinity | to_nice_yaml(indent=2) | indent(8) }} containers: - name: coredns image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}" imagePullPolicy: {{ k8s_image_pull_policy }} resources: # TODO: Set memory limits when we've profiled the container for large # clusters, then set request = limit to keep this container in # guaranteed class. Currently, this container falls into the # "burstable" category so the kubelet doesn't backoff from restarting it. limits: {% if dns_cpu_limit is defined %} cpu: {{ dns_cpu_limit }} {% endif %} memory: {{ dns_memory_limit }} requests: cpu: {{ dns_cpu_requests }} memory: {{ dns_memory_requests }} args: [ "-conf", "/etc/coredns/Corefile" ] volumeMounts: - name: config-volume mountPath: /etc/coredns ports: - containerPort: 53 name: dns protocol: UDP - containerPort: 53 name: dns-tcp protocol: TCP - containerPort: 9153 name: metrics protocol: TCP securityContext: allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - all readOnlyRootFilesystem: true livenessProbe: httpGet: path: /health port: 8080 scheme: HTTP timeoutSeconds: 5 successThreshold: 1 failureThreshold: 10 readinessProbe: httpGet: path: /ready port: 8181 scheme: HTTP timeoutSeconds: 5 successThreshold: 1 failureThreshold: 10 dnsPolicy: Default volumes: - name: config-volume configMap: name: coredns items: - key: Corefile path: Corefile {% if dns_etchosts | default(None) %} - key: hosts path: hosts {% endif %} ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 ================================================ apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: coredns{{ coredns_ordinal_suffix }} spec: maxUnavailable: {{ coredns_pod_disruption_budget_max_unavailable }} selector: matchLabels: k8s-app: kube-dns{{ coredns_ordinal_suffix }} ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: coredns namespace: kube-system labels: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile ================================================ FILE: roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 ================================================ --- apiVersion: v1 kind: Service metadata: name: {{ coredns_svc_name }}{{ coredns_ordinal_suffix }} namespace: kube-system labels: k8s-app: kube-dns{{ coredns_ordinal_suffix }} kubernetes.io/name: "coredns{{ coredns_ordinal_suffix }}" addonmanager.kubernetes.io/mode: Reconcile annotations: prometheus.io/port: "9153" prometheus.io/scrape: "true" createdby: 'kubespray' spec: selector: k8s-app: kube-dns{{ coredns_ordinal_suffix }} clusterIP: {{ clusterIP }} ports: - name: dns port: 53 protocol: UDP - name: dns-tcp port: 53 protocol: TCP - name: metrics port: 9153 protocol: TCP ================================================ FILE: roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2 ================================================ --- # Copyright 2016 The Kubernetes Authors. All rights reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:dns-autoscaler labels: addonmanager.kubernetes.io/mode: Reconcile rules: - apiGroups: [""] resources: ["nodes"] verbs: ["list", "watch"] - apiGroups: [""] resources: ["replicationcontrollers/scale"] verbs: ["get", "update"] - apiGroups: ["extensions", "apps"] resources: ["deployments/scale", "replicasets/scale"] verbs: ["get", "update"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "create"] ================================================ FILE: roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2 ================================================ --- # Copyright 2016 The Kubernetes Authors. All rights reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:dns-autoscaler labels: addonmanager.kubernetes.io/mode: Reconcile subjects: - kind: ServiceAccount name: dns-autoscaler namespace: kube-system roleRef: kind: ClusterRole name: system:dns-autoscaler apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2 ================================================ --- # Copyright 2016 The Kubernetes Authors. All rights reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. kind: ServiceAccount apiVersion: v1 metadata: name: dns-autoscaler namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile ================================================ FILE: roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 ================================================ --- # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: apps/v1 kind: Deployment metadata: name: dns-autoscaler{{ coredns_ordinal_suffix }} namespace: kube-system labels: k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} template: metadata: labels: k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} annotations: spec: nodeSelector: {{ dns_autoscaler_deployment_nodeselector }} priorityClassName: system-cluster-critical securityContext: seccompProfile: type: RuntimeDefault supplementalGroups: [ 65534 ] fsGroup: 65534 nodeSelector: kubernetes.io/os: linux tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane {% if dns_autoscaler_extra_tolerations is defined %} {{ dns_autoscaler_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} {% endif %} affinity: {{ dns_autoscaler_affinity | to_nice_yaml(indent=2) | indent(8) }} containers: - name: autoscaler image: "{{ dnsautoscaler_image_repo }}:{{ dnsautoscaler_image_tag }}" resources: requests: cpu: {{ dns_autoscaler_cpu_requests }} memory: {{ dns_autoscaler_memory_requests }} readinessProbe: httpGet: path: /healthz port: 8080 scheme: HTTP command: - /cluster-proportional-autoscaler - --namespace=kube-system - --default-params={"linear":{"preventSinglePointFailure":{{ dns_prevent_single_point_failure }},"coresPerReplica":{{ dns_cores_per_replica }},"nodesPerReplica":{{ dns_nodes_per_replica }},"min":{{ dns_min_replicas }}}} - --logtostderr=true - --v=2 - --configmap=dns-autoscaler{{ coredns_ordinal_suffix }} - --target=Deployment/coredns{{ coredns_ordinal_suffix }} serviceAccountName: dns-autoscaler ================================================ FILE: roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2 ================================================ apiVersion: v1 kind: Endpoints metadata: name: etcd-metrics namespace: kube-system labels: k8s-app: etcd app.kubernetes.io/managed-by: Kubespray subsets: {% for etcd_metrics_address, etcd_host in etcd_metrics_addresses.split(',') | zip(etcd_hosts) %} - addresses: - ip: {{ etcd_metrics_address | urlsplit('hostname') }} targetRef: kind: Node name: {{ etcd_host }} ports: - name: http-metrics port: {{ etcd_metrics_address | urlsplit('port') }} protocol: TCP {% endfor %} ================================================ FILE: roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2 ================================================ apiVersion: v1 kind: Service metadata: name: etcd-metrics namespace: kube-system labels: {{ etcd_metrics_service_labels | to_yaml(indent=2, width=1337) | indent(width=4) }} spec: ports: - name: http-metrics protocol: TCP port: {{ etcd_metrics_port }} # targetPort: ================================================ FILE: roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 ================================================ apiVersion: v1 kind: ConfigMap metadata: name: nodelocaldns namespace: kube-system labels: addonmanager.kubernetes.io/mode: EnsureExists data: Corefile: | {% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %} {% for block in nodelocaldns_external_zones %} {{ block['zones'] | join(' ') }} { errors cache {{ block['cache'] | default(30) }} reload {% if block['rewrite'] is defined and block['rewrite'] | length > 0 %} {% for rewrite_match in block['rewrite'] %} rewrite {{ rewrite_match }} {% endfor %} {% endif %} loop bind {{ nodelocaldns_ip }} forward . {{ block['nameservers'] | join(' ') }} prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} log {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } {% endfor %} {% endif %} {{ ([dns_domain] + old_dns_domains) | join(' ') }}:53 { errors cache { success 9984 30 denial 9984 5 } reload loop bind {{ nodelocaldns_ip }} forward . {{ forwardTarget }} { force_tcp } prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} health {{ nodelocaldns_ip | ansible.utils.ipwrap }}:{{ nodelocaldns_health_port }} {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } in-addr.arpa:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} forward . {{ forwardTarget }} { force_tcp } prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} } ip6.arpa:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} forward . {{ forwardTarget }} { force_tcp } prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} } .:53 { {% if nodelocaldns_additional_configs is defined %} {{ nodelocaldns_additional_configs | indent(width=8, first=False) }} {% endif %} errors cache 30 reload loop bind {{ nodelocaldns_ip }} forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} { {% for optname, optvalue in dns_upstream_forward_extra_opts.items() %} {{ (optname ~ ' ' ~ optvalue) | trim }} {# do not add a trailing space when optvalue == '' workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #} {% endfor %} }{% endif %} prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } {% if enable_nodelocaldns_secondary %} Corefile-second: | {% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %} {% for block in nodelocaldns_external_zones %} {{ block['zones'] | join(' ') }} { errors cache {{ block['cache'] | default(30) }} reload loop bind {{ nodelocaldns_ip }} forward . {{ block['nameservers'] | join(' ') }} prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} log {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } {% endfor %} {% endif %} {{ dns_domain }}:53 { errors cache { success 9984 30 denial 9984 5 } reload loop bind {{ nodelocaldns_ip }} forward . {{ forwardTarget }} { force_tcp } prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} health {{ nodelocaldns_ip | ansible.utils.ipwrap }}:{{ nodelocaldns_second_health_port }} {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } in-addr.arpa:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} forward . {{ forwardTarget }} { force_tcp } prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} } ip6.arpa:53 { errors cache 30 reload loop bind {{ nodelocaldns_ip }} forward . {{ forwardTarget }} { force_tcp } prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} } .:53 { {% if nodelocaldns_additional_configs is defined %} {{ nodelocaldns_additional_configs | indent(width=8, first=False) }} {% endif %} errors cache 30 reload loop bind {{ nodelocaldns_ip }} forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} { {% for optname, optvalue in dns_upstream_forward_extra_opts.items() %} {{ (optname ~ ' ' ~ optvalue) | trim }} {# do not add a trailing space when optvalue == '' workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #} {% endfor %} }{% endif %} prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } {% endif %} {% if dns_etchosts | default(None) %} hosts: | {{ dns_etchosts | indent(width=4, first=False) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: nodelocaldns namespace: kube-system labels: k8s-app: kube-dns addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: node-local-dns template: metadata: labels: k8s-app: node-local-dns annotations: prometheus.io/scrape: 'true' prometheus.io/port: '{{ nodelocaldns_prometheus_port }}' checksum/config: "{{ lookup('template', 'nodelocaldns-config.yml.j2') | checksum }}" spec: nodeSelector: {{ nodelocaldns_ds_nodeselector }} priorityClassName: system-node-critical serviceAccountName: nodelocaldns hostNetwork: true dnsPolicy: Default # Don't use cluster DNS. tolerations: - effect: NoSchedule operator: "Exists" - effect: NoExecute operator: "Exists" containers: - name: node-cache image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}" resources: limits: memory: {{ nodelocaldns_memory_limit }} requests: cpu: {{ nodelocaldns_cpu_requests }} memory: {{ nodelocaldns_memory_requests }} args: - -localip - {{ nodelocaldns_ip }} - -conf - /etc/coredns/Corefile - -upstreamsvc - coredns {% if enable_nodelocaldns_secondary %} - -skipteardown {% endif %} ports: - containerPort: 53 name: dns protocol: UDP - containerPort: 53 name: dns-tcp protocol: TCP - containerPort: {{ nodelocaldns_prometheus_port }} name: metrics protocol: TCP securityContext: capabilities: add: - NET_ADMIN {% if nodelocaldns_bind_metrics_host_ip %} env: - name: MY_HOST_IP valueFrom: fieldRef: fieldPath: status.hostIP {% endif %} livenessProbe: httpGet: host: {{ nodelocaldns_ip }} path: /health port: {{ nodelocaldns_health_port }} scheme: HTTP timeoutSeconds: 5 successThreshold: 1 failureThreshold: 10 readinessProbe: httpGet: host: {{ nodelocaldns_ip }} path: /health port: {{ nodelocaldns_health_port }} scheme: HTTP timeoutSeconds: 5 successThreshold: 1 failureThreshold: 10 volumeMounts: - name: config-volume mountPath: /etc/coredns - name: xtables-lock mountPath: /run/xtables.lock volumes: - name: config-volume configMap: name: nodelocaldns items: - key: Corefile path: Corefile {% if dns_etchosts | default(None) %} - key: hosts path: hosts {% endif %} - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate ================================================ FILE: roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: nodelocaldns namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile ================================================ FILE: roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: nodelocaldns-second namespace: kube-system labels: k8s-app: kube-dns addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: node-local-dns-second template: metadata: labels: k8s-app: node-local-dns-second annotations: prometheus.io/scrape: 'true' prometheus.io/port: '{{ nodelocaldns_secondary_prometheus_port }}' checksum/config: "{{ lookup('template', 'nodelocaldns-config.yml.j2') | checksum }}" spec: nodeSelector: {{ nodelocaldns_ds_nodeselector }} priorityClassName: system-cluster-critical serviceAccountName: nodelocaldns hostNetwork: true dnsPolicy: Default # Don't use cluster DNS. tolerations: - effect: NoSchedule operator: "Exists" - effect: NoExecute operator: "Exists" containers: - name: node-cache image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}" resources: limits: memory: {{ nodelocaldns_memory_limit }} requests: cpu: {{ nodelocaldns_cpu_requests }} memory: {{ nodelocaldns_memory_requests }} args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns", "-skipteardown" ] ports: - containerPort: {{ nodelocaldns_secondary_prometheus_port }} name: metrics protocol: TCP securityContext: capabilities: add: - NET_ADMIN {% if nodelocaldns_bind_metrics_host_ip %} env: - name: MY_HOST_IP valueFrom: fieldRef: fieldPath: status.hostIP {% endif %} livenessProbe: httpGet: host: {{ nodelocaldns_ip }} path: /health port: {{ nodelocaldns_health_port }} scheme: HTTP timeoutSeconds: 5 successThreshold: 1 failureThreshold: 10 readinessProbe: httpGet: host: {{ nodelocaldns_ip }} path: /health port: {{ nodelocaldns_health_port }} scheme: HTTP timeoutSeconds: 5 successThreshold: 1 failureThreshold: 10 volumeMounts: - name: config-volume mountPath: /etc/coredns - name: xtables-lock mountPath: /run/xtables.lock lifecycle: preStop: exec: command: - sh - -c - sleep {{ nodelocaldns_secondary_skew_seconds }} && kill -9 1 volumes: - name: config-volume configMap: name: nodelocaldns items: - key: Corefile-second path: Corefile {% if dns_etchosts | default(None) %} - key: hosts path: hosts {% endif %} - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate # Implement a time skew between the main nodelocaldns and this secondary. # Since the two nodelocaldns instances share the :53 port, we want to keep # at least one running at any time even if the manifests are replaced simultaneously terminationGracePeriodSeconds: {{ nodelocaldns_secondary_skew_seconds }} updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate ================================================ FILE: roles/kubernetes-apps/ansible/vars/main.yml ================================================ --- dns_autoscaler_manifests: - dns-autoscaler-sa.yml.j2 - dns-autoscaler.yml.j2 - dns-autoscaler-clusterrole.yml.j2 - dns-autoscaler-clusterrolebinding.yml.j2 coredns_manifests: - coredns-clusterrole.yml.j2 - coredns-clusterrolebinding.yml.j2 - coredns-config.yml.j2 - coredns-deployment.yml.j2 - coredns-sa.yml.j2 - coredns-svc.yml.j2 - "{{ dns_autoscaler_manifests if enable_dns_autoscaler else [] }}" - "{{ 'coredns-poddisruptionbudget.yml.j2' if coredns_pod_disruption_budget else [] }}" nodelocaldns_manifests: - nodelocaldns-config.yml.j2 - nodelocaldns-daemonset.yml.j2 - nodelocaldns-sa.yml.j2 - "{{ 'nodelocaldns-second-daemonset.yml.j2' if enable_nodelocaldns_secondary else [] }}" ================================================ FILE: roles/kubernetes-apps/argocd/defaults/main.yml ================================================ --- argocd_enabled: false argocd_version: 2.14.5 argocd_namespace: argocd # argocd_admin_password: ================================================ FILE: roles/kubernetes-apps/argocd/tasks/main.yml ================================================ --- - name: Kubernetes Apps | Download yq include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.yq) }}" - name: Kubernetes Apps | Copy yq binary from download dir ansible.posix.synchronize: src: "{{ downloads.yq.dest }}" dest: "{{ bin_dir }}/yq" compress: false perms: true owner: false group: false delegate_to: "{{ inventory_hostname }}" - name: Kubernetes Apps | Set ArgoCD template list set_fact: argocd_templates: - name: namespace file: argocd-namespace.yml - name: install file: "{{ downloads.argocd_install.dest | basename }}" namespace: "{{ argocd_namespace }}" download: "{{ downloads.argocd_install }}" when: - "inventory_hostname == groups['kube_control_plane'][0]" - name: Kubernetes Apps | Download ArgoCD remote manifests include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(item.download) }}" with_items: "{{ argocd_templates | selectattr('download', 'defined') | list }}" loop_control: label: "{{ item.file }}" when: - "inventory_hostname == groups['kube_control_plane'][0]" - name: Kubernetes Apps | Copy ArgoCD remote manifests from download dir ansible.posix.synchronize: src: "{{ local_release_dir }}/{{ item.file }}" dest: "{{ kube_config_dir }}/{{ item.file }}" compress: false perms: true owner: false group: false delegate_to: "{{ inventory_hostname }}" with_items: "{{ argocd_templates | selectattr('download', 'defined') | list }}" when: - "inventory_hostname == groups['kube_control_plane'][0]" - name: Kubernetes Apps | Set ArgoCD namespace for remote manifests become: true command: | {{ bin_dir }}/yq eval-all -i '.metadata.namespace="{{ argocd_namespace }}"' {{ kube_config_dir }}/{{ item.file }} with_items: "{{ argocd_templates | selectattr('download', 'defined') | list }}" loop_control: label: "{{ item.file }}" when: - "inventory_hostname == groups['kube_control_plane'][0]" - name: Kubernetes Apps | Create ArgoCD manifests from templates become: true template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: "{{ argocd_templates | selectattr('download', 'undefined') | list }}" loop_control: label: "{{ item.file }}" when: - "inventory_hostname == groups['kube_control_plane'][0]" - name: Kubernetes Apps | Install ArgoCD become: true kube: name: ArgoCD kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.file }}" state: latest with_items: "{{ argocd_templates }}" when: - "inventory_hostname == groups['kube_control_plane'][0]" # https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it - name: Kubernetes Apps | Set ArgoCD custom admin password become: true shell: | {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n {{ argocd_namespace }} patch secret argocd-secret -p \ '{ "stringData": { "admin.password": "{{ argocd_admin_password | password_hash('bcrypt') }}", "admin.passwordMtime": "'$(date +%FT%T%Z)'" } }' when: - argocd_admin_password is defined - "inventory_hostname == groups['kube_control_plane'][0]" ================================================ FILE: roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ argocd_namespace }} labels: app: argocd ================================================ FILE: roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml ================================================ --- apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: k8s-cluster-critical value: 1000000000 globalDefault: false description: "This priority class should only be used by the pods installed using kubespray." ================================================ FILE: roles/kubernetes-apps/cluster_roles/tasks/main.yml ================================================ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: url: "{{ kube_apiserver_endpoint }}/healthz" validate_certs: false client_cert: "{{ kube_apiserver_client_cert }}" client_key: "{{ kube_apiserver_client_key }}" register: result until: result.status == 200 retries: 10 delay: 6 when: inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes template: src: "node-crb.yml.j2" dest: "{{ kube_config_dir }}/node-crb.yml" mode: "0640" register: node_crb_manifest when: - rbac_enabled - inventory_hostname == groups['kube_control_plane'][0] - name: Apply workaround to allow all nodes with cert O=system:nodes to register kube: name: "kubespray:system:node" kubectl: "{{ bin_dir }}/kubectl" resource: "clusterrolebinding" filename: "{{ kube_config_dir }}/node-crb.yml" state: latest register: result until: result is succeeded retries: 10 delay: 6 when: - rbac_enabled - node_crb_manifest.changed - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Apps | Remove old webhook ClusterRole kube: name: "system:node-webhook" kubectl: "{{ bin_dir }}/kubectl" resource: "clusterrole" state: absent when: - rbac_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: node-webhook - name: Kubernetes Apps | Remove old webhook ClusterRoleBinding kube: name: "system:node-webhook" kubectl: "{{ bin_dir }}/kubectl" resource: "clusterrolebinding" state: absent when: - rbac_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: node-webhook - name: PriorityClass | Copy k8s-cluster-critical-pc.yml file copy: src: k8s-cluster-critical-pc.yml dest: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml" mode: "0640" when: inventory_hostname == groups['kube_control_plane'] | last - name: PriorityClass | Create k8s-cluster-critical kube: name: k8s-cluster-critical kubectl: "{{ bin_dir }}/kubectl" resource: "PriorityClass" filename: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml" state: latest register: result until: result is succeeded retries: 10 delay: 6 when: inventory_hostname == groups['kube_control_plane'] | last ================================================ FILE: roles/kubernetes-apps/cluster_roles/templates/namespace.j2 ================================================ apiVersion: v1 kind: Namespace metadata: name: "kube-system" ================================================ FILE: roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: kubernetes.io/bootstrapping: rbac-defaults name: kubespray:system:node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:node subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes ================================================ FILE: roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:vsphere-cloud-provider rules: - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:vsphere-cloud-provider roleRef: kind: ClusterRole name: system:vsphere-cloud-provider apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: vsphere-cloud-provider namespace: kube-system ================================================ FILE: roles/kubernetes-apps/common_crds/gateway_api/defaults/main.yml ================================================ --- gateway_api_enabled: false # `gateway_api_channel` default is "standard". # "standard" release channel includes all resources that have graduated to GA or beta, including GatewayClass, Gateway, HTTPRoute, and ReferenceGrant. # "experimental" for some experimental resources and fields. Note that future releases of the API could include breaking changes to experimental resources and fields. For example, any experimental resource or field could be removed in a future release. # https://gateway-api.sigs.k8s.io/guides/#install-experimental-channel gateway_api_channel: "standard" ================================================ FILE: roles/kubernetes-apps/common_crds/gateway_api/tasks/main.yml ================================================ --- - name: Gateway API | Download YAML include_tasks: "../../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.gateway_api_crds) }}" - name: Gateway API | Create addon dir file: path: "{{ kube_config_dir }}/addons/gateway_api" state: directory owner: root group: root mode: "0755" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Gateway API | Copy YAML from download dir copy: src: "{{ local_release_dir }}/gateway-api-{{ gateway_api_channel }}-install.yaml" dest: "{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml" mode: "0644" remote_src: true when: - "inventory_hostname == groups['kube_control_plane'][0]" - name: Gateway API | Install Gateway API kube: name: Gateway API kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml" state: latest when: - "inventory_hostname == groups['kube_control_plane'][0]" ================================================ FILE: roles/kubernetes-apps/common_crds/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/common_crds/gateway_api when: gateway_api_enabled tags: - gateway_api - role: kubernetes-apps/common_crds/prometheus_operator_crds when: prometheus_operator_crds_enabled tags: - prometheus_operator_crds ================================================ FILE: roles/kubernetes-apps/common_crds/prometheus_operator_crds/tasks/main.yml ================================================ --- - name: Prometheus Operator CRDs | Download YAML include_tasks: "../../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.prometheus_operator_crds) }}" - name: Prometheus Operator CRDs | Install command: cmd: "{{ bin_dir }}/kubectl apply -f {{ local_release_dir }}/prometheus-operator-crds.yaml" when: - "inventory_hostname == groups['kube_control_plane'][0]" ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/container_engine_accelerator/nvidia_gpu when: nvidia_accelerator_enabled tags: - apps - nvidia_gpu - container_engine_accelerator ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml ================================================ --- nvidia_accelerator_enabled: false nvidia_driver_version: "390.87" nvidia_gpu_tesla_base_url: https://us.download.nvidia.com/tesla/ nvidia_gpu_gtx_base_url: http://us.download.nvidia.com/XFree86/Linux-x86_64/ nvidia_gpu_flavor: tesla nvidia_url_end: "{{ nvidia_driver_version }}/NVIDIA-Linux-x86_64-{{ nvidia_driver_version }}.run" nvidia_driver_install_container: false nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2 nvidia_driver_install_ubuntu_container: registry.k8s.io/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63 nvidia_driver_install_supported: false nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" nvidia_gpu_nodes: [] nvidia_gpu_device_plugin_memory: 30Mi ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml ================================================ --- - name: Container Engine Acceleration Nvidia GPU | gather os specific variables include_vars: "{{ item }}" with_first_found: - files: - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}.yml" - "{{ ansible_os_family | lower }}.yml" skip: true - name: Container Engine Acceleration Nvidia GPU | Set fact of download url Tesla set_fact: nvidia_driver_download_url_default: "{{ nvidia_gpu_tesla_base_url }}{{ nvidia_url_end }}" when: nvidia_gpu_flavor | lower == "tesla" - name: Container Engine Acceleration Nvidia GPU | Set fact of download url GTX set_fact: nvidia_driver_download_url_default: "{{ nvidia_gpu_gtx_base_url }}{{ nvidia_url_end }}" when: nvidia_gpu_flavor | lower == "gtx" - name: Container Engine Acceleration Nvidia GPU | Create addon dir file: path: "{{ kube_config_dir }}/addons/container_engine_accelerator" owner: root group: root mode: "0755" recurse: true - name: Container Engine Acceleration Nvidia GPU | Create manifests for nvidia accelerators template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.file }}" mode: "0644" with_items: - { name: nvidia-driver-install-daemonset, file: nvidia-driver-install-daemonset.yml, type: daemonset } - { name: k8s-device-plugin-nvidia-daemonset, file: k8s-device-plugin-nvidia-daemonset.yml, type: daemonset } register: container_engine_accelerator_manifests when: - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container - name: Container Engine Acceleration Nvidia GPU | Apply manifests for nvidia accelerators kube: name: "{{ item.item.name }}" namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.item.file }}" state: "latest" with_items: - "{{ container_engine_accelerator_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container and nvidia_driver_install_supported ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: nvidia-gpu-device-plugin namespace: kube-system labels: k8s-app: nvidia-gpu-device-plugin addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: nvidia-gpu-device-plugin template: metadata: labels: k8s-app: nvidia-gpu-device-plugin spec: priorityClassName: system-node-critical affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "nvidia.com/gpu" operator: Exists tolerations: - operator: "Exists" effect: "NoExecute" - operator: "Exists" effect: "NoSchedule" hostNetwork: true dnsPolicy: ClusterFirstWithHostNet hostPID: true volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins - name: dev hostPath: path: /dev containers: - image: "{{ nvidia_gpu_device_plugin_container }}" command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] name: nvidia-gpu-device-plugin resources: requests: cpu: 50m memory: {{ nvidia_gpu_device_plugin_memory }} limits: cpu: 50m memory: {{ nvidia_gpu_device_plugin_memory }} securityContext: privileged: true volumeMounts: - name: device-plugin mountPath: /device-plugin - name: dev mountPath: /dev updateStrategy: type: RollingUpdate ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 ================================================ # Copyright 2017 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: apps/v1 kind: DaemonSet metadata: name: nvidia-driver-installer namespace: kube-system spec: selector: matchLabels: name: nvidia-driver-installer template: metadata: labels: name: nvidia-driver-installer spec: priorityClassName: system-node-critical affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "nvidia.com/gpu" operator: Exists tolerations: - key: "nvidia.com/gpu" effect: "NoSchedule" operator: "Exists" hostNetwork: true dnsPolicy: ClusterFirstWithHostNet hostPID: true volumes: - name: dev hostPath: path: /dev - name: nvidia-install-dir-host hostPath: path: /home/kubernetes/bin/nvidia - name: root-mount hostPath: path: / initContainers: - image: "{{ nvidia_driver_install_container }}" name: nvidia-driver-installer resources: requests: cpu: 0.15 securityContext: privileged: true env: - name: NVIDIA_INSTALL_DIR_HOST value: /home/kubernetes/bin/nvidia - name: NVIDIA_INSTALL_DIR_CONTAINER value: /usr/local/nvidia - name: ROOT_MOUNT_DIR value: /root - name: NVIDIA_DRIVER_VERSION value: "{{ nvidia_driver_version }}" - name: NVIDIA_DRIVER_DOWNLOAD_URL value: "{{ nvidia_driver_download_url_default }}" volumeMounts: - name: nvidia-install-dir-host mountPath: /usr/local/nvidia - name: dev mountPath: /dev - name: root-mount mountPath: /root containers: - image: "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" name: pause ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml ================================================ --- nvidia_driver_install_container: "{{ nvidia_driver_install_ubuntu_container }}" nvidia_driver_install_supported: true ================================================ FILE: roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml ================================================ --- nvidia_driver_install_container: "{{ nvidia_driver_install_ubuntu_container }}" nvidia_driver_install_supported: true ================================================ FILE: roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml ================================================ --- - name: Crun | Copy runtime class manifest template: src: runtimeclass-crun.yml dest: "{{ kube_config_dir }}/runtimeclass-crun.yml" mode: "0664" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Crun | Apply manifests kube: name: "runtimeclass-crun" kubectl: "{{ bin_dir }}/kubectl" resource: "runtimeclass" filename: "{{ kube_config_dir }}/runtimeclass-crun.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml ================================================ --- kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: name: crun handler: crun ================================================ FILE: roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml ================================================ --- - name: GVisor | Create addon dir file: path: "{{ kube_config_dir }}/addons/gvisor" owner: root group: root mode: "0755" recurse: true - name: GVisor | Templates List set_fact: gvisor_templates: - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass } - name: GVisort | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/gvisor/{{ item.file }}" mode: "0644" with_items: "{{ gvisor_templates }}" register: gvisor_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: GVisor | Apply manifests kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}" state: "latest" with_items: "{{ gvisor_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 ================================================ --- kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: name: gvisor handler: runsc ================================================ FILE: roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml ================================================ --- kata_containers_qemu_overhead: true kata_containers_qemu_overhead_fixed_cpu: 250m kata_containers_qemu_overhead_fixed_memory: 160Mi ================================================ FILE: roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml ================================================ --- - name: Kata Containers | Create addon dir file: path: "{{ kube_config_dir }}/addons/kata_containers" owner: root group: root mode: "0755" recurse: true - name: Kata Containers | Templates list set_fact: kata_containers_templates: - { name: runtimeclass-kata-qemu, file: runtimeclass-kata-qemu.yml, type: runtimeclass } - name: Kata Containers | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/kata_containers/{{ item.file }}" mode: "0644" with_items: "{{ kata_containers_templates }}" register: kata_containers_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kata Containers | Apply manifests kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/kata_containers/{{ item.item.file }}" state: "latest" with_items: "{{ kata_containers_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2 ================================================ --- kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: name: kata-qemu handler: kata-qemu {% if kata_containers_qemu_overhead %} overhead: podFixed: cpu: {{ kata_containers_qemu_overhead_fixed_cpu }} memory: {{ kata_containers_qemu_overhead_fixed_memory }} {% endif %} ================================================ FILE: roles/kubernetes-apps/container_runtimes/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/container_runtimes/kata_containers when: kata_containers_enabled tags: - apps - kata-containers - container-runtimes - role: kubernetes-apps/container_runtimes/gvisor when: gvisor_enabled tags: - apps - gvisor - container-runtimes - role: kubernetes-apps/container_runtimes/crun when: crun_enabled tags: - apps - crun - container-runtimes - role: kubernetes-apps/container_runtimes/youki when: - youki_enabled - container_manager == 'crio' tags: - apps - youki - container-runtimes ================================================ FILE: roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml ================================================ --- - name: Youki | Copy runtime class manifest template: src: runtimeclass-youki.yml dest: "{{ kube_config_dir }}/runtimeclass-youki.yml" mode: "0664" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Youki | Apply manifests kube: name: "runtimeclass-youki" kubectl: "{{ bin_dir }}/kubectl" resource: "runtimeclass" filename: "{{ kube_config_dir }}/runtimeclass-youki.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml ================================================ --- kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: name: youki handler: youki ================================================ FILE: roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml ================================================ --- aws_ebs_csi_enable_volume_scheduling: true aws_ebs_csi_enable_volume_snapshot: false aws_ebs_csi_enable_volume_resizing: false aws_ebs_csi_controller_replicas: 1 aws_ebs_csi_plugin_image_tag: latest # Add annotions to ebs_csi_controller. Useful if using kube2iam for role assumption # aws_ebs_csi_annotations: # - key: iam.amazonaws.com/role # value: your-ebs-role-arn ================================================ FILE: roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml ================================================ --- - name: AWS CSI Driver | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml} - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml} - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml} - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml} register: aws_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: AWS CSI Driver | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ aws_csi_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 ================================================ # Controller Service apiVersion: v1 kind: ServiceAccount metadata: name: ebs-csi-controller-sa namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-external-provisioner-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-csi-provisioner-binding subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: ebs-external-provisioner-role apiGroup: rbac.authorization.k8s.io --- # The permissions in this ClusterRole are tightly coupled with the version of csi-attacher used. More information about this can be found in kubernetes-csi/external-attacher. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-external-attacher-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-csi-attacher-binding subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: ebs-external-attacher-role apiGroup: rbac.authorization.k8s.io {% if aws_ebs_csi_enable_volume_snapshot %} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-external-snapshotter-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list", "watch", "delete"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-csi-snapshotter-binding subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: ebs-external-snapshotter-role apiGroup: rbac.authorization.k8s.io {% endif %} {% if aws_ebs_csi_enable_volume_resizing %} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-external-resizer-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-csi-resizer-binding subjects: - kind: ServiceAccount name: ebs-csi-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: ebs-external-resizer-role apiGroup: rbac.authorization.k8s.io {% endif %} ================================================ FILE: roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 ================================================ --- kind: Deployment apiVersion: apps/v1 metadata: name: ebs-csi-controller namespace: kube-system spec: replicas: {{ aws_ebs_csi_controller_replicas }} selector: matchLabels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver template: metadata: labels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver {% if aws_ebs_csi_annotations is defined %} annotations: {% for annotation in aws_ebs_csi_annotations %} {{ annotation.key }}: {{ annotation.value }} {% endfor %} {% endif %} spec: nodeSelector: kubernetes.io/os: linux serviceAccountName: ebs-csi-controller-sa priorityClassName: system-cluster-critical containers: - name: ebs-plugin image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }} args: - --endpoint=$(CSI_ENDPOINT) {% if aws_ebs_csi_extra_volume_tags is defined %} - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }} {% endif %} - --logtostderr - --v=5 env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: name: aws-secret key: key_id optional: true - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: aws-secret key: access_key optional: true volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ ports: - name: healthz containerPort: 9808 protocol: TCP livenessProbe: httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 - name: csi-provisioner image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} args: - --csi-address=$(ADDRESS) - --v=5 {% if aws_ebs_csi_enable_volume_scheduling %} - --feature-gates=Topology=true {% endif %} - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-attacher image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} args: - --csi-address=$(ADDRESS) - --v=5 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ {% if aws_ebs_csi_enable_volume_snapshot %} - name: csi-snapshotter image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} args: - --csi-address=$(ADDRESS) - --timeout=15s env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ {% endif %} {% if aws_ebs_csi_enable_volume_resizing %} - name: csi-resizer image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - --csi-address=$(ADDRESS) - --v=5 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ {% endif %} - name: liveness-probe image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} args: - --csi-address=/csi/csi.sock volumeMounts: - name: socket-dir mountPath: /csi volumes: - name: socket-dir emptyDir: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 ================================================ --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: ebs.csi.aws.com spec: attachRequired: true podInfoOnMount: false ================================================ FILE: roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 ================================================ --- # Node Service kind: DaemonSet apiVersion: apps/v1 metadata: name: ebs-csi-node namespace: kube-system spec: selector: matchLabels: app: ebs-csi-node app.kubernetes.io/name: aws-ebs-csi-driver template: metadata: labels: app: ebs-csi-node app.kubernetes.io/name: aws-ebs-csi-driver spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true priorityClassName: system-node-critical containers: - name: ebs-plugin securityContext: privileged: true image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }} args: - --endpoint=$(CSI_ENDPOINT) {% if aws_ebs_csi_extra_volume_tags is defined %} - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }} {% endif %} - --logtostderr - --v=5 env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi - name: device-dir mountPath: /dev ports: - name: healthz containerPort: 9808 protocol: TCP livenessProbe: httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 - name: node-driver-registrar image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=5 lifecycle: preStop: exec: command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock volumeMounts: - name: plugin-dir mountPath: /csi - name: registration-dir mountPath: /registration - name: liveness-probe image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} args: - --csi-address=/csi/csi.sock volumeMounts: - name: plugin-dir mountPath: /csi volumes: - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ type: DirectoryOrCreate - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry/ type: Directory - name: device-dir hostPath: path: /dev type: Directory ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml ================================================ --- azure_csi_use_instance_metadata: true azure_csi_controller_replicas: 2 azure_csi_plugin_image_tag: latest azure_csi_controller_affinity: {} azure_csi_node_affinity: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml ================================================ --- - name: Azure CSI Driver | check azure_csi_tenant_id value fail: msg: "azure_csi_tenant_id is missing" when: azure_csi_tenant_id is not defined or not azure_csi_tenant_id - name: Azure CSI Driver | check azure_csi_subscription_id value fail: msg: "azure_csi_subscription_id is missing" when: azure_csi_subscription_id is not defined or not azure_csi_subscription_id - name: Azure CSI Driver | check azure_csi_aad_client_id value fail: msg: "azure_csi_aad_client_id is missing" when: azure_csi_aad_client_id is not defined or not azure_csi_aad_client_id - name: Azure CSI Driver | check azure_csi_aad_client_secret value fail: msg: "azure_csi_aad_client_secret is missing" when: azure_csi_aad_client_secret is not defined or not azure_csi_aad_client_secret - name: Azure CSI Driver | check azure_csi_resource_group value fail: msg: "azure_csi_resource_group is missing" when: azure_csi_resource_group is not defined or not azure_csi_resource_group - name: Azure CSI Driver | check azure_csi_location value fail: msg: "azure_csi_location is missing" when: azure_csi_location is not defined or not azure_csi_location - name: Azure CSI Driver | check azure_csi_subnet_name value fail: msg: "azure_csi_subnet_name is missing" when: azure_csi_subnet_name is not defined or not azure_csi_subnet_name - name: Azure CSI Driver | check azure_csi_security_group_name value fail: msg: "azure_csi_security_group_name is missing" when: azure_csi_security_group_name is not defined or not azure_csi_security_group_name - name: Azure CSI Driver | check azure_csi_vnet_name value fail: msg: "azure_csi_vnet_name is missing" when: azure_csi_vnet_name is not defined or not azure_csi_vnet_name - name: Azure CSI Driver | check azure_csi_vnet_resource_group value fail: msg: "azure_csi_vnet_resource_group is missing" when: azure_csi_vnet_resource_group is not defined or not azure_csi_vnet_resource_group - name: "Azure CSI Driver | check azure_csi_use_instance_metadata is a bool" assert: that: azure_csi_use_instance_metadata | type_debug == 'bool' ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml ================================================ --- - name: Azure CSI Driver | Check Azure credentials include_tasks: azure-credential-check.yml - name: Azure CSI Driver | Write Azure CSI cloud-config template: src: "azure-csi-cloud-config.j2" dest: "{{ kube_config_dir }}/azure_csi_cloud_config" group: "{{ kube_cert_group }}" mode: "0640" when: inventory_hostname == groups['kube_control_plane'][0] - name: Azure CSI Driver | Get base64 cloud-config slurp: src: "{{ kube_config_dir }}/azure_csi_cloud_config" register: cloud_config_secret when: inventory_hostname == groups['kube_control_plane'][0] - name: Azure CSI Driver | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml} - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml} - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller-rbac.yml} - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller.yml} - {name: azure-csi-azuredisk-node-rbac, file: azure-csi-azuredisk-node-rbac.yml} - {name: azure-csi-azuredisk-node, file: azure-csi-azuredisk-node.yml} register: azure_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: Azure CSI Driver | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ azure_csi_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: csi-azuredisk-controller-sa namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-provisioner-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-provisioner-binding subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: azuredisk-external-provisioner-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-attacher-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["csi.storage.k8s.io"] resources: ["csinodeinfos"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-attacher-binding subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: azuredisk-external-attacher-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-cluster-driver-registrar-role rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list", "watch", "delete"] - apiGroups: ["csi.storage.k8s.io"] resources: ["csidrivers"] verbs: ["create", "delete"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-driver-registrar-binding subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: azuredisk-cluster-driver-registrar-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-snapshotter-role rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create", "list", "watch", "delete"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-snapshotter-binding subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: azuredisk-external-snapshotter-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-external-resizer-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: azuredisk-csi-resizer-role subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: azuredisk-external-resizer-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-azuredisk-controller-secret-role rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-azuredisk-controller-secret-binding subjects: - kind: ServiceAccount name: csi-azuredisk-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-azuredisk-controller-secret-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 ================================================ --- kind: Deployment apiVersion: apps/v1 metadata: name: csi-azuredisk-controller namespace: kube-system spec: replicas: {{ azure_csi_controller_replicas }} selector: matchLabels: app: csi-azuredisk-controller template: metadata: labels: app: csi-azuredisk-controller spec: hostNetwork: true serviceAccountName: csi-azuredisk-controller-sa nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical tolerations: - key: "node-role.kubernetes.io/control-plane" effect: "NoSchedule" {% if azure_csi_controller_affinity %} affinity: {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }} {% endif %} containers: - name: csi-provisioner image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--feature-gates=Topology=true" - "--csi-address=$(ADDRESS)" - "--v=2" - "--timeout=15s" - "--leader-election" - "--worker-threads=40" - "--extra-create-metadata=true" - "--strict-topology=true" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir resources: limits: memory: 500Mi requests: cpu: 10m memory: 20Mi - name: csi-attacher image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "-v=2" - "-csi-address=$(ADDRESS)" - "-timeout=600s" - "-leader-election" - "-worker-threads=500" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir resources: limits: memory: 500Mi requests: cpu: 10m memory: 20Mi - name: csi-snapshotter image: {{ azure_csi_image_repo }}/csi-snapshotter:{{ azure_csi_snapshotter_image_tag }} args: - "-csi-address=$(ADDRESS)" - "-leader-election" - "-v=2" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - name: socket-dir mountPath: /csi resources: limits: memory: 100Mi requests: cpu: 10m memory: 20Mi - name: csi-resizer image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }} args: - "-csi-address=$(ADDRESS)" - "-v=2" - "-leader-election" - '-handle-volume-inuse-error=false' - "-timeout=60s" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - name: socket-dir mountPath: /csi resources: limits: memory: 500Mi requests: cpu: 10m memory: 20Mi - name: liveness-probe image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} args: - --csi-address=/csi/csi.sock - --probe-timeout=3s - --health-port=29602 - --v=2 volumeMounts: - name: socket-dir mountPath: /csi resources: limits: memory: 100Mi requests: cpu: 10m memory: 20Mi - name: azuredisk image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--metrics-address=0.0.0.0:29604" - "--disable-avset-nodes=true" - "--drivername=disk.csi.azure.com" - "--cloud-config-secret-name=cloud-config" - "--cloud-config-secret-namespace=kube-system" ports: - containerPort: 29602 name: healthz protocol: TCP - containerPort: 29604 name: metrics protocol: TCP livenessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 env: - name: AZURE_CREDENTIAL_FILE value: "/etc/kubernetes/azure.json" - name: CSI_ENDPOINT value: unix:///csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir - mountPath: /etc/kubernetes/ name: azure-cred readOnly: true resources: limits: memory: 500Mi requests: cpu: 10m memory: 20Mi volumes: - name: socket-dir emptyDir: {} - name: azure-cred secret: secretName: cloud-config ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 ================================================ --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: disk.csi.azure.com spec: attachRequired: true podInfoOnMount: true volumeLifecycleModes: # added in Kubernetes 1.16 - Persistent ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: csi-azuredisk-node-sa namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-azuredisk-node-secret-role rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-azuredisk-node-secret-binding subjects: - kind: ServiceAccount name: csi-azuredisk-node-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-azuredisk-node-secret-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 ================================================ --- kind: DaemonSet apiVersion: apps/v1 metadata: name: csi-azuredisk-node namespace: kube-system spec: updateStrategy: rollingUpdate: maxUnavailable: 1 type: RollingUpdate selector: matchLabels: app: csi-azuredisk-node template: metadata: labels: app: csi-azuredisk-node spec: hostNetwork: true dnsPolicy: Default serviceAccountName: csi-azuredisk-node-sa nodeSelector: kubernetes.io/os: linux {% if azure_csi_node_affinity %} affinity: {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }} {% endif %} priorityClassName: system-node-critical tolerations: - operator: Exists containers: - name: liveness-probe volumeMounts: - mountPath: /csi name: socket-dir image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - --csi-address=/csi/csi.sock - --probe-timeout=3s - --health-port=29603 - --v=2 resources: limits: memory: 100Mi requests: cpu: 10m memory: 20Mi - name: node-driver-registrar image: {{ azure_csi_image_repo }}/csi-node-driver-registrar:{{ azure_csi_node_registrar_image_tag }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v=2 livenessProbe: exec: command: - /csi-node-driver-registrar - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --mode=kubelet-registration-probe initialDelaySeconds: 30 timeoutSeconds: 15 env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock volumeMounts: - name: socket-dir mountPath: /csi - name: registration-dir mountPath: /registration resources: limits: memory: 100Mi requests: cpu: 10m memory: 20Mi - name: azuredisk image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - "--metrics-address=0.0.0.0:29605" - "--enable-perf-optimization=true" - "--drivername=disk.csi.azure.com" - "--volume-attach-limit=-1" - "--cloud-config-secret-name=cloud-config" - "--cloud-config-secret-namespace=kube-system" ports: - containerPort: 29603 name: healthz protocol: TCP - containerPort: 29605 name: metrics protocol: TCP livenessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 env: - name: AZURE_CREDENTIAL_FILE value: "/etc/kubernetes/azure.json" - name: CSI_ENDPOINT value: unix:///csi/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName securityContext: privileged: true volumeMounts: - mountPath: /csi name: socket-dir - mountPath: /var/lib/kubelet/ mountPropagation: Bidirectional name: mountpoint-dir - mountPath: /etc/kubernetes/ name: azure-cred - mountPath: /dev name: device-dir - mountPath: /sys/bus/scsi/devices name: sys-devices-dir - mountPath: /sys/class/scsi_host/ name: scsi-host-dir resources: limits: memory: 200Mi requests: cpu: 10m memory: 20Mi volumes: - hostPath: path: /var/lib/kubelet/plugins/disk.csi.azure.com type: DirectoryOrCreate name: socket-dir - hostPath: path: /var/lib/kubelet/ type: DirectoryOrCreate name: mountpoint-dir - hostPath: path: /var/lib/kubelet/plugins_registry/ type: DirectoryOrCreate name: registration-dir - secret: defaultMode: 0644 secretName: cloud-config name: azure-cred - hostPath: path: /dev type: Directory name: device-dir - hostPath: path: /sys/bus/scsi/devices type: Directory name: sys-devices-dir - hostPath: path: /sys/class/scsi_host/ type: Directory name: scsi-host-dir ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 ================================================ kind: Secret apiVersion: v1 metadata: name: cloud-config namespace: kube-system data: azure.json: {{ cloud_config_secret.content }} ================================================ FILE: roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 ================================================ { "cloud":"AzurePublicCloud", "tenantId": "{{ azure_csi_tenant_id }}", "subscriptionId": "{{ azure_csi_subscription_id }}", "aadClientId": "{{ azure_csi_aad_client_id }}", "aadClientSecret": "{{ azure_csi_aad_client_secret }}", "location": "{{ azure_csi_location }}", "resourceGroup": "{{ azure_csi_resource_group }}", "vnetName": "{{ azure_csi_vnet_name }}", "vnetResourceGroup": "{{ azure_csi_vnet_resource_group }}", "subnetName": "{{ azure_csi_subnet_name }}", "securityGroupName": "{{ azure_csi_security_group_name }}", "useInstanceMetadata": {{ azure_csi_use_instance_metadata }}, } ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml ================================================ --- cinder_csi_attacher_image_tag: "v4.4.2" cinder_csi_provisioner_image_tag: "v3.6.2" cinder_csi_snapshotter_image_tag: "v6.3.2" cinder_csi_resizer_image_tag: "v1.9.2" cinder_csi_livenessprobe_image_tag: "v2.11.0" # To access Cinder, the CSI controller will need credentials to access # openstack apis. Per default this values will be # read from the environment. cinder_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}" cinder_username: "{{ lookup('env', 'OS_USERNAME') }}" cinder_password: "{{ lookup('env', 'OS_PASSWORD') }}" cinder_application_credential_id: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}" cinder_application_credential_name: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}" cinder_application_credential_secret: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" cinder_region: "{{ lookup('env', 'OS_REGION_NAME') }}" cinder_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}" cinder_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}" cinder_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}" cinder_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}" cinder_cacert: "{{ lookup('env', 'OS_CACERT') }}" # For now, only Cinder v3 is supported in Cinder CSI driver cinder_blockstorage_version: "v3" cinder_csi_controller_replicas: 1 # Optional. Set to true, to rescan block device and verify its size before expanding # the filesystem. # Not all hypervizors have a /sys/class/block/XXX/device/rescan location, therefore if # you enable this option and your hypervizor doesn't support this, you'll get a warning # log on resize event. It is recommended to disable this option in this case. # Defaults to false # cinder_csi_rescan_on_resize: true cinder_tolerations: [] ## Dictionaries of extra arguments to add to the cinder CSI plugin containers ## Format: ## cinder_csi_attacher_extra_args: ## arg1: "value1" ## arg2: "value2" cinder_csi_attacher_extra_args: {} cinder_csi_provisioner_extra_args: {} cinder_csi_snapshotter_extra_args: {} cinder_csi_resizer_extra_args: {} cinder_csi_plugin_extra_args: {} cinder_liveness_probe_extra_args: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml ================================================ --- - name: Cinder CSI Driver | check cinder_auth_url value fail: msg: "cinder_auth_url is missing" when: cinder_auth_url is not defined or not cinder_auth_url - name: Cinder CSI Driver | check cinder_username value cinder_application_credential_name value fail: msg: "you must either set cinder_username or cinder_application_credential_name" when: - cinder_username is not defined or not cinder_username - cinder_application_credential_name is not defined or not cinder_application_credential_name - name: Cinder CSI Driver | check cinder_application_credential_id value fail: msg: "cinder_application_credential_id is missing" when: - cinder_application_credential_name is defined - cinder_application_credential_name | length > 0 - cinder_application_credential_id is not defined or not cinder_application_credential_id - name: Cinder CSI Driver | check cinder_application_credential_secret value fail: msg: "cinder_application_credential_secret is missing" when: - cinder_application_credential_name is defined - cinder_application_credential_name | length > 0 - cinder_application_credential_secret is not defined or not cinder_application_credential_secret - name: Cinder CSI Driver | check cinder_password value fail: msg: "cinder_password is missing" when: - cinder_username is defined - cinder_username | length > 0 - cinder_application_credential_name is not defined or not cinder_application_credential_name - cinder_application_credential_secret is not defined or not cinder_application_credential_secret - cinder_password is not defined or not cinder_password - name: Cinder CSI Driver | check cinder_region value fail: msg: "cinder_region is missing" when: cinder_region is not defined or not cinder_region - name: Cinder CSI Driver | check cinder_tenant_id value fail: msg: "one of cinder_tenant_id or cinder_tenant_name must be specified" when: - cinder_tenant_id is not defined or not cinder_tenant_id - cinder_tenant_name is not defined or not cinder_tenant_name - cinder_application_credential_name is not defined or not cinder_application_credential_name - name: Cinder CSI Driver | check cinder_domain_id value fail: msg: "one of cinder_domain_id or cinder_domain_name must be specified" when: - cinder_domain_id is not defined or not cinder_domain_id - cinder_domain_name is not defined or not cinder_domain_name - cinder_application_credential_name is not defined or not cinder_application_credential_name ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml ================================================ --- # include to workaround mitogen issue # https://github.com/dw/mitogen/issues/663 - name: Cinder CSI Driver | Write cacert file copy: src: "{{ cinder_cacert }}" dest: "{{ kube_config_dir }}/cinder-cacert.pem" group: "{{ kube_cert_group }}" mode: "0640" delegate_to: "{{ delegate_host_to_write_cacert }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml ================================================ --- - name: Cinder CSI Driver | Check Cinder credentials include_tasks: cinder-credential-check.yml - name: Cinder CSI Driver | Write cacert file include_tasks: cinder-write-cacert.yml run_once: true loop: "{{ groups['k8s_cluster'] }}" loop_control: loop_var: delegate_host_to_write_cacert when: - ('k8s_cluster' in group_names) - cinder_cacert is defined - cinder_cacert | length > 0 - name: Cinder CSI Driver | Write Cinder cloud-config template: src: "cinder-csi-cloud-config.j2" dest: "{{ kube_config_dir }}/cinder_cloud_config" group: "{{ kube_cert_group }}" mode: "0640" when: inventory_hostname == groups['kube_control_plane'][0] - name: Cinder CSI Driver | Get base64 cloud-config slurp: src: "{{ kube_config_dir }}/cinder_cloud_config" register: cloud_config_secret when: inventory_hostname == groups['kube_control_plane'][0] - name: Cinder CSI Driver | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: cinder-csi-driver, file: cinder-csi-driver.yml} - {name: cinder-csi-cloud-config-secret, file: cinder-csi-cloud-config-secret.yml} - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin-rbac.yml} - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin.yml} - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin-rbac.yml} - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin.yml} - {name: cinder-csi-poddisruptionbudget, file: cinder-csi-poddisruptionbudget.yml} register: cinder_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: Cinder CSI Driver | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ cinder_csi_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 ================================================ # This YAML file contains secret objects, # which are necessary to run csi cinder plugin. kind: Secret apiVersion: v1 metadata: name: cloud-config namespace: kube-system data: cloud.conf: {{ cloud_config_secret.content }} ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 ================================================ [Global] auth-url="{{ cinder_auth_url }}" {% if cinder_application_credential_id|length == 0 and cinder_application_credential_name|length == 0 %} username="{{ cinder_username }}" password="{{ cinder_password }}" {% endif %} {% if cinder_application_credential_id|length > 0 %} application-credential-id={{ cinder_application_credential_id }} {% endif %} {% if cinder_application_credential_name|length > 0 %} application-credential-name={{ cinder_application_credential_name }} {% endif %} {% if cinder_application_credential_secret|length > 0 %} application-credential-secret={{ cinder_application_credential_secret }} {% endif %} region="{{ cinder_region }}" {% if cinder_tenant_id|length > 0 %} tenant-id="{{ cinder_tenant_id }}" {% endif %} {% if cinder_tenant_name|length > 0 %} tenant-name="{{ cinder_tenant_name }}" {% endif %} {% if cinder_domain_name|length > 0 %} domain-name="{{ cinder_domain_name }}" {% elif cinder_domain_id|length > 0 %} domain-id ="{{ cinder_domain_id }}" {% endif %} {% if cinder_cacert|length > 0 %} ca-file="{{ kube_config_dir }}/cinder-cacert.pem" {% endif %} [BlockStorage] {% if cinder_blockstorage_version is defined %} bs-version={{ cinder_blockstorage_version }} {% endif %} {% if cinder_csi_ignore_volume_az is defined %} ignore-volume-az={{ cinder_csi_ignore_volume_az | bool }} {% endif %} {% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %} node-volume-attach-limit="{{ node_volume_attach_limit }}" {% endif %} {% if cinder_csi_rescan_on_resize is defined %} rescan-on-resize={{ cinder_csi_rescan_on_resize | bool }} {% endif %} ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 ================================================ # This YAML file contains RBAC API objects, # which are necessary to run csi controller plugin apiVersion: v1 kind: ServiceAccount metadata: name: csi-cinder-controller-sa namespace: kube-system --- # external attacher kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-binding subjects: - kind: ServiceAccount name: csi-cinder-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-attacher-role apiGroup: rbac.authorization.k8s.io --- # external Provisioner kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-binding subjects: - kind: ServiceAccount name: csi-cinder-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-provisioner-role apiGroup: rbac.authorization.k8s.io --- # external snapshotter kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-snapshotter-role rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-snapshotter-binding subjects: - kind: ServiceAccount name: csi-cinder-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-snapshotter-role apiGroup: rbac.authorization.k8s.io --- # External Resizer kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-role rules: # The following rule should be uncommented for plugins that require secrets # for provisioning. # - apiGroups: [""] # resources: ["secrets"] # verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-binding subjects: - kind: ServiceAccount name: csi-cinder-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-resizer-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 ================================================ # This YAML file contains CSI Controller Plugin Sidecars # external-attacher, external-provisioner, external-snapshotter # external-resize, liveness-probe --- kind: Deployment apiVersion: apps/v1 metadata: name: csi-cinder-controllerplugin namespace: kube-system spec: replicas: {{ cinder_csi_controller_replicas }} selector: matchLabels: app: csi-cinder-controllerplugin template: metadata: labels: app: csi-cinder-controllerplugin spec: serviceAccount: csi-cinder-controller-sa containers: - name: csi-attacher image: {{ csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" - "--timeout=3m" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - --leader-election=true {% endif %} - "--default-fstype=ext4" {% for key, value in cinder_csi_attacher_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-provisioner image: {{ csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" - "--timeout=3m" - "--default-fstype=ext4" - "--extra-create-metadata" {% if cinder_topology is defined and cinder_topology %} - --feature-gates=Topology=true {% endif %} {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - "--leader-election=true" {% endif %} {% for key, value in cinder_csi_provisioner_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-snapshotter image: {{ csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" - "--timeout=3m" - "--extra-create-metadata" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - --leader-election=true {% endif %} {% for key, value in cinder_csi_snapshotter_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - name: csi-resizer image: {{ csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" - "--timeout=3m" - "--handle-volume-inuse-error=false" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - --leader-election=true {% endif %} {% for key, value in cinder_csi_resizer_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: liveness-probe image: {{ csi_livenessprobe_image_repo }}:{{ cinder_csi_livenessprobe_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" {% for key, value in cinder_liveness_probe_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - name: cinder-csi-plugin image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - /bin/cinder-csi-plugin - "--endpoint=$(CSI_ENDPOINT)" - "--cloud-config=$(CLOUD_CONFIG)" - "--cluster=$(CLUSTER_NAME)" {% for key, value in cinder_csi_plugin_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} env: - name: CSI_ENDPOINT value: unix://csi/csi.sock - name: CLOUD_CONFIG value: /etc/config/cloud.conf - name: CLUSTER_NAME value: {{ cluster_name }} ports: - containerPort: 9808 name: healthz protocol: TCP livenessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 10 periodSeconds: 60 volumeMounts: - name: socket-dir mountPath: /csi - name: secret-cinderplugin mountPath: /etc/config readOnly: true - name: ca-certs mountPath: /etc/ssl/certs readOnly: true {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} mountPath: {{ dir }} readOnly: true {% endfor %} {% endif %} {% if cinder_cacert is defined and cinder_cacert != "" %} - name: cinder-cacert mountPath: {{ kube_config_dir }}/cinder-cacert.pem readOnly: true {% endif %} volumes: - name: socket-dir emptyDir: - name: secret-cinderplugin secret: secretName: cloud-config - name: ca-certs hostPath: path: /etc/ssl/certs type: DirectoryOrCreate {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} hostPath: path: {{ dir }} type: DirectoryOrCreate {% endfor %} {% endif %} {% if cinder_cacert is defined and cinder_cacert != "" %} - name: cinder-cacert hostPath: path: {{ kube_config_dir }}/cinder-cacert.pem type: FileOrCreate {% endif %} ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 ================================================ apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: cinder.csi.openstack.org spec: attachRequired: true podInfoOnMount: true volumeLifecycleModes: - Persistent - Ephemeral ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 ================================================ # This YAML defines all API objects to create RBAC roles for csi node plugin. apiVersion: v1 kind: ServiceAccount metadata: name: csi-cinder-node-sa namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-nodeplugin-role rules: - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-nodeplugin-binding subjects: - kind: ServiceAccount name: csi-cinder-node-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-nodeplugin-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 ================================================ # This YAML file contains driver-registrar & csi driver nodeplugin API objects, # which are necessary to run csi nodeplugin for cinder. kind: DaemonSet apiVersion: apps/v1 metadata: name: csi-cinder-nodeplugin namespace: kube-system spec: selector: matchLabels: app: csi-cinder-nodeplugin template: metadata: labels: app: csi-cinder-nodeplugin spec: tolerations: - operator: Exists serviceAccountName: csi-cinder-node-sa hostNetwork: true dnsPolicy: ClusterFirstWithHostNet containers: - name: node-driver-registrar image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - name: socket-dir mountPath: /csi - name: registration-dir mountPath: /registration - name: liveness-probe image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} args: - "--csi-address=/csi/csi.sock" volumeMounts: - name: socket-dir mountPath: /csi - name: cinder-csi-plugin securityContext: privileged: true capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - /bin/cinder-csi-plugin - "--endpoint=$(CSI_ENDPOINT)" - "--cloud-config=$(CLOUD_CONFIG)" env: - name: CSI_ENDPOINT value: unix://csi/csi.sock - name: CLOUD_CONFIG value: /etc/config/cloud.conf ports: - containerPort: 9808 name: healthz protocol: TCP livenessProbe: failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 10 volumeMounts: - name: socket-dir mountPath: /csi - name: kubelet-dir mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - name: pods-probe-dir mountPath: /dev mountPropagation: "HostToContainer" - name: secret-cinderplugin mountPath: /etc/config readOnly: true - name: ca-certs mountPath: /etc/ssl/certs readOnly: true {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} mountPath: {{ dir }} readOnly: true {% endfor %} {% endif %} {% if cinder_cacert is defined and cinder_cacert != "" %} - name: cinder-cacert mountPath: {{ kube_config_dir }}/cinder-cacert.pem readOnly: true {% endif %} volumes: - name: socket-dir hostPath: path: /var/lib/kubelet/plugins/cinder.csi.openstack.org type: DirectoryOrCreate - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry/ type: Directory - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory - name: pods-probe-dir hostPath: path: /dev type: Directory - name: secret-cinderplugin secret: secretName: cloud-config - name: ca-certs hostPath: path: /etc/ssl/certs type: DirectoryOrCreate {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} hostPath: path: {{ dir }} type: DirectoryOrCreate {% endfor %} {% endif %} {% if cinder_cacert is defined and cinder_cacert != "" %} - name: cinder-cacert hostPath: path: {{ kube_config_dir }}/cinder-cacert.pem type: FileOrCreate {% endif %} {% if cinder_tolerations %} tolerations: {{ cinder_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2 ================================================ apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: cinder-csi-pdb namespace: kube-system spec: {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} minAvailable: 1 {% else %} minAvailable: 0 {% endif %} selector: matchLabels: app: csi-cinder-controllerplugin ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml ================================================ --- - name: CSI CRD | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: volumegroupsnapshotclasses, file: volumegroupsnapshotclasses.yml} - {name: volumegroupsnapshotcontents, file: volumegroupsnapshotcontents.yml} - {name: volumegroupsnapshots, file: volumegroupsnapshots.yml} - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml} - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml} - {name: volumesnapshots, file: volumesnapshots.yml} register: csi_crd_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: CSI CRD | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" wait: true with_items: - "{{ csi_crd_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/templates/volumegroupsnapshotclasses.yml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150" controller-gen.kubebuilder.io/version: v0.15.0 name: volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io spec: group: groupsnapshot.storage.k8s.io names: kind: VolumeGroupSnapshotClass listKind: VolumeGroupSnapshotClassList plural: volumegroupsnapshotclasses shortNames: - vgsclass - vgsclasses singular: volumegroupsnapshotclass scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .driver name: Driver type: string - description: Determines whether a VolumeGroupSnapshotContent created through the VolumeGroupSnapshotClass should be deleted when its bound VolumeGroupSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta1 schema: openAPIV3Schema: description: |- VolumeGroupSnapshotClass specifies parameters that a underlying storage system uses when creating a volume group snapshot. A specific VolumeGroupSnapshotClass is used by specifying its name in a VolumeGroupSnapshot object. VolumeGroupSnapshotClasses are non-namespaced. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string deletionPolicy: description: |- DeletionPolicy determines whether a VolumeGroupSnapshotContent created through the VolumeGroupSnapshotClass should be deleted when its bound VolumeGroupSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeGroupSnapshotContent and its physical group snapshot on underlying storage system are kept. "Delete" means that the VolumeGroupSnapshotContent and its physical group snapshot on underlying storage system are deleted. Required. enum: - Delete - Retain type: string driver: description: |- Driver is the name of the storage driver expected to handle this VolumeGroupSnapshotClass. Required. type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object parameters: additionalProperties: type: string description: |- Parameters is a key-value map with storage driver specific parameters for creating group snapshots. These values are opaque to Kubernetes and are passed directly to the driver. type: object required: - deletionPolicy - driver type: object served: true storage: true subresources: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/templates/volumegroupsnapshotcontents.yml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150" controller-gen.kubebuilder.io/version: v0.15.0 name: volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io spec: group: groupsnapshot.storage.k8s.io names: kind: VolumeGroupSnapshotContent listKind: VolumeGroupSnapshotContentList plural: volumegroupsnapshotcontents shortNames: - vgsc - vgscs singular: volumegroupsnapshotcontent scope: Cluster versions: - additionalPrinterColumns: - description: Indicates if all the individual snapshots in the group are ready to be used to restore a group of volumes. jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: Determines whether this VolumeGroupSnapshotContent and its physical group snapshot on the underlying storage system should be deleted when its bound VolumeGroupSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - description: Name of the CSI driver used to create the physical group snapshot on the underlying storage system. jsonPath: .spec.driver name: Driver type: string - description: Name of the VolumeGroupSnapshotClass from which this group snapshot was (or will be) created. jsonPath: .spec.volumeGroupSnapshotClassName name: VolumeGroupSnapshotClass type: string - description: Namespace of the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent object is bound. jsonPath: .spec.volumeGroupSnapshotRef.namespace name: VolumeGroupSnapshotNamespace type: string - description: Name of the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent object is bound. jsonPath: .spec.volumeGroupSnapshotRef.name name: VolumeGroupSnapshot type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta1 schema: openAPIV3Schema: description: |- VolumeGroupSnapshotContent represents the actual "on-disk" group snapshot object in the underlying storage system properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: |- Spec defines properties of a VolumeGroupSnapshotContent created by the underlying storage system. Required. properties: deletionPolicy: description: |- DeletionPolicy determines whether this VolumeGroupSnapshotContent and the physical group snapshot on the underlying storage system should be deleted when the bound VolumeGroupSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeGroupSnapshotContent and its physical group snapshot on underlying storage system are kept. "Delete" means that the VolumeGroupSnapshotContent and its physical group snapshot on underlying storage system are deleted. For dynamically provisioned group snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeGroupSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeGroupSnapshotContent object. Required. enum: - Delete - Retain type: string driver: description: |- Driver is the name of the CSI driver used to create the physical group snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. type: string source: description: |- Source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. properties: groupSnapshotHandles: description: |- GroupSnapshotHandles specifies the CSI "group_snapshot_id" of a pre-existing group snapshot and a list of CSI "snapshot_id" of pre-existing snapshots on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. properties: volumeGroupSnapshotHandle: description: |- VolumeGroupSnapshotHandle specifies the CSI "group_snapshot_id" of a pre-existing group snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. Required. type: string volumeSnapshotHandles: description: |- VolumeSnapshotHandles is a list of CSI "snapshot_id" of pre-existing snapshots on the underlying storage system for which Kubernetes objects representation were (or should be) created. This field is immutable. Required. items: type: string type: array required: - volumeGroupSnapshotHandle - volumeSnapshotHandles type: object x-kubernetes-validations: - message: groupSnapshotHandles is immutable rule: self == oldSelf volumeHandles: description: |- VolumeHandles is a list of volume handles on the backend to be snapshotted together. It is specified for dynamic provisioning of the VolumeGroupSnapshot. This field is immutable. items: type: string type: array x-kubernetes-validations: - message: volumeHandles is immutable rule: self == oldSelf type: object x-kubernetes-validations: - message: volumeHandles is required once set rule: '!has(oldSelf.volumeHandles) || has(self.volumeHandles)' - message: groupSnapshotHandles is required once set rule: '!has(oldSelf.groupSnapshotHandles) || has(self.groupSnapshotHandles)' - message: exactly one of volumeHandles and groupSnapshotHandles must be set rule: (has(self.volumeHandles) && !has(self.groupSnapshotHandles)) || (!has(self.volumeHandles) && has(self.groupSnapshotHandles)) volumeGroupSnapshotClassName: description: |- VolumeGroupSnapshotClassName is the name of the VolumeGroupSnapshotClass from which this group snapshot was (or will be) created. Note that after provisioning, the VolumeGroupSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. For dynamic provisioning, this field must be set. This field may be unset for pre-provisioned snapshots. type: string volumeGroupSnapshotRef: description: |- VolumeGroupSnapshotRef specifies the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent object is bound. VolumeGroupSnapshot.Spec.VolumeGroupSnapshotContentName field must reference to this VolumeGroupSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeGroupSnapshotContent object, name and namespace of the VolumeGroupSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: |- If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future. type: string kind: description: |- Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: description: |- Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: description: |- UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic x-kubernetes-validations: - message: both volumeGroupSnapshotRef.name and volumeGroupSnapshotRef.namespace must be set rule: has(self.name) && has(self.__namespace__) required: - deletionPolicy - driver - source - volumeGroupSnapshotRef type: object status: description: status represents the current information of a group snapshot. properties: creationTime: description: |- CreationTime is the timestamp when the point-in-time group snapshot is taken by the underlying storage system. If not specified, it indicates the creation time is unknown. If not specified, it means the readiness of a group snapshot is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command date +%s%N returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. This field is the source for the CreationTime field in VolumeGroupSnapshotStatus format: date-time type: string error: description: |- Error is the last observed error during group snapshot creation, if any. Upon success after retry, this error field will be cleared. properties: message: description: |- message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information. type: string time: description: time is the timestamp when the error was encountered. format: date-time type: string type: object readyToUse: description: |- ReadyToUse indicates if all the individual snapshots in the group are ready to be used to restore a group of volumes. ReadyToUse becomes true when ReadyToUse of all individual snapshots become true. type: boolean volumeGroupSnapshotHandle: description: |- VolumeGroupSnapshotHandle is a unique id returned by the CSI driver to identify the VolumeGroupSnapshot on the storage system. If a storage system does not provide such an id, the CSI driver can choose to return the VolumeGroupSnapshot name. type: string volumeSnapshotHandlePairList: description: |- VolumeSnapshotHandlePairList is a list of CSI "volume_id" and "snapshot_id" pair returned by the CSI driver to identify snapshots and their source volumes on the storage system. items: description: VolumeSnapshotHandlePair defines a pair of a source volume handle and a snapshot handle properties: snapshotHandle: description: |- SnapshotHandle is a unique id returned by the CSI driver to identify a volume snapshot on the storage system Required. type: string volumeHandle: description: |- VolumeHandle is a unique id returned by the CSI driver to identify a volume on the storage system Required. type: string required: - snapshotHandle - volumeHandle type: object type: array type: object required: - spec type: object served: true storage: true subresources: status: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/templates/volumegroupsnapshots.yml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150" controller-gen.kubebuilder.io/version: v0.15.0 name: volumegroupsnapshots.groupsnapshot.storage.k8s.io spec: group: groupsnapshot.storage.k8s.io names: kind: VolumeGroupSnapshot listKind: VolumeGroupSnapshotList plural: volumegroupsnapshots shortNames: - vgs singular: volumegroupsnapshot scope: Namespaced versions: - additionalPrinterColumns: - description: Indicates if all the individual snapshots in the group are ready to be used to restore a group of volumes. jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: The name of the VolumeGroupSnapshotClass requested by the VolumeGroupSnapshot. jsonPath: .spec.volumeGroupSnapshotClassName name: VolumeGroupSnapshotClass type: string - description: Name of the VolumeGroupSnapshotContent object to which the VolumeGroupSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeGroupSnapshot and VolumeGroupSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. jsonPath: .status.boundVolumeGroupSnapshotContentName name: VolumeGroupSnapshotContent type: string - description: Timestamp when the point-in-time group snapshot was taken by the underlying storage system. jsonPath: .status.creationTime name: CreationTime type: date - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta1 schema: openAPIV3Schema: description: |- VolumeGroupSnapshot is a user's request for creating either a point-in-time group snapshot or binding to a pre-existing group snapshot. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: |- Spec defines the desired characteristics of a group snapshot requested by a user. Required. properties: source: description: |- Source specifies where a group snapshot will be created from. This field is immutable after creation. Required. properties: selector: description: |- Selector is a label query over persistent volume claims that are to be grouped together for snapshotting. This labelSelector will be used to match the label added to a PVC. If the label is added or removed to a volume after a group snapshot is created, the existing group snapshots won't be modified. Once a VolumeGroupSnapshotContent is created and the sidecar starts to process it, the volume list will not change with retries. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array x-kubernetes-list-type: atomic required: - key - operator type: object type: array x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic x-kubernetes-validations: - message: selector is immutable rule: self == oldSelf volumeGroupSnapshotContentName: description: |- VolumeGroupSnapshotContentName specifies the name of a pre-existing VolumeGroupSnapshotContent object representing an existing volume group snapshot. This field should be set if the volume group snapshot already exists and only needs a representation in Kubernetes. This field is immutable. type: string x-kubernetes-validations: - message: volumeGroupSnapshotContentName is immutable rule: self == oldSelf type: object x-kubernetes-validations: - message: selector is required once set rule: '!has(oldSelf.selector) || has(self.selector)' - message: volumeGroupSnapshotContentName is required once set rule: '!has(oldSelf.volumeGroupSnapshotContentName) || has(self.volumeGroupSnapshotContentName)' - message: exactly one of selector and volumeGroupSnapshotContentName must be set rule: (has(self.selector) && !has(self.volumeGroupSnapshotContentName)) || (!has(self.selector) && has(self.volumeGroupSnapshotContentName)) volumeGroupSnapshotClassName: description: |- VolumeGroupSnapshotClassName is the name of the VolumeGroupSnapshotClass requested by the VolumeGroupSnapshot. VolumeGroupSnapshotClassName may be left nil to indicate that the default class will be used. Empty string is not allowed for this field. type: string x-kubernetes-validations: - message: volumeGroupSnapshotClassName must not be the empty string when set rule: size(self) > 0 required: - source type: object status: description: |- Status represents the current information of a group snapshot. Consumers must verify binding between VolumeGroupSnapshot and VolumeGroupSnapshotContent objects is successful (by validating that both VolumeGroupSnapshot and VolumeGroupSnapshotContent point to each other) before using this object. properties: boundVolumeGroupSnapshotContentName: description: |- BoundVolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent object to which this VolumeGroupSnapshot object intends to bind to. If not specified, it indicates that the VolumeGroupSnapshot object has not been successfully bound to a VolumeGroupSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeGroupSnapshot and VolumeGroupSnapshotContent objects is successful (by validating that both VolumeGroupSnapshot and VolumeGroupSnapshotContent point at each other) before using this object. type: string creationTime: description: |- CreationTime is the timestamp when the point-in-time group snapshot is taken by the underlying storage system. If not specified, it may indicate that the creation time of the group snapshot is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command date +%s%N returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. This field is updated based on the CreationTime field in VolumeGroupSnapshotContentStatus format: date-time type: string error: description: |- Error is the last observed error during group snapshot creation, if any. This field could be helpful to upper level controllers (i.e., application controller) to decide whether they should continue on waiting for the group snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the group snapshot creation. Upon success, this error field will be cleared. properties: message: description: |- message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information. type: string time: description: time is the timestamp when the error was encountered. format: date-time type: string type: object readyToUse: description: |- ReadyToUse indicates if all the individual snapshots in the group are ready to be used to restore a group of volumes. ReadyToUse becomes true when ReadyToUse of all individual snapshots become true. If not specified, it means the readiness of a group snapshot is unknown. type: boolean type: object required: - spec type: object served: true storage: true subresources: status: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" controller-gen.kubebuilder.io/version: v0.15.0 name: volumesnapshotclasses.snapshot.storage.k8s.io spec: group: snapshot.storage.k8s.io names: kind: VolumeSnapshotClass listKind: VolumeSnapshotClassList plural: volumesnapshotclasses shortNames: - vsclass - vsclasses singular: volumesnapshotclass scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .driver name: Driver type: string - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: description: |- VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string deletionPolicy: description: |- deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. enum: - Delete - Retain type: string driver: description: |- driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object parameters: additionalProperties: type: string description: |- parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. type: object required: - deletionPolicy - driver type: object served: true storage: true subresources: {} - additionalPrinterColumns: - jsonPath: .driver name: Driver type: string - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta1 # This indicates the v1beta1 version of the custom resource is deprecated. # API requests to this version receive a warning in the server response. deprecated: true # This overrides the default warning returned to clients making v1beta1 API requests. deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" schema: openAPIV3Schema: description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string deletionPolicy: description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. enum: - Delete - Retain type: string driver: description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string parameters: additionalProperties: type: string description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. type: object required: - deletionPolicy - driver type: object served: false storage: false subresources: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/955" name: volumesnapshotcontents.snapshot.storage.k8s.io spec: group: snapshot.storage.k8s.io names: kind: VolumeSnapshotContent listKind: VolumeSnapshotContentList plural: volumesnapshotcontents shortNames: - vsc - vscs singular: volumesnapshotcontent scope: Cluster versions: - additionalPrinterColumns: - description: Indicates if the snapshot is ready to be used to restore a volume. jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: Represents the complete size of the snapshot in bytes jsonPath: .status.restoreSize name: RestoreSize type: integer - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. jsonPath: .spec.driver name: Driver type: string - description: Name of the VolumeSnapshotClass to which this snapshot belongs. jsonPath: .spec.volumeSnapshotClassName name: VolumeSnapshotClass type: string - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. jsonPath: .spec.volumeSnapshotRef.namespace name: VolumeSnapshotNamespace type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: description: |- VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: |- spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. properties: deletionPolicy: description: |- deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. enum: - Delete - Retain type: string driver: description: |- driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. type: string source: description: |- source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. properties: snapshotHandle: description: |- snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. type: string x-kubernetes-validations: - message: snapshotHandle is immutable rule: self == oldSelf volumeHandle: description: |- volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. type: string x-kubernetes-validations: - message: volumeHandle is immutable rule: self == oldSelf type: object x-kubernetes-validations: - message: volumeHandle is required once set rule: '!has(oldSelf.volumeHandle) || has(self.volumeHandle)' - message: snapshotHandle is required once set rule: '!has(oldSelf.snapshotHandle) || has(self.snapshotHandle)' - message: exactly one of volumeHandle and snapshotHandle must be set rule: (has(self.volumeHandle) && !has(self.snapshotHandle)) || (!has(self.volumeHandle) && has(self.snapshotHandle)) sourceVolumeMode: description: |- SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, it indicates the source volume's mode is unknown. This field is immutable. This field is an alpha field. type: string x-kubernetes-validations: - message: sourceVolumeMode is immutable rule: self == oldSelf volumeSnapshotClassName: description: |- name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. type: string volumeSnapshotRef: description: |- volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: |- If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future. type: string kind: description: |- Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: description: |- Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: description: |- Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: description: |- UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic x-kubernetes-validations: - message: both spec.volumeSnapshotRef.name and spec.volumeSnapshotRef.namespace must be set rule: has(self.name) && has(self.__namespace__) required: - deletionPolicy - driver - source - volumeSnapshotRef type: object x-kubernetes-validations: - message: sourceVolumeMode is required once set rule: '!has(oldSelf.sourceVolumeMode) || has(self.sourceVolumeMode)' status: description: status represents the current information of a snapshot. properties: creationTime: description: |- creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. format: int64 type: integer error: description: |- error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. properties: message: description: |- message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information. type: string time: description: time is the timestamp when the error was encountered. format: date-time type: string type: object readyToUse: description: |- readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. type: boolean restoreSize: description: |- restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: description: |- snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. type: string volumeGroupSnapshotHandle: description: |- VolumeGroupSnapshotHandle is the CSI "group_snapshot_id" of a group snapshot on the underlying storage system. type: string type: object required: - spec type: object served: true storage: true subresources: status: {} - additionalPrinterColumns: - description: Indicates if the snapshot is ready to be used to restore a volume. jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: Represents the complete size of the snapshot in bytes jsonPath: .status.restoreSize name: RestoreSize type: integer - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. jsonPath: .spec.driver name: Driver type: string - description: Name of the VolumeSnapshotClass to which this snapshot belongs. jsonPath: .spec.volumeSnapshotClassName name: VolumeSnapshotClass type: string - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. jsonPath: .spec.volumeSnapshotRef.namespace name: VolumeSnapshotNamespace type: string - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta1 # This indicates the v1beta1 version of the custom resource is deprecated. # API requests to this version receive a warning in the server response. deprecated: true # This overrides the default warning returned to clients making v1beta1 API requests. deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" schema: openAPIV3Schema: description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. properties: deletionPolicy: description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. enum: - Delete - Retain type: string driver: description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. type: string source: description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. properties: snapshotHandle: description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. type: string volumeHandle: description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. type: string type: object volumeSnapshotClassName: description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. type: string volumeSnapshotRef: description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object required: - deletionPolicy - driver - source - volumeSnapshotRef type: object status: description: status represents the current information of a snapshot. properties: creationTime: description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. format: int64 type: integer error: description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. properties: message: description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. format: date-time type: string type: object readyToUse: description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. type: boolean restoreSize: description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. type: string type: object required: - spec type: object served: false storage: false subresources: status: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] ================================================ FILE: roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.15.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" name: volumesnapshots.snapshot.storage.k8s.io spec: group: snapshot.storage.k8s.io names: kind: VolumeSnapshot listKind: VolumeSnapshotList plural: volumesnapshots shortNames: - vs singular: volumesnapshot scope: Namespaced versions: - additionalPrinterColumns: - description: Indicates if the snapshot is ready to be used to restore a volume. jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. jsonPath: .spec.source.persistentVolumeClaimName name: SourcePVC type: string - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. jsonPath: .spec.source.volumeSnapshotContentName name: SourceSnapshotContent type: string - description: Represents the minimum size of volume required to rehydrate from this snapshot. jsonPath: .status.restoreSize name: RestoreSize type: string - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. jsonPath: .spec.volumeSnapshotClassName name: SnapshotClass type: string - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. jsonPath: .status.boundVolumeSnapshotContentName name: SnapshotContent type: string - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. jsonPath: .status.creationTime name: CreationTime type: date - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: description: |- VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: |- spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required. properties: source: description: |- source specifies where a snapshot will be created from. This field is immutable after creation. Required. properties: persistentVolumeClaimName: description: |- persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. type: string x-kubernetes-validations: - message: persistentVolumeClaimName is immutable rule: self == oldSelf volumeSnapshotContentName: description: |- volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. type: string x-kubernetes-validations: - message: volumeSnapshotContentName is immutable rule: self == oldSelf type: object x-kubernetes-validations: - message: persistentVolumeClaimName is required once set rule: '!has(oldSelf.persistentVolumeClaimName) || has(self.persistentVolumeClaimName)' - message: volumeSnapshotContentName is required once set rule: '!has(oldSelf.volumeSnapshotContentName) || has(self.volumeSnapshotContentName)' - message: exactly one of volumeSnapshotContentName and persistentVolumeClaimName must be set rule: (has(self.volumeSnapshotContentName) && !has(self.persistentVolumeClaimName)) || (!has(self.volumeSnapshotContentName) && has(self.persistentVolumeClaimName)) volumeSnapshotClassName: description: |- VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field. type: string x-kubernetes-validations: - message: volumeSnapshotClassName must not be the empty string when set rule: size(self) > 0 required: - source type: object status: description: |- status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. properties: boundVolumeSnapshotContentName: description: |- boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. type: string creationTime: description: |- creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. format: date-time type: string error: description: |- error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. properties: message: description: |- message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information. type: string time: description: time is the timestamp when the error was encountered. format: date-time type: string type: object readyToUse: description: |- readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string description: |- restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true volumeGroupSnapshotName: description: |- VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this VolumeSnapshot is a part of. type: string type: object required: - spec type: object served: true storage: true subresources: status: {} - additionalPrinterColumns: - description: Indicates if the snapshot is ready to be used to restore a volume. jsonPath: .status.readyToUse name: ReadyToUse type: boolean - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. jsonPath: .spec.source.persistentVolumeClaimName name: SourcePVC type: string - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. jsonPath: .spec.source.volumeSnapshotContentName name: SourceSnapshotContent type: string - description: Represents the minimum size of volume required to rehydrate from this snapshot. jsonPath: .status.restoreSize name: RestoreSize type: string - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. jsonPath: .spec.volumeSnapshotClassName name: SnapshotClass type: string - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. jsonPath: .status.boundVolumeSnapshotContentName name: SnapshotContent type: string - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. jsonPath: .status.creationTime name: CreationTime type: date - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1beta1 # This indicates the v1beta1 version of the custom resource is deprecated. # API requests to this version receive a warning in the server response. deprecated: true # This overrides the default warning returned to clients making v1beta1 API requests. deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" schema: openAPIV3Schema: description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' properties: source: description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. properties: persistentVolumeClaimName: description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. type: string volumeSnapshotContentName: description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. type: string type: object volumeSnapshotClassName: description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' type: string required: - source type: object status: description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. properties: boundVolumeSnapshotContentName: description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' type: string creationTime: description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. format: date-time type: string error: description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. properties: message: description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. format: date-time type: string type: object readyToUse: description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object required: - spec type: object served: false storage: false subresources: status: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml ================================================ --- gcp_pd_csi_controller_replicas: 1 ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml ================================================ --- - name: GCP PD CSI Driver | Check if cloud-sa.json exists fail: msg: "Credentials file cloud-sa.json is mandatory" when: gcp_pd_csi_sa_cred_file is not defined or not gcp_pd_csi_sa_cred_file - name: GCP PD CSI Driver | Copy GCP credentials file copy: src: "{{ gcp_pd_csi_sa_cred_file }}" dest: "{{ kube_config_dir }}/cloud-sa.json" group: "{{ kube_cert_group }}" mode: "0640" when: inventory_hostname == groups['kube_control_plane'][0] - name: GCP PD CSI Driver | Get base64 cloud-sa.json slurp: src: "{{ kube_config_dir }}/cloud-sa.json" register: gcp_cred_secret when: inventory_hostname == groups['kube_control_plane'][0] - name: GCP PD CSI Driver | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: gcp-pd-csi-cred-secret, file: gcp-pd-csi-cred-secret.yml} - {name: gcp-pd-csi-setup, file: gcp-pd-csi-setup.yml} - {name: gcp-pd-csi-controller, file: gcp-pd-csi-controller.yml} - {name: gcp-pd-csi-node, file: gcp-pd-csi-node.yml} - {name: gcp-pd-csi-sc-regional, file: gcp-pd-csi-sc-regional.yml} - {name: gcp-pd-csi-sc-zonal, file: gcp-pd-csi-sc-zonal.yml} register: gcp_pd_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: GCP PD CSI Driver | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ gcp_pd_csi_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2 ================================================ kind: Deployment apiVersion: apps/v1 metadata: name: csi-gce-pd-controller namespace: kube-system spec: replicas: {{ gcp_pd_csi_controller_replicas }} selector: matchLabels: app: gcp-compute-persistent-disk-csi-driver template: metadata: labels: app: gcp-compute-persistent-disk-csi-driver spec: # Host network must be used for interaction with Workload Identity in GKE # since it replaces GCE Metadata Server with GKE Metadata Server. Remove # this requirement when issue is resolved and before any exposure of # metrics ports hostNetwork: true nodeSelector: kubernetes.io/os: linux serviceAccountName: csi-gce-pd-controller-sa priorityClassName: csi-gce-pd-controller containers: - name: csi-provisioner image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--feature-gates=Topology=true" - "--http-endpoint=:22011" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - "--timeout=250s" - "--extra-create-metadata" # - "--run-controller-service=false" # disable the controller service of the CSI driver # - "--run-node-service=false" # disable the node service of the CSI driver - "--leader-election" - "--default-fstype=ext4" - "--controller-publish-readonly=true" env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace ports: - containerPort: 22011 name: http-endpoint protocol: TCP livenessProbe: failureThreshold: 1 httpGet: path: /healthz/leader-election port: http-endpoint initialDelaySeconds: 10 timeoutSeconds: 10 periodSeconds: 20 volumeMounts: - name: socket-dir mountPath: /csi - name: csi-attacher image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--http-endpoint=:22012" - "--leader-election" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - "--timeout=250s" env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace ports: - containerPort: 22012 name: http-endpoint protocol: TCP livenessProbe: failureThreshold: 1 httpGet: path: /healthz/leader-election port: http-endpoint initialDelaySeconds: 10 timeoutSeconds: 10 periodSeconds: 20 volumeMounts: - name: socket-dir mountPath: /csi - name: csi-resizer image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--http-endpoint=:22013" - "--leader-election" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - "--handle-volume-inuse-error=false" env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace ports: - containerPort: 22013 name: http-endpoint protocol: TCP livenessProbe: failureThreshold: 1 httpGet: path: /healthz/leader-election port: http-endpoint initialDelaySeconds: 10 timeoutSeconds: 10 periodSeconds: 20 volumeMounts: - name: socket-dir mountPath: /csi - name: csi-snapshotter image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--metrics-address=:22014" - "--leader-election" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - "--timeout=300s" env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace volumeMounts: - name: socket-dir mountPath: /csi - name: gce-pd-driver # Don't change base image without changing pdImagePlaceholder in # test/k8s-integration/main.go image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }} args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" env: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/cloud-sa/cloud-sa.json" volumeMounts: - name: socket-dir mountPath: /csi - name: cloud-sa-volume readOnly: true mountPath: "/etc/cloud-sa" volumes: - name: socket-dir emptyDir: {} - name: cloud-sa-volume secret: secretName: cloud-sa --- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: pd.csi.storage.gke.io spec: attachRequired: true podInfoOnMount: false ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2 ================================================ --- kind: Secret apiVersion: v1 metadata: name: cloud-sa namespace: kube-system data: cloud-sa.json: {{ gcp_cred_secret.content }} ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2 ================================================ kind: DaemonSet apiVersion: apps/v1 metadata: name: csi-gce-pd-node namespace: kube-system spec: selector: matchLabels: app: gcp-compute-persistent-disk-csi-driver template: metadata: labels: app: gcp-compute-persistent-disk-csi-driver spec: # Host network must be used for interaction with Workload Identity in GKE # since it replaces GCE Metadata Server with GKE Metadata Server. Remove # this requirement when issue is resolved and before any exposure of # metrics ports. hostNetwork: true priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa containers: - name: csi-driver-registrar image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} args: - "--v=5" - "--csi-address=/csi/csi.sock" - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock" lifecycle: preStop: exec: command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"] env: - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - name: plugin-dir mountPath: /csi - name: registration-dir mountPath: /registration - name: gce-pd-driver securityContext: privileged: true # Don't change base image without changing pdImagePlaceholder in # test/k8s-integration/main.go image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }} args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - "--run-controller-service=false" volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi - name: device-dir mountPath: /dev # The following mounts are required to trigger host udevadm from # container - name: udev-rules-etc mountPath: /etc/udev - name: udev-rules-lib mountPath: /lib/udev - name: udev-socket mountPath: /run/udev - name: sys mountPath: /sys nodeSelector: kubernetes.io/os: linux volumes: - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry/ type: Directory - name: kubelet-dir hostPath: path: /var/lib/kubelet type: Directory - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/ type: DirectoryOrCreate - name: device-dir hostPath: path: /dev type: Directory # The following mounts are required to trigger host udevadm from # container - name: udev-rules-etc hostPath: path: /etc/udev type: Directory - name: udev-rules-lib hostPath: path: /lib/udev type: Directory - name: udev-socket hostPath: path: /run/udev type: Directory - name: sys hostPath: path: /sys type: Directory # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ # See "special case". This will tolerate everything. Node component should # be scheduled on all nodes. tolerations: - operator: Exists ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2 ================================================ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-gce-pd-regional provisioner: pd.csi.storage.gke.io parameters: type: pd-balanced replication-type: regional-pd volumeBindingMode: WaitForFirstConsumer ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2 ================================================ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-gce-pd-zonal provisioner: pd.csi.storage.gke.io parameters: type: pd-balanced volumeBindingMode: WaitForFirstConsumer ================================================ FILE: roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2 ================================================ ##### Node Service Account, Roles, RoleBindings apiVersion: v1 kind: ServiceAccount metadata: name: csi-gce-pd-node-sa namespace: kube-system --- ##### Controller Service Account, Roles, Rolebindings apiVersion: v1 kind: ServiceAccount metadata: name: csi-gce-pd-controller-sa namespace: kube-system --- # xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-provisioner-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] # Access to volumeattachments is only needed when the CSI driver # has the PUBLISH_UNPUBLISH_VOLUME controller capability. # In that case, external-provisioner will watch volumeattachments # to determine when it is safe to delete a volume. - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-controller-provisioner-binding subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-gce-pd-provisioner-role apiGroup: rbac.authorization.k8s.io --- # xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-attacher-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-controller-attacher-binding subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-gce-pd-attacher-role apiGroup: rbac.authorization.k8s.io --- apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: csi-gce-pd-controller value: 900000000 globalDefault: false description: "This priority class should be used for the GCE PD CSI driver controller deployment only." --- apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: csi-gce-pd-node value: 900001000 globalDefault: false description: "This priority class should be used for the GCE PD CSI driver node deployment only." --- # Resizer must be able to work with PVCs, PVs, SCs. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-resizer-role rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] # If handle-volume-inuse-error=true, the pod specific rbac is needed - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-resizer-binding subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-gce-pd-resizer-role apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: csi-gce-pd-controller roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: csi-gce-pd-node-deploy subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: csi-gce-pd-snapshotter-role rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] # Secrets resource omitted since GCE PD snapshots does not require them - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-controller-snapshotter-binding subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-gce-pd-snapshotter-role apiGroup: rbac.authorization.k8s.io --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-leaderelection-role namespace: kube-system labels: k8s-app: gcp-compute-persistent-disk-csi-driver rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-gce-pd-controller-leaderelection-binding namespace: kube-system labels: k8s-app: gcp-compute-persistent-disk-csi-driver subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa namespace: kube-system roleRef: kind: Role name: csi-gce-pd-leaderelection-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml ================================================ --- upcloud_csi_controller_replicas: 1 upcloud_csi_provisioner_image_tag: "v3.1.0" upcloud_csi_attacher_image_tag: "v3.4.0" upcloud_csi_resizer_image_tag: "v1.4.0" upcloud_csi_plugin_image_tag: "v1.1.0" upcloud_csi_node_image_tag: "v2.5.0" upcloud_username: "{{ lookup('env', 'UPCLOUD_USERNAME') }}" upcloud_password: "{{ lookup('env', 'UPCLOUD_PASSWORD') }}" upcloud_tolerations: [] upcloud_csi_enable_volume_snapshot: false upcloud_csi_snapshot_controller_replicas: 2 upcloud_csi_snapshotter_image_tag: "v4.2.1" upcloud_csi_snapshot_controller_image_tag: "v4.2.1" upcloud_csi_snapshot_validation_webhook_image_tag: "v4.2.1" upcloud_cacert: "{{ lookup('env', 'OS_CACERT') }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml ================================================ --- - name: UpCloud CSI Driver | Check if UPCLOUD_USERNAME exists fail: msg: "UpCloud username is missing. Env UPCLOUD_USERNAME is mandatory" when: upcloud_username is not defined or not upcloud_username - name: UpCloud CSI Driver | Check if UPCLOUD_PASSWORD exists fail: msg: "UpCloud password is missing. Env UPCLOUD_PASSWORD is mandatory" when: - upcloud_username is defined - upcloud_username | length > 0 - upcloud_password is not defined or not upcloud_password - name: UpCloud CSI Driver | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: upcloud-csi-cred-secret, file: upcloud-csi-cred-secret.yml} - {name: upcloud-csi-setup, file: upcloud-csi-setup.yml} - {name: upcloud-csi-controller, file: upcloud-csi-controller.yml} - {name: upcloud-csi-node, file: upcloud-csi-node.yml} - {name: upcloud-csi-driver, file: upcloud-csi-driver.yml} register: upcloud_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: UpCloud CSI Driver | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ upcloud_csi_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2 ================================================ kind: StatefulSet apiVersion: apps/v1 metadata: name: csi-upcloud-controller namespace: kube-system spec: serviceName: "csi-upcloud" replicas: {{ upcloud_csi_controller_replicas }} selector: matchLabels: app: csi-upcloud-controller template: metadata: labels: app: csi-upcloud-controller role: csi-upcloud spec: priorityClassName: system-cluster-critical serviceAccount: csi-upcloud-controller-sa containers: - name: csi-provisioner image: registry.k8s.io/sig-storage/csi-provisioner:{{ upcloud_csi_provisioner_image_tag }} args: - "--csi-address=$(ADDRESS)" - "--v=5" - "--timeout=600s" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-attacher image: registry.k8s.io/sig-storage/csi-attacher:{{ upcloud_csi_attacher_image_tag }} args: - "--v=5" - "--csi-address=$(ADDRESS)" - "--timeout=120s" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-resizer image: registry.k8s.io/sig-storage/csi-resizer:{{ upcloud_csi_resizer_image_tag }} args: - "--v=5" - "--timeout=120s" - "--csi-address=$(ADDRESS)" - "--handle-volume-inuse-error=true" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-snapshotter image: k8s.gcr.io/sig-storage/csi-snapshotter:{{ upcloud_csi_snapshotter_image_tag }} args: - "--csi-address=$(ADDRESS)" - "--v=5" - "--timeout=600s" - "--leader-election=false" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-upcloud-plugin image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }} args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodehost=$(NODE_ID)" - "--mode=monolith" env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - name: UPCLOUD_USERNAME valueFrom: secretKeyRef: name: upcloud key: username - name: UPCLOUD_PASSWORD valueFrom: secretKeyRef: name: upcloud key: password - name: NODE_ID valueFrom: fieldRef: fieldPath: spec.nodeName imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ imagePullSecrets: - name: upcloud volumes: - name: socket-dir emptyDir: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2 ================================================ --- apiVersion: v1 kind: Secret metadata: name: upcloud namespace: kube-system stringData: username: {{ upcloud_username }} password: {{ upcloud_password }} ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2 ================================================ apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: storage.csi.upcloud.com spec: attachRequired: true podInfoOnMount: true fsGroupPolicy: File ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2 ================================================ kind: DaemonSet apiVersion: apps/v1 metadata: name: csi-upcloud-node namespace: kube-system spec: selector: matchLabels: app: csi-upcloud-node template: metadata: labels: app: csi-upcloud-node role: csi-upcloud spec: priorityClassName: system-node-critical serviceAccount: csi-upcloud-node-sa hostNetwork: true containers: - name: csi-node-driver-registrar image: registry.k8s.io/sig-storage/csi-node-driver-registrar:{{ upcloud_csi_node_image_tag }} args: - "--v=5" - "--csi-address=$(ADDRESS)" - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/storage.csi.upcloud.com/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - name: plugin-dir mountPath: /csi/ - name: registration-dir mountPath: /registration/ - name: csi-upcloud-plugin image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }} args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodehost=$(NODE_ID)" - "--mode=monolith" env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock - name: UPCLOUD_USERNAME valueFrom: secretKeyRef: name: upcloud key: username - name: UPCLOUD_PASSWORD valueFrom: secretKeyRef: name: upcloud key: password - name: NODE_ID valueFrom: fieldRef: fieldPath: spec.nodeName imagePullPolicy: "Always" securityContext: privileged: true capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true volumeMounts: - name: plugin-dir mountPath: /csi - name: pods-mount-dir mountPath: /var/lib/kubelet # needed so that any mounts setup inside this container are # propagated back to the host machine. mountPropagation: "Bidirectional" - name: device-dir mountPath: /dev imagePullSecrets: - name: upcloud volumes: - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry/ type: DirectoryOrCreate - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/storage.csi.upcloud.com type: DirectoryOrCreate - name: pods-mount-dir hostPath: path: /var/lib/kubelet type: Directory - name: device-dir hostPath: path: /dev {% if upcloud_tolerations %} tolerations: {{ upcloud_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2 ================================================ kind: ServiceAccount apiVersion: v1 metadata: name: csi-upcloud-controller-sa namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: csi-upcloud-node-sa namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-node-driver-registrar-role namespace: kube-system rules: - apiGroups: [ "" ] resources: [ "events" ] verbs: [ "get", "list", "watch", "create", "update", "patch" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-node-driver-registrar-binding subjects: - kind: ServiceAccount name: csi-upcloud-node-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-upcloud-node-driver-registrar-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-provisioner-role rules: - apiGroups: [ "" ] resources: [ "secrets" ] verbs: [ "get", "list" ] - apiGroups: [ "" ] resources: [ "persistentvolumes" ] verbs: [ "get", "list", "watch", "create", "delete" ] - apiGroups: [ "" ] resources: [ "persistentvolumeclaims" ] verbs: [ "get", "list", "watch", "update" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "storageclasses" ] verbs: [ "get", "list", "watch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "csinodes" ] verbs: [ "get", "list", "watch" ] - apiGroups: [ "" ] resources: [ "events" ] verbs: [ "list", "watch", "create", "update", "patch" ] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] verbs: [ "get", "list" ] - apiGroups: [ "" ] resources: [ "nodes" ] verbs: [ "get", "list", "watch" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-provisioner-binding subjects: - kind: ServiceAccount name: csi-upcloud-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-upcloud-provisioner-role apiGroup: rbac.authorization.k8s.io --- # Attacher must be able to work with PVs, nodes and VolumeAttachments kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-attacher-role rules: - apiGroups: [ "" ] resources: [ "persistentvolumes" ] verbs: [ "get", "list", "watch", "update", "patch" ] - apiGroups: [ "" ] resources: [ "nodes" ] verbs: [ "get", "list", "watch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "csinodes" ] verbs: [ "get", "list", "watch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments" ] verbs: [ "get", "list", "watch", "update", "patch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments/status" ] verbs: [ "get", "list", "watch", "update", "patch" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-attacher-binding subjects: - kind: ServiceAccount name: csi-upcloud-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-upcloud-attacher-role apiGroup: rbac.authorization.k8s.io --- # Provisioner must be able to work with endpoints and leases in current namespace # if (and only if) leadership election is enabled kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: namespace: kube-system name: csi-upcloud-provisioner-cfg-role rules: - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "watch", "list", "delete", "update", "create"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-role-cfg-binding namespace: kube-system subjects: - kind: ServiceAccount name: csi-upcloud-controller-sa namespace: kube-system roleRef: kind: Role name: csi-upcloud-provisioner-cfg-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-resizer-role rules: - apiGroups: [ "" ] resources: [ "persistentvolumes" ] verbs: [ "get", "list", "watch", "update", "patch" ] - apiGroups: [ "" ] resources: [ "persistentvolumeclaims" ] verbs: [ "get", "list", "watch" ] - apiGroups: [ "" ] resources: [ "persistentvolumeclaims/status" ] verbs: [ "update", "patch" ] - apiGroups: [ "" ] resources: [ "events" ] verbs: [ "list", "watch", "create", "update", "patch" ] - apiGroups: [ "" ] resources: [ "pods" ] verbs: [ "watch", "list" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-resizer-binding subjects: - kind: ServiceAccount name: csi-upcloud-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-upcloud-resizer-role apiGroup: rbac.authorization.k8s.io --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-snapshotter-role rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-upcloud-snapshotter-binding subjects: - kind: ServiceAccount name: csi-upcloud-controller-sa namespace: kube-system roleRef: kind: ClusterRole name: csi-upcloud-snapshotter-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml ================================================ --- external_vsphere_vcenter_port: "443" external_vsphere_insecure: "true" external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" external_vsphere_version: "7.0u1" vsphere_syncer_image_tag: "v3.3.1" vsphere_csi_attacher_image_tag: "v4.3.0" vsphere_csi_controller: "v3.3.1" vsphere_csi_liveness_probe_image_tag: "v2.10.0" vsphere_csi_provisioner_image_tag: "v3.5.0" vsphere_csi_snapshotter_image_tag: "v6.2.2" vsphere_csi_node_driver_registrar_image_tag: "v2.8.0" vsphere_csi_driver_image_tag: "v3.3.1" vsphere_csi_resizer_tag: "v1.8.0" # Set to kube-system for backward compatibility, should be change to vmware-system-csi on the long run vsphere_csi_namespace: "kube-system" vsphere_csi_controller_replicas: 1 csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}' vsphere_csi_aggressive_node_drain: false vsphere_csi_aggressive_node_unreachable_timeout: 300 vsphere_csi_aggressive_node_not_ready_timeout: 300 vsphere_csi_node_affinity: {} # https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/docs/book/features/volume_snapshot.md#how-to-enable-volume-snapshot--restore-feature-in-vsphere-csi- # according to the above link , we can controler the block-volume-snapshot parameter vsphere_csi_block_volume_snapshot: false external_vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" external_vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" # Controller resources vsphere_csi_snapshotter_resources: {} vsphere_csi_provisioner_resources: {} vsphere_syncer_resources: {} vsphere_csi_liveness_probe_controller_resources: {} vsphere_csi_resources: {} vsphere_csi_resizer_resources: {} vsphere_csi_attacher_resources: {} # DaemonSet node resources vsphere_csi_node_driver_registrar_resources: {} vsphere_csi_driver_resources: {} vsphere_csi_liveness_probe_ds_resources: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml ================================================ --- - name: VSphere CSI Driver | Check vsphare credentials include_tasks: vsphere-credentials-check.yml - name: VSphere CSI Driver | Generate CSI cloud-config template: src: "{{ item }}.j2" dest: "{{ kube_config_dir }}/{{ item }}" mode: "0640" with_items: - vsphere-csi-cloud-config when: inventory_hostname == groups['kube_control_plane'][0] - name: VSphere CSI Driver | Generate Manifests template: src: "{{ item }}.j2" dest: "{{ kube_config_dir }}/{{ item }}" mode: "0644" with_items: - vsphere-csi-namespace.yml - vsphere-csi-driver.yml - vsphere-csi-controller-rbac.yml - vsphere-csi-node-rbac.yml - vsphere-csi-controller-config.yml - vsphere-csi-controller-deployment.yml - vsphere-csi-controller-service.yml - vsphere-csi-node.yml register: vsphere_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: VSphere CSI Driver | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item }}" state: "latest" with_items: - "{{ vsphere_csi_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item }}" - name: VSphere CSI Driver | Generate a CSI secret manifest command: "{{ kubectl }} create secret generic vsphere-config-secret --from-file=csi-vsphere.conf={{ kube_config_dir }}/vsphere-csi-cloud-config -n {{ vsphere_csi_namespace }} --dry-run --save-config -o yaml" register: vsphere_csi_secret_manifest when: inventory_hostname == groups['kube_control_plane'][0] no_log: "{{ not (unsafe_show_logs | bool) }}" - name: VSphere CSI Driver | Apply a CSI secret manifest command: cmd: "{{ kubectl }} apply -f -" stdin: "{{ vsphere_csi_secret_manifest.stdout }}" when: inventory_hostname == groups['kube_control_plane'][0] no_log: "{{ not (unsafe_show_logs | bool) }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml ================================================ --- - name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value fail: msg: "external_vsphere_vcenter_ip is missing" when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip - name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value fail: msg: "external_vsphere_vcenter_port is missing" when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port - name: External vSphere Cloud Provider | check external_vsphere_insecure value fail: msg: "external_vsphere_insecure is missing" when: external_vsphere_insecure is not defined or not external_vsphere_insecure - name: External vSphere Cloud Provider | check external_vsphere_user value fail: msg: "external_vsphere_user is missing" when: external_vsphere_user is not defined or not external_vsphere_user - name: External vSphere Cloud Provider | check external_vsphere_password value fail: msg: "external_vsphere_password is missing" when: - external_vsphere_password is not defined or not external_vsphere_password - name: External vSphere Cloud Provider | check external_vsphere_datacenter value fail: msg: "external_vsphere_datacenter is missing" when: - external_vsphere_datacenter is not defined or not external_vsphere_datacenter - name: External vSphere Cloud Provider | check external_vsphere_kubernetes_cluster_id value fail: msg: "external_vsphere_kubernetes_cluster_id is missing" when: - external_vsphere_kubernetes_cluster_id is not defined or not external_vsphere_kubernetes_cluster_id ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2 ================================================ [Global] cluster-id = "{{ external_vsphere_kubernetes_cluster_id }}" [VirtualCenter "{{ external_vsphere_vcenter_ip }}"] insecure-flag = "{{ external_vsphere_insecure }}" user = "{{ external_vsphere_user }}" password = "{{ external_vsphere_password }}" port = "{{ external_vsphere_vcenter_port }}" datacenters = "{{ external_vsphere_datacenter }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 ================================================ apiVersion: v1 data: {% if external_vsphere_version >= "7.0" %} "csi-auth-check": "true" {% else %} "csi-auth-check": "false" {% endif %} "csi-auth-check": "true" "online-volume-extend": "true" "trigger-csi-fullsync": "false" "async-query-volume": "true" "block-volume-snapshot": "true" "csi-windows-support": "false" "list-volumes": "true" "pv-to-backingdiskobjectid-mapping": "false" "cnsmgr-suspend-create-volume": "true" "topology-preferential-datastores": "true" "max-pvscsi-targets-per-vm": "true" "multi-vcenter-csi-topology": "true" "csi-internal-generated-cluster-id": "true" "listview-tasks": "true" "improved-csi-idempotency": "true" "improved-volume-topology": "true" "use-csinode-id": "true" "list-volumes": "false" kind: ConfigMap metadata: name: internal-feature-states.csi.vsphere.vmware.com namespace: "{{ vsphere_csi_namespace }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 ================================================ kind: Deployment apiVersion: apps/v1 metadata: name: vsphere-csi-controller namespace: "{{ vsphere_csi_namespace }}" spec: replicas: {{ vsphere_csi_controller_replicas }} strategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 maxSurge: 0 selector: matchLabels: app: vsphere-csi-controller template: metadata: labels: app: vsphere-csi-controller role: vsphere-csi spec: priorityClassName: system-cluster-critical # Guarantees scheduling for critical system pods affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: "app" operator: In values: - vsphere-csi-controller topologyKey: "kubernetes.io/hostname" serviceAccountName: vsphere-csi-controller nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - operator: "Exists" key: node-role.kubernetes.io/control-plane effect: NoSchedule {% if vsphere_csi_aggressive_node_drain %} # set below toleration if you need an aggressive pod eviction in case when # node becomes not-ready or unreachable. Default is 300 seconds if not specified. - key: node.kubernetes.io/not-ready operator: Exists effect: NoExecute tolerationSeconds: {{ vsphere_csi_aggressive_node_not_ready_timeout }} - key: node.kubernetes.io/unreachable operator: Exists effect: NoExecute tolerationSeconds: {{ vsphere_csi_aggressive_node_unreachable_timeout }} {% endif %} dnsPolicy: "Default" containers: - name: csi-attacher image: {{ kube_image_repo }}/sig-storage/csi-attacher:{{ vsphere_csi_attacher_image_tag }} args: - "--v=4" - "--timeout=300s" - "--csi-address=$(ADDRESS)" - "--leader-election" - "--leader-election-lease-duration=120s" - "--leader-election-renew-deadline=60s" - "--leader-election-retry-period=30s" - "--kube-api-qps=100" - "--kube-api-burst=100" {% if vsphere_csi_attacher_resources | length > 0 %} resources: {{ vsphere_csi_attacher_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} {% endif %} env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir {% if external_vsphere_version >= "7.0" %} - name: csi-resizer image: {{ kube_image_repo }}/sig-storage/csi-resizer:{{ vsphere_csi_resizer_tag }} args: - "--v=4" - "--timeout=300s" - "--csi-address=$(ADDRESS)" - "--handle-volume-inuse-error=false" - "--kube-api-qps=100" - "--kube-api-burst=100" - "--leader-election" - "--leader-election-lease-duration=120s" - "--leader-election-renew-deadline=60s" - "--leader-election-retry-period=30s" {% if vsphere_csi_resizer_resources | length > 0 %} resources: {{ vsphere_csi_resizer_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} {% endif %} env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir {% endif %} - name: vsphere-csi-controller image: {{ kube_image_repo }}/csi-vsphere/driver:{{ vsphere_csi_controller }} args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace={{ vsphere_csi_namespace }}" {% if vsphere_csi_resources | length > 0 %} resources: {{ vsphere_csi_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} {% endif %} imagePullPolicy: {{ k8s_image_pull_policy }} env: - name: CSI_ENDPOINT value: unix://{{ csi_endpoint }}/csi.sock - name: X_CSI_MODE value: "controller" - name: X_CSI_SPEC_DISABLE_LEN_CHECK value: "true" - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT value: 3m - name: VSPHERE_CSI_CONFIG value: "/etc/cloud/csi-vsphere.conf" - name: LOGGER_LEVEL value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION {% if external_vsphere_version >= "7.0u1" %} - name: INCLUSTER_CLIENT_QPS value: "100" - name: INCLUSTER_CLIENT_BURST value: "100" {% endif %} volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - mountPath: {{ csi_endpoint }} name: socket-dir securityContext: runAsNonRoot: true runAsUser: 65532 runAsGroup: 65532 ports: - name: healthz containerPort: 9808 protocol: TCP - name: prometheus containerPort: 2112 protocol: TCP livenessProbe: httpGet: path: /healthz port: healthz initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 180 failureThreshold: 3 - name: liveness-probe image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} args: - "--v=4" - "--csi-address=$(ADDRESS)" {% if vsphere_csi_liveness_probe_controller_resources | length > 0 %} resources: {{ vsphere_csi_liveness_probe_controller_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} {% endif %} env: - name: ADDRESS value: {{ csi_endpoint }}/csi.sock volumeMounts: - name: socket-dir mountPath: {{ csi_endpoint }} - name: vsphere-syncer image: {{ kube_image_repo }}/csi-vsphere/syncer:{{ vsphere_syncer_image_tag }} args: - "--leader-election" - "--leader-election-lease-duration=30s" - "--leader-election-renew-deadline=20s" - "--leader-election-retry-period=10s" - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace={{ vsphere_csi_namespace }}" imagePullPolicy: {{ k8s_image_pull_policy }} securityContext: runAsNonRoot: true runAsUser: 65532 runAsGroup: 65532 ports: - containerPort: 2113 name: prometheus protocol: TCP {% if vsphere_syncer_resources | length > 0 %} resources: {{ vsphere_syncer_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} {% endif %} env: - name: FULL_SYNC_INTERVAL_MINUTES value: "30" - name: VSPHERE_CSI_CONFIG value: "/etc/cloud/csi-vsphere.conf" - name: LOGGER_LEVEL value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION {% if external_vsphere_version >= "7.0u1" %} - name: INCLUSTER_CLIENT_QPS value: "100" - name: INCLUSTER_CLIENT_BURST value: "100" {% endif %} volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - name: csi-provisioner image: {{ kube_image_repo }}/sig-storage/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }} args: - "--v=4" - "--timeout=300s" - "--csi-address=$(ADDRESS)" - "--kube-api-qps=100" - "--kube-api-burst=100" - "--leader-election" - "--leader-election-lease-duration=120s" - "--leader-election-renew-deadline=60s" - "--leader-election-retry-period=30s" - "--default-fstype=ext4" - "--leader-election" - "--default-fstype=ext4" # needed only for topology aware setup #- "--feature-gates=Topology=true" #- "--strict-topology" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir {% if vsphere_csi_provisioner_resources | length > 0 %} resources: {{ vsphere_csi_provisioner_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} {% endif %} - name: csi-snapshotter image: {{ kube_image_repo }}/sig-storage/csi-snapshotter:{{ vsphere_csi_snapshotter_image_tag }} args: - "--v=4" - "--kube-api-qps=100" - "--kube-api-burst=100" - "--timeout=300s" - "--csi-address=$(ADDRESS)" - "--leader-election" - "--leader-election-lease-duration=120s" - "--leader-election-renew-deadline=60s" - "--leader-election-retry-period=30s" env: - name: ADDRESS value: /csi/csi.sock volumeMounts: - mountPath: /csi name: socket-dir volumes: - name: vsphere-config-volume secret: secretName: vsphere-config-secret - name: socket-dir emptyDir: {} ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 ================================================ kind: ServiceAccount apiVersion: v1 metadata: name: vsphere-csi-controller namespace: "{{ vsphere_csi_namespace }}" --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vsphere-csi-controller-role rules: - apiGroups: [""] resources: ["nodes", "pods"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] {% if external_vsphere_version >= "7.0" %} - apiGroups: [""] resources: ["persistentvolumeclaims/status"] {% if external_vsphere_version >= "7.0u1" %} verbs: ["patch"] {% else %} verbs: ["update", "patch"] {% endif %} {% endif %} - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses","csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "patch", "update"] - apiGroups: ["cns.vmware.com"] resources: ["triggercsifullsyncs"] verbs: ["create", "get", "update", "watch", "list"] - apiGroups: ["cns.vmware.com"] resources: ["cnsvspherevolumemigrations"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "create", "update"] - apiGroups: ["cns.vmware.com"] resources: ["cnsvolumeoperationrequests"] verbs: ["create", "get", "list", "update", "delete"] - apiGroups: [ "cns.vmware.com" ] resources: [ "csinodetopologies" ] verbs: ["get", "update", "watch", "list"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotcontents" ] verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotcontents/status" ] verbs: [ "update", "patch" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vsphere-csi-controller-binding subjects: - kind: ServiceAccount name: vsphere-csi-controller namespace: "{{ vsphere_csi_namespace }}" roleRef: kind: ClusterRole name: vsphere-csi-controller-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2 ================================================ apiVersion: v1 kind: Service metadata: name: vsphere-csi-controller namespace: "{{ vsphere_csi_namespace }}" labels: app: vsphere-csi-controller spec: ports: - name: ctlr port: 2112 targetPort: 2112 protocol: TCP - name: syncer port: 2113 targetPort: 2113 protocol: TCP selector: app: vsphere-csi-controller ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2 ================================================ apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.vsphere.vmware.com spec: attachRequired: true podInfoOnMount: false ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2 ================================================ apiVersion: v1 kind: Namespace metadata: name: "{{ vsphere_csi_namespace }}" ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2 ================================================ --- kind: ServiceAccount apiVersion: v1 metadata: name: vsphere-csi-node namespace: "{{ vsphere_csi_namespace }}" --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vsphere-csi-node-cluster-role rules: - apiGroups: ["cns.vmware.com"] resources: ["csinodetopologies"] verbs: ["create", "watch", "get", "patch" ] - apiGroups: [""] resources: ["nodes"] verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vsphere-csi-node-cluster-role-binding subjects: - kind: ServiceAccount name: vsphere-csi-node namespace: "{{ vsphere_csi_namespace }}" roleRef: kind: ClusterRole name: vsphere-csi-node-cluster-role apiGroup: rbac.authorization.k8s.io --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vsphere-csi-node-role namespace: "{{ vsphere_csi_namespace }}" rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: vsphere-csi-node-binding namespace: "{{ vsphere_csi_namespace }}" subjects: - kind: ServiceAccount name: vsphere-csi-node namespace: "{{ vsphere_csi_namespace }}" roleRef: kind: Role name: vsphere-csi-node-role apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 ================================================ kind: DaemonSet apiVersion: apps/v1 metadata: name: vsphere-csi-node namespace: "{{ vsphere_csi_namespace }}" spec: selector: matchLabels: app: vsphere-csi-node updateStrategy: type: "RollingUpdate" rollingUpdate: maxUnavailable: 1 template: metadata: labels: app: vsphere-csi-node role: vsphere-csi spec: priorityClassName: system-node-critical nodeSelector: kubernetes.io/os: linux {% if vsphere_csi_node_affinity %} affinity: {{ vsphere_csi_node_affinity | to_nice_yaml | indent(width=8) }} {% endif %} serviceAccountName: vsphere-csi-node hostNetwork: true dnsPolicy: "ClusterFirstWithHostNet" containers: - name: node-driver-registrar image: {{ kube_image_repo }}/sig-storage/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }} {% if external_vsphere_version < "7.0u1" %} lifecycle: preStop: exec: command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock"] {% endif %} args: - "--v=5" - "--csi-address=$(ADDRESS)" - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" {% if vsphere_csi_node_driver_registrar_resources | length > 0 %} resources: {{ vsphere_csi_node_driver_registrar_resources | default({}) | to_nice_yaml | trim | indent(width=10) }} {% endif %} env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock volumeMounts: - name: plugin-dir mountPath: /csi - name: registration-dir mountPath: /registration livenessProbe: exec: command: - /csi-node-driver-registrar - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - --mode=kubelet-registration-probe initialDelaySeconds: 3 - name: vsphere-csi-node image: {{ kube_image_repo }}/csi-vsphere/driver:{{ vsphere_csi_driver_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace={{ vsphere_csi_namespace }}" imagePullPolicy: "Always" {% if vsphere_csi_driver_resources | length > 0 %} resources: {{ vsphere_csi_driver_resources | default({}) | to_nice_yaml | trim | indent(width=10) }} {% endif %} env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock - name: MAX_VOLUMES_PER_NODE value: "59" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node. - name: X_CSI_MODE value: "node" - name: X_CSI_SPEC_REQ_VALIDATION value: "false" - name: X_CSI_DEBUG value: "true" - name: X_CSI_SPEC_DISABLE_LEN_CHECK value: "true" - name: LOGGER_LEVEL value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - name: GODEBUG value: x509sha1=1 - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" securityContext: privileged: true capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true volumeMounts: - name: plugin-dir mountPath: /csi - name: pods-mount-dir mountPath: /var/lib/kubelet # needed so that any mounts setup inside this container are # propagated back to the host machine. mountPropagation: "Bidirectional" - name: device-dir mountPath: /dev - name: blocks-dir mountPath: /sys/block - name: sys-devices-dir mountPath: /sys/devices ports: - containerPort: 9808 name: healthz livenessProbe: httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 5 periodSeconds: 5 failureThreshold: 3 - name: liveness-probe image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} args: {% if external_vsphere_version >= "7.0u1" %} - "--v=4" {% endif %} - "--csi-address=/csi/csi.sock" {% if vsphere_csi_liveness_probe_ds_resources | length > 0 %} resources: {{ vsphere_csi_liveness_probe_ds_resources | default({}) | to_nice_yaml | trim | indent(width=10) }} {% endif %} volumeMounts: - name: plugin-dir mountPath: /csi volumes: - name: registration-dir hostPath: path: /var/lib/kubelet/plugins_registry type: Directory - name: plugin-dir hostPath: path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com type: DirectoryOrCreate - name: pods-mount-dir hostPath: path: /var/lib/kubelet type: Directory - name: device-dir hostPath: path: /dev - name: blocks-dir hostPath: path: /sys/block type: Directory - name: sys-devices-dir hostPath: path: /sys/devices type: Directory tolerations: - effect: NoExecute operator: Exists - effect: NoSchedule operator: Exists ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml ================================================ --- external_hcloud_cloud: hcloud_api_token: "" token_secret_name: hcloud service_account_name: cloud-controller-manager controller_image_tag: "latest" ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset ## Format: ## external_hcloud_cloud.controller_extra_args: ## arg1: "value1" ## arg2: "value2" controller_extra_args: {} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml ================================================ --- - name: External Hcloud Cloud Controller | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" group: "{{ kube_cert_group }}" mode: "0640" with_items: - {name: external-hcloud-cloud-secret, file: external-hcloud-cloud-secret.yml} - {name: external-hcloud-cloud-service-account, file: external-hcloud-cloud-service-account.yml} - {name: external-hcloud-cloud-role-bindings, file: external-hcloud-cloud-role-bindings.yml} - {name: "{{ 'external-hcloud-cloud-controller-manager-ds-with-networks' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds' }}", file: "{{ 'external-hcloud-cloud-controller-manager-ds-with-networks.yml' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds.yml' }}"} register: external_hcloud_manifests when: inventory_hostname == groups['kube_control_plane'][0] tags: external-hcloud - name: External Hcloud Cloud Controller | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ external_hcloud_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" tags: external-hcloud ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 ================================================ --- apiVersion: apps/v1 kind: DaemonSet metadata: name: hcloud-cloud-controller-manager namespace: kube-system labels: k8s-app: hcloud-cloud-controller-manger spec: selector: matchLabels: app: hcloud-cloud-controller-manager template: metadata: labels: app: hcloud-cloud-controller-manager annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: {{ external_hcloud_cloud.service_account_name }} dnsPolicy: Default tolerations: - key: "node.cloudprovider.kubernetes.io/uninitialized" value: "true" effect: "NoSchedule" - key: "CriticalAddonsOnly" operator: "Exists" - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule operator: Exists - key: "node.kubernetes.io/not-ready" effect: "NoSchedule" hostNetwork: true containers: - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }} name: hcloud-cloud-controller-manager command: - "/bin/hcloud-cloud-controller-manager" - "--cloud-provider=hcloud" - "--leader-elect=false" - "--allow-untagged-cloud" - "--allocate-node-cidrs=true" - "--cluster-cidr={{ kube_pods_subnet }}" {% if external_hcloud_cloud.controller_extra_args is defined %} args: {% for key, value in external_hcloud_cloud.controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} {% endif %} resources: requests: cpu: 100m memory: 50Mi env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: HCLOUD_TOKEN valueFrom: secretKeyRef: name: {{ external_hcloud_cloud.token_secret_name }} key: token - name: HCLOUD_NETWORK valueFrom: secretKeyRef: name: {{ external_hcloud_cloud.token_secret_name }} key: network {% if external_hcloud_cloud.network_routes_enabled is defined %} - name: HCLOUD_NETWORK_ROUTES_ENABLED value: "{{ external_hcloud_cloud.network_routes_enabled }}" {% endif %} {% if external_hcloud_cloud.load_balancers_location is defined %} - name: HCLOUD_LOAD_BALANCERS_LOCATION value: "{{ external_hcloud_cloud.load_balancers_location }}" {% endif %} {% if external_hcloud_cloud.load_balancers_network_zone is defined %} - name: HCLOUD_LOAD_BALANCERS_NETWORK_ZONE value: "{{ external_hcloud_cloud.load_balancers_network_zone }}" {% endif %} {% if external_hcloud_cloud.load_balancers_disable_private_ingress is defined %} - name: HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS value: "{{ external_hcloud_cloud.load_balancers_disable_private_ingress }}" {% endif %} {% if external_hcloud_cloud.load_balancers_use_private_ip is defined %} - name: HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP value: "{{ external_hcloud_cloud.load_balancers_use_private_ip }}" {% endif %} {% if external_hcloud_cloud.load_balancers_enabled is defined %} - name: HCLOUD_LOAD_BALANCERS_ENABLED value: "{{ external_hcloud_cloud.load_balancers_enabled }}" {% endif %} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 ================================================ --- apiVersion: apps/v1 kind: DaemonSet metadata: name: hcloud-cloud-controller-manager namespace: kube-system labels: k8s-app: hcloud-cloud-controller-manger spec: selector: matchLabels: app: hcloud-cloud-controller-manager updateStrategy: type: RollingUpdate template: metadata: labels: app: hcloud-cloud-controller-manager annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: {{ external_hcloud_cloud.service_account_name }} dnsPolicy: Default tolerations: - key: "node.cloudprovider.kubernetes.io/uninitialized" value: "true" effect: "NoSchedule" - key: "CriticalAddonsOnly" operator: "Exists" - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule - key: "node.kubernetes.io/not-ready" effect: "NoSchedule" containers: - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }} name: hcloud-cloud-controller-manager command: - "/bin/hcloud-cloud-controller-manager" - "--cloud-provider=hcloud" - "--leader-elect=false" - "--allow-untagged-cloud" {% if external_hcloud_cloud.controller_extra_args is defined %} args: {% for key, value in external_hcloud_cloud.controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} {% endif %} resources: requests: cpu: 100m memory: 50Mi env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: HCLOUD_TOKEN valueFrom: secretKeyRef: name: {{ external_hcloud_cloud.token_secret_name }} key: token {% if external_hcloud_cloud.network_name is defined %} - name: HCLOUD_NETWORK valueFrom: secretKeyRef: name: {{ external_hcloud_cloud.token_secret_name }} key: network {% endif %} {% if external_hcloud_cloud.network_routes_enabled is defined %} - name: HCLOUD_NETWORK_ROUTES_ENABLED value: "{{ external_hcloud_cloud.network_routes_enabled }}" {% endif %} {% if external_hcloud_cloud.load_balancers_location is defined %} - name: HCLOUD_LOAD_BALANCERS_LOCATION value: "{{ external_hcloud_cloud.load_balancers_location }}" {% endif %} {% if external_hcloud_cloud.load_balancers_network_zone is defined %} - name: HCLOUD_LOAD_BALANCERS_NETWORK_ZONE value: "{{ external_hcloud_cloud.load_balancers_network_zone }}" {% endif %} {% if external_hcloud_cloud.load_balancers_disable_private_ingress is defined %} - name: HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS value: "{{ external_hcloud_cloud.load_balancers_disable_private_ingress }}" {% endif %} {% if external_hcloud_cloud.load_balancers_use_private_ip is defined %} - name: HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP value: "{{ external_hcloud_cloud.load_balancers_use_private_ip }}" {% endif %} {% if external_hcloud_cloud.load_balancers_enabled is defined %} - name: HCLOUD_LOAD_BALANCERS_ENABLED value: "{{ external_hcloud_cloud.load_balancers_enabled }}" {% endif %} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 ================================================ --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: {{ external_hcloud_cloud.service_account_name }} namespace: kube-system ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 ================================================ --- apiVersion: v1 kind: Secret metadata: name: "{{ external_hcloud_cloud.token_secret_name }}" namespace: kube-system data: token: "{{ external_hcloud_cloud.hcloud_api_token | b64encode }}" {% if external_hcloud_cloud.with_networks or external_hcloud_cloud.network_name is defined %} {% if network_id is defined%} network: "{{ network_id | b64encode }}" {% else %} network: "{{ external_hcloud_cloud.network_name | b64encode }}" {% endif %} {% endif %} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: {{ external_hcloud_cloud.service_account_name }} namespace: kube-system ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml ================================================ --- # The external cloud controller will need credentials to access # openstack apis. Per default these values will be # read from the environment. external_huaweicloud_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}" external_huaweicloud_access_key: "{{ lookup('env', 'OS_ACCESS_KEY') }}" external_huaweicloud_secret_key: "{{ lookup('env', 'OS_SECRET_KEY') }}" external_huaweicloud_region: "{{ lookup('env', 'OS_REGION_NAME') }}" external_huaweicloud_project_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}" external_huaweicloud_cloud: "{{ lookup('env', 'OS_CLOUD') }}" ## A dictionary of extra arguments to add to the huawei cloud controller manager deployment ## Format: ## external_huawei_cloud_controller_extra_args: ## arg1: "value1" ## arg2: "value2" external_huawei_cloud_controller_extra_args: {} external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" external_huawei_cloud_controller_image_tag: "v0.26.8" ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml ================================================ --- - name: External Huawei Cloud Controller | check external_huaweicloud_auth_url value fail: msg: "external_huaweicloud_auth_url is missing" when: external_huaweicloud_auth_url is not defined or not external_huaweicloud_auth_url - name: External Huawei Cloud Controller | check external_huaweicloud_access_key value fail: msg: "you must set external_huaweicloud_access_key" when: - external_huaweicloud_access_key is not defined or not external_huaweicloud_access_key - name: External Huawei Cloud Controller | check external_huaweicloud_secret_key value fail: msg: "external_huaweicloud_secret_key is missing" when: - external_huaweicloud_access_key is defined - external_huaweicloud_access_key|length > 0 - external_huaweicloud_secret_key is not defined or not external_huaweicloud_secret_key - name: External Huawei Cloud Controller | check external_huaweicloud_region value fail: msg: "external_huaweicloud_region is missing" when: external_huaweicloud_region is not defined or not external_huaweicloud_region - name: External Huawei Cloud Controller | check external_huaweicloud_project_id value fail: msg: "one of external_huaweicloud_project_id must be specified" when: - external_huaweicloud_project_id is not defined or not external_huaweicloud_project_id ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml ================================================ --- - name: External Huawei Cloud Controller | Check Huawei credentials include_tasks: huaweicloud-credential-check.yml tags: external-huaweicloud - name: External huaweicloud Cloud Controller | Get base64 cacert slurp: src: "{{ external_huaweicloud_cacert }}" register: external_huaweicloud_cacert_b64 when: - inventory_hostname == groups['kube_control_plane'][0] - external_huaweicloud_cacert is defined - external_huaweicloud_cacert | length > 0 tags: external-huaweicloud - name: External huaweicloud Cloud Controller | Get base64 cloud-config set_fact: external_huawei_cloud_config_secret: "{{ lookup('template', 'external-huawei-cloud-config.j2') | b64encode }}" when: inventory_hostname == groups['kube_control_plane'][0] tags: external-huaweicloud - name: External Huawei Cloud Controller | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" group: "{{ kube_cert_group }}" mode: "0640" with_items: - {name: external-huawei-cloud-config-secret, file: external-huawei-cloud-config-secret.yml} - {name: external-huawei-cloud-controller-manager-roles, file: external-huawei-cloud-controller-manager-roles.yml} - {name: external-huawei-cloud-controller-manager-role-bindings, file: external-huawei-cloud-controller-manager-role-bindings.yml} - {name: external-huawei-cloud-controller-manager-ds, file: external-huawei-cloud-controller-manager-ds.yml} register: external_huaweicloud_manifests when: inventory_hostname == groups['kube_control_plane'][0] tags: external-huaweicloud - name: External Huawei Cloud Controller | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ external_huaweicloud_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" tags: external-huaweicloud ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 ================================================ # This YAML file contains secret objects, # which are necessary to run external huaweicloud cloud controller. kind: Secret apiVersion: v1 metadata: name: external-huawei-cloud-config namespace: kube-system data: cloud-config: {{ external_huawei_cloud_config_secret }} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 ================================================ [Global] auth-url="{{ external_huaweicloud_auth_url }}" {% if external_huaweicloud_access_key is defined and external_huaweicloud_access_key != "" %} access-key={{ external_huaweicloud_access_key }} {% endif %} {% if external_huaweicloud_secret_key is defined and external_huaweicloud_secret_key != "" %} secret-key={{ external_huaweicloud_secret_key }} {% endif %} region="{{ external_huaweicloud_region }}" {% if external_huaweicloud_project_id is defined and external_huaweicloud_project_id != "" %} project-id="{{ external_huaweicloud_project_id }}" {% endif %} {% if external_huaweicloud_cloud is defined and external_huaweicloud_cloud != "" %} cloud="{{ external_huaweicloud_cloud }}" {% endif %} [VPC] {% if external_huaweicloud_lbaas_subnet_id is defined %} subnet-id={{ external_huaweicloud_lbaas_subnet_id }} {% endif %} {% if external_huaweicloud_lbaas_network_id is defined %} id={{ external_huaweicloud_lbaas_network_id }} {% endif %} {% if external_huaweicloud_security_group_id is defined %} security-group-id={{ external_huaweicloud_security_group_id }} {% endif %} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 ================================================ kind: Namespace apiVersion: v1 metadata: name: huawei-cloud-provider --- apiVersion: v1 kind: ServiceAccount metadata: name: cloud-controller-manager namespace: kube-system --- apiVersion: apps/v1 kind: DaemonSet metadata: name: huawei-cloud-controller-manager namespace: kube-system labels: k8s-app: huawei-cloud-controller-manager spec: selector: matchLabels: k8s-app: huawei-cloud-controller-manager updateStrategy: type: RollingUpdate template: metadata: labels: k8s-app: huawei-cloud-controller-manager spec: nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: runAsUser: 1001 tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule serviceAccountName: cloud-controller-manager containers: - name: huawei-cloud-controller-manager image: {{ external_huawei_cloud_controller_image_repo }}/k8s-cloudprovider/huawei-cloud-controller-manager:{{ external_huawei_cloud_controller_image_tag }} args: - /bin/huawei-cloud-controller-manager - --v=1 - --cloud-config=$(CLOUD_CONFIG) - --cloud-provider=huaweicloud - --use-service-account-credentials=true - --node-status-update-frequency=5s - --node-monitor-period=5s - --leader-elect-lease-duration=30s - --leader-elect-renew-deadline=20s - --leader-elect-retry-period=2s {% for key, value in external_huawei_cloud_controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} volumeMounts: - mountPath: /etc/kubernetes name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs name: ca-certs readOnly: true - mountPath: /etc/config name: cloud-config-volume readOnly: true {% if kubelet_flexvolumes_plugins_dir is defined %} - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec name: flexvolume-dir {% endif %} resources: requests: cpu: 200m env: - name: CLOUD_CONFIG value: /etc/config/cloud-config hostNetwork: true volumes: {% if kubelet_flexvolumes_plugins_dir is defined %} - name: flexvolume-dir hostPath: path: "{{ kubelet_flexvolumes_plugins_dir }}" type: DirectoryOrCreate {% endif %} - name: k8s-certs hostPath: path: /etc/kubernetes type: DirectoryOrCreate - name: ca-certs hostPath: path: /etc/ssl/certs type: DirectoryOrCreate - name: cloud-config-volume secret: secretName: external-huawei-cloud-config ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 ================================================ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - kind: ServiceAccount name: cloud-controller-manager namespace: kube-system ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:cloud-controller-manager rules: - resources: - tokenreviews verbs: - get - list - watch - create - update - patch apiGroups: - authentication.k8s.io - resources: - configmaps - endpoints - pods - services - secrets - serviceaccounts - serviceaccounts/token verbs: - get - list - watch - create - update - patch apiGroups: - '' - resources: - nodes verbs: - get - list - watch - delete - patch - update apiGroups: - '' - resources: - services/status - pods/status verbs: - update - patch apiGroups: - '' - resources: - nodes/status verbs: - patch - update apiGroups: - '' - resources: - events - endpoints verbs: - create - patch - update apiGroups: - '' - resources: - leases verbs: - get - update - create - delete apiGroups: - coordination.k8s.io - resources: - customresourcedefinitions verbs: - get - update - create - delete apiGroups: - apiextensions.k8s.io - resources: - ingresses verbs: - get - list - watch - update - create - patch - delete apiGroups: - networking.k8s.io - resources: - ingresses/status verbs: - update - patch apiGroups: - networking.k8s.io - resources: - endpointslices verbs: - get - list - watch apiGroups: - discovery.k8s.io ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/external_cloud_controller/openstack when: - cloud_provider == "external" - external_cloud_provider == "openstack" - inventory_hostname == groups['kube_control_plane'][0] tags: - external-cloud-controller - external-openstack - role: kubernetes-apps/external_cloud_controller/vsphere when: - cloud_provider == "external" - external_cloud_provider == "vsphere" - inventory_hostname == groups['kube_control_plane'][0] tags: - external-cloud-controller - external-vsphere - role: kubernetes-apps/external_cloud_controller/hcloud when: - cloud_provider == "external" - external_cloud_provider == "hcloud" - inventory_hostname == groups['kube_control_plane'][0] tags: - external-cloud-controller - external-hcloud - role: kubernetes-apps/external_cloud_controller/huaweicloud when: - cloud_provider == "external" - external_cloud_provider == "huaweicloud" - inventory_hostname == groups['kube_control_plane'][0] tags: - external-cloud-controller - external-huaweicloud - role: kubernetes-apps/external_cloud_controller/oci when: - cloud_provider == "external" - external_cloud_provider == "oci" - inventory_hostname == groups['kube_control_plane'][0] tags: - external-cloud-controller - external-oci ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml ================================================ --- ## External Oracle Cloud Controller Manager ## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml external_oracle_auth_region: "" external_oracle_auth_tenancy: "" external_oracle_auth_user: "" external_oracle_auth_key: "" external_oracle_auth_passphrase: "" external_oracle_auth_fingerprint: "" external_oracle_auth_use_instance_principals: false external_oracle_compartment: "" external_oracle_vcn: "" external_oracle_load_balancer_subnet1: "" external_oracle_load_balancer_subnet2: "" external_oracle_load_balancer_security_list_management_mode: All external_oracle_load_balancer_security_lists: {} external_oracle_ratelimiter_qps_read: 20.0 external_oracle_ratelimiter_bucket_read: 5 external_oracle_ratelimiter_qps_write: 20.0 external_oracle_ratelimiter_bucket_write: 5 external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci external_oracle_cloud_controller_image_tag: "v1.29.0" ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml ================================================ --- - name: "External OCI Cloud Controller Manager | Check credentials" ansible.builtin.assert: that: - external_oracle_auth_key | length > 0 - external_oracle_auth_region | length > 0 - external_oracle_auth_tenancy | length > 0 - external_oracle_auth_user | length > 0 - external_oracle_auth_fingerprint | length > 0 when: not external_oracle_auth_use_instance_principals - name: "External OCI Cloud Controller Manager | Check settings" ansible.builtin.assert: that: - external_oracle_compartment | length > 0 - external_oracle_vcn | length > 0 - external_oracle_load_balancer_subnet1 | length > 0 - external_oracle_load_balancer_subnet2 | length > 0 - external_oracle_load_balancer_security_list_management_mode in ["All", "Frontend", "None"] - name: External OCI Cloud Controller Manager | Get base64 cloud-config set_fact: external_oracle_cloud_config_secret: "{{ lookup('template', 'external-oci-cloud-config.j2') | b64encode }}" when: inventory_hostname == groups['kube_control_plane'][0] tags: external-oci - name: External OCI Cloud Controller Manager | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" group: "{{ kube_cert_group }}" mode: "0640" with_items: - {name: external-oci-cloud-config-secret, file: external-oci-cloud-config-secret.yml} - {name: external-oci-cloud-controller-manager-rbac, file: external-oci-cloud-controller-manager-rbac.yml} - {name: external-oci-cloud-controller-manager, file: external-oci-cloud-controller-manager.yml} register: external_oracle_manifests when: inventory_hostname == groups['kube_control_plane'][0] tags: external-oci - name: External OCI Cloud Controller Manager | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ external_oracle_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" tags: external-oci ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 ================================================ # This YAML file contains secret objects, # which are necessary to run external oci cloud controller. kind: Secret apiVersion: v1 metadata: name: oci-cloud-controller-manager namespace: kube-system data: cloud-provider.yaml: {{ external_oracle_cloud_config_secret }} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 ================================================ {% if external_oracle_auth_use_instance_principals %} useInstancePrincipals: true {% endif %} auth: {% if external_oracle_auth_use_instance_principals %} useInstancePrincipals: true {% else %} useInstancePrincipals: false region: {{ external_oracle_auth_region }} tenancy: {{ external_oracle_auth_tenancy }} user: {{ external_oracle_auth_user }} key: | {{ external_oracle_auth_key }} {% if external_oracle_auth_passphrase is defined %} # Omit if there is not a password for the key passphrase: {{ external_oracle_auth_passphrase }} {% endif %} fingerprint: {{ external_oracle_auth_fingerprint }} {% endif %} compartment: {{ external_oracle_compartment }} vcn: {{ external_oracle_vcn }} loadBalancer: subnet1: {{ external_oracle_load_balancer_subnet1 }} subnet2: {{ external_oracle_load_balancer_subnet2 }} securityListManagementMode: {{ external_oracle_load_balancer_security_list_management_mode }} {% if external_oracle_security_lists is defined and external_oracle_security_lists | length > 0 %} # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off. securityLists: {% for subnet_ocid, list_ocid in external_oracle_load_balancer_security_lists.items() %} {{ subnet_ocid }}: {{ list_ocid }} {% endfor %} {% endif %} # Optional rate limit controls for accessing OCI API rateLimiter: rateLimitQPSRead: {{ external_oracle_ratelimiter_qps_read }} rateLimitBucketRead: {{ external_oracle_ratelimiter_bucket_read }} rateLimitQPSWrite: {{ external_oracle_ratelimiter_qps_write }} rateLimitBucketWrite: {{ external_oracle_ratelimiter_bucket_write }} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: cloud-controller-manager namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:cloud-controller-manager labels: kubernetes.io/cluster-service: "true" rules: - apiGroups: - "" resources: - nodes verbs: - '*' - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - "" resources: - services verbs: - list - watch - patch - get - apiGroups: - "" resources: - services/status verbs: - patch - get - update - apiGroups: - "" resources: - configmaps resourceNames: - "extension-apiserver-authentication" verbs: - get - apiGroups: - "" resources: - events verbs: - list - watch - create - patch - update # For leader election - apiGroups: - "" resources: - endpoints verbs: - create - apiGroups: - "" resources: - endpoints resourceNames: - "cloud-controller-manager" verbs: - get - list - watch - update - apiGroups: - "" resources: - configmaps verbs: - create - apiGroups: - "coordination.k8s.io" resources: - leases verbs: - get - create - update - delete - patch - watch - apiGroups: - "" resources: - configmaps resourceNames: - "cloud-controller-manager" verbs: - get - update - apiGroups: - "" resources: - configmaps resourceNames: - "extension-apiserver-authentication" verbs: - get - list - watch - apiGroups: - "" resources: - serviceaccounts verbs: - create - list - get - watch - apiGroups: - "" resources: - secrets verbs: - get - list # For the PVL - apiGroups: - "" resources: - persistentvolumes verbs: - list - watch - patch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: oci-cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - kind: ServiceAccount name: cloud-controller-manager namespace: kube-system ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 ================================================ --- apiVersion: apps/v1 kind: DaemonSet metadata: name: oci-cloud-controller-manager namespace: kube-system labels: k8s-app: oci-cloud-controller-manager spec: selector: matchLabels: component: oci-cloud-controller-manager tier: control-plane updateStrategy: type: RollingUpdate template: metadata: labels: component: oci-cloud-controller-manager tier: control-plane spec: serviceAccountName: cloud-controller-manager hostNetwork: true nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule - key: node-role.kubernetes.io/master operator: Exists effect: NoSchedule volumes: - name: cfg secret: secretName: oci-cloud-controller-manager - name: kubernetes hostPath: path: /etc/kubernetes containers: - name: oci-cloud-controller-manager image: {{ external_oracle_cloud_controller_image_repo }}:{{ external_oracle_cloud_controller_image_tag }} command: ["/usr/local/bin/oci-cloud-controller-manager"] args: - --cloud-config=/etc/oci/cloud-provider.yaml - --cloud-provider=oci - --leader-elect-resource-lock=leases - --concurrent-service-syncs=3 - --v=2 volumeMounts: - name: cfg mountPath: /etc/oci readOnly: true - name: kubernetes mountPath: /etc/kubernetes readOnly: true ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml ================================================ --- # The external cloud controller will need credentials to access # openstack apis. Per default these values will be # read from the environment. external_openstack_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}" external_openstack_username: "{{ lookup('env', 'OS_USERNAME') }}" external_openstack_password: "{{ lookup('env', 'OS_PASSWORD') }}" external_openstack_application_credential_id: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}" external_openstack_application_credential_name: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}" external_openstack_application_credential_secret: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" external_openstack_region: "{{ lookup('env', 'OS_REGION_NAME') }}" external_openstack_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}" external_openstack_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}" external_openstack_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}" external_openstack_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}" external_openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}" ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset ## Format: ## external_openstack_cloud_controller_extra_args: ## arg1: "value1" ## arg2: "value2" external_openstack_cloud_controller_extra_args: {} external_openstack_cloud_controller_image_tag: "v1.35.0" external_openstack_cloud_controller_bind_address: 127.0.0.1 external_openstack_cloud_controller_dns_policy: ClusterFirst external_openstack_lbaas_member_subnet_id: "{{ external_openstack_lbaas_subnet_id }}" ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml ================================================ --- - name: External OpenStack Cloud Controller | Check OpenStack credentials include_tasks: openstack-credential-check.yml tags: external-openstack - name: External OpenStack Cloud Controller | Get base64 cacert slurp: src: "{{ external_openstack_cacert }}" register: external_openstack_cacert_b64 when: - inventory_hostname == groups['kube_control_plane'][0] - external_openstack_cacert is defined - external_openstack_cacert | length > 0 tags: external-openstack - name: External OpenStack Cloud Controller | Get base64 cloud-config set_fact: external_openstack_cloud_config_secret: "{{ lookup('template', 'external-openstack-cloud-config.j2') | b64encode }}" when: inventory_hostname == groups['kube_control_plane'][0] tags: external-openstack - name: External OpenStack Cloud Controller | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" group: "{{ kube_cert_group }}" mode: "0640" with_items: - {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml} - {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml} - {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml} - {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml} register: external_openstack_manifests when: inventory_hostname == groups['kube_control_plane'][0] tags: external-openstack - name: External OpenStack Cloud Controller | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ external_openstack_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" tags: external-openstack ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml ================================================ --- - name: External OpenStack Cloud Controller | check external_openstack_auth_url value fail: msg: "external_openstack_auth_url is missing" when: external_openstack_auth_url is not defined or not external_openstack_auth_url - name: External OpenStack Cloud Controller | check external_openstack_username or external_openstack_application_credential_name value fail: msg: "you must either set external_openstack_username or external_openstack_application_credential_name" when: - external_openstack_username is not defined or not external_openstack_username - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name - name: External OpenStack Cloud Controller | check external_openstack_application_credential_id value fail: msg: "external_openstack_application_credential_id is missing" when: - external_openstack_application_credential_name is defined - external_openstack_application_credential_name | length > 0 - external_openstack_application_credential_id is not defined or not external_openstack_application_credential_id - name: External OpenStack Cloud Controller | check external_openstack_application_credential_secret value fail: msg: "external_openstack_application_credential_secret is missing" when: - external_openstack_application_credential_name is defined - external_openstack_application_credential_name | length > 0 - external_openstack_application_credential_secret is not defined or not external_openstack_application_credential_secret - name: External OpenStack Cloud Controller | check external_openstack_password value fail: msg: "external_openstack_password is missing" when: - external_openstack_username is defined - external_openstack_username | length > 0 - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name - external_openstack_application_credential_secret is not defined or not external_openstack_application_credential_secret - external_openstack_password is not defined or not external_openstack_password - name: External OpenStack Cloud Controller | check external_openstack_region value fail: msg: "external_openstack_region is missing" when: external_openstack_region is not defined or not external_openstack_region - name: External OpenStack Cloud Controller | check external_openstack_tenant_id value fail: msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified" when: - external_openstack_tenant_id is not defined or not external_openstack_tenant_id - external_openstack_tenant_name is not defined or not external_openstack_tenant_name - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name - name: External OpenStack Cloud Controller | check external_openstack_domain_id value fail: msg: "one of external_openstack_domain_id or external_openstack_domain_name must be specified" when: - external_openstack_domain_id is not defined or not external_openstack_domain_id - external_openstack_domain_name is not defined or not external_openstack_domain_name - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2 ================================================ # This YAML file contains secret objects, # which are necessary to run external openstack cloud controller. kind: Secret apiVersion: v1 metadata: name: external-openstack-cloud-config namespace: kube-system data: cloud.conf: {{ external_openstack_cloud_config_secret }} {% if external_openstack_cacert_b64.content is defined %} ca.cert: {{ external_openstack_cacert_b64.content }} {% endif %} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2 ================================================ [Global] auth-url="{{ external_openstack_auth_url }}" {% if external_openstack_application_credential_id == "" and external_openstack_application_credential_name == "" %} username="{{ external_openstack_username }}" password="{{ external_openstack_password }}" {% endif %} {% if external_openstack_application_credential_id is defined and external_openstack_application_credential_id != "" %} application-credential-id={{ external_openstack_application_credential_id }} {% endif %} {% if external_openstack_application_credential_name is defined and external_openstack_application_credential_name != "" %} application-credential-name={{ external_openstack_application_credential_name }} {% endif %} {% if external_openstack_application_credential_secret is defined and external_openstack_application_credential_secret != "" %} application-credential-secret={{ external_openstack_application_credential_secret }} {% endif %} region="{{ external_openstack_region }}" {% if external_openstack_tenant_id is defined and external_openstack_tenant_id != "" %} tenant-id="{{ external_openstack_tenant_id }}" {% endif %} {% if external_openstack_tenant_name is defined and external_openstack_tenant_name != "" %} tenant-name="{{ external_openstack_tenant_name }}" {% endif %} {% if external_openstack_domain_name is defined and external_openstack_domain_name != "" %} domain-name="{{ external_openstack_domain_name }}" {% elif external_openstack_domain_id is defined and external_openstack_domain_id != "" %} domain-id ="{{ external_openstack_domain_id }}" {% endif %} {% if external_openstack_cacert is defined and external_openstack_cacert != "" %} ca-file="{{ kube_config_dir }}/external-openstack-cacert.pem" {% endif %} [LoadBalancer] enabled={{ external_openstack_lbaas_enabled | string | lower }} {% if external_openstack_lbaas_floating_network_id is defined %} floating-network-id={{ external_openstack_lbaas_floating_network_id }} {% endif %} {% if external_openstack_lbaas_floating_subnet_id is defined %} floating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }} {% endif %} {% if external_openstack_lbaas_method is defined %} lb-method={{ external_openstack_lbaas_method }} {% endif %} {% if external_openstack_lbaas_provider is defined %} lb-provider={{ external_openstack_lbaas_provider }} {% endif %} {% if external_openstack_lbaas_subnet_id is defined %} subnet-id={{ external_openstack_lbaas_subnet_id }} member-subnet-id={{ external_openstack_lbaas_member_subnet_id }} {% endif %} {% if external_openstack_lbaas_network_id is defined %} network-id={{ external_openstack_lbaas_network_id }} {% endif %} {% if external_openstack_lbaas_manage_security_groups is defined %} manage-security-groups={{ external_openstack_lbaas_manage_security_groups }} {% endif %} {% if external_openstack_lbaas_create_monitor is defined %} create-monitor={{ external_openstack_lbaas_create_monitor }} {% endif %} {% if external_openstack_lbaas_monitor_delay is defined %} monitor-delay={{ external_openstack_lbaas_monitor_delay }} {% endif %} {% if external_openstack_lbaas_monitor_max_retries is defined %} monitor-max-retries={{ external_openstack_lbaas_monitor_max_retries }} {% endif %} {% if external_openstack_lbaas_monitor_timeout is defined %} monitor-timeout={{ external_openstack_lbaas_monitor_timeout }} {% endif %} {% if external_openstack_lbaas_internal_lb is defined %} internal-lb={{ external_openstack_lbaas_internal_lb }} {% endif %} {% if external_openstack_enable_ingress_hostname is defined %} enable-ingress-hostname={{ external_openstack_enable_ingress_hostname | string | lower }} {% endif %} {% if external_openstack_ingress_hostname_suffix is defined %} ingress-hostname-suffix={{ external_openstack_ingress_hostname_suffix | string | lower }} {% endif %} {% if external_openstack_max_shared_lb is defined %} max-shared-lb={{ external_openstack_max_shared_lb }} {% endif %} [Networking] ipv6-support-disabled={{ external_openstack_network_ipv6_disabled | string | lower }} {% for network_name in external_openstack_network_internal_networks %} internal-network-name="{{ network_name }}" {% endfor %} {% for network_name in external_openstack_network_public_networks %} public-network-name="{{ network_name }}" {% endfor %} [Metadata] {% if external_openstack_metadata_search_order is defined %} search-order="{{ external_openstack_metadata_search_order }}" {% endif %} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: cloud-controller-manager namespace: kube-system --- apiVersion: apps/v1 kind: DaemonSet metadata: name: openstack-cloud-controller-manager namespace: kube-system labels: k8s-app: openstack-cloud-controller-manager spec: selector: matchLabels: k8s-app: openstack-cloud-controller-manager updateStrategy: type: RollingUpdate template: metadata: labels: k8s-app: openstack-cloud-controller-manager spec: nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: runAsUser: 999 tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule serviceAccountName: cloud-controller-manager containers: - name: openstack-cloud-controller-manager image: {{ external_openstack_cloud_controller_image_repo }}:{{ external_openstack_cloud_controller_image_tag }} args: - /bin/openstack-cloud-controller-manager - --v=1 - --cloud-config=$(CLOUD_CONFIG) - --cloud-provider=openstack - --cluster-name={{ cluster_name }} - --use-service-account-credentials=true - --bind-address={{ external_openstack_cloud_controller_bind_address }} {% for key, value in external_openstack_cloud_controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} volumeMounts: - mountPath: /etc/kubernetes/pki name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs name: ca-certs readOnly: true {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} mountPath: {{ dir }} readOnly: true {% endfor %} {% endif %} - mountPath: /etc/config/cloud.conf name: cloud-config-volume readOnly: true subPath: cloud.conf - mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem name: cloud-config-volume readOnly: true subPath: ca.cert {% if kubelet_flexvolumes_plugins_dir is defined %} - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec name: flexvolume-dir {% endif %} resources: requests: cpu: 200m env: - name: CLOUD_CONFIG value: /etc/config/cloud.conf hostNetwork: true {% if external_openstack_cloud_controller_dns_policy is defined %} dnsPolicy: {{ external_openstack_cloud_controller_dns_policy }} {% endif %} volumes: {% if kubelet_flexvolumes_plugins_dir is defined %} - name: flexvolume-dir hostPath: path: "{{ kubelet_flexvolumes_plugins_dir }}" type: DirectoryOrCreate {% endif %} - name: k8s-certs hostPath: path: /etc/kubernetes/pki type: DirectoryOrCreate - name: ca-certs hostPath: path: /etc/ssl/certs type: DirectoryOrCreate {% if ssl_ca_dirs | length %} {% for dir in ssl_ca_dirs %} - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} hostPath: path: {{ dir }} type: DirectoryOrCreate {% endfor %} {% endif %} - name: cloud-config-volume secret: secretName: external-openstack-cloud-config ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2 ================================================ apiVersion: v1 items: - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - kind: ServiceAccount name: cloud-controller-manager namespace: kube-system kind: List metadata: {} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2 ================================================ apiVersion: v1 items: - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:cloud-controller-manager rules: - apiGroups: - coordination.k8s.io resources: - leases verbs: - get - create - update - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - "" resources: - nodes verbs: - '*' - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - "" resources: - services verbs: - list - patch - update - watch - apiGroups: - "" resources: - services/status verbs: - patch - apiGroups: - "" resources: - serviceaccounts/token verbs: - create - apiGroups: - "" resources: - serviceaccounts verbs: - create - get - apiGroups: - "" resources: - persistentvolumes verbs: - '*' - apiGroups: - "" resources: - endpoints verbs: - create - get - list - watch - update - apiGroups: - "" resources: - configmaps verbs: - get - list - watch - apiGroups: - "" resources: - secrets verbs: - list - get - watch - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create kind: List metadata: {} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml ================================================ --- external_vsphere_vcenter_port: "443" external_vsphere_insecure: "true" ## A dictionary of extra arguments to add to the vsphere cloud controller manager daemonset ## Format: ## external_vsphere_cloud_controller_extra_args: ## arg1: "value1" ## arg2: "value2" external_vsphere_cloud_controller_extra_args: {} external_vsphere_cloud_controller_image_tag: "v1.31.0" external_vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" external_vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml ================================================ --- - name: External vSphere Cloud Controller | Check vsphere credentials include_tasks: vsphere-credentials-check.yml - name: External vSphere Cloud Controller | Generate CPI cloud-config template: src: "{{ item }}.j2" dest: "{{ kube_config_dir }}/{{ item }}" mode: "0640" with_items: - external-vsphere-cpi-cloud-config when: inventory_hostname == groups['kube_control_plane'][0] - name: External vSphere Cloud Controller | Generate Manifests template: src: "{{ item }}.j2" dest: "{{ kube_config_dir }}/{{ item }}" mode: "0644" with_items: - external-vsphere-cpi-cloud-config-secret.yml - external-vsphere-cloud-controller-manager-roles.yml - external-vsphere-cloud-controller-manager-role-bindings.yml - external-vsphere-cloud-controller-manager-ds.yml register: external_vsphere_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: External vSphere Cloud Provider Interface | Create a CPI configMap manifest command: "{{ bin_dir }}/kubectl create configmap cloud-config --from-file=vsphere.conf={{ kube_config_dir }}/external-vsphere-cpi-cloud-config -n kube-system --dry-run --save-config -o yaml" register: external_vsphere_configmap_manifest when: inventory_hostname == groups['kube_control_plane'][0] - name: External vSphere Cloud Provider Interface | Apply a CPI configMap manifest command: cmd: "{{ bin_dir }}/kubectl apply -f -" stdin: "{{ external_vsphere_configmap_manifest.stdout }}" when: inventory_hostname == groups['kube_control_plane'][0] - name: External vSphere Cloud Controller | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item }}" state: "latest" with_items: - "{{ external_vsphere_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item }}" ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/vsphere-credentials-check.yml ================================================ --- - name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value fail: msg: "external_vsphere_vcenter_ip is missing" when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip - name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value fail: msg: "external_vsphere_vcenter_port is missing" when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port - name: External vSphere Cloud Provider | check external_vsphere_insecure value fail: msg: "external_vsphere_insecure is missing" when: external_vsphere_insecure is not defined or not external_vsphere_insecure - name: External vSphere Cloud Provider | check external_vsphere_user value fail: msg: "external_vsphere_user is missing" when: external_vsphere_user is not defined or not external_vsphere_user - name: External vSphere Cloud Provider | check external_vsphere_password value fail: msg: "external_vsphere_password is missing" when: - external_vsphere_password is not defined or not external_vsphere_password - name: External vSphere Cloud Provider | check external_vsphere_datacenter value fail: msg: "external_vsphere_datacenter is missing" when: - external_vsphere_datacenter is not defined or not external_vsphere_datacenter ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: cloud-controller-manager namespace: kube-system --- apiVersion: apps/v1 kind: DaemonSet metadata: name: vsphere-cloud-controller-manager namespace: kube-system labels: k8s-app: vsphere-cloud-controller-manager spec: selector: matchLabels: k8s-app: vsphere-cloud-controller-manager updateStrategy: type: RollingUpdate template: metadata: labels: k8s-app: vsphere-cloud-controller-manager spec: nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: runAsUser: 0 tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule serviceAccountName: cloud-controller-manager containers: - name: vsphere-cloud-controller-manager image: {{ kube_image_repo }}/cloud-pv-vsphere/cloud-provider-vsphere:{{ external_vsphere_cloud_controller_image_tag }} args: - --v=2 - --cloud-provider=vsphere - --cloud-config=/etc/cloud/vsphere.conf {% for key, value in external_vsphere_cloud_controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true resources: requests: cpu: 200m hostNetwork: true volumes: - name: vsphere-config-volume configMap: name: cloud-config --- apiVersion: v1 kind: Service metadata: labels: component: cloud-controller-manager name: vsphere-cloud-controller-manager namespace: kube-system spec: type: NodePort ports: - port: 43001 protocol: TCP targetPort: 43001 selector: component: cloud-controller-manager ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-role-bindings.yml.j2 ================================================ apiVersion: v1 items: - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: servicecatalog.k8s.io:apiserver-authentication-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - apiGroup: "" kind: ServiceAccount name: cloud-controller-manager namespace: kube-system - apiGroup: "" kind: User name: cloud-controller-manager - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - kind: ServiceAccount name: cloud-controller-manager namespace: kube-system - kind: User name: cloud-controller-manager kind: List metadata: {} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-roles.yml.j2 ================================================ apiVersion: v1 items: - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:cloud-controller-manager rules: - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - "" resources: - nodes verbs: - '*' - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - "" resources: - services verbs: - list - patch - update - watch - apiGroups: - "" resources: - services/status verbs: - patch - apiGroups: - "" resources: - serviceaccounts verbs: - create - get - list - watch - update - apiGroups: - "" resources: - persistentvolumes verbs: - get - list - update - watch - apiGroups: - "" resources: - endpoints verbs: - create - get - list - watch - update - apiGroups: - "" resources: - secrets verbs: - get - list - watch - apiGroups: - "coordination.k8s.io" resources: - leases verbs: - get - list - watch - create - update kind: List metadata: {} ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config-secret.yml.j2 ================================================ # This YAML file contains secret objects, # which are necessary to run external vsphere cloud controller. apiVersion: v1 kind: Secret metadata: name: cpi-global-secret namespace: kube-system stringData: {{ external_vsphere_vcenter_ip }}.username: "{{ external_vsphere_user }}" {{ external_vsphere_vcenter_ip }}.password: "{{ external_vsphere_password }}" ================================================ FILE: roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config.j2 ================================================ [Global] port = "{{ external_vsphere_vcenter_port }}" insecure-flag = "{{ external_vsphere_insecure }}" secret-name = "cpi-global-secret" secret-namespace = "kube-system" [VirtualCenter "{{ external_vsphere_vcenter_ip }}"] datacenters = "{{ external_vsphere_datacenter }}" ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml ================================================ --- local_path_provisioner_enabled: false local_path_provisioner_namespace: "local-path-storage" local_path_provisioner_storage_class: "local-path" local_path_provisioner_reclaim_policy: Delete local_path_provisioner_claim_root: /opt/local-path-provisioner/ local_path_provisioner_is_default_storageclass: "true" local_path_provisioner_debug: false local_path_provisioner_helper_image_repo: "busybox" local_path_provisioner_helper_image_tag: "latest" local_path_provisioner_resources: {} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml ================================================ --- - name: Local Path Provisioner | Create addon dir file: path: "{{ kube_config_dir }}/addons/local_path_provisioner" state: directory owner: root group: root mode: "0755" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Local Path Provisioner | Create claim root dir file: path: "{{ local_path_provisioner_claim_root }}" state: directory mode: "0755" - name: Local Path Provisioner | Render Template set_fact: local_path_provisioner_templates: - { name: local-path-storage-ns, file: local-path-storage-ns.yml, type: ns } - { name: local-path-storage-sa, file: local-path-storage-sa.yml, type: sa } - { name: local-path-storage-cr, file: local-path-storage-cr.yml, type: cr } - { name: local-path-storage-clusterrolebinding, file: local-path-storage-clusterrolebinding.yml, type: clusterrolebinding } - { name: local-path-storage-cm, file: local-path-storage-cm.yml, type: cm } - { name: local-path-storage-deployment, file: local-path-storage-deployment.yml, type: deployment } - { name: local-path-storage-sc, file: local-path-storage-sc.yml, type: sc } - name: Local Path Provisioner | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.file }}" mode: "0644" with_items: "{{ local_path_provisioner_templates }}" register: local_path_provisioner_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: Local Path Provisioner | Apply manifests kube: name: "{{ item.item.name }}" namespace: "{{ local_path_provisioner_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.item.file }}" state: "latest" with_items: "{{ local_path_provisioner_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: local-path-provisioner-bind roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: local-path-provisioner-role subjects: - kind: ServiceAccount name: local-path-provisioner-service-account namespace: {{ local_path_provisioner_namespace }} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2 ================================================ --- kind: ConfigMap apiVersion: v1 metadata: name: local-path-config namespace: {{ local_path_provisioner_namespace }} data: config.json: |- { "nodePathMap":[ { "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", "paths":["{{ local_path_provisioner_claim_root }}"] } ] } setup: |- #!/bin/sh set -eu mkdir -m 0777 -p "$VOL_DIR" teardown: |- #!/bin/sh set -eu rm -rf "$VOL_DIR" helperPod.yaml: |- apiVersion: v1 kind: Pod metadata: name: helper-pod spec: containers: - name: helper-pod image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}" imagePullPolicy: IfNotPresent ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: local-path-provisioner-role rules: - apiGroups: [ "" ] resources: [ "nodes", "persistentvolumeclaims", "configmaps" ] verbs: [ "get", "list", "watch" ] - apiGroups: [ "" ] resources: [ "endpoints", "persistentvolumes", "pods" ] verbs: [ "*" ] - apiGroups: [ "" ] resources: [ "events" ] verbs: [ "create", "patch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "storageclasses" ] verbs: [ "get", "list", "watch" ] ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2 ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: local-path-provisioner namespace: {{ local_path_provisioner_namespace }} spec: replicas: 1 selector: matchLabels: app: local-path-provisioner template: metadata: labels: app: local-path-provisioner spec: serviceAccountName: local-path-provisioner-service-account containers: - name: local-path-provisioner image: {{ local_path_provisioner_image_repo }}:{{ local_path_provisioner_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: - local-path-provisioner - start - --config - /etc/config/config.json {% if local_path_provisioner_debug | default(false) %} - --debug {% endif %} volumeMounts: - name: config-volume mountPath: /etc/config/ env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {% if local_path_provisioner_resources %} resources: {{ local_path_provisioner_resources | to_nice_yaml | indent(10) | trim }} {% endif %} volumes: - name: config-volume configMap: name: local-path-config ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ local_path_provisioner_namespace }} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: local-path-provisioner-service-account namespace: {{ local_path_provisioner_namespace }} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2 ================================================ --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: {{ local_path_provisioner_storage_class }} annotations: storageclass.kubernetes.io/is-default-class: "{{ local_path_provisioner_is_default_storageclass }}" provisioner: rancher.io/local-path volumeBindingMode: WaitForFirstConsumer reclaimPolicy: {{ local_path_provisioner_reclaim_policy }} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml ================================================ --- local_volume_provisioner_namespace: "kube-system" # List of node labels to be copied to the PVs created by the provisioner local_volume_provisioner_nodelabels: [] # - kubernetes.io/hostname # - topology.kubernetes.io/region # - topology.kubernetes.io/zone local_volume_provisioner_tolerations: [] local_volume_provisioner_use_node_name_only: false # Leverages Ansible's string to Python datatype casting. Otherwise the dict_key isn't substituted. # see https://github.com/ansible/ansible/issues/17324 local_volume_provisioner_storage_classes: | { "{{ local_volume_provisioner_storage_class | default('local-storage') }}": { "host_dir": "{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}", "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}", "volume_mode": "Filesystem", "fs_type": "ext4" } } local_volume_provisioner_log_level: 2 ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml ================================================ --- # include to workaround mitogen issue # https://github.com/dw/mitogen/issues/663 - name: "Local Volume Provisioner | Ensure base dir {{ delegate_host_base_dir.1 }} is created on {{ delegate_host_base_dir.0 }}" file: path: "{{ local_volume_provisioner_storage_classes[delegate_host_base_dir.1].host_dir }}" state: directory owner: root group: root mode: "{{ local_volume_provisioner_directory_mode }}" delegate_to: "{{ delegate_host_base_dir.0 }}" ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml ================================================ --- - name: Local Volume Provisioner | Ensure base dir is created on all hosts include_tasks: basedirs.yml loop_control: loop_var: delegate_host_base_dir loop: "{{ groups['k8s_cluster'] | product(local_volume_provisioner_storage_classes.keys()) | list }}" - name: Local Volume Provisioner | Create addon dir file: path: "{{ kube_config_dir }}/addons/local_volume_provisioner" state: directory owner: root group: root mode: "0755" - name: Local Volume Provisioner | Templates list set_fact: local_volume_provisioner_templates: - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns } - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa } - { name: local-volume-provisioner-clusterrole, file: local-volume-provisioner-clusterrole.yml, type: clusterrole } - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type: clusterrolebinding } - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type: cm } - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type: ds } - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type: sc } - name: Local Volume Provisioner | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}" mode: "0644" with_items: "{{ local_volume_provisioner_templates }}" register: local_volume_provisioner_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: Local Volume Provisioner | Apply manifests kube: name: "{{ item.item.name }}" namespace: "{{ local_volume_provisioner_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.item.file }}" state: "latest" with_items: "{{ local_volume_provisioner_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: local-volume-provisioner-node-clusterrole namespace: {{ local_volume_provisioner_namespace }} rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["watch"] - apiGroups: ["", "events.k8s.io"] resources: ["events"] verbs: ["create", "update", "patch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get"] ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: local-volume-provisioner-system-node namespace: {{ local_volume_provisioner_namespace }} subjects: - kind: ServiceAccount name: local-volume-provisioner namespace: {{ local_volume_provisioner_namespace }} roleRef: kind: ClusterRole name: local-volume-provisioner-node-clusterrole apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 ================================================ # Macro to convert camelCase dictionary keys to snake_case keys {% macro convert_keys(mydict) -%} {% for key in mydict.keys() | list -%} {% set key_split = key.split('_') -%} {% set new_key = key_split[0] + key_split[1:] | map('capitalize') | join -%} {% set value = mydict.pop(key) -%} {{ mydict.__setitem__(new_key, value) -}} {{ convert_keys(value) if value is mapping else None -}} {% endfor -%} {% endmacro -%} --- apiVersion: v1 kind: ConfigMap metadata: name: local-volume-provisioner namespace: {{ local_volume_provisioner_namespace }} data: {% if local_volume_provisioner_nodelabels | length > 0 %} nodeLabelsForPV: | {% for nodelabel in local_volume_provisioner_nodelabels %} - {{ nodelabel }} {% endfor %} {% endif %} {% if local_volume_provisioner_use_node_name_only %} useNodeNameOnly: "true" {% endif %} storageClassMap: | {% for class_name, storage_class in local_volume_provisioner_storage_classes.items() %} {{ class_name }}: {{- convert_keys(storage_class) }} {{ storage_class | to_nice_yaml(indent=2) | indent(6) }} {%- endfor %} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 ================================================ --- apiVersion: apps/v1 kind: DaemonSet metadata: name: local-volume-provisioner namespace: {{ local_volume_provisioner_namespace }} labels: k8s-app: local-volume-provisioner version: {{ local_volume_provisioner_image_tag }} spec: selector: matchLabels: k8s-app: local-volume-provisioner version: {{ local_volume_provisioner_image_tag }} template: metadata: labels: k8s-app: local-volume-provisioner version: {{ local_volume_provisioner_image_tag }} spec: priorityClassName: {% if local_volume_provisioner_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} serviceAccountName: local-volume-provisioner nodeSelector: kubernetes.io/os: linux {% if local_volume_provisioner_tolerations %} tolerations: {{ local_volume_provisioner_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} containers: - name: provisioner image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "-v={{ local_volume_provisioner_log_level }}" securityContext: privileged: true env: - name: MY_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: MY_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace volumeMounts: - name: local-volume-provisioner mountPath: /etc/provisioner/config readOnly: true - mountPath: /dev name: provisioner-dev {% for class_name, class_config in local_volume_provisioner_storage_classes.items() %} - name: local-volume-provisioner-hostpath-{{ class_name }} mountPath: {{ class_config.mount_dir }} mountPropagation: "HostToContainer" {% endfor %} volumes: - name: local-volume-provisioner configMap: name: local-volume-provisioner - name: provisioner-dev hostPath: path: /dev {% for class_name, class_config in local_volume_provisioner_storage_classes.items() %} - name: local-volume-provisioner-hostpath-{{ class_name }} hostPath: path: {{ class_config.host_dir }} {% endfor %} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ local_volume_provisioner_namespace }} labels: name: {{ local_volume_provisioner_namespace }} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: local-volume-provisioner namespace: {{ local_volume_provisioner_namespace }} ================================================ FILE: roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 ================================================ {% for class_name, class_config in local_volume_provisioner_storage_classes.items() %} --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: {{ class_name }} provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer {% if class_config.reclaim_policy is defined %} reclaimPolicy: {{ class_config.reclaim_policy }} {% endif %} {% endfor %} ================================================ FILE: roles/kubernetes-apps/external_provisioner/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/external_provisioner/local_volume_provisioner when: - local_volume_provisioner_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - apps - local-volume-provisioner - external-provisioner - role: kubernetes-apps/external_provisioner/local_path_provisioner when: local_path_provisioner_enabled tags: - apps - local-path-provisioner - external-provisioner ================================================ FILE: roles/kubernetes-apps/helm/.gitkeep ================================================ ================================================ FILE: roles/kubernetes-apps/helm/defaults/main.yml ================================================ --- helm_enabled: false ================================================ FILE: roles/kubernetes-apps/helm/tasks/main.yml ================================================ --- - name: Helm | Gather os specific variables include_vars: "{{ item }}" with_first_found: - files: - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}.yml" - "{{ ansible_os_family | lower }}.yml" - defaults.yml paths: - ../vars skip: true - name: Helm | Install PyYaml package: name: "{{ pyyaml_package }}" state: present when: pyyaml_package is defined - name: Helm | Install PyYaml [flatcar] include_tasks: pyyaml-flatcar.yml when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - name: Helm | Download helm include_tasks: "../../../download/tasks/download_file.yml" vars: download: "{{ download_defaults | combine(downloads.helm) }}" - name: Helm | Copy helm binary from download dir copy: src: "{{ local_release_dir }}/helm-{{ helm_version }}/linux-{{ image_arch }}/helm" dest: "{{ bin_dir }}/helm" mode: "0755" remote_src: true - name: Helm | Get helm completion command: "{{ bin_dir }}/helm completion bash" changed_when: false register: helm_completion check_mode: false - name: Helm | Install helm completion copy: dest: /etc/bash_completion.d/helm.sh content: "{{ helm_completion.stdout }}" mode: "0755" become: true ================================================ FILE: roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml ================================================ --- - name: Get installed pip version command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip --version" register: pip_version_output ignore_errors: true changed_when: false - name: Get installed PyYAML version command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip show PyYAML" register: pyyaml_version_output ignore_errors: true changed_when: false - name: Install pip command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m ensurepip --upgrade" when: (pyyaml_version_output is failed) and (pip_version_output is failed) - name: Install PyYAML ansible.builtin.pip: name: - PyYAML when: (pyyaml_version_output is failed) ================================================ FILE: roles/kubernetes-apps/helm/vars/amazon.yml ================================================ --- pyyaml_package: PyYAML ================================================ FILE: roles/kubernetes-apps/helm/vars/centos-7.yml ================================================ --- pyyaml_package: PyYAML ================================================ FILE: roles/kubernetes-apps/helm/vars/centos.yml ================================================ --- pyyaml_package: python3-pyyaml ================================================ FILE: roles/kubernetes-apps/helm/vars/debian.yml ================================================ --- pyyaml_package: python3-yaml ================================================ FILE: roles/kubernetes-apps/helm/vars/fedora.yml ================================================ --- pyyaml_package: python3-pyyaml ================================================ FILE: roles/kubernetes-apps/helm/vars/redhat-7.yml ================================================ --- pyyaml_package: PyYAML ================================================ FILE: roles/kubernetes-apps/helm/vars/redhat.yml ================================================ --- pyyaml_package: python3-pyyaml ================================================ FILE: roles/kubernetes-apps/helm/vars/suse.yml ================================================ --- pyyaml_package: python3-PyYAML ================================================ FILE: roles/kubernetes-apps/helm/vars/ubuntu.yml ================================================ --- pyyaml_package: python3-yaml ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/defaults/main.yml ================================================ --- alb_ingress_controller_namespace: kube-system alb_ingress_aws_region: "us-east-1" # Enables logging on all outbound requests sent to the AWS API. # If logging is desired, set to true. alb_ingress_aws_debug: "false" ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml ================================================ --- - name: ALB Ingress Controller | Create addon dir file: path: "{{ kube_config_dir }}/addons/alb_ingress" state: directory owner: root group: root mode: "0755" - name: ALB Ingress Controller | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/alb_ingress/{{ item.file }}" mode: "0644" with_items: - { name: alb-ingress-clusterrole, file: alb-ingress-clusterrole.yml, type: clusterrole } - { name: alb-ingress-clusterrolebinding, file: alb-ingress-clusterrolebinding.yml, type: clusterrolebinding } - { name: alb-ingress-ns, file: alb-ingress-ns.yml, type: ns } - { name: alb-ingress-sa, file: alb-ingress-sa.yml, type: sa } - { name: alb-ingress-deploy, file: alb-ingress-deploy.yml, type: deploy } register: alb_ingress_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: ALB Ingress Controller | Apply manifests kube: name: "{{ item.item.name }}" namespace: "{{ alb_ingress_controller_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/alb_ingress/{{ item.item.file }}" state: "latest" with_items: "{{ alb_ingress_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrole.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: alb-ingress namespace: {{ alb_ingress_controller_namespace }} rules: - apiGroups: ["", "extensions"] resources: ["configmaps", "endpoints", "nodes", "pods", "secrets", "events", "ingresses", "ingresses/status", "services"] verbs: ["list", "create", "get", "update", "watch", "patch"] - apiGroups: ["", "extensions"] resources: ["nodes", "pods", "secrets", "services", "namespaces"] verbs: ["get", "list", "watch"] ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrolebinding.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: alb-ingress namespace: {{ alb_ingress_controller_namespace }} subjects: - kind: ServiceAccount name: alb-ingress namespace: {{ alb_ingress_controller_namespace }} roleRef: kind: ClusterRole name: alb-ingress apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-deploy.yml.j2 ================================================ # Application Load Balancer (ALB) Ingress Controller Deployment Manifest. # This manifest details sensible defaults for deploying an ALB Ingress Controller. # GitHub: https://github.com/coreos/alb-ingress-controller --- apiVersion: apps/v1 kind: Deployment metadata: name: alb-ingress-controller labels: k8s-app: alb-ingress-controller # Namespace the ALB Ingress Controller should run in. Does not impact which # namespaces it's able to resolve ingress resource for. For limiting ingress # namespace scope, see --watch-namespace. namespace: {{ alb_ingress_controller_namespace }} spec: replicas: 1 selector: matchLabels: k8s-app: alb-ingress-controller strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: metadata: creationTimestamp: null labels: k8s-app: alb-ingress-controller spec: containers: - args: # Limit the namespace where this ALB Ingress Controller deployment will # resolve ingress resources. If left commented, all namespaces are used. #- --watch-namespace=your-k8s-namespace # Setting the ingress-class flag below will ensure that only ingress resources with the # annotation kubernetes.io/ingress.class: "alb" are respected by the controller. You may # choose any class you'd like for this controller to respect. - --ingress-class=alb # Name of your cluster. Used when naming resources created # by the ALB Ingress Controller, providing distinction between # clusters. - --cluster-name={{ cluster_name }} # Enables logging on all outbound requests sent to the AWS API. # If logging is desired, set to true. # - ---aws-api-debug {% if alb_ingress_aws_debug %} - --aws-api-debug {% endif %} # Maximum number of times to retry the aws calls. # defaults to 10. # - --aws-max-retries=10 # AWS region this ingress controller will operate in. # If unspecified, it will be discovered from ec2metadata. # List of regions: http://docs.aws.amazon.com/general/latest/gr/rande.html#vpc_region {% if alb_ingress_aws_region is defined %} - --aws-region={{ alb_ingress_aws_region }} {% endif %} image: "{{ alb_ingress_image_repo }}:{{ alb_ingress_image_tag }}" imagePullPolicy: {{ k8s_image_pull_policy }} name: server resources: {} terminationMessagePath: /dev/termination-log dnsPolicy: ClusterFirst restartPolicy: Always securityContext: {} terminationGracePeriodSeconds: 30 {% if rbac_enabled %} serviceAccountName: alb-ingress {% endif %} ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-ns.yml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ alb_ingress_controller_namespace }} labels: name: {{ alb_ingress_controller_namespace }} ================================================ FILE: roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-sa.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: alb-ingress namespace: {{ alb_ingress_controller_namespace }} ================================================ FILE: roles/kubernetes-apps/ingress_controller/cert_manager/defaults/main.yml ================================================ --- cert_manager_namespace: "cert-manager" cert_manager_user: 1001 cert_manager_tolerations: [] cert_manager_affinity: {} cert_manager_nodeselector: {} cert_manager_dns_policy: "ClusterFirst" cert_manager_dns_config: {} cert_manager_controller_extra_args: [] ## Allow http_proxy, https_proxy and no_proxy environment variables ## Details https://github.com/kubernetes-sigs/kubespray/blob/master/docs/proxy.md cert_manager_http_proxy: "{{ http_proxy | default('') }}" cert_manager_https_proxy: "{{ https_proxy | default('') }}" cert_manager_no_proxy: "{{ no_proxy | default('') }}" ## Change leader election namespace when deploying on GKE Autopilot that forbid the changes on kube-system namespace. ## See https://github.com/jetstack/cert-manager/issues/3717 cert_manager_leader_election_namespace: kube-system ================================================ FILE: roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml ================================================ --- - name: Cert Manager | Remove legacy addon dir and manifests file: path: "{{ kube_config_dir }}/addons/cert_manager" state: absent when: - inventory_hostname == groups['kube_control_plane'][0] tags: - upgrade - name: Cert Manager | Remove legacy namespace command: > {{ kubectl }} delete namespace {{ cert_manager_namespace }} ignore_errors: true # noqa ignore-errors when: - inventory_hostname == groups['kube_control_plane'][0] tags: - upgrade - name: Cert Manager | Create addon dir file: path: "{{ kube_config_dir }}/addons/cert_manager" state: directory owner: root group: root mode: "0755" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Cert Manager | Templates list set_fact: cert_manager_templates: - { name: cert-manager, file: cert-manager.yml, type: all } - { name: cert-manager.crds, file: cert-manager.crds.yml, type: crd } - name: Cert Manager | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/cert_manager/{{ item.file }}" mode: "0644" with_items: "{{ cert_manager_templates }}" register: cert_manager_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Cert Manager | Apply manifests kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/cert_manager/{{ item.item.file }}" state: "latest" with_items: "{{ cert_manager_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 ================================================ # Copyright 2022 The cert-manager Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Source: cert-manager/deploy/crds/crd-clusterissuers.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clusterissuers.cert-manager.io labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' app.kubernetes.io/instance: "cert-manager" # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: group: cert-manager.io names: kind: ClusterIssuer listKind: ClusterIssuerList plural: clusterissuers singular: clusterissuer categories: - cert-manager scope: Cluster versions: - name: v1 subresources: status: {} additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Status priority: 1 type: string - jsonPath: .metadata.creationTimestamp description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. name: Age type: date schema: openAPIV3Schema: description: |- A ClusterIssuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is similar to an Issuer, however it is cluster-scoped and therefore can be referenced by resources that exist in *any* namespace, not just the same namespace as the referent. type: object required: - spec properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: Desired state of the ClusterIssuer resource. type: object properties: acme: description: |- ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates. type: object required: - privateKeySecretRef - server properties: caBundle: description: |- Base64-encoded bundle of PEM CAs which can be used to validate the certificate chain presented by the ACME server. Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various kinds of security vulnerabilities. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. type: string format: byte disableAccountKeyGeneration: description: |- Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false. type: boolean email: description: |- Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered. type: string enableDurationFeature: description: |- Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it, it will create an error on the Order. Defaults to false. type: boolean externalAccountBinding: description: |- ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account. type: object required: - keyID - keySecretRef properties: keyAlgorithm: description: |- Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme. type: string enum: - HS256 - HS384 - HS512 keyID: description: keyID is the ID of the CA key that the External Account is bound to. type: string keySecretRef: description: |- keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string preferredChain: description: |- PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let's Encrypt's DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let's Encrypt root CA. This value picks the first certificate bundle in the combined set of ACME default and alternative chains that has a root-most certificate with this value as its issuer's commonname. type: string maxLength: 64 privateKeySecretRef: description: |- PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string server: description: |- Server is the URL used to access the ACME server's 'directory' endpoint. For example, for Let's Encrypt's staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported. type: string skipTLSVerify: description: |- INSECURE: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have the TLS certificate chain validated. Mutually exclusive with CABundle; prefer using CABundle to prevent various kinds of security vulnerabilities. Only enable this option in development environments. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. Defaults to false. type: boolean solvers: description: |- Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/ type: array items: description: |- An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided. type: object properties: dns01: description: |- Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. type: object properties: acmeDNS: description: |- Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. type: object required: - accountSecretRef - host properties: accountSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string host: type: string akamai: description: Use the Akamai DNS zone management API to manage DNS01 challenge records. type: object required: - accessTokenSecretRef - clientSecretSecretRef - clientTokenSecretRef - serviceConsumerDomain properties: accessTokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientSecretSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientTokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceConsumerDomain: type: string azureDNS: description: Use the Microsoft Azure DNS API to manage DNS01 challenge records. type: object required: - resourceGroupName - subscriptionID properties: clientID: description: |- Auth: Azure Service Principal: The ClientID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientSecret and TenantID must also be set. type: string clientSecretSecretRef: description: |- Auth: Azure Service Principal: A reference to a Secret containing the password associated with the Service Principal. If set, ClientID and TenantID must also be set. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string environment: description: name of the Azure environment (default AzurePublicCloud) type: string enum: - AzurePublicCloud - AzureChinaCloud - AzureGermanCloud - AzureUSGovernmentCloud hostedZoneName: description: name of the DNS zone that should be used type: string managedIdentity: description: |- Auth: Azure Workload Identity or Azure Managed Service Identity: Settings to enable Azure Workload Identity or Azure Managed Service Identity If set, ClientID, ClientSecret and TenantID must not be set. type: object properties: clientID: description: client ID of the managed identity, can not be used at the same time as resourceID type: string resourceID: description: |- resource ID of the managed identity, can not be used at the same time as clientID Cannot be used for Azure Managed Service Identity type: string resourceGroupName: description: resource group the DNS zone is located in type: string subscriptionID: description: ID of the Azure subscription type: string tenantID: description: |- Auth: Azure Service Principal: The TenantID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientID and ClientSecret must also be set. type: string cloudDNS: description: Use the Google Cloud DNS API to manage DNS01 challenge records. type: object required: - project properties: hostedZoneName: description: |- HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. type: string project: type: string serviceAccountSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string cloudflare: description: Use the Cloudflare API to manage DNS01 challenge records. type: object properties: apiKeySecretRef: description: |- API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string apiTokenSecretRef: description: API token used to authenticate with Cloudflare. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string email: description: Email of the account, only required when using API key based authentication. type: string cnameStrategy: description: |- CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. type: string enum: - None - Follow digitalocean: description: Use the DigitalOcean DNS API to manage DNS01 challenge records. type: object required: - tokenSecretRef properties: tokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string rfc2136: description: |- Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. type: object required: - nameserver properties: nameserver: description: |- The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. type: string tsigAlgorithm: description: |- The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``. type: string tsigKeyName: description: |- The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. type: string tsigSecretSecretRef: description: |- The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string route53: description: Use the AWS Route53 API to manage DNS01 challenge records. type: object required: - region properties: accessKeyID: description: |- The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: string accessKeyIDSecretRef: description: |- The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string auth: description: Auth configures how cert-manager authenticates. type: object required: - kubernetes properties: kubernetes: description: |- Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity by passing a bound ServiceAccount token. type: object required: - serviceAccountRef properties: serviceAccountRef: description: |- A reference to a service account that will be used to request a bound token (also known as "projected token"). To use this field, you must configure an RBAC rule to let cert-manager request a token. type: object required: - name properties: audiences: description: |- TokenAudiences is an optional list of audiences to include in the token passed to AWS. The default token consisting of the issuer's namespace and name is always included. If unset the audience defaults to `sts.amazonaws.com`. type: array items: type: string name: description: Name of the ServiceAccount used to request a token. type: string hostedZoneID: description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. type: string region: description: Always set the region when using AccessKeyID and SecretAccessKey type: string role: description: |- Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata type: string secretAccessKeySecretRef: description: |- The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string webhook: description: |- Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. type: object required: - groupName - solverName properties: config: description: |- Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. x-kubernetes-preserve-unknown-fields: true groupName: description: |- The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. type: string solverName: description: |- The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. type: string http01: description: |- Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. type: object properties: gatewayHTTPRoute: description: |- The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. type: object properties: labels: description: |- Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. type: object additionalProperties: type: string parentRefs: description: |- When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways type: array items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with "Core" support: * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, ClusterIP Services only) This API may be extended in the future to support additional kinds of parent resources. The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid. type: object required: - name properties: group: description: |- Group is the group of the referent. When unspecified, "gateway.networking.k8s.io" is inferred. To set the core API group (such as for a "Service" kind referent), Group must be explicitly set to "" (empty string). Support: Core type: string default: gateway.networking.k8s.io maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: description: |- Kind is kind of the referent. There are two kinds of parent resources with "Core" support: * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, ClusterIP Services only) Support for other resources is Implementation-Specific. type: string default: Gateway maxLength: 63 minLength: 1 pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ name: description: |- Name is the name of the referent. Support: Core type: string maxLength: 253 minLength: 1 namespace: description: |- Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. ParentRefs from a Route to a Service in the same namespace are "producer" routes, which apply default routing rules to inbound connections from any namespace to the Service. ParentRefs from a Route to a Service in a different namespace are "consumer" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. Support: Core type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: description: |- Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. Support: Extended type: integer format: int32 maximum: 65535 minimum: 1 sectionName: description: |- SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: * Gateway: Listener name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. Support: Core type: string maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ serviceType: description: |- Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. type: string ingress: description: |- The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. type: object properties: class: description: |- This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressClassName: description: |- This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: description: |- Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. type: object properties: metadata: description: |- ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. type: object properties: annotations: description: Annotations that should be added to the created ACME HTTP01 solver ingress. type: object additionalProperties: type: string labels: description: Labels that should be added to the created ACME HTTP01 solver ingress. type: object additionalProperties: type: string name: description: |- The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. type: string podTemplate: description: |- Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. type: object properties: metadata: description: |- ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. type: object properties: annotations: description: Annotations that should be added to the create ACME HTTP01 solver pods. type: object additionalProperties: type: string labels: description: Labels that should be added to the created ACME HTTP01 solver pods. type: object additionalProperties: type: string spec: description: |- PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. type: object properties: affinity: description: If specified, the pod's scheduling constraints type: object properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. type: array items: description: |- An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). type: object required: - preference - weight properties: preference: description: A node selector term, associated with the corresponding weight. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. type: object required: - nodeSelectorTerms properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. type: array items: description: |- A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) type: object required: - podAffinityTerm - weight properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string weight: description: |- weight associated with matching the corresponding podAffinityTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: description: |- Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string x-kubernetes-list-type: atomic podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) type: object required: - podAffinityTerm - weight properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string weight: description: |- weight associated with matching the corresponding podAffinityTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: description: |- Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string x-kubernetes-list-type: atomic imagePullSecrets: description: If specified, the pod's imagePullSecrets type: array items: description: |- LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. type: object properties: name: description: |- Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string default: "" x-kubernetes-map-type: atomic nodeSelector: description: |- NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object additionalProperties: type: string priorityClassName: description: If specified, the pod's priorityClassName. type: string serviceAccountName: description: If specified, the pod's service account type: string tolerations: description: If specified, the pod's tolerations. type: array items: description: |- The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . type: object properties: effect: description: |- Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: |- Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: description: |- Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. type: string tolerationSeconds: description: |- TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. type: integer format: int64 value: description: |- Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string serviceType: description: |- Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. type: string selector: description: |- Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. type: object properties: dnsNames: description: |- List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. type: array items: type: string dnsZones: description: |- List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. type: array items: type: string matchLabels: description: |- A label selector that is used to refine the set of certificate's that this challenge solver will apply to. type: object additionalProperties: type: string ca: description: |- CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager. type: object required: - secretName properties: crlDistributionPoints: description: |- The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set. type: array items: type: string issuingCertificateURLs: description: |- IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details. As an example, such a URL might be "http://ca.domain.com/ca.crt". type: array items: type: string ocspServers: description: |- The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". type: array items: type: string secretName: description: |- SecretName is the name of the secret used to sign Certificates issued by this Issuer. type: string selfSigned: description: |- SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object. type: object properties: crlDistributionPoints: description: |- The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings. type: array items: type: string vault: description: |- Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend. type: object required: - auth - path - server properties: auth: description: Auth configures how cert-manager authenticates with the Vault server. type: object properties: appRole: description: |- AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource. type: object required: - path - roleId - secretRef properties: path: description: |- Path where the App Role authentication backend is mounted in Vault, e.g: "approle" type: string roleId: description: |- RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault. type: string secretRef: description: |- Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string kubernetes: description: |- Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server. type: object required: - role properties: mountPath: description: |- The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. type: string role: description: |- A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies. type: string secretRef: description: |- The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceAccountRef: description: |- A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. type: object required: - name properties: audiences: description: |- TokenAudiences is an optional list of extra audiences to include in the token passed to Vault. The default token consisting of the issuer's namespace and name is always included. type: array items: type: string name: description: Name of the ServiceAccount used to request a token. type: string tokenSecretRef: description: TokenSecretRef authenticates with Vault by presenting a token. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string caBundle: description: |- Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by Vault. Only used if using HTTPS to connect to Vault and ignored for HTTP connections. Mutually exclusive with CABundleSecretRef. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. type: string format: byte caBundleSecretRef: description: |- Reference to a Secret containing a bundle of PEM-encoded CAs to use when verifying the certificate chain presented by Vault when using HTTPS. Mutually exclusive with CABundle. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. If no key for the Secret is specified, cert-manager will default to 'ca.crt'. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientCertSecretRef: description: |- Reference to a Secret containing a PEM-encoded Client Certificate to use when the Vault server requires mTLS. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientKeySecretRef: description: |- Reference to a Secret containing a PEM-encoded Client Private Key to use when the Vault server requires mTLS. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces type: string path: description: |- Path is the mount path of the Vault PKI backend's `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name". type: string server: description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".' type: string venafi: description: |- Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone. type: object required: - zone properties: cloud: description: |- Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified. type: object required: - apiTokenSecretRef properties: apiTokenSecretRef: description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: description: |- URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1". type: string tpp: description: |- TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified. type: object required: - credentialsRef - url properties: caBundle: description: |- Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. If undefined, the certificate bundle in the cert-manager controller container is used to validate the chain. type: string format: byte credentialsRef: description: |- CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'. type: object required: - name properties: name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: description: |- URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk". type: string zone: description: |- Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required. type: string status: description: Status of the ClusterIssuer. This is set and managed automatically. type: object properties: acme: description: |- ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. type: object properties: lastPrivateKeyHash: description: |- LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer type: string lastRegisteredEmail: description: |- LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer type: string uri: description: |- URI is the unique account identifier, which can also be used to retrieve account details from the CA type: string conditions: description: |- List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`. type: array items: description: IssuerCondition contains condition information for an Issuer. type: object required: - status - type properties: lastTransitionTime: description: |- LastTransitionTime is the timestamp corresponding to the last status change of this condition. type: string format: date-time message: description: |- Message is a human readable description of the details of the last transition, complementing reason. type: string observedGeneration: description: |- If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer. type: integer format: int64 reason: description: |- Reason is a brief machine readable explanation for the condition's last transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). type: string enum: - "True" - "False" - Unknown type: description: Type of the condition, known values are (`Ready`). type: string x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map served: true storage: true --- # Source: cert-manager/deploy/crds/crd-challenges.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: challenges.acme.cert-manager.io labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' app.kubernetes.io/instance: 'cert-manager' # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: group: acme.cert-manager.io names: kind: Challenge listKind: ChallengeList plural: challenges singular: challenge categories: - cert-manager - cert-manager-acme scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .status.state name: State type: string - jsonPath: .spec.dnsName name: Domain type: string - jsonPath: .status.reason name: Reason priority: 1 type: string - description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: description: Challenge is a type to represent a Challenge request with an ACME server type: object required: - metadata - spec properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: type: object required: - authorizationURL - dnsName - issuerRef - key - solver - token - type - url properties: authorizationURL: description: |- The URL to the ACME Authorization resource that this challenge is a part of. type: string dnsName: description: |- dnsName is the identifier that this challenge is for, e.g. example.com. If the requested DNSName is a 'wildcard', this field MUST be set to the non-wildcard domain, e.g. for `*.example.com`, it must be `example.com`. type: string issuerRef: description: |- References a properly configured ACME-type Issuer which should be used to create this Challenge. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Challenge will be marked as failed. type: object required: - name properties: group: description: Group of the resource being referred to. type: string kind: description: Kind of the resource being referred to. type: string name: description: Name of the resource being referred to. type: string key: description: |- The ACME challenge key for this challenge For HTTP01 challenges, this is the value that must be responded with to complete the HTTP01 challenge in the format: `.`. For DNS01 challenges, this is the base64 encoded SHA256 sum of the `.` text that must be set as the TXT record content. type: string solver: description: |- Contains the domain solving configuration that should be used to solve this challenge resource. type: object properties: dns01: description: |- Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. type: object properties: acmeDNS: description: |- Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. type: object required: - accountSecretRef - host properties: accountSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string host: type: string akamai: description: Use the Akamai DNS zone management API to manage DNS01 challenge records. type: object required: - accessTokenSecretRef - clientSecretSecretRef - clientTokenSecretRef - serviceConsumerDomain properties: accessTokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientSecretSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientTokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceConsumerDomain: type: string azureDNS: description: Use the Microsoft Azure DNS API to manage DNS01 challenge records. type: object required: - resourceGroupName - subscriptionID properties: clientID: description: |- Auth: Azure Service Principal: The ClientID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientSecret and TenantID must also be set. type: string clientSecretSecretRef: description: |- Auth: Azure Service Principal: A reference to a Secret containing the password associated with the Service Principal. If set, ClientID and TenantID must also be set. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string environment: description: name of the Azure environment (default AzurePublicCloud) type: string enum: - AzurePublicCloud - AzureChinaCloud - AzureGermanCloud - AzureUSGovernmentCloud hostedZoneName: description: name of the DNS zone that should be used type: string managedIdentity: description: |- Auth: Azure Workload Identity or Azure Managed Service Identity: Settings to enable Azure Workload Identity or Azure Managed Service Identity If set, ClientID, ClientSecret and TenantID must not be set. type: object properties: clientID: description: client ID of the managed identity, can not be used at the same time as resourceID type: string resourceID: description: |- resource ID of the managed identity, can not be used at the same time as clientID Cannot be used for Azure Managed Service Identity type: string resourceGroupName: description: resource group the DNS zone is located in type: string subscriptionID: description: ID of the Azure subscription type: string tenantID: description: |- Auth: Azure Service Principal: The TenantID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientID and ClientSecret must also be set. type: string cloudDNS: description: Use the Google Cloud DNS API to manage DNS01 challenge records. type: object required: - project properties: hostedZoneName: description: |- HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. type: string project: type: string serviceAccountSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string cloudflare: description: Use the Cloudflare API to manage DNS01 challenge records. type: object properties: apiKeySecretRef: description: |- API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string apiTokenSecretRef: description: API token used to authenticate with Cloudflare. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string email: description: Email of the account, only required when using API key based authentication. type: string cnameStrategy: description: |- CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. type: string enum: - None - Follow digitalocean: description: Use the DigitalOcean DNS API to manage DNS01 challenge records. type: object required: - tokenSecretRef properties: tokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string rfc2136: description: |- Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. type: object required: - nameserver properties: nameserver: description: |- The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. type: string tsigAlgorithm: description: |- The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``. type: string tsigKeyName: description: |- The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. type: string tsigSecretSecretRef: description: |- The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string route53: description: Use the AWS Route53 API to manage DNS01 challenge records. type: object required: - region properties: accessKeyID: description: |- The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: string accessKeyIDSecretRef: description: |- The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string auth: description: Auth configures how cert-manager authenticates. type: object required: - kubernetes properties: kubernetes: description: |- Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity by passing a bound ServiceAccount token. type: object required: - serviceAccountRef properties: serviceAccountRef: description: |- A reference to a service account that will be used to request a bound token (also known as "projected token"). To use this field, you must configure an RBAC rule to let cert-manager request a token. type: object required: - name properties: audiences: description: |- TokenAudiences is an optional list of audiences to include in the token passed to AWS. The default token consisting of the issuer's namespace and name is always included. If unset the audience defaults to `sts.amazonaws.com`. type: array items: type: string name: description: Name of the ServiceAccount used to request a token. type: string hostedZoneID: description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. type: string region: description: Always set the region when using AccessKeyID and SecretAccessKey type: string role: description: |- Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata type: string secretAccessKeySecretRef: description: |- The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string webhook: description: |- Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. type: object required: - groupName - solverName properties: config: description: |- Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. x-kubernetes-preserve-unknown-fields: true groupName: description: |- The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. type: string solverName: description: |- The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. type: string http01: description: |- Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. type: object properties: gatewayHTTPRoute: description: |- The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. type: object properties: labels: description: |- Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. type: object additionalProperties: type: string parentRefs: description: |- When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways type: array items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with "Core" support: * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, ClusterIP Services only) This API may be extended in the future to support additional kinds of parent resources. The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid. type: object required: - name properties: group: description: |- Group is the group of the referent. When unspecified, "gateway.networking.k8s.io" is inferred. To set the core API group (such as for a "Service" kind referent), Group must be explicitly set to "" (empty string). Support: Core type: string default: gateway.networking.k8s.io maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: description: |- Kind is kind of the referent. There are two kinds of parent resources with "Core" support: * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, ClusterIP Services only) Support for other resources is Implementation-Specific. type: string default: Gateway maxLength: 63 minLength: 1 pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ name: description: |- Name is the name of the referent. Support: Core type: string maxLength: 253 minLength: 1 namespace: description: |- Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. ParentRefs from a Route to a Service in the same namespace are "producer" routes, which apply default routing rules to inbound connections from any namespace to the Service. ParentRefs from a Route to a Service in a different namespace are "consumer" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. Support: Core type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: description: |- Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. Support: Extended type: integer format: int32 maximum: 65535 minimum: 1 sectionName: description: |- SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: * Gateway: Listener name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. Support: Core type: string maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ serviceType: description: |- Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. type: string ingress: description: |- The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. type: object properties: class: description: |- This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressClassName: description: |- This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: description: |- Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. type: object properties: metadata: description: |- ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. type: object properties: annotations: description: Annotations that should be added to the created ACME HTTP01 solver ingress. type: object additionalProperties: type: string labels: description: Labels that should be added to the created ACME HTTP01 solver ingress. type: object additionalProperties: type: string name: description: |- The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. type: string podTemplate: description: |- Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. type: object properties: metadata: description: |- ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. type: object properties: annotations: description: Annotations that should be added to the create ACME HTTP01 solver pods. type: object additionalProperties: type: string labels: description: Labels that should be added to the created ACME HTTP01 solver pods. type: object additionalProperties: type: string spec: description: |- PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. type: object properties: affinity: description: If specified, the pod's scheduling constraints type: object properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. type: array items: description: |- An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). type: object required: - preference - weight properties: preference: description: A node selector term, associated with the corresponding weight. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. type: object required: - nodeSelectorTerms properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. type: array items: description: |- A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) type: object required: - podAffinityTerm - weight properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string weight: description: |- weight associated with matching the corresponding podAffinityTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: description: |- Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string x-kubernetes-list-type: atomic podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) type: object required: - podAffinityTerm - weight properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string weight: description: |- weight associated with matching the corresponding podAffinityTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: description: |- Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string x-kubernetes-list-type: atomic imagePullSecrets: description: If specified, the pod's imagePullSecrets type: array items: description: |- LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. type: object properties: name: description: |- Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string default: "" x-kubernetes-map-type: atomic nodeSelector: description: |- NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object additionalProperties: type: string priorityClassName: description: If specified, the pod's priorityClassName. type: string serviceAccountName: description: If specified, the pod's service account type: string tolerations: description: If specified, the pod's tolerations. type: array items: description: |- The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . type: object properties: effect: description: |- Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: |- Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: description: |- Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. type: string tolerationSeconds: description: |- TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. type: integer format: int64 value: description: |- Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string serviceType: description: |- Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. type: string selector: description: |- Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. type: object properties: dnsNames: description: |- List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. type: array items: type: string dnsZones: description: |- List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. type: array items: type: string matchLabels: description: |- A label selector that is used to refine the set of certificate's that this challenge solver will apply to. type: object additionalProperties: type: string token: description: |- The ACME challenge token for this challenge. This is the raw value returned from the ACME server. type: string type: description: |- The type of ACME challenge this resource represents. One of "HTTP-01" or "DNS-01". type: string enum: - HTTP-01 - DNS-01 url: description: |- The URL of the ACME Challenge resource for this challenge. This can be used to lookup details about the status of this challenge. type: string wildcard: description: |- wildcard will be true if this challenge is for a wildcard identifier, for example '*.example.com'. type: boolean status: type: object properties: presented: description: |- presented will be set to true if the challenge values for this challenge are currently 'presented'. This *does not* imply the self check is passing. Only that the values have been 'submitted' for the appropriate challenge mechanism (i.e. the DNS01 TXT record has been presented, or the HTTP01 configuration has been configured). type: boolean processing: description: |- Used to denote whether this challenge should be processed or not. This field will only be set to true by the 'scheduling' component. It will only be set to false by the 'challenges' controller, after the challenge has reached a final state or timed out. If this field is set to false, the challenge controller will not take any more action. type: boolean reason: description: |- Contains human readable information on why the Challenge is in the current state. type: string state: description: |- Contains the current 'state' of the challenge. If not set, the state of the challenge is unknown. type: string enum: - valid - ready - pending - processing - invalid - expired - errored served: true storage: true subresources: status: {} --- # Source: cert-manager/deploy/crds/crd-certificaterequests.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: certificaterequests.cert-manager.io labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' app.kubernetes.io/instance: 'cert-manager' # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: group: cert-manager.io names: kind: CertificateRequest listKind: CertificateRequestList plural: certificaterequests shortNames: - cr - crs singular: certificaterequest categories: - cert-manager scope: Namespaced versions: - name: v1 subresources: status: {} additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Approved")].status name: Approved type: string - jsonPath: .status.conditions[?(@.type=="Denied")].status name: Denied type: string - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready type: string - jsonPath: .spec.issuerRef.name name: Issuer type: string - jsonPath: .spec.username name: Requestor type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Status priority: 1 type: string - jsonPath: .metadata.creationTimestamp description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. name: Age type: date schema: openAPIV3Schema: description: |- A CertificateRequest is used to request a signed certificate from one of the configured issuers. All fields within the CertificateRequest's `spec` are immutable after creation. A CertificateRequest will either succeed or fail, as denoted by its `Ready` status condition and its `status.failureTime` field. A CertificateRequest is a one-shot resource, meaning it represents a single point in time request for a certificate and cannot be re-used. type: object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: |- Specification of the desired state of the CertificateRequest resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object required: - issuerRef - request properties: duration: description: |- Requested 'duration' (i.e. lifetime) of the Certificate. Note that the issuer may choose to ignore the requested duration, just like any other requested attribute. type: string extra: description: |- Extra contains extra attributes of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. type: object additionalProperties: type: array items: type: string groups: description: |- Groups contains group membership of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. type: array items: type: string x-kubernetes-list-type: atomic isCA: description: |- Requested basic constraints isCA value. Note that the issuer may choose to ignore the requested isCA value, just like any other requested attribute. NOTE: If the CSR in the `Request` field has a BasicConstraints extension, it must have the same isCA value as specified here. If true, this will automatically add the `cert sign` usage to the list of requested `usages`. type: boolean issuerRef: description: |- Reference to the issuer responsible for issuing the certificate. If the issuer is namespace-scoped, it must be in the same namespace as the Certificate. If the issuer is cluster-scoped, it can be used from any namespace. The `name` field of the reference must always be specified. type: object required: - name properties: group: description: Group of the resource being referred to. type: string kind: description: Kind of the resource being referred to. type: string name: description: Name of the resource being referred to. type: string request: description: |- The PEM-encoded X.509 certificate signing request to be submitted to the issuer for signing. If the CSR has a BasicConstraints extension, its isCA attribute must match the `isCA` value of this CertificateRequest. If the CSR has a KeyUsage extension, its key usages must match the key usages in the `usages` field of this CertificateRequest. If the CSR has a ExtKeyUsage extension, its extended key usages must match the extended key usages in the `usages` field of this CertificateRequest. type: string format: byte uid: description: |- UID contains the uid of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. type: string usages: description: |- Requested key usages and extended key usages. NOTE: If the CSR in the `Request` field has uses the KeyUsage or ExtKeyUsage extension, these extensions must have the same values as specified here without any additional values. If unset, defaults to `digital signature` and `key encipherment`. type: array items: description: |- KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 Valid KeyUsage values are as follows: "signing", "digital signature", "content commitment", "key encipherment", "key agreement", "data encipherment", "cert sign", "crl sign", "encipher only", "decipher only", "any", "server auth", "client auth", "code signing", "email protection", "s/mime", "ipsec end system", "ipsec tunnel", "ipsec user", "timestamping", "ocsp signing", "microsoft sgc", "netscape sgc" type: string enum: - signing - digital signature - content commitment - key encipherment - key agreement - data encipherment - cert sign - crl sign - encipher only - decipher only - any - server auth - client auth - code signing - email protection - s/mime - ipsec end system - ipsec tunnel - ipsec user - timestamping - ocsp signing - microsoft sgc - netscape sgc username: description: |- Username contains the name of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. type: string status: description: |- Status of the CertificateRequest. This is set and managed automatically. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object properties: ca: description: |- The PEM encoded X.509 certificate of the signer, also known as the CA (Certificate Authority). This is set on a best-effort basis by different issuers. If not set, the CA is assumed to be unknown/not available. type: string format: byte certificate: description: |- The PEM encoded X.509 certificate resulting from the certificate signing request. If not set, the CertificateRequest has either not been completed or has failed. More information on failure can be found by checking the `conditions` field. type: string format: byte conditions: description: |- List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`, `InvalidRequest`, `Approved` and `Denied`. type: array items: description: CertificateRequestCondition contains condition information for a CertificateRequest. type: object required: - status - type properties: lastTransitionTime: description: |- LastTransitionTime is the timestamp corresponding to the last status change of this condition. type: string format: date-time message: description: |- Message is a human readable description of the details of the last transition, complementing reason. type: string reason: description: |- Reason is a brief machine readable explanation for the condition's last transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). type: string enum: - "True" - "False" - Unknown type: description: |- Type of the condition, known values are (`Ready`, `InvalidRequest`, `Approved`, `Denied`). type: string x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map failureTime: description: |- FailureTime stores the time that this CertificateRequest failed. This is used to influence garbage collection and back-off. type: string format: date-time served: true storage: true --- # Source: cert-manager/deploy/crds/crd-issuers.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: issuers.cert-manager.io labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' app.kubernetes.io/instance: "cert-manager" # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: group: cert-manager.io names: kind: Issuer listKind: IssuerList plural: issuers singular: issuer categories: - cert-manager scope: Namespaced versions: - name: v1 subresources: status: {} additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Status priority: 1 type: string - jsonPath: .metadata.creationTimestamp description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. name: Age type: date schema: openAPIV3Schema: description: |- An Issuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is scoped to a single namespace and can therefore only be referenced by resources within the same namespace. type: object required: - spec properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: Desired state of the Issuer resource. type: object properties: acme: description: |- ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates. type: object required: - privateKeySecretRef - server properties: caBundle: description: |- Base64-encoded bundle of PEM CAs which can be used to validate the certificate chain presented by the ACME server. Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various kinds of security vulnerabilities. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. type: string format: byte disableAccountKeyGeneration: description: |- Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false. type: boolean email: description: |- Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered. type: string enableDurationFeature: description: |- Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it, it will create an error on the Order. Defaults to false. type: boolean externalAccountBinding: description: |- ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account. type: object required: - keyID - keySecretRef properties: keyAlgorithm: description: |- Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme. type: string enum: - HS256 - HS384 - HS512 keyID: description: keyID is the ID of the CA key that the External Account is bound to. type: string keySecretRef: description: |- keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string preferredChain: description: |- PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let's Encrypt's DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let's Encrypt root CA. This value picks the first certificate bundle in the combined set of ACME default and alternative chains that has a root-most certificate with this value as its issuer's commonname. type: string maxLength: 64 privateKeySecretRef: description: |- PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string server: description: |- Server is the URL used to access the ACME server's 'directory' endpoint. For example, for Let's Encrypt's staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported. type: string skipTLSVerify: description: |- INSECURE: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have the TLS certificate chain validated. Mutually exclusive with CABundle; prefer using CABundle to prevent various kinds of security vulnerabilities. Only enable this option in development environments. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. Defaults to false. type: boolean solvers: description: |- Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/ type: array items: description: |- An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided. type: object properties: dns01: description: |- Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. type: object properties: acmeDNS: description: |- Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. type: object required: - accountSecretRef - host properties: accountSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string host: type: string akamai: description: Use the Akamai DNS zone management API to manage DNS01 challenge records. type: object required: - accessTokenSecretRef - clientSecretSecretRef - clientTokenSecretRef - serviceConsumerDomain properties: accessTokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientSecretSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientTokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceConsumerDomain: type: string azureDNS: description: Use the Microsoft Azure DNS API to manage DNS01 challenge records. type: object required: - resourceGroupName - subscriptionID properties: clientID: description: |- Auth: Azure Service Principal: The ClientID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientSecret and TenantID must also be set. type: string clientSecretSecretRef: description: |- Auth: Azure Service Principal: A reference to a Secret containing the password associated with the Service Principal. If set, ClientID and TenantID must also be set. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string environment: description: name of the Azure environment (default AzurePublicCloud) type: string enum: - AzurePublicCloud - AzureChinaCloud - AzureGermanCloud - AzureUSGovernmentCloud hostedZoneName: description: name of the DNS zone that should be used type: string managedIdentity: description: |- Auth: Azure Workload Identity or Azure Managed Service Identity: Settings to enable Azure Workload Identity or Azure Managed Service Identity If set, ClientID, ClientSecret and TenantID must not be set. type: object properties: clientID: description: client ID of the managed identity, can not be used at the same time as resourceID type: string resourceID: description: |- resource ID of the managed identity, can not be used at the same time as clientID Cannot be used for Azure Managed Service Identity type: string resourceGroupName: description: resource group the DNS zone is located in type: string subscriptionID: description: ID of the Azure subscription type: string tenantID: description: |- Auth: Azure Service Principal: The TenantID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientID and ClientSecret must also be set. type: string cloudDNS: description: Use the Google Cloud DNS API to manage DNS01 challenge records. type: object required: - project properties: hostedZoneName: description: |- HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. type: string project: type: string serviceAccountSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string cloudflare: description: Use the Cloudflare API to manage DNS01 challenge records. type: object properties: apiKeySecretRef: description: |- API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string apiTokenSecretRef: description: API token used to authenticate with Cloudflare. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string email: description: Email of the account, only required when using API key based authentication. type: string cnameStrategy: description: |- CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. type: string enum: - None - Follow digitalocean: description: Use the DigitalOcean DNS API to manage DNS01 challenge records. type: object required: - tokenSecretRef properties: tokenSecretRef: description: |- A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string rfc2136: description: |- Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. type: object required: - nameserver properties: nameserver: description: |- The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. type: string tsigAlgorithm: description: |- The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``. type: string tsigKeyName: description: |- The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. type: string tsigSecretSecretRef: description: |- The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string route53: description: Use the AWS Route53 API to manage DNS01 challenge records. type: object required: - region properties: accessKeyID: description: |- The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: string accessKeyIDSecretRef: description: |- The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string auth: description: Auth configures how cert-manager authenticates. type: object required: - kubernetes properties: kubernetes: description: |- Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity by passing a bound ServiceAccount token. type: object required: - serviceAccountRef properties: serviceAccountRef: description: |- A reference to a service account that will be used to request a bound token (also known as "projected token"). To use this field, you must configure an RBAC rule to let cert-manager request a token. type: object required: - name properties: audiences: description: |- TokenAudiences is an optional list of audiences to include in the token passed to AWS. The default token consisting of the issuer's namespace and name is always included. If unset the audience defaults to `sts.amazonaws.com`. type: array items: type: string name: description: Name of the ServiceAccount used to request a token. type: string hostedZoneID: description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. type: string region: description: Always set the region when using AccessKeyID and SecretAccessKey type: string role: description: |- Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata type: string secretAccessKeySecretRef: description: |- The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string webhook: description: |- Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. type: object required: - groupName - solverName properties: config: description: |- Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. x-kubernetes-preserve-unknown-fields: true groupName: description: |- The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. type: string solverName: description: |- The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. type: string http01: description: |- Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. type: object properties: gatewayHTTPRoute: description: |- The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. type: object properties: labels: description: |- Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. type: object additionalProperties: type: string parentRefs: description: |- When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways type: array items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with "Core" support: * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, ClusterIP Services only) This API may be extended in the future to support additional kinds of parent resources. The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid. type: object required: - name properties: group: description: |- Group is the group of the referent. When unspecified, "gateway.networking.k8s.io" is inferred. To set the core API group (such as for a "Service" kind referent), Group must be explicitly set to "" (empty string). Support: Core type: string default: gateway.networking.k8s.io maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: description: |- Kind is kind of the referent. There are two kinds of parent resources with "Core" support: * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, ClusterIP Services only) Support for other resources is Implementation-Specific. type: string default: Gateway maxLength: 63 minLength: 1 pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ name: description: |- Name is the name of the referent. Support: Core type: string maxLength: 253 minLength: 1 namespace: description: |- Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. ParentRefs from a Route to a Service in the same namespace are "producer" routes, which apply default routing rules to inbound connections from any namespace to the Service. ParentRefs from a Route to a Service in a different namespace are "consumer" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. Support: Core type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: description: |- Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. Support: Extended type: integer format: int32 maximum: 65535 minimum: 1 sectionName: description: |- SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: * Gateway: Listener name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. Support: Core type: string maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ serviceType: description: |- Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. type: string ingress: description: |- The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. type: object properties: class: description: |- This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressClassName: description: |- This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: description: |- Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. type: object properties: metadata: description: |- ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. type: object properties: annotations: description: Annotations that should be added to the created ACME HTTP01 solver ingress. type: object additionalProperties: type: string labels: description: Labels that should be added to the created ACME HTTP01 solver ingress. type: object additionalProperties: type: string name: description: |- The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. type: string podTemplate: description: |- Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. type: object properties: metadata: description: |- ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. type: object properties: annotations: description: Annotations that should be added to the create ACME HTTP01 solver pods. type: object additionalProperties: type: string labels: description: Labels that should be added to the created ACME HTTP01 solver pods. type: object additionalProperties: type: string spec: description: |- PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. type: object properties: affinity: description: If specified, the pod's scheduling constraints type: object properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. type: array items: description: |- An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). type: object required: - preference - weight properties: preference: description: A node selector term, associated with the corresponding weight. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. type: object required: - nodeSelectorTerms properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. type: array items: description: |- A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: The label key that the selector applies to. type: string operator: description: |- Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: |- An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) type: object required: - podAffinityTerm - weight properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string weight: description: |- weight associated with matching the corresponding podAffinityTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: description: |- Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string x-kubernetes-list-type: atomic podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) type: object required: - podAffinityTerm - weight properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string weight: description: |- weight associated with matching the corresponding podAffinityTerm, in the range 1-100. type: integer format: int32 x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: description: |- If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: description: |- Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: description: |- A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic matchLabelKeys: description: |- MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic mismatchLabelKeys: description: |- MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. type: array items: type: string x-kubernetes-list-type: atomic namespaceSelector: description: |- A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. type: object required: - key - operator properties: key: description: key is the label key that the selector applies to. type: string operator: description: |- operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: |- values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. type: array items: type: string x-kubernetes-list-type: atomic x-kubernetes-list-type: atomic matchLabels: description: |- matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: description: |- namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string x-kubernetes-list-type: atomic topologyKey: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string x-kubernetes-list-type: atomic imagePullSecrets: description: If specified, the pod's imagePullSecrets type: array items: description: |- LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. type: object properties: name: description: |- Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string default: "" x-kubernetes-map-type: atomic nodeSelector: description: |- NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object additionalProperties: type: string priorityClassName: description: If specified, the pod's priorityClassName. type: string serviceAccountName: description: If specified, the pod's service account type: string tolerations: description: If specified, the pod's tolerations. type: array items: description: |- The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . type: object properties: effect: description: |- Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: |- Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: description: |- Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. type: string tolerationSeconds: description: |- TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. type: integer format: int64 value: description: |- Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string serviceType: description: |- Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. type: string selector: description: |- Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. type: object properties: dnsNames: description: |- List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. type: array items: type: string dnsZones: description: |- List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. type: array items: type: string matchLabels: description: |- A label selector that is used to refine the set of certificate's that this challenge solver will apply to. type: object additionalProperties: type: string ca: description: |- CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager. type: object required: - secretName properties: crlDistributionPoints: description: |- The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set. type: array items: type: string issuingCertificateURLs: description: |- IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details. As an example, such a URL might be "http://ca.domain.com/ca.crt". type: array items: type: string ocspServers: description: |- The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". type: array items: type: string secretName: description: |- SecretName is the name of the secret used to sign Certificates issued by this Issuer. type: string selfSigned: description: |- SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object. type: object properties: crlDistributionPoints: description: |- The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings. type: array items: type: string vault: description: |- Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend. type: object required: - auth - path - server properties: auth: description: Auth configures how cert-manager authenticates with the Vault server. type: object properties: appRole: description: |- AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource. type: object required: - path - roleId - secretRef properties: path: description: |- Path where the App Role authentication backend is mounted in Vault, e.g: "approle" type: string roleId: description: |- RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault. type: string secretRef: description: |- Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string kubernetes: description: |- Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server. type: object required: - role properties: mountPath: description: |- The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. type: string role: description: |- A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies. type: string secretRef: description: |- The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceAccountRef: description: |- A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. type: object required: - name properties: audiences: description: |- TokenAudiences is an optional list of extra audiences to include in the token passed to Vault. The default token consisting of the issuer's namespace and name is always included. type: array items: type: string name: description: Name of the ServiceAccount used to request a token. type: string tokenSecretRef: description: TokenSecretRef authenticates with Vault by presenting a token. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string caBundle: description: |- Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by Vault. Only used if using HTTPS to connect to Vault and ignored for HTTP connections. Mutually exclusive with CABundleSecretRef. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. type: string format: byte caBundleSecretRef: description: |- Reference to a Secret containing a bundle of PEM-encoded CAs to use when verifying the certificate chain presented by Vault when using HTTPS. Mutually exclusive with CABundle. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. If no key for the Secret is specified, cert-manager will default to 'ca.crt'. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientCertSecretRef: description: |- Reference to a Secret containing a PEM-encoded Client Certificate to use when the Vault server requires mTLS. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientKeySecretRef: description: |- Reference to a Secret containing a PEM-encoded Client Private Key to use when the Vault server requires mTLS. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: description: |- Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces type: string path: description: |- Path is the mount path of the Vault PKI backend's `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name". type: string server: description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".' type: string venafi: description: |- Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone. type: object required: - zone properties: cloud: description: |- Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified. type: object required: - apiTokenSecretRef properties: apiTokenSecretRef: description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: description: |- URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1". type: string tpp: description: |- TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified. type: object required: - credentialsRef - url properties: caBundle: description: |- Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. If undefined, the certificate bundle in the cert-manager controller container is used to validate the chain. type: string format: byte credentialsRef: description: |- CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'. type: object required: - name properties: name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: description: |- URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk". type: string zone: description: |- Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required. type: string status: description: Status of the Issuer. This is set and managed automatically. type: object properties: acme: description: |- ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. type: object properties: lastPrivateKeyHash: description: |- LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer type: string lastRegisteredEmail: description: |- LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer type: string uri: description: |- URI is the unique account identifier, which can also be used to retrieve account details from the CA type: string conditions: description: |- List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`. type: array items: description: IssuerCondition contains condition information for an Issuer. type: object required: - status - type properties: lastTransitionTime: description: |- LastTransitionTime is the timestamp corresponding to the last status change of this condition. type: string format: date-time message: description: |- Message is a human readable description of the details of the last transition, complementing reason. type: string observedGeneration: description: |- If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer. type: integer format: int64 reason: description: |- Reason is a brief machine readable explanation for the condition's last transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). type: string enum: - "True" - "False" - Unknown type: description: Type of the condition, known values are (`Ready`). type: string x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map served: true storage: true --- # Source: cert-manager/deploy/crds/crd-certificates.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: certificates.cert-manager.io labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' app.kubernetes.io/instance: 'cert-manager' # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: group: cert-manager.io names: kind: Certificate listKind: CertificateList plural: certificates shortNames: - cert - certs singular: certificate categories: - cert-manager scope: Namespaced versions: - name: v1 subresources: status: {} additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready type: string - jsonPath: .spec.secretName name: Secret type: string - jsonPath: .spec.issuerRef.name name: Issuer priority: 1 type: string - jsonPath: .status.conditions[?(@.type=="Ready")].message name: Status priority: 1 type: string - jsonPath: .metadata.creationTimestamp description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. name: Age type: date schema: openAPIV3Schema: description: |- A Certificate resource should be created to ensure an up to date and signed X.509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`). type: object properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: |- Specification of the desired state of the Certificate resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object required: - issuerRef - secretName properties: additionalOutputFormats: description: |- Defines extra output formats of the private key and signed certificate chain to be written to this Certificate's target Secret. This is a Beta Feature enabled by default. It can be disabled with the `--feature-gates=AdditionalCertificateOutputFormats=false` option set on both the controller and webhook components. type: array items: description: |- CertificateAdditionalOutputFormat defines an additional output format of a Certificate resource. These contain supplementary data formats of the signed certificate chain and paired private key. type: object required: - type properties: type: description: |- Type is the name of the format type that should be written to the Certificate's target Secret. type: string enum: - DER - CombinedPEM commonName: description: |- Requested common name X509 certificate subject attribute. More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 NOTE: TLS clients will ignore this value when any subject alternative name is set (see https://tools.ietf.org/html/rfc6125#section-6.4.4). Should have a length of 64 characters or fewer to avoid generating invalid CSRs. Cannot be set if the `literalSubject` field is set. type: string dnsNames: description: Requested DNS subject alternative names. type: array items: type: string duration: description: |- Requested 'duration' (i.e. lifetime) of the Certificate. Note that the issuer may choose to ignore the requested duration, just like any other requested attribute. If unset, this defaults to 90 days. Minimum accepted duration is 1 hour. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration. type: string emailAddresses: description: Requested email subject alternative names. type: array items: type: string encodeUsagesInRequest: description: |- Whether the KeyUsage and ExtKeyUsage extensions should be set in the encoded CSR. This option defaults to true, and should only be disabled if the target issuer does not support CSRs with these X509 KeyUsage/ ExtKeyUsage extensions. type: boolean ipAddresses: description: Requested IP address subject alternative names. type: array items: type: string isCA: description: |- Requested basic constraints isCA value. The isCA value is used to set the `isCA` field on the created CertificateRequest resources. Note that the issuer may choose to ignore the requested isCA value, just like any other requested attribute. If true, this will automatically add the `cert sign` usage to the list of requested `usages`. type: boolean issuerRef: description: |- Reference to the issuer responsible for issuing the certificate. If the issuer is namespace-scoped, it must be in the same namespace as the Certificate. If the issuer is cluster-scoped, it can be used from any namespace. The `name` field of the reference must always be specified. type: object required: - name properties: group: description: Group of the resource being referred to. type: string kind: description: Kind of the resource being referred to. type: string name: description: Name of the resource being referred to. type: string keystores: description: Additional keystore output formats to be stored in the Certificate's Secret. type: object properties: jks: description: |- JKS configures options for storing a JKS keystore in the `spec.secretName` Secret resource. type: object required: - create - passwordSecretRef properties: alias: description: |- Alias specifies the alias of the key in the keystore, required by the JKS format. If not provided, the default alias `certificate` will be used. type: string create: description: |- Create enables JKS keystore creation for the Certificate. If true, a file named `keystore.jks` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.jks` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority type: boolean passwordSecretRef: description: |- PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the JKS keystore. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string pkcs12: description: |- PKCS12 configures options for storing a PKCS12 keystore in the `spec.secretName` Secret resource. type: object required: - create - passwordSecretRef properties: create: description: |- Create enables PKCS12 keystore creation for the Certificate. If true, a file named `keystore.p12` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.p12` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority type: boolean passwordSecretRef: description: |- PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the PKCS12 keystore. type: object required: - name properties: key: description: |- The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. type: string name: description: |- Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string profile: description: |- Profile specifies the key and certificate encryption algorithms and the HMAC algorithm used to create the PKCS12 keystore. Default value is `LegacyRC2` for backward compatibility. If provided, allowed values are: `LegacyRC2`: Deprecated. Not supported by default in OpenSSL 3 or Java 20. `LegacyDES`: Less secure algorithm. Use this option for maximal compatibility. `Modern2023`: Secure algorithm. Use this option in case you have to always use secure algorithms (eg. because of company policy). Please note that the security of the algorithm is not that important in reality, because the unencrypted certificate and private key are also stored in the Secret. type: string enum: - LegacyRC2 - LegacyDES - Modern2023 literalSubject: description: |- Requested X.509 certificate subject, represented using the LDAP "String Representation of a Distinguished Name" [1]. Important: the LDAP string format also specifies the order of the attributes in the subject, this is important when issuing certs for LDAP authentication. Example: `CN=foo,DC=corp,DC=example,DC=com` More info [1]: https://datatracker.ietf.org/doc/html/rfc4514 More info: https://github.com/cert-manager/cert-manager/issues/3203 More info: https://github.com/cert-manager/cert-manager/issues/4424 Cannot be set if the `subject` or `commonName` field is set. type: string nameConstraints: description: |- x.509 certificate NameConstraint extension which MUST NOT be used in a non-CA certificate. More Info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.10 This is an Alpha Feature and is only enabled with the `--feature-gates=NameConstraints=true` option set on both the controller and webhook components. type: object properties: critical: description: if true then the name constraints are marked critical. type: boolean excluded: description: |- Excluded contains the constraints which must be disallowed. Any name matching a restriction in the excluded field is invalid regardless of information appearing in the permitted type: object properties: dnsDomains: description: DNSDomains is a list of DNS domains that are permitted or excluded. type: array items: type: string emailAddresses: description: EmailAddresses is a list of Email Addresses that are permitted or excluded. type: array items: type: string ipRanges: description: |- IPRanges is a list of IP Ranges that are permitted or excluded. This should be a valid CIDR notation. type: array items: type: string uriDomains: description: URIDomains is a list of URI domains that are permitted or excluded. type: array items: type: string permitted: description: Permitted contains the constraints in which the names must be located. type: object properties: dnsDomains: description: DNSDomains is a list of DNS domains that are permitted or excluded. type: array items: type: string emailAddresses: description: EmailAddresses is a list of Email Addresses that are permitted or excluded. type: array items: type: string ipRanges: description: |- IPRanges is a list of IP Ranges that are permitted or excluded. This should be a valid CIDR notation. type: array items: type: string uriDomains: description: URIDomains is a list of URI domains that are permitted or excluded. type: array items: type: string otherNames: description: |- `otherNames` is an escape hatch for SAN that allows any type. We currently restrict the support to string like otherNames, cf RFC 5280 p 37 Any UTF8 String valued otherName can be passed with by setting the keys oid: x.x.x.x and UTF8Value: somevalue for `otherName`. Most commonly this would be UPN set with oid: 1.3.6.1.4.1.311.20.2.3 You should ensure that any OID passed is valid for the UTF8String type as we do not explicitly validate this. type: array items: type: object properties: oid: description: |- OID is the object identifier for the otherName SAN. The object identifier must be expressed as a dotted string, for example, "1.2.840.113556.1.4.221". type: string utf8Value: description: |- utf8Value is the string value of the otherName SAN. The utf8Value accepts any valid UTF8 string to set as value for the otherName SAN. type: string privateKey: description: |- Private key options. These include the key algorithm and size, the used encoding and the rotation policy. type: object properties: algorithm: description: |- Algorithm is the private key algorithm of the corresponding private key for this certificate. If provided, allowed values are either `RSA`, `ECDSA` or `Ed25519`. If `algorithm` is specified and `size` is not provided, key size of 2048 will be used for `RSA` key algorithm and key size of 256 will be used for `ECDSA` key algorithm. key size is ignored when using the `Ed25519` key algorithm. type: string enum: - RSA - ECDSA - Ed25519 encoding: description: |- The private key cryptography standards (PKCS) encoding for this certificate's private key to be encoded in. If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 and PKCS#8, respectively. Defaults to `PKCS1` if not specified. type: string enum: - PKCS1 - PKCS8 rotationPolicy: description: |- RotationPolicy controls how private keys should be regenerated when a re-issuance is being processed. If set to `Never`, a private key will only be generated if one does not already exist in the target `spec.secretName`. If one does exists but it does not have the correct algorithm or size, a warning will be raised to await user intervention. If set to `Always`, a private key matching the specified requirements will be generated whenever a re-issuance occurs. Default is `Never` for backward compatibility. type: string enum: - Never - Always size: description: |- Size is the key bit size of the corresponding private key for this certificate. If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, and will default to `2048` if not specified. If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, and will default to `256` if not specified. If `algorithm` is set to `Ed25519`, Size is ignored. No other values are allowed. type: integer renewBefore: description: |- How long before the currently issued certificate's expiry cert-manager should renew the certificate. For example, if a certificate is valid for 60 minutes, and `renewBefore=10m`, cert-manager will begin to attempt to renew the certificate 50 minutes after it was issued (i.e. when there are 10 minutes remaining until the certificate is no longer valid). NOTE: The actual lifetime of the issued certificate is used to determine the renewal time. If an issuer returns a certificate with a different lifetime than the one requested, cert-manager will use the lifetime of the issued certificate. If unset, this defaults to 1/3 of the issued certificate's lifetime. Minimum accepted value is 5 minutes. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration. type: string revisionHistoryLimit: description: |- The maximum number of CertificateRequest revisions that are maintained in the Certificate's history. Each revision represents a single `CertificateRequest` created by this Certificate, either when it was created, renewed, or Spec was changed. Revisions will be removed by oldest first if the number of revisions exceeds this number. If set, revisionHistoryLimit must be a value of `1` or greater. If unset (`nil`), revisions will not be garbage collected. Default value is `nil`. type: integer format: int32 secretName: description: |- Name of the Secret resource that will be automatically created and managed by this Certificate resource. It will be populated with a private key and certificate, signed by the denoted issuer. The Secret resource lives in the same namespace as the Certificate resource. type: string secretTemplate: description: |- Defines annotations and labels to be copied to the Certificate's Secret. Labels and annotations on the Secret will be changed as they appear on the SecretTemplate when added or removed. SecretTemplate annotations are added in conjunction with, and cannot overwrite, the base set of annotations cert-manager sets on the Certificate's Secret. type: object properties: annotations: description: Annotations is a key value map to be copied to the target Kubernetes Secret. type: object additionalProperties: type: string labels: description: Labels is a key value map to be copied to the target Kubernetes Secret. type: object additionalProperties: type: string subject: description: |- Requested set of X509 certificate subject attributes. More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 The common name attribute is specified separately in the `commonName` field. Cannot be set if the `literalSubject` field is set. type: object properties: countries: description: Countries to be used on the Certificate. type: array items: type: string localities: description: Cities to be used on the Certificate. type: array items: type: string organizationalUnits: description: Organizational Units to be used on the Certificate. type: array items: type: string organizations: description: Organizations to be used on the Certificate. type: array items: type: string postalCodes: description: Postal codes to be used on the Certificate. type: array items: type: string provinces: description: State/Provinces to be used on the Certificate. type: array items: type: string serialNumber: description: Serial number to be used on the Certificate. type: string streetAddresses: description: Street addresses to be used on the Certificate. type: array items: type: string uris: description: Requested URI subject alternative names. type: array items: type: string usages: description: |- Requested key usages and extended key usages. These usages are used to set the `usages` field on the created CertificateRequest resources. If `encodeUsagesInRequest` is unset or set to `true`, the usages will additionally be encoded in the `request` field which contains the CSR blob. If unset, defaults to `digital signature` and `key encipherment`. type: array items: description: |- KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 Valid KeyUsage values are as follows: "signing", "digital signature", "content commitment", "key encipherment", "key agreement", "data encipherment", "cert sign", "crl sign", "encipher only", "decipher only", "any", "server auth", "client auth", "code signing", "email protection", "s/mime", "ipsec end system", "ipsec tunnel", "ipsec user", "timestamping", "ocsp signing", "microsoft sgc", "netscape sgc" type: string enum: - signing - digital signature - content commitment - key encipherment - key agreement - data encipherment - cert sign - crl sign - encipher only - decipher only - any - server auth - client auth - code signing - email protection - s/mime - ipsec end system - ipsec tunnel - ipsec user - timestamping - ocsp signing - microsoft sgc - netscape sgc status: description: |- Status of the Certificate. This is set and managed automatically. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object properties: conditions: description: |- List of status conditions to indicate the status of certificates. Known condition types are `Ready` and `Issuing`. type: array items: description: CertificateCondition contains condition information for an Certificate. type: object required: - status - type properties: lastTransitionTime: description: |- LastTransitionTime is the timestamp corresponding to the last status change of this condition. type: string format: date-time message: description: |- Message is a human readable description of the details of the last transition, complementing reason. type: string observedGeneration: description: |- If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Certificate. type: integer format: int64 reason: description: |- Reason is a brief machine readable explanation for the condition's last transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). type: string enum: - "True" - "False" - Unknown type: description: Type of the condition, known values are (`Ready`, `Issuing`). type: string x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map failedIssuanceAttempts: description: |- The number of continuous failed issuance attempts up till now. This field gets removed (if set) on a successful issuance and gets set to 1 if unset and an issuance has failed. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). type: integer lastFailureTime: description: |- LastFailureTime is set only if the lastest issuance for this Certificate failed and contains the time of the failure. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). If the latest issuance has succeeded this field will be unset. type: string format: date-time nextPrivateKeySecretName: description: |- The name of the Secret resource containing the private key to be used for the next certificate iteration. The keymanager controller will automatically set this field if the `Issuing` condition is set to `True`. It will automatically unset this field when the Issuing condition is not set or False. type: string notAfter: description: |- The expiration time of the certificate stored in the secret named by this resource in `spec.secretName`. type: string format: date-time notBefore: description: |- The time after which the certificate stored in the secret named by this resource in `spec.secretName` is valid. type: string format: date-time renewalTime: description: |- RenewalTime is the time at which the certificate will be next renewed. If not set, no upcoming renewal is scheduled. type: string format: date-time revision: description: |- The current 'revision' of the certificate as issued. When a CertificateRequest resource is created, it will have the `cert-manager.io/certificate-revision` set to one greater than the current value of this field. Upon issuance, this field will be set to the value of the annotation on the CertificateRequest resource used to issue the certificate. Persisting the value on the CertificateRequest resource allows the certificates controller to know whether a request is part of an old issuance or if it is part of the ongoing revision's issuance by checking if the revision value in the annotation is greater than this field. type: integer served: true storage: true --- # Source: cert-manager/deploy/crds/crd-orders.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: orders.acme.cert-manager.io labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' app.kubernetes.io/instance: 'cert-manager' # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: group: acme.cert-manager.io names: kind: Order listKind: OrderList plural: orders singular: order categories: - cert-manager - cert-manager-acme scope: Namespaced versions: - name: v1 subresources: status: {} additionalPrinterColumns: - jsonPath: .status.state name: State type: string - jsonPath: .spec.issuerRef.name name: Issuer priority: 1 type: string - jsonPath: .status.reason name: Reason priority: 1 type: string - jsonPath: .metadata.creationTimestamp description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. name: Age type: date schema: openAPIV3Schema: description: Order is a type to represent an Order with an ACME server type: object required: - metadata - spec properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: type: object required: - issuerRef - request properties: commonName: description: |- CommonName is the common name as specified on the DER encoded CSR. If specified, this value must also be present in `dnsNames` or `ipAddresses`. This field must match the corresponding field on the DER encoded CSR. type: string dnsNames: description: |- DNSNames is a list of DNS names that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR. type: array items: type: string duration: description: |- Duration is the duration for the not after date for the requested certificate. this is set on order creation as pe the ACME spec. type: string ipAddresses: description: |- IPAddresses is a list of IP addresses that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR. type: array items: type: string issuerRef: description: |- IssuerRef references a properly configured ACME-type Issuer which should be used to create this Order. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Order will be marked as failed. type: object required: - name properties: group: description: Group of the resource being referred to. type: string kind: description: Kind of the resource being referred to. type: string name: description: Name of the resource being referred to. type: string request: description: |- Certificate signing request bytes in DER encoding. This will be used when finalizing the order. This field must be set on the order. type: string format: byte status: type: object properties: authorizations: description: |- Authorizations contains data returned from the ACME server on what authorizations must be completed in order to validate the DNS names specified on the Order. type: array items: description: |- ACMEAuthorization contains data returned from the ACME server on an authorization that must be completed in order validate a DNS name on an ACME Order resource. type: object required: - url properties: challenges: description: |- Challenges specifies the challenge types offered by the ACME server. One of these challenge types will be selected when validating the DNS name and an appropriate Challenge resource will be created to perform the ACME challenge process. type: array items: description: |- Challenge specifies a challenge offered by the ACME server for an Order. An appropriate Challenge resource can be created to perform the ACME challenge process. type: object required: - token - type - url properties: token: description: |- Token is the token that must be presented for this challenge. This is used to compute the 'key' that must also be presented. type: string type: description: |- Type is the type of challenge being offered, e.g. 'http-01', 'dns-01', 'tls-sni-01', etc. This is the raw value retrieved from the ACME server. Only 'http-01' and 'dns-01' are supported by cert-manager, other values will be ignored. type: string url: description: |- URL is the URL of this challenge. It can be used to retrieve additional metadata about the Challenge from the ACME server. type: string identifier: description: Identifier is the DNS name to be validated as part of this authorization type: string initialState: description: |- InitialState is the initial state of the ACME authorization when first fetched from the ACME server. If an Authorization is already 'valid', the Order controller will not create a Challenge resource for the authorization. This will occur when working with an ACME server that enables 'authz reuse' (such as Let's Encrypt's production endpoint). If not set and 'identifier' is set, the state is assumed to be pending and a Challenge will be created. type: string enum: - valid - ready - pending - processing - invalid - expired - errored url: description: URL is the URL of the Authorization that must be completed type: string wildcard: description: |- Wildcard will be true if this authorization is for a wildcard DNS name. If this is true, the identifier will be the *non-wildcard* version of the DNS name. For example, if '*.example.com' is the DNS name being validated, this field will be 'true' and the 'identifier' field will be 'example.com'. type: boolean certificate: description: |- Certificate is a copy of the PEM encoded certificate for this Order. This field will be populated after the order has been successfully finalized with the ACME server, and the order has transitioned to the 'valid' state. type: string format: byte failureTime: description: |- FailureTime stores the time that this order failed. This is used to influence garbage collection and back-off. type: string format: date-time finalizeURL: description: |- FinalizeURL of the Order. This is used to obtain certificates for this order once it has been completed. type: string reason: description: |- Reason optionally provides more information about a why the order is in the current state. type: string state: description: |- State contains the current state of this Order resource. States 'success' and 'expired' are 'final' type: string enum: - valid - ready - pending - processing - invalid - expired - errored url: description: |- URL of the Order. This will initially be empty when the resource is first created. The Order controller will populate this field when the Order is first processed. This field will be immutable after it is initially set. type: string served: true storage: true ================================================ FILE: roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 ================================================ # Copyright 2022 The cert-manager Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. --- apiVersion: v1 kind: Namespace metadata: name: {{ cert_manager_namespace }} --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount automountServiceAccountToken: true metadata: name: cert-manager-cainjector namespace: {{ cert_manager_namespace }} labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" --- # Source: cert-manager/deploy/charts/cert-manager/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount automountServiceAccountToken: true metadata: name: cert-manager namespace: {{ cert_manager_namespace }} labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount automountServiceAccountToken: true metadata: name: cert-manager-webhook namespace: {{ cert_manager_namespace }} labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" --- # Source: cert-manager/deploy/charts/cert-manager/templates/controller-config.yaml apiVersion: v1 kind: ConfigMap metadata: name: cert-manager namespace: {{ cert_manager_namespace }} labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" data: --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-config.yaml apiVersion: v1 kind: ConfigMap metadata: name: cert-manager-webhook namespace: {{ cert_manager_namespace }} labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" data: --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-cainjector labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["cert-manager.io"] resources: ["certificates"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "create", "update", "patch"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["apiregistration.k8s.io"] resources: ["apiservices"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "list", "watch", "update", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Issuer controller role apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-issuers labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["cert-manager.io"] resources: ["issuers", "issuers/status"] verbs: ["update", "patch"] - apiGroups: ["cert-manager.io"] resources: ["issuers"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch", "create", "update", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # ClusterIssuer controller role apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-clusterissuers labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["cert-manager.io"] resources: ["clusterissuers", "clusterissuers/status"] verbs: ["update", "patch"] - apiGroups: ["cert-manager.io"] resources: ["clusterissuers"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch", "create", "update", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Certificates controller role apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-certificates labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificates/status", "certificaterequests", "certificaterequests/status"] verbs: ["update", "patch"] - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificaterequests", "clusterissuers", "issuers"] verbs: ["get", "list", "watch"] # We require these rules to support users with the OwnerReferencesPermissionEnforcement # admission controller enabled: # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement - apiGroups: ["cert-manager.io"] resources: ["certificates/finalizers", "certificaterequests/finalizers"] verbs: ["update"] - apiGroups: ["acme.cert-manager.io"] resources: ["orders"] verbs: ["create", "delete", "get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Orders controller role apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-orders labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["acme.cert-manager.io"] resources: ["orders", "orders/status"] verbs: ["update", "patch"] - apiGroups: ["acme.cert-manager.io"] resources: ["orders", "challenges"] verbs: ["get", "list", "watch"] - apiGroups: ["cert-manager.io"] resources: ["clusterissuers", "issuers"] verbs: ["get", "list", "watch"] - apiGroups: ["acme.cert-manager.io"] resources: ["challenges"] verbs: ["create", "delete"] # We require these rules to support users with the OwnerReferencesPermissionEnforcement # admission controller enabled: # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement - apiGroups: ["acme.cert-manager.io"] resources: ["orders/finalizers"] verbs: ["update"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Challenges controller role apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-challenges labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: # Use to update challenge resource status - apiGroups: ["acme.cert-manager.io"] resources: ["challenges", "challenges/status"] verbs: ["update", "patch"] # Used to watch challenge resources - apiGroups: ["acme.cert-manager.io"] resources: ["challenges"] verbs: ["get", "list", "watch"] # Used to watch challenges, issuer and clusterissuer resources - apiGroups: ["cert-manager.io"] resources: ["issuers", "clusterissuers"] verbs: ["get", "list", "watch"] # Need to be able to retrieve ACME account private key to complete challenges - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] # Used to create events - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] # HTTP01 rules - apiGroups: [""] resources: ["pods", "services"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["networking.k8s.io"] resources: ["ingresses"] verbs: ["get", "list", "watch", "create", "delete", "update"] - apiGroups: [ "gateway.networking.k8s.io" ] resources: [ "httproutes" ] verbs: ["get", "list", "watch", "create", "delete", "update"] # We require the ability to specify a custom hostname when we are creating # new ingress resources. # See: https://github.com/openshift/origin/blob/21f191775636f9acadb44fa42beeb4f75b255532/pkg/route/apiserver/admission/ingress_admission.go#L84-L148 - apiGroups: ["route.openshift.io"] resources: ["routes/custom-host"] verbs: ["create"] # We require these rules to support users with the OwnerReferencesPermissionEnforcement # admission controller enabled: # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement - apiGroups: ["acme.cert-manager.io"] resources: ["challenges/finalizers"] verbs: ["update"] # DNS01 rules (duplicated above) - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # ingress-shim controller role apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-ingress-shim labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificaterequests"] verbs: ["create", "update", "delete"] - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificaterequests", "issuers", "clusterissuers"] verbs: ["get", "list", "watch"] - apiGroups: ["networking.k8s.io"] resources: ["ingresses"] verbs: ["get", "list", "watch"] # We require these rules to support users with the OwnerReferencesPermissionEnforcement # admission controller enabled: # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement - apiGroups: ["networking.k8s.io"] resources: ["ingresses/finalizers"] verbs: ["update"] - apiGroups: ["gateway.networking.k8s.io"] resources: ["gateways", "httproutes"] verbs: ["get", "list", "watch"] - apiGroups: ["gateway.networking.k8s.io"] resources: ["gateways/finalizers", "httproutes/finalizers"] verbs: ["update"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-cluster-view labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" rules: - apiGroups: ["cert-manager.io"] resources: ["clusterissuers"] verbs: ["get", "list", "watch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-view labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rbac.authorization.k8s.io/aggregate-to-view: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-admin: "true" rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" rules: - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificaterequests", "issuers"] verbs: ["get", "list", "watch"] - apiGroups: ["acme.cert-manager.io"] resources: ["challenges", "orders"] verbs: ["get", "list", "watch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-edit labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-admin: "true" rules: - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificaterequests", "issuers"] verbs: ["create", "delete", "deletecollection", "patch", "update"] - apiGroups: ["cert-manager.io"] resources: ["certificates/status"] verbs: ["update"] - apiGroups: ["acme.cert-manager.io"] resources: ["challenges", "orders"] verbs: ["create", "delete", "deletecollection", "patch", "update"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Permission to approve CertificateRequests referencing cert-manager.io Issuers and ClusterIssuers apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-approve:cert-manager-io labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cert-manager" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["cert-manager.io"] resources: ["signers"] verbs: ["approve"] resourceNames: - "issuers.cert-manager.io/*" - "clusterissuers.cert-manager.io/*" --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Permission to: # - Update and sign CertificatSigningeRequests referencing cert-manager.io Issuers and ClusterIssuers # - Perform SubjectAccessReviews to test whether users are able to reference Namespaced Issuers apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-controller-certificatesigningrequests labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cert-manager" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["certificates.k8s.io"] resources: ["certificatesigningrequests"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["certificates.k8s.io"] resources: ["certificatesigningrequests/status"] verbs: ["update", "patch"] - apiGroups: ["certificates.k8s.io"] resources: ["signers"] resourceNames: ["issuers.cert-manager.io/*", "clusterissuers.cert-manager.io/*"] verbs: ["sign"] - apiGroups: ["authorization.k8s.io"] resources: ["subjectaccessreviews"] verbs: ["create"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cert-manager-webhook:subjectaccessreviews labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["authorization.k8s.io"] resources: ["subjectaccessreviews"] verbs: ["create"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-cainjector labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-cainjector subjects: - name: cert-manager-cainjector namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-issuers labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-issuers subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-clusterissuers labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-clusterissuers subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-certificates labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-certificates subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-orders labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-orders subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-challenges labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-challenges subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-ingress-shim labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-ingress-shim subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-approve:cert-manager-io labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cert-manager" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-approve:cert-manager-io subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-controller-certificatesigningrequests labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cert-manager" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-controller-certificatesigningrequests subjects: - name: cert-manager namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cert-manager-webhook:subjectaccessreviews labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cert-manager-webhook:subjectaccessreviews subjects: - apiGroup: "" kind: ServiceAccount name: cert-manager-webhook namespace: {{ cert_manager_namespace }} --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml # leader election rules apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cert-manager-cainjector:leaderelection namespace: {{ cert_manager_leader_election_namespace }} labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: # Used for leader election by the controller # cert-manager-cainjector-leader-election is used by the CertificateBased injector controller # see cmd/cainjector/start.go#L113 # cert-manager-cainjector-leader-election-core is used by the SecretBased injector controller # see cmd/cainjector/start.go#L137 - apiGroups: ["coordination.k8s.io"] resources: ["leases"] resourceNames: ["cert-manager-cainjector-leader-election", "cert-manager-cainjector-leader-election-core"] verbs: ["get", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cert-manager:leaderelection namespace: {{ cert_manager_leader_election_namespace }} labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] resourceNames: ["cert-manager-controller"] verbs: ["get", "update", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cert-manager-webhook:dynamic-serving namespace: {{ cert_manager_namespace }} labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" rules: - apiGroups: [""] resources: ["secrets"] resourceNames: - 'cert-manager-webhook-ca' verbs: ["get", "list", "watch", "update"] # It's not possible to grant CREATE permission on a single resourceName. - apiGroups: [""] resources: ["secrets"] verbs: ["create"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml # grant cert-manager permission to manage the leaderelection configmap in the # leader election namespace apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cert-manager-cainjector:leaderelection namespace: {{ cert_manager_leader_election_namespace }} labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cert-manager-cainjector:leaderelection subjects: - kind: ServiceAccount name: cert-manager-cainjector namespace: {{ cert_manager_namespace }} --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # grant cert-manager permission to manage the leaderelection configmap in the # leader election namespace apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cert-manager:leaderelection namespace: {{ cert_manager_leader_election_namespace }} labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cert-manager:leaderelection subjects: - apiGroup: "" kind: ServiceAccount name: cert-manager namespace: {{ cert_manager_namespace }} --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cert-manager-webhook:dynamic-serving namespace: {{ cert_manager_namespace }} labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cert-manager-webhook:dynamic-serving subjects: - apiGroup: "" kind: ServiceAccount name: cert-manager-webhook namespace: {{ cert_manager_namespace }} --- # Source: cert-manager/deploy/charts/cert-manager/templates/service.yaml apiVersion: v1 kind: Service metadata: name: cert-manager namespace: {{ cert_manager_namespace }} labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: type: ClusterIP ports: - protocol: TCP port: 9402 name: tcp-prometheus-servicemonitor targetPort: 9402 selector: app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-service.yaml apiVersion: v1 kind: Service metadata: name: cert-manager-webhook namespace: {{ cert_manager_namespace }} labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: type: ClusterIP ports: - name: https port: 443 protocol: TCP targetPort: "https" selector: app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: cert-manager-cainjector namespace: {{ cert_manager_namespace }} labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" template: metadata: labels: app: cainjector app.kubernetes.io/name: cainjector app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "cainjector" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: serviceAccountName: cert-manager-cainjector enableServiceLinks: false securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault containers: - name: cert-manager-cainjector image: "{{ cert_manager_cainjector_image_repo }}:{{ cert_manager_cainjector_image_tag }}" imagePullPolicy: {{ k8s_image_pull_policy }} args: - --v=2 - --leader-election-namespace={{ cert_manager_leader_election_namespace }} env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {% if cert_manager_http_proxy is defined and cert_manager_http_proxy != "" %} - name: HTTP_PROXY value: "{{ cert_manager_http_proxy }}" {% endif %} {% if cert_manager_https_proxy is defined and cert_manager_https_proxy != "" %} - name: HTTPS_PROXY value: "{{ cert_manager_https_proxy }}" {% endif %} {% if cert_manager_no_proxy is defined and cert_manager_no_proxy != "" %} - name: NO_PROXY value: "{{ cert_manager_no_proxy }}" {% endif %} securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true {% if cert_manager_tolerations %} tolerations: {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} {% if cert_manager_nodeselector %} nodeSelector: {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }} {% endif %} {% if cert_manager_affinity %} affinity: {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }} {% endif %} --- {% if cert_manager_trusted_internal_ca is defined %} apiVersion: v1 data: internal-ca.pem: | {{ cert_manager_trusted_internal_ca | indent(width=4, first=False) }} kind: ConfigMap metadata: name: ca-internal-truststore namespace: {{ cert_manager_namespace }} --- {% endif %} # Source: cert-manager/deploy/charts/cert-manager/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: cert-manager namespace: {{ cert_manager_namespace }} labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" template: metadata: labels: app: cert-manager app.kubernetes.io/name: cert-manager app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "controller" app.kubernetes.io/version: "{{ cert_manager_version }}" annotations: prometheus.io/path: "/metrics" prometheus.io/scrape: 'true' prometheus.io/port: '9402' spec: serviceAccountName: cert-manager enableServiceLinks: false securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault containers: - name: cert-manager-controller image: "{{ cert_manager_controller_image_repo }}:{{ cert_manager_controller_image_tag }}" imagePullPolicy: {{ k8s_image_pull_policy }} args: - --v=2 - --cluster-resource-namespace=$(POD_NAMESPACE) - --leader-election-namespace={{ cert_manager_leader_election_namespace }} {% for extra_arg in cert_manager_controller_extra_args %} - {{ extra_arg }} {% endfor %} ports: - containerPort: 9402 name: http-metrics protocol: TCP - containerPort: 9403 name: http-healthz protocol: TCP securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {% if cert_manager_http_proxy is defined and cert_manager_http_proxy != "" %} - name: HTTP_PROXY value: "{{ cert_manager_http_proxy }}" {% endif %} {% if cert_manager_https_proxy is defined and cert_manager_https_proxy != "" %} - name: HTTPS_PROXY value: "{{ cert_manager_https_proxy }}" {% endif %} {% if cert_manager_no_proxy is defined and cert_manager_no_proxy != "" %} - name: NO_PROXY value: "{{ cert_manager_no_proxy }}" {% endif %} livenessProbe: httpGet: port: http-healthz path: /livez scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 15 successThreshold: 1 failureThreshold: 8 {% if cert_manager_trusted_internal_ca is defined %} volumeMounts: - mountPath: /etc/ssl/certs/internal-ca.pem name: ca-internal-truststore subPath: internal-ca.pem volumes: - configMap: defaultMode: 420 name: ca-internal-truststore name: ca-internal-truststore {% endif %} {% if cert_manager_tolerations %} tolerations: {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} {% if cert_manager_nodeselector %} nodeSelector: {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }} {% endif %} {% if cert_manager_affinity %} affinity: {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }} {% endif %} {% if cert_manager_dns_policy %} dnsPolicy: {{ cert_manager_dns_policy }} {% endif %} {% if cert_manager_dns_config %} dnsConfig: {{ cert_manager_dns_config | to_nice_yaml | indent(width=8) }} {% endif %} --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: cert-manager-webhook namespace: {{ cert_manager_namespace }} labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" template: metadata: labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" spec: serviceAccountName: cert-manager-webhook enableServiceLinks: false securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault containers: - name: cert-manager-webhook image: "{{ cert_manager_webhook_image_repo }}:{{ cert_manager_webhook_image_tag }}" imagePullPolicy: {{ k8s_image_pull_policy }} args: - --v=2 - --secure-port=10250 - --dynamic-serving-ca-secret-namespace=$(POD_NAMESPACE) - --dynamic-serving-ca-secret-name=cert-manager-webhook-ca - --dynamic-serving-dns-names=cert-manager-webhook - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE) - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE).svc ports: - name: https protocol: TCP containerPort: 10250 - name: healthcheck protocol: TCP containerPort: 6080 livenessProbe: httpGet: path: /livez port: 6080 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 1 successThreshold: 1 failureThreshold: 3 readinessProbe: httpGet: path: /healthz port: 6080 scheme: HTTP initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 1 successThreshold: 1 failureThreshold: 3 securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {% if cert_manager_http_proxy is defined and cert_manager_http_proxy != "" %} - name: HTTP_PROXY value: "{{ cert_manager_http_proxy }}" {% endif %} {% if cert_manager_https_proxy is defined and cert_manager_https_proxy != "" %} - name: HTTPS_PROXY value: "{{ cert_manager_https_proxy }}" {% endif %} {% if cert_manager_no_proxy is defined and cert_manager_no_proxy != "" %} - name: NO_PROXY value: "{{ cert_manager_no_proxy }}" {% endif %} {% if cert_manager_tolerations %} tolerations: {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} {% if cert_manager_nodeselector %} nodeSelector: {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }} {% endif %} {% if cert_manager_affinity %} affinity: {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }} {% endif %} --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-mutating-webhook.yaml apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: cert-manager-webhook labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" annotations: cert-manager.io/inject-ca-from-secret: "{{ cert_manager_namespace }}/cert-manager-webhook-ca" webhooks: - name: webhook.cert-manager.io rules: - apiGroups: - "cert-manager.io" apiVersions: - "v1" operations: - CREATE resources: - "certificaterequests" admissionReviewVersions: ["v1"] # This webhook only accepts v1 cert-manager resources. # Equivalent matchPolicy ensures that non-v1 resource requests are sent to # this webhook (after the resources have been converted to v1). matchPolicy: Equivalent timeoutSeconds: 30 failurePolicy: Fail # Only include 'sideEffects' field in Kubernetes 1.12+ sideEffects: None clientConfig: service: name: cert-manager-webhook namespace: {{ cert_manager_namespace }} path: /mutate --- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-validating-webhook.yaml apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: cert-manager-webhook labels: app: webhook app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" annotations: cert-manager.io/inject-ca-from-secret: "{{ cert_manager_namespace }}/cert-manager-webhook-ca" webhooks: - name: webhook.cert-manager.io namespaceSelector: matchExpressions: - key: cert-manager.io/disable-validation operator: NotIn values: - "true" rules: - apiGroups: - "cert-manager.io" - "acme.cert-manager.io" apiVersions: - "v1" operations: - CREATE - UPDATE resources: - "*/*" admissionReviewVersions: ["v1"] # This webhook only accepts v1 cert-manager resources. # Equivalent matchPolicy ensures that non-v1 resource requests are sent to # this webhook (after the resources have been converted to v1). matchPolicy: Equivalent timeoutSeconds: 30 failurePolicy: Fail sideEffects: None clientConfig: service: name: cert-manager-webhook namespace: {{ cert_manager_namespace }} path: /validate ================================================ FILE: roles/kubernetes-apps/ingress_controller/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/ingress_controller/cert_manager when: cert_manager_enabled tags: - apps - ingress-controller - cert-manager - role: kubernetes-apps/ingress_controller/alb_ingress_controller when: ingress_alb_enabled tags: - apps - ingress-controller - ingress_alb ================================================ FILE: roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml ================================================ --- kubelet_csr_approver_enabled: "{{ kubelet_rotate_server_certificates }}" kubelet_csr_approver_namespace: kube-system kubelet_csr_approver_repository_name: kubelet-csr-approver kubelet_csr_approver_repository_url: https://postfinance.github.io/kubelet-csr-approver kubelet_csr_approver_chart_ref: "{{ kubelet_csr_approver_repository_name }}/kubelet-csr-approver" kubelet_csr_approver_chart_version: 1.1.0 # Fill values override here # See upstream https://github.com/postfinance/kubelet-csr-approver kubelet_csr_approver_values: {} ================================================ FILE: roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml ================================================ --- dependencies: - role: helm-apps when: - inventory_hostname == groups['kube_control_plane'][0] - kubelet_csr_approver_enabled environment: http_proxy: "{{ http_proxy | default('') }}" https_proxy: "{{ https_proxy | default('') }}" release_common_opts: {} releases: - name: kubelet-csr-approver namespace: "{{ kubelet_csr_approver_namespace }}" chart_ref: "{{ kubelet_csr_approver_chart_ref }}" chart_version: "{{ kubelet_csr_approver_chart_version }}" wait: "{{ kube_network_plugin != 'cni' }}" atomic: "{{ kube_network_plugin != 'cni' }}" values: "{{ kubelet_csr_approver_values }}" repositories: - name: "{{ kubelet_csr_approver_repository_name }}" url: "{{ kubelet_csr_approver_repository_url }}" ================================================ FILE: roles/kubernetes-apps/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/utils - role: kubernetes-apps/ansible when: - inventory_hostname == groups['kube_control_plane'][0] - role: kubernetes-apps/helm when: - helm_enabled tags: - helm - role: kubernetes-apps/registry when: - registry_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - registry - role: kubernetes-apps/metrics_server when: - metrics_server_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - metrics_server - role: kubernetes-apps/csi_driver/csi_crd when: - cinder_csi_enabled or csi_snapshot_controller_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - csi-driver - role: kubernetes-apps/csi_driver/cinder when: - cinder_csi_enabled tags: - cinder-csi-driver - csi-driver - role: kubernetes-apps/csi_driver/aws_ebs when: - aws_ebs_csi_enabled tags: - aws-ebs-csi-driver - csi-driver - role: kubernetes-apps/csi_driver/azuredisk when: - azure_csi_enabled tags: - azure-csi-driver - csi-driver - role: kubernetes-apps/csi_driver/gcp_pd when: - gcp_pd_csi_enabled tags: - gcp-pd-csi-driver - csi-driver - role: kubernetes-apps/csi_driver/upcloud when: - upcloud_csi_enabled tags: - upcloud-csi-driver - csi-driver - role: kubernetes-apps/csi_driver/vsphere when: - vsphere_csi_enabled tags: - vsphere-csi-driver - csi-driver - role: kubernetes-apps/persistent_volumes when: - persistent_volumes_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - persistent_volumes - role: kubernetes-apps/snapshots when: inventory_hostname == groups['kube_control_plane'][0] tags: - snapshots - csi-driver - role: kubernetes-apps/container_runtimes when: - inventory_hostname == groups['kube_control_plane'][0] tags: - container-runtimes - role: kubernetes-apps/container_engine_accelerator when: nvidia_accelerator_enabled tags: - container_engine_accelerator - role: kubernetes-apps/kubelet-csr-approver when: - kubelet_csr_approver_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - kubelet-csr-approver - role: kubernetes-apps/metallb when: - metallb_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - metallb - role: kubernetes-apps/argocd when: - argocd_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - argocd - role: kubernetes-apps/scheduler_plugins when: - scheduler_plugins_enabled - kube_major_version is version('1.29', '<') - inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins - role: kubernetes-apps/node_feature_discovery when: - node_feature_discovery_enabled - inventory_hostname == groups['kube_control_plane'][0] tags: - node_feature_discovery ================================================ FILE: roles/kubernetes-apps/metallb/defaults/main.yml ================================================ --- metallb_enabled: false metallb_log_level: info metallb_namespace: "metallb-system" metallb_port: "7472" metallb_memberlist_port: "7946" metallb_speaker_enabled: "{{ metallb_enabled }}" metallb_speaker_nodeselector: kubernetes.io/os: "linux" metallb_controller_nodeselector: kubernetes.io/os: "linux" metallb_speaker_tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists metallb_controller_tolerations: [] metallb_loadbalancer_class: "" ================================================ FILE: roles/kubernetes-apps/metallb/tasks/main.yml ================================================ --- - name: Kubernetes Apps | Check cluster settings for MetalLB fail: msg: "MetalLB require kube_proxy_strict_arp = true, see https://github.com/danderson/metallb/issues/153#issuecomment-518651132" when: - "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp" - name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore fail: msg: "'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead" when: - matallb_auto_assign is defined - name: Kubernetes Apps | Lay Down MetalLB become: true template: src: "metallb.yaml.j2" dest: "{{ kube_config_dir }}/metallb.yaml" mode: "0644" register: metallb_rendering when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Apps | Install and configure MetalLB kube: name: "MetalLB" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/metallb.yaml" state: "{{ metallb_rendering.changed | ternary('latest', 'present') }}" wait: true become: true when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Apps | Wait for MetalLB controller to be running command: "{{ bin_dir }}/kubectl rollout status -n {{ metallb_namespace }} deployment -l app=metallb,component=controller --timeout=2m" become: true when: - inventory_hostname == groups['kube_control_plane'][0] - name: MetalLB | Address pools when: - inventory_hostname == groups['kube_control_plane'][0] - metallb_config.address_pools is defined block: - name: MetalLB | Layout address pools template ansible.builtin.template: src: pools.yaml.j2 dest: "{{ kube_config_dir }}/pools.yaml" mode: "0644" register: pools_rendering - name: MetalLB | Create address pools configuration kube: name: "MetalLB" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/pools.yaml" state: "{{ pools_rendering.changed | ternary('latest', 'present') }}" become: true - name: MetalLB | Layer2 when: - inventory_hostname == groups['kube_control_plane'][0] - metallb_config.layer2 is defined block: - name: MetalLB | Layout layer2 template ansible.builtin.template: src: layer2.yaml.j2 dest: "{{ kube_config_dir }}/layer2.yaml" mode: "0644" register: layer2_rendering - name: MetalLB | Create layer2 configuration kube: name: "MetalLB" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/layer2.yaml" state: "{{ layer2_rendering.changed | ternary('latest', 'present') }}" become: true - name: MetalLB | Layer3 when: - inventory_hostname == groups['kube_control_plane'][0] - metallb_config.layer3 is defined block: - name: MetalLB | Layout layer3 template ansible.builtin.template: src: layer3.yaml.j2 dest: "{{ kube_config_dir }}/layer3.yaml" mode: "0644" register: layer3_rendering - name: MetalLB | Create layer3 configuration kube: name: "MetalLB" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/layer3.yaml" state: "{{ layer3_rendering.changed | ternary('latest', 'present') }}" become: true - name: Kubernetes Apps | Delete MetalLB ConfigMap kube: name: config kubectl: "{{ bin_dir }}/kubectl" resource: ConfigMap namespace: "{{ metallb_namespace }}" state: absent ================================================ FILE: roles/kubernetes-apps/metallb/templates/layer2.yaml.j2 ================================================ #jinja2: trim_blocks: True, lstrip_blocks: True # yamllint disable-file --- # Create layer2 configuration {% for entry in metallb_config.layer2 %} --- # L2 Configuration apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: "{{ entry }}" namespace: "{{ metallb_namespace }}" spec: ipAddressPools: - "{{ entry }}" {% endfor %} ================================================ FILE: roles/kubernetes-apps/metallb/templates/layer3.yaml.j2 ================================================ #jinja2: trim_blocks: True, lstrip_blocks: True # yamllint disable-file --- # Create layer3 configuration {% if metallb_config.layer3.communities is defined %} {% for community_name, community in metallb_config.layer3.communities.items() %} --- apiVersion: metallb.io/v1beta1 kind: Community metadata: name: "{{ community_name }}" namespace: "{{ metallb_namespace }}" spec: communities: - name: "{{ community_name }}" value: "{{ community }}" {% endfor %} {% endif %} --- apiVersion: metallb.io/v1beta1 kind: Community metadata: name: well-known namespace: "{{ metallb_namespace }}" spec: communities: - name: no-export value: 65535:65281 - name: no-advertise value: 65535:65282 - name: local-as value: 65535:65283 - name: nopeer value: 65535:65284 # BGPAdvertisement is used to advertise address pools to the BGP peer. Specific pools can be listed to be advertised. # Local BGP Advertisement specifies that the IP specified in the address pool will be used as remote source address for traffic entering your cluster from the remote peer. # When using this option, be sure to use a subnet and routable IP for your address pool. # This is good: 10.0.0.10/24. This is also good: 10.0.0.129/25. This is bad: 10.0.0.0/24. This is also bad: 10.0.0.128/25. # In this example, 10.0.0.10 will be used as the remote source address. # This is also bad: 10.0.0.10-10.0.0.25. Remember: you are working with aggregationLength, which specifies a subnet, not an IP range! # The no-advertise community is set on the local advertisement to prevent this route from being published to the BGP peer. # Your aggregationLength ideally is the same size as your address pool. {% for peer_name, peer in metallb_config.layer3.metallb_peers.items() %} {% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %} --- apiVersion: metallb.io/v1beta1 kind: BGPAdvertisement metadata: name: "{{ peer_name }}-local" namespace: "{{ metallb_namespace }}" spec: aggregationLength: 32 aggregationLengthV6: 128 communities: - no-advertise localpref: "{{ peer.localpref | default("100") }}" ipAddressPools: {% for address_pool in peer.address_pool %} - "{{ address_pool }}" {% endfor %} {% endif %} # External BGP Advertisement. The IP range specied in the address pool is advertised to the BGP peer. --- apiVersion: metallb.io/v1beta1 kind: BGPAdvertisement metadata: name: "{{ peer_name }}-external" namespace: "{{ metallb_namespace }}" spec: {% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %} aggregationLength: {{ peer.aggregation_length }} {% endif %} ipAddressPools: {% for address_pool in peer.address_pool %} - "{{ address_pool }}" {% endfor %} {% if peer.communities is defined %} {% for community in peer.communities %} communities: - "{{ community }}" {% endfor %} {% endif %} # Configuration for the BGP peer. --- apiVersion: metallb.io/v1beta2 kind: BGPPeer metadata: name: "{{ peer_name }}" namespace: "{{ metallb_namespace }}" spec: myASN: {{ peer.my_asn }} peerASN: {{ peer.peer_asn }} peerAddress: {{ peer.peer_address }} {% if peer.peer_port is defined %} peerPort: {{ peer.peer_port }} {% else %} peerPort: {{ metallb_config.layer3.defaults.peer_port }} {% endif -%} {% if peer.password is defined %} password: "{{ peer.password }}" {% endif -%} {% if peer.router_id is defined %} routerID: "{{ peer.router_id }}" {% endif -%} {% if peer.hold_time is defined %} holdTime: {{ peer.hold_time }} {% elif metallb_config.layer3.defaults.hold_time is defined %} holdTime: {{ metallb_config.layer3.defaults.hold_time }} {% endif -%} {% if peer.multihop is defined %} ebgpMultiHop: {{ peer.multihop }} {% endif -%} {% endfor %} ================================================ FILE: roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: labels: pod-security.kubernetes.io/audit: privileged pod-security.kubernetes.io/enforce: privileged pod-security.kubernetes.io/warn: privileged name: {{ metallb_namespace }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 name: addresspools.metallb.io spec: conversion: strategy: Webhook webhook: clientConfig: caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /convert conversionReviewVersions: - v1alpha1 - v1beta1 group: metallb.io names: kind: AddressPool listKind: AddressPoolList plural: addresspools singular: addresspool scope: Namespaced versions: - deprecated: true deprecationWarning: metallb.io v1alpha1 AddressPool is deprecated name: v1alpha1 schema: openAPIV3Schema: description: AddressPool is the Schema for the addresspools API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: AddressPoolSpec defines the desired state of AddressPool. properties: addresses: description: A list of IP address ranges over which MetalLB has authority. You can list multiple ranges in a single pool, they will all share the same settings. Each range can be either a CIDR prefix, or an explicit start-end range of IPs. items: type: string type: array autoAssign: default: true description: AutoAssign flag used to prevent MetallB from automatic allocation for a pool. type: boolean bgpAdvertisements: description: When an IP is allocated from this pool, how should it be translated into BGP announcements? items: properties: aggregationLength: default: 32 description: The aggregation-length advertisement option lets you “roll up” the /32s into a larger prefix. format: int32 minimum: 1 type: integer aggregationLengthV6: default: 128 description: Optional, defaults to 128 (i.e. no aggregation) if not specified. format: int32 type: integer communities: description: BGP communities items: type: string type: array localPref: description: BGP LOCAL_PREF attribute which is used by BGP best path algorithm, Path with higher localpref is preferred over one with lower localpref. format: int32 type: integer type: object type: array protocol: description: Protocol can be used to select how the announcement is done. enum: - layer2 - bgp type: string required: - addresses - protocol type: object status: description: AddressPoolStatus defines the observed state of AddressPool. type: object required: - spec type: object served: true storage: false subresources: status: {} - deprecated: true deprecationWarning: metallb.io v1beta1 AddressPool is deprecated, consider using IPAddressPool name: v1beta1 schema: openAPIV3Schema: description: AddressPool represents a pool of IP addresses that can be allocated to LoadBalancer services. AddressPool is deprecated and being replaced by IPAddressPool. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: AddressPoolSpec defines the desired state of AddressPool. properties: addresses: description: A list of IP address ranges over which MetalLB has authority. You can list multiple ranges in a single pool, they will all share the same settings. Each range can be either a CIDR prefix, or an explicit start-end range of IPs. items: type: string type: array autoAssign: default: true description: AutoAssign flag used to prevent MetallB from automatic allocation for a pool. type: boolean bgpAdvertisements: description: Drives how an IP allocated from this pool should translated into BGP announcements. items: properties: aggregationLength: default: 32 description: The aggregation-length advertisement option lets you “roll up” the /32s into a larger prefix. format: int32 minimum: 1 type: integer aggregationLengthV6: default: 128 description: Optional, defaults to 128 (i.e. no aggregation) if not specified. format: int32 type: integer communities: description: BGP communities to be associated with the given advertisement. items: type: string type: array localPref: description: BGP LOCAL_PREF attribute which is used by BGP best path algorithm, Path with higher localpref is preferred over one with lower localpref. format: int32 type: integer type: object type: array protocol: description: Protocol can be used to select how the announcement is done. enum: - layer2 - bgp type: string required: - addresses - protocol type: object status: description: AddressPoolStatus defines the observed state of AddressPool. type: object required: - spec type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: bfdprofiles.metallb.io spec: group: metallb.io names: kind: BFDProfile listKind: BFDProfileList plural: bfdprofiles singular: bfdprofile scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.passiveMode name: Passive Mode type: boolean - jsonPath: .spec.transmitInterval name: Transmit Interval type: integer - jsonPath: .spec.receiveInterval name: Receive Interval type: integer - jsonPath: .spec.detectMultiplier name: Multiplier type: integer name: v1beta1 schema: openAPIV3Schema: description: BFDProfile represents the settings of the bfd session that can be optionally associated with a BGP session. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BFDProfileSpec defines the desired state of BFDProfile. properties: detectMultiplier: description: Configures the detection multiplier to determine packet loss. The remote transmission interval will be multiplied by this value to determine the connection loss detection timer. format: int32 maximum: 255 minimum: 2 type: integer echoInterval: description: Configures the minimal echo receive transmission interval that this system is capable of handling in milliseconds. Defaults to 50ms format: int32 maximum: 60000 minimum: 10 type: integer echoMode: description: Enables or disables the echo transmission mode. This mode is disabled by default, and not supported on multi hops setups. type: boolean minimumTtl: description: 'For multi hop sessions only: configure the minimum expected TTL for an incoming BFD control packet.' format: int32 maximum: 254 minimum: 1 type: integer passiveMode: description: 'Mark session as passive: a passive session will not attempt to start the connection and will wait for control packets from peer before it begins replying.' type: boolean receiveInterval: description: The minimum interval that this system is capable of receiving control packets in milliseconds. Defaults to 300ms. format: int32 maximum: 60000 minimum: 10 type: integer transmitInterval: description: The minimum transmission interval (less jitter) that this system wants to use to send BFD control packets in milliseconds. Defaults to 300ms format: int32 maximum: 60000 minimum: 10 type: integer type: object status: description: BFDProfileStatus defines the observed state of BFDProfile. type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: bgpadvertisements.metallb.io spec: group: metallb.io names: kind: BGPAdvertisement listKind: BGPAdvertisementList plural: bgpadvertisements singular: bgpadvertisement scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.ipAddressPools name: IPAddressPools type: string - jsonPath: .spec.ipAddressPoolSelectors name: IPAddressPool Selectors type: string - jsonPath: .spec.peers name: Peers type: string - jsonPath: .spec.nodeSelectors name: Node Selectors priority: 10 type: string name: v1beta1 schema: openAPIV3Schema: description: BGPAdvertisement allows to advertise the IPs coming from the selected IPAddressPools via BGP, setting the parameters of the BGP Advertisement. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPAdvertisementSpec defines the desired state of BGPAdvertisement. properties: aggregationLength: default: 32 description: The aggregation-length advertisement option lets you “roll up” the /32s into a larger prefix. Defaults to 32. Works for IPv4 addresses. format: int32 minimum: 1 type: integer aggregationLengthV6: default: 128 description: The aggregation-length advertisement option lets you “roll up” the /128s into a larger prefix. Defaults to 128. Works for IPv6 addresses. format: int32 type: integer communities: description: The BGP communities to be associated with the announcement. Each item can be a community of the form 1234:1234 or the name of an alias defined in the Community CRD. items: type: string type: array ipAddressPoolSelectors: description: A selector for the IPAddressPools which would get advertised via this advertisement. If no IPAddressPool is selected by this or by the list, the advertisement is applied to all the IPAddressPools. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array ipAddressPools: description: The list of IPAddressPools to advertise via this advertisement, selected by name. items: type: string type: array localPref: description: The BGP LOCAL_PREF attribute which is used by BGP best path algorithm, Path with higher localpref is preferred over one with lower localpref. format: int32 type: integer nodeSelectors: description: NodeSelectors allows to limit the nodes to announce as next hops for the LoadBalancer IP. When empty, all the nodes having are announced as next hops. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array peers: description: Peers limits the bgppeer to advertise the ips of the selected pools to. When empty, the loadbalancer IP is announced to all the BGPPeers configured. items: type: string type: array type: object status: description: BGPAdvertisementStatus defines the observed state of BGPAdvertisement. type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 name: bgppeers.metallb.io spec: conversion: strategy: Webhook webhook: clientConfig: caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /convert conversionReviewVersions: - v1beta1 - v1beta2 group: metallb.io names: kind: BGPPeer listKind: BGPPeerList plural: bgppeers singular: bgppeer scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.peerAddress name: Address type: string - jsonPath: .spec.peerASN name: ASN type: string - jsonPath: .spec.bfdProfile name: BFD Profile type: string - jsonPath: .spec.ebgpMultiHop name: Multi Hops type: string name: v1beta1 schema: openAPIV3Schema: description: BGPPeer is the Schema for the peers API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPPeerSpec defines the desired state of Peer. properties: bfdProfile: type: string ebgpMultiHop: description: EBGP peer is multi-hops away type: boolean holdTime: description: Requested BGP hold time, per RFC4271. type: string keepaliveTime: description: Requested BGP keepalive time, per RFC4271. type: string myASN: description: AS number to use for the local end of the session. format: int32 maximum: 4294967295 minimum: 0 type: integer nodeSelectors: description: Only connect to this peer on nodes that match one of these selectors. items: properties: matchExpressions: items: properties: key: type: string operator: type: string values: items: type: string minItems: 1 type: array required: - key - operator - values type: object type: array matchLabels: additionalProperties: type: string type: object type: object type: array password: description: Authentication password for routers enforcing TCP MD5 authenticated sessions type: string peerASN: description: AS number to expect from the remote end of the session. format: int32 maximum: 4294967295 minimum: 0 type: integer peerAddress: description: Address to dial when establishing the session. type: string peerPort: description: Port to dial when establishing the session. maximum: 16384 minimum: 0 type: integer routerID: description: BGP router ID to advertise to the peer type: string sourceAddress: description: Source address to use when establishing the session. type: string required: - myASN - peerASN - peerAddress type: object status: description: BGPPeerStatus defines the observed state of Peer. type: object type: object served: true storage: false subresources: status: {} - additionalPrinterColumns: - jsonPath: .spec.peerAddress name: Address type: string - jsonPath: .spec.peerASN name: ASN type: string - jsonPath: .spec.bfdProfile name: BFD Profile type: string - jsonPath: .spec.ebgpMultiHop name: Multi Hops type: string name: v1beta2 schema: openAPIV3Schema: description: BGPPeer is the Schema for the peers API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BGPPeerSpec defines the desired state of Peer. properties: bfdProfile: description: The name of the BFD Profile to be used for the BFD session associated to the BGP session. If not set, the BFD session won't be set up. type: string ebgpMultiHop: description: To set if the BGPPeer is multi-hops away. Needed for FRR mode only. type: boolean holdTime: description: Requested BGP hold time, per RFC4271. type: string keepaliveTime: description: Requested BGP keepalive time, per RFC4271. type: string myASN: description: AS number to use for the local end of the session. format: int32 maximum: 4294967295 minimum: 0 type: integer nodeSelectors: description: Only connect to this peer on nodes that match one of these selectors. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array password: description: Authentication password for routers enforcing TCP MD5 authenticated sessions type: string passwordSecret: description: passwordSecret is name of the authentication secret for BGP Peer. the secret must be of type "kubernetes.io/basic-auth", and created in the same namespace as the MetalLB deployment. The password is stored in the secret as the key "password". properties: name: description: name is unique within a namespace to reference a secret resource. type: string namespace: description: namespace defines the space within which the secret name must be unique. type: string type: object x-kubernetes-map-type: atomic peerASN: description: AS number to expect from the remote end of the session. format: int32 maximum: 4294967295 minimum: 0 type: integer peerAddress: description: Address to dial when establishing the session. type: string peerPort: default: 179 description: Port to dial when establishing the session. maximum: 16384 minimum: 0 type: integer routerID: description: BGP router ID to advertise to the peer type: string sourceAddress: description: Source address to use when establishing the session. type: string vrf: description: To set if we want to peer with the BGPPeer using an interface belonging to a host vrf type: string required: - myASN - peerASN - peerAddress type: object status: description: BGPPeerStatus defines the observed state of Peer. type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: communities.metallb.io spec: group: metallb.io names: kind: Community listKind: CommunityList plural: communities singular: community scope: Namespaced versions: - name: v1beta1 schema: openAPIV3Schema: description: Community is a collection of aliases for communities. Users can define named aliases to be used in the BGPPeer CRD. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: CommunitySpec defines the desired state of Community. properties: communities: items: properties: name: description: The name of the alias for the community. type: string value: description: The BGP community value corresponding to the given name. type: string type: object type: array type: object status: description: CommunityStatus defines the observed state of Community. type: object type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: ipaddresspools.metallb.io spec: group: metallb.io names: kind: IPAddressPool listKind: IPAddressPoolList plural: ipaddresspools singular: ipaddresspool scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.autoAssign name: Auto Assign type: boolean - jsonPath: .spec.avoidBuggyIPs name: Avoid Buggy IPs type: boolean - jsonPath: .spec.addresses name: Addresses type: string name: v1beta1 schema: openAPIV3Schema: description: IPAddressPool represents a pool of IP addresses that can be allocated to LoadBalancer services. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: IPAddressPoolSpec defines the desired state of IPAddressPool. properties: addresses: description: A list of IP address ranges over which MetalLB has authority. You can list multiple ranges in a single pool, they will all share the same settings. Each range can be either a CIDR prefix, or an explicit start-end range of IPs. items: type: string type: array autoAssign: default: true description: AutoAssign flag used to prevent MetallB from automatic allocation for a pool. type: boolean avoidBuggyIPs: default: false description: AvoidBuggyIPs prevents addresses ending with .0 and .255 to be used by a pool. type: boolean serviceAllocation: description: AllocateTo makes ip pool allocation to specific namespace and/or service. The controller will use the pool with lowest value of priority in case of multiple matches. A pool with no priority set will be used only if the pools with priority can't be used. If multiple matching IPAddressPools are available it will check for the availability of IPs sorting the matching IPAddressPools by priority, starting from the highest to the lowest. If multiple IPAddressPools have the same priority, choice will be random. properties: namespaceSelectors: description: NamespaceSelectors list of label selectors to select namespace(s) for ip pool, an alternative to using namespace list. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array namespaces: description: Namespaces list of namespace(s) on which ip pool can be attached. items: type: string type: array priority: description: Priority priority given for ip pool while ip allocation on a service. type: integer serviceSelectors: description: ServiceSelectors list of label selector to select service(s) for which ip pool can be used for ip allocation. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array type: object required: - addresses type: object status: description: IPAddressPoolStatus defines the observed state of IPAddressPool. type: object required: - spec type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: l2advertisements.metallb.io spec: group: metallb.io names: kind: L2Advertisement listKind: L2AdvertisementList plural: l2advertisements singular: l2advertisement scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.ipAddressPools name: IPAddressPools type: string - jsonPath: .spec.ipAddressPoolSelectors name: IPAddressPool Selectors type: string - jsonPath: .spec.interfaces name: Interfaces type: string - jsonPath: .spec.nodeSelectors name: Node Selectors priority: 10 type: string name: v1beta1 schema: openAPIV3Schema: description: L2Advertisement allows to advertise the LoadBalancer IPs provided by the selected pools via L2. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: L2AdvertisementSpec defines the desired state of L2Advertisement. properties: interfaces: description: A list of interfaces to announce from. The LB IP will be announced only from these interfaces. If the field is not set, we advertise from all the interfaces on the host. items: type: string type: array ipAddressPoolSelectors: description: A selector for the IPAddressPools which would get advertised via this advertisement. If no IPAddressPool is selected by this or by the list, the advertisement is applied to all the IPAddressPools. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array ipAddressPools: description: The list of IPAddressPools to advertise via this advertisement, selected by name. items: type: string type: array nodeSelectors: description: NodeSelectors allows to limit the nodes to announce as next hops for the LoadBalancer IP. When empty, all the nodes having are announced as next hops. items: description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic type: array type: object status: description: L2AdvertisementStatus defines the observed state of L2Advertisement. type: object type: object served: true storage: true subresources: status: {} --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: metallb pod-security.kubernetes.io/audit: privileged pod-security.kubernetes.io/enforce: privileged pod-security.kubernetes.io/warn: privileged name: controller namespace: "{{ metallb_namespace }}" {% if metallb_speaker_enabled %} --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: metallb name: speaker namespace: "{{ metallb_namespace }}" {% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: labels: app: metallb name: controller namespace: "{{ metallb_namespace }}" rules: - apiGroups: - "" resources: - secrets verbs: - create - delete - get - list - patch - update - watch - apiGroups: - "" resourceNames: - memberlist resources: - secrets verbs: - list - apiGroups: - apps resourceNames: - controller resources: - deployments verbs: - get - apiGroups: - metallb.io resources: - bgppeers verbs: - get - list - apiGroups: - metallb.io resources: - addresspools verbs: - get - list - watch - apiGroups: - metallb.io resources: - bfdprofiles verbs: - get - list - watch - apiGroups: - metallb.io resources: - ipaddresspools verbs: - get - list - watch - apiGroups: - metallb.io resources: - bgpadvertisements verbs: - get - list - watch - apiGroups: - metallb.io resources: - l2advertisements verbs: - get - list - watch - apiGroups: - metallb.io resources: - communities verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: labels: app: metallb name: pod-lister namespace: "{{ metallb_namespace }}" rules: - apiGroups: - "" resources: - pods verbs: - list - apiGroups: - "" resources: - secrets verbs: - get - list - watch - apiGroups: - metallb.io resources: - addresspools verbs: - get - list - watch - apiGroups: - metallb.io resources: - bfdprofiles verbs: - get - list - watch - apiGroups: - metallb.io resources: - bgppeers verbs: - get - list - watch - apiGroups: - metallb.io resources: - l2advertisements verbs: - get - list - watch - apiGroups: - metallb.io resources: - bgpadvertisements verbs: - get - list - watch - apiGroups: - metallb.io resources: - ipaddresspools verbs: - get - list - watch - apiGroups: - metallb.io resources: - communities verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: metallb name: {{ metallb_namespace }}:controller rules: - apiGroups: - "" resources: - services - namespaces verbs: - get - list - watch - apiGroups: - "" resources: - services/status verbs: - update - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - admissionregistration.k8s.io resourceNames: - metallb-webhook-configuration resources: - validatingwebhookconfigurations - mutatingwebhookconfigurations verbs: - create - delete - get - list - patch - update - watch - apiGroups: - admissionregistration.k8s.io resources: - validatingwebhookconfigurations - mutatingwebhookconfigurations verbs: - list - watch - apiGroups: - apiextensions.k8s.io resourceNames: - addresspools.metallb.io - bfdprofiles.metallb.io - bgpadvertisements.metallb.io - bgppeers.metallb.io - ipaddresspools.metallb.io - l2advertisements.metallb.io - communities.metallb.io resources: - customresourcedefinitions verbs: - create - delete - get - list - patch - update - watch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - list - watch --- {% if metallb_speaker_enabled %} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: metallb name: {{ metallb_namespace }}:speaker rules: - apiGroups: - "" resources: - services - endpoints - nodes - namespaces verbs: - get - list - watch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch {% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: app: metallb name: controller namespace: "{{ metallb_namespace }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: controller subjects: - kind: ServiceAccount name: controller namespace: "{{ metallb_namespace }}" --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: app: metallb name: pod-lister namespace: "{{ metallb_namespace }}" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: pod-lister subjects: - kind: ServiceAccount name: speaker namespace: "{{ metallb_namespace }}" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: metallb name: {{ metallb_namespace }}:controller roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ metallb_namespace }}:controller subjects: - kind: ServiceAccount name: controller namespace: "{{ metallb_namespace }}" {% if metallb_speaker_enabled %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: metallb name: {{ metallb_namespace }}:speaker roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ metallb_namespace }}:speaker subjects: - kind: ServiceAccount name: speaker namespace: "{{ metallb_namespace }}" {% endif %} --- apiVersion: v1 kind: Secret metadata: name: webhook-server-cert namespace: "{{ metallb_namespace }}" --- apiVersion: v1 kind: Service metadata: name: webhook-service namespace: "{{ metallb_namespace }}" spec: ports: - port: 443 targetPort: 9443 selector: component: controller --- apiVersion: apps/v1 kind: Deployment metadata: labels: app: metallb component: controller name: controller namespace: "{{ metallb_namespace }}" spec: revisionHistoryLimit: 3 selector: matchLabels: app: metallb component: controller template: metadata: annotations: prometheus.io/port: '{{ metallb_port }}' prometheus.io/scrape: 'true' labels: app: metallb component: controller spec: priorityClassName: system-cluster-critical containers: - args: - --port={{ metallb_port }} - --log-level={{ metallb_log_level }} {% if metallb_loadbalancer_class != "" %} - --lb-class={{ metallb_loadbalancer_class }} {% endif %} env: - name: METALLB_ML_SECRET_NAME value: memberlist - name: METALLB_DEPLOYMENT value: controller image: "{{ metallb_controller_image_repo }}:{{ metallb_image_tag }}" livenessProbe: failureThreshold: 3 httpGet: path: /metrics port: monitoring initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 name: controller ports: - containerPort: {{ metallb_port }} name: monitoring - containerPort: 9443 name: webhook-server protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /metrics port: monitoring initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 securityContext: allowPrivilegeEscalation: false capabilities: drop: - all readOnlyRootFilesystem: true volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true {% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %} tolerations: {{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {%- endif %} nodeSelector: {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}} {% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %} {{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}} {%- endif %} securityContext: fsGroup: 65534 runAsNonRoot: true runAsUser: 65534 serviceAccountName: controller terminationGracePeriodSeconds: 0 volumes: - name: cert secret: defaultMode: 420 secretName: webhook-server-cert --- {% if metallb_speaker_enabled %} apiVersion: apps/v1 kind: DaemonSet metadata: labels: app: metallb component: speaker name: speaker namespace: "{{ metallb_namespace }}" spec: selector: matchLabels: app: metallb component: speaker template: metadata: annotations: prometheus.io/port: '{{ metallb_port }}' prometheus.io/scrape: 'true' labels: app: metallb component: speaker spec: containers: - args: - --port={{ metallb_port }} - --log-level={{ metallb_log_level }} {% if metallb_loadbalancer_class != "" %} - --lb-class={{ metallb_loadbalancer_class }} {% endif %} env: - name: METALLB_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: METALLB_HOST valueFrom: fieldRef: fieldPath: status.hostIP - name: METALLB_ML_BIND_ADDR valueFrom: fieldRef: fieldPath: status.podIP - name: METALLB_ML_LABELS value: app=metallb,component=speaker - name: METALLB_ML_SECRET_KEY valueFrom: secretKeyRef: key: secretkey name: memberlist image: "{{ metallb_speaker_image_repo }}:{{ metallb_image_tag }}" livenessProbe: failureThreshold: 3 httpGet: path: /metrics port: monitoring initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 name: speaker ports: - containerPort: {{ metallb_port }} name: monitoring - containerPort: {{ metallb_memberlist_port }} name: memberlist-tcp - containerPort: {{ metallb_memberlist_port }} name: memberlist-udp protocol: UDP readinessProbe: failureThreshold: 3 httpGet: path: /metrics port: monitoring initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 securityContext: allowPrivilegeEscalation: false capabilities: add: - NET_RAW drop: - ALL readOnlyRootFilesystem: true hostNetwork: true nodeSelector: {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}} {% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %} {{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}} {%- endif %} serviceAccountName: speaker terminationGracePeriodSeconds: 2 tolerations: {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}} {% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %} {{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}} {% endif %} {% endif %} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: creationTimestamp: null name: metallb-webhook-configuration webhooks: - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta2-bgppeer failurePolicy: Fail name: bgppeersvalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta2 operations: - CREATE - UPDATE resources: - bgppeers sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta1-addresspool failurePolicy: Fail name: addresspoolvalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta1 operations: - CREATE - UPDATE resources: - addresspools sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta1-bfdprofile failurePolicy: Fail name: bfdprofilevalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta1 operations: - CREATE - DELETE resources: - bfdprofiles sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta1-bgpadvertisement failurePolicy: Fail name: bgpadvertisementvalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta1 operations: - CREATE - UPDATE resources: - bgpadvertisements sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta1-community failurePolicy: Fail name: communityvalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta1 operations: - CREATE - UPDATE resources: - communities sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta1-ipaddresspool failurePolicy: Fail name: ipaddresspoolvalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta1 operations: - CREATE - UPDATE resources: - ipaddresspools sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: webhook-service namespace: "{{ metallb_namespace }}" path: /validate-metallb-io-v1beta1-l2advertisement failurePolicy: Fail name: l2advertisementvalidationwebhook.metallb.io rules: - apiGroups: - metallb.io apiVersions: - v1beta1 operations: - CREATE - UPDATE resources: - l2advertisements sideEffects: None ================================================ FILE: roles/kubernetes-apps/metallb/templates/pools.yaml.j2 ================================================ #jinja2: trim_blocks: True, lstrip_blocks: True # yamllint disable-file --- # Create all pools {% for pool_name, pool in metallb_config.address_pools.items() %} --- apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: namespace: "{{ metallb_namespace }}" name: "{{ pool_name }}" spec: addresses: {% for ip_range in pool.ip_range %} - "{{ ip_range }}" {% endfor %} autoAssign: {{ pool.auto_assign | default(true) }} avoidBuggyIPs: {{ pool.avoid_buggy_ips | default(false) }} {% endfor %} ================================================ FILE: roles/kubernetes-apps/metrics_server/defaults/main.yml ================================================ --- metrics_server_container_port: 10250 metrics_server_kubelet_insecure_tls: true metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname" metrics_server_metric_resolution: 15s metrics_server_limits_cpu: 100m metrics_server_limits_memory: 200Mi metrics_server_requests_cpu: 100m metrics_server_requests_memory: 200Mi metrics_server_host_network: false metrics_server_replicas: 1 metrics_server_extra_tolerations: [] metrics_server_extra_affinity: {} metrics_server_nodeselector: {} ================================================ FILE: roles/kubernetes-apps/metrics_server/tasks/main.yml ================================================ --- - name: Metrics Server | Delete addon dir file: path: "{{ kube_config_dir }}/addons/metrics_server" state: absent when: - inventory_hostname == groups['kube_control_plane'][0] tags: - upgrade - name: Metrics Server | Create addon dir file: path: "{{ kube_config_dir }}/addons/metrics_server" state: directory owner: root group: root mode: "0755" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Metrics Server | Templates list set_fact: metrics_server_templates: - { name: auth-delegator, file: auth-delegator.yaml, type: clusterrolebinding } - { name: auth-reader, file: auth-reader.yaml, type: rolebinding } - { name: metrics-server-sa, file: metrics-server-sa.yaml, type: sa } - { name: metrics-server-deployment, file: metrics-server-deployment.yaml, type: deploy } - { name: metrics-server-service, file: metrics-server-service.yaml, type: service } - { name: metrics-apiservice, file: metrics-apiservice.yaml, type: service } - { name: resource-reader-clusterrolebinding, file: resource-reader-clusterrolebinding.yaml, type: clusterrolebinding } - { name: resource-reader, file: resource-reader.yaml, type: clusterrole } - name: Metrics Server | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/metrics_server/{{ item.file }}" mode: "0644" with_items: "{{ metrics_server_templates }}" register: metrics_server_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Metrics Server | Apply manifests kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/metrics_server/{{ item.item.file }}" state: "latest" with_items: "{{ metrics_server_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: metrics-server:system:auth-delegator labels: addonmanager.kubernetes.io/mode: Reconcile roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: metrics-server-auth-reader namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 ================================================ apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: v1beta1.metrics.k8s.io labels: addonmanager.kubernetes.io/mode: Reconcile spec: service: name: metrics-server namespace: kube-system group: metrics.k8s.io version: v1beta1 insecureSkipTLSVerify: {{ metrics_server_kubelet_insecure_tls }} groupPriorityMinimum: 100 versionPriority: 100 ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: metrics-server namespace: kube-system labels: app.kubernetes.io/name: metrics-server addonmanager.kubernetes.io/mode: Reconcile version: {{ metrics_server_version }} spec: replicas: {{ metrics_server_replicas }} selector: matchLabels: app.kubernetes.io/name: metrics-server version: {{ metrics_server_version }} strategy: rollingUpdate: maxUnavailable: 0 template: metadata: name: metrics-server labels: app.kubernetes.io/name: metrics-server version: {{ metrics_server_version }} spec: priorityClassName: system-cluster-critical serviceAccountName: metrics-server hostNetwork: {{ metrics_server_host_network | default(false) }} containers: - name: metrics-server image: {{ metrics_server_image_repo }}:{{ metrics_server_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - --cert-dir=/tmp - --secure-port={{ metrics_server_container_port }} {% if metrics_server_kubelet_preferred_address_types %} - --kubelet-preferred-address-types={{ metrics_server_kubelet_preferred_address_types }} {% endif %} - --kubelet-use-node-status-port {% if metrics_server_kubelet_insecure_tls %} - --kubelet-insecure-tls=true {% endif %} - --metric-resolution={{ metrics_server_metric_resolution }} ports: - containerPort: {{ metrics_server_container_port }} name: https protocol: TCP volumeMounts: - name: tmp mountPath: /tmp livenessProbe: httpGet: path: /livez port: https scheme: HTTPS periodSeconds: 10 failureThreshold: 3 initialDelaySeconds: 40 readinessProbe: httpGet: path: /readyz port: https scheme: HTTPS periodSeconds: 10 failureThreshold: 3 initialDelaySeconds: 40 securityContext: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 allowPrivilegeEscalation: false seccompProfile: type: RuntimeDefault capabilities: drop: - ALL resources: limits: cpu: {{ metrics_server_limits_cpu }} memory: {{ metrics_server_limits_memory }} requests: cpu: {{ metrics_server_requests_cpu }} memory: {{ metrics_server_requests_memory }} volumes: - name: tmp emptyDir: {} tolerations: - key: node-role.kubernetes.io/control-plane effect: NoSchedule {% if metrics_server_extra_tolerations %} {{ metrics_server_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} {%- endif %} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: app.kubernetes.io/name operator: In values: - metrics-server topologyKey: kubernetes.io/hostname namespaces: - kube-system {% if metrics_server_extra_affinity %} {{ metrics_server_extra_affinity | to_nice_yaml(indent=2) | indent(8) }} {%- endif %} {% if metrics_server_nodeselector %} nodeSelector: {{ metrics_server_nodeselector | to_nice_yaml(indent=2) | indent(8) }} {%- endif %} ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: metrics-server namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 ================================================ apiVersion: v1 kind: Service metadata: name: metrics-server namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile app.kubernetes.io/name: "metrics-server" spec: type: ClusterIP selector: app.kubernetes.io/name: metrics-server ports: - name: https port: 443 protocol: TCP targetPort: https ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:metrics-server labels: addonmanager.kubernetes.io/mode: Reconcile roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:metrics-server subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system ================================================ FILE: roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:metrics-server labels: addonmanager.kubernetes.io/mode: Reconcile rules: - apiGroups: - "" resources: - nodes/metrics verbs: - get - apiGroups: - "" resources: - pods - nodes verbs: - get - list - watch ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/defaults/main.yml ================================================ --- node_feature_discovery_enabled: false node_feature_discovery_namespace: node-feature-discovery node_feature_discovery_enable_nodefeature_api: true node_feature_discovery_gc_replicas: 1 node_feature_discovery_gc_interval: 1h node_feature_discovery_gc_sa_name: node-feature-discovery-gc node_feature_discovery_gc_sa_create: true node_feature_discovery_master_replicas: 1 node_feature_discovery_master_crd_controller: null node_feature_discovery_master_instance: null node_feature_discovery_master_config: null node_feature_discovery_worker_sa_name: node-feature-discovery-worker node_feature_discovery_worker_sa_create: true node_feature_discovery_worker_config: null node_feature_discovery_worker_tolerations: null ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/tasks/main.yml ================================================ --- - name: Node Feature Discovery | Create addon dir file: path: "{{ kube_config_dir }}/addons/node_feature_discovery" state: directory owner: root group: root mode: "0755" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Node Feature Discovery | Templates list set_fact: node_feature_discovery_templates: - { name: nfd-ns, file: nfd-ns.yaml, type: ns } - { name: nfd-api-crd, file: nfd-api-crds.yaml, type: crd } - { name: nfd-serviceaccount, file: nfd-serviceaccount.yaml, type: sa } - { name: nfd-role, file: nfd-role.yaml, type: role } - { name: nfd-clusterrole, file: nfd-clusterrole.yaml, type: clusterrole } - { name: nfd-rolebinding, file: nfd-rolebinding.yaml, type: rolebinding } - { name: nfd-clusterrolebinding, file: nfd-clusterrolebinding.yaml, type: clusterrolebinding } - { name: nfd-master-conf, file: nfd-master-conf.yaml, type: cm } - { name: nfd-worker-conf, file: nfd-worker-conf.yaml, type: cm } - { name: nfd-topologyupdater-conf, file: nfd-topologyupdater-conf.yaml, type: cm } - { name: nfd-gc, file: nfd-gc.yaml, type: deploy } - { name: nfd-master, file: nfd-master.yaml, type: deploy } - { name: nfd-worker, file: nfd-worker.yaml, type: ds } - { name: nfd-service, file: nfd-service.yaml, type: srv } - name: Node Feature Discovery | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.file }}" mode: "0644" with_items: "{{ node_feature_discovery_templates }}" register: node_feature_discovery_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Node Feature Discovery | Apply manifests kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.item.file }}" state: "latest" with_items: "{{ node_feature_discovery_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.14.0 name: nodefeatures.nfd.k8s-sigs.io spec: group: nfd.k8s-sigs.io names: kind: NodeFeature listKind: NodeFeatureList plural: nodefeatures singular: nodefeature scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: |- NodeFeature resource holds the features discovered for one node in the cluster. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: Specification of the NodeFeature, containing features discovered for a node. properties: features: description: Features is the full "raw" features data that has been discovered. properties: attributes: additionalProperties: description: AttributeFeatureSet is a set of features having string value. properties: elements: additionalProperties: type: string description: Individual features of the feature set. type: object required: - elements type: object description: Attributes contains all the attribute-type features of the node. type: object flags: additionalProperties: description: FlagFeatureSet is a set of simple features only containing names without values. properties: elements: additionalProperties: description: Nil is a dummy empty struct for protobuf compatibility type: object description: Individual features of the feature set. type: object required: - elements type: object description: Flags contains all the flag-type features of the node. type: object instances: additionalProperties: description: InstanceFeatureSet is a set of features each of which is an instance having multiple attributes. properties: elements: description: Individual features of the feature set. items: description: InstanceFeature represents one instance of a complex features, e.g. a device. properties: attributes: additionalProperties: type: string description: Attributes of the instance feature. type: object required: - attributes type: object type: array required: - elements type: object description: Instances contains all the instance-type features of the node. type: object type: object labels: additionalProperties: type: string description: Labels is the set of node labels that are requested to be created. type: object type: object required: - spec type: object served: true storage: true --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.14.0 name: nodefeaturegroups.nfd.k8s-sigs.io spec: group: nfd.k8s-sigs.io names: kind: NodeFeatureGroup listKind: NodeFeatureGroupList plural: nodefeaturegroups shortNames: - nfg singular: nodefeaturegroup scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: NodeFeatureGroup resource holds Node pools by featureGroup properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: Spec defines the rules to be evaluated. properties: featureGroupRules: description: List of rules to evaluate to determine nodes that belong in this group. items: description: GroupRule defines a rule for nodegroup filtering. properties: matchAny: description: MatchAny specifies a list of matchers one of which must match. items: description: MatchAnyElem specifies one sub-matcher of MatchAny. properties: matchFeatures: description: MatchFeatures specifies a set of matcher terms all of which must match. items: description: |- FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. properties: feature: description: Feature is the name of the feature set to match against. type: string matchExpressions: additionalProperties: description: |- MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object description: |- MatchExpressions is the set of per-element expressions evaluated. These match against the value of the specified elements. type: object matchName: description: |- MatchName in an expression that is matched against the name of each element in the feature set. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object required: - feature type: object type: array required: - matchFeatures type: object type: array matchFeatures: description: MatchFeatures specifies a set of matcher terms all of which must match. items: description: |- FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. properties: feature: description: Feature is the name of the feature set to match against. type: string matchExpressions: additionalProperties: description: |- MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object description: |- MatchExpressions is the set of per-element expressions evaluated. These match against the value of the specified elements. type: object matchName: description: |- MatchName in an expression that is matched against the name of each element in the feature set. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object required: - feature type: object type: array name: description: Name of the rule. type: string required: - name type: object type: array required: - featureGroupRules type: object status: description: |- Status of the NodeFeatureGroup after the most recent evaluation of the specification. properties: nodes: description: Nodes is a list of FeatureGroupNode in the cluster that match the featureGroupRules items: properties: name: description: Name of the node. type: string required: - name type: object type: array x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map type: object required: - spec type: object served: true storage: true subresources: status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.14.0 name: nodefeaturerules.nfd.k8s-sigs.io spec: group: nfd.k8s-sigs.io names: kind: NodeFeatureRule listKind: NodeFeatureRuleList plural: nodefeaturerules shortNames: - nfr singular: nodefeaturerule scope: Cluster versions: - name: v1alpha1 schema: openAPIV3Schema: description: |- NodeFeatureRule resource specifies a configuration for feature-based customization of node objects, such as node labeling. properties: apiVersion: description: |- APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: description: |- Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: Spec defines the rules to be evaluated. properties: rules: description: Rules is a list of node customization rules. items: description: Rule defines a rule for node customization such as labeling. properties: annotations: additionalProperties: type: string description: Annotations to create if the rule matches. type: object extendedResources: additionalProperties: type: string description: ExtendedResources to create if the rule matches. type: object labels: additionalProperties: type: string description: Labels to create if the rule matches. type: object labelsTemplate: description: |- LabelsTemplate specifies a template to expand for dynamically generating multiple labels. Data (after template expansion) must be keys with an optional value ([=]) separated by newlines. type: string matchAny: description: MatchAny specifies a list of matchers one of which must match. items: description: MatchAnyElem specifies one sub-matcher of MatchAny. properties: matchFeatures: description: MatchFeatures specifies a set of matcher terms all of which must match. items: description: |- FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. properties: feature: description: Feature is the name of the feature set to match against. type: string matchExpressions: additionalProperties: description: |- MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object description: |- MatchExpressions is the set of per-element expressions evaluated. These match against the value of the specified elements. type: object matchName: description: |- MatchName in an expression that is matched against the name of each element in the feature set. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object required: - feature type: object type: array required: - matchFeatures type: object type: array matchFeatures: description: MatchFeatures specifies a set of matcher terms all of which must match. items: description: |- FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. properties: feature: description: Feature is the name of the feature set to match against. type: string matchExpressions: additionalProperties: description: |- MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object description: |- MatchExpressions is the set of per-element expressions evaluated. These match against the value of the specified elements. type: object matchName: description: |- MatchName in an expression that is matched against the name of each element in the feature set. properties: op: description: Op is the operator to be applied. enum: - In - NotIn - InRegexp - Exists - DoesNotExist - Gt - Lt - GtLt - IsTrue - IsFalse type: string value: description: |- Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. items: type: string type: array required: - op type: object required: - feature type: object type: array name: description: Name of the rule. type: string taints: description: Taints to create if the rule matches. items: description: |- The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. properties: effect: description: |- Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: Required. The taint key to be applied to a node. type: string timeAdded: description: |- TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. format: date-time type: string value: description: The taint value corresponding to the taint key. type: string required: - effect - key type: object type: array vars: additionalProperties: type: string description: |- Vars is the variables to store if the rule matches. Variables do not directly inflict any changes in the node object. However, they can be referenced from other rules enabling more complex rule hierarchies, without exposing intermediary output values as labels. type: object varsTemplate: description: |- VarsTemplate specifies a template to expand for dynamically generating multiple variables. Data (after template expansion) must be keys with an optional value ([=]) separated by newlines. type: string required: - name type: object type: array required: - rules type: object required: - spec type: object served: true storage: true ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: node-feature-discovery rules: - apiGroups: - "" resources: - nodes - nodes/status verbs: - get - patch - update - list - apiGroups: - nfd.k8s-sigs.io resources: - nodefeatures - nodefeaturerules - nodefeaturegroups verbs: - get - list - watch - apiGroups: - nfd.k8s-sigs.io resources: - nodefeaturegroup/status verbs: - patch - update - apiGroups: - coordination.k8s.io resources: - leases verbs: - create - apiGroups: - coordination.k8s.io resources: - leases resourceNames: - "nfd-master.nfd.kubernetes.io" verbs: - get - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: node-feature-discovery-gc rules: - apiGroups: - "" resources: - nodes verbs: - list - watch - apiGroups: - topology.node.k8s.io resources: - noderesourcetopologies verbs: - delete - list - apiGroups: - nfd.k8s-sigs.io resources: - nodefeatures verbs: - delete - list ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: node-feature-discovery roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: node-feature-discovery subjects: - kind: ServiceAccount name: node-feature-discovery namespace: {{ node_feature_discovery_namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: node-feature-discovery-gc roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: node-feature-discovery-gc subjects: - kind: ServiceAccount name: {{ node_feature_discovery_gc_sa_name }} namespace: {{ node_feature_discovery_namespace }} ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2 ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: node-feature-discovery-gc namespace: {{ node_feature_discovery_namespace }} labels: app.kubernetes.io/name: node-feature-discovery role: gc spec: replicas: {{ node_feature_discovery_gc_replicas }} selector: matchLabels: app.kubernetes.io/name: node-feature-discovery role: gc template: metadata: labels: app.kubernetes.io/name: node-feature-discovery role: gc spec: serviceAccountName: {{ node_feature_discovery_gc_sa_name }} dnsPolicy: ClusterFirstWithHostNet containers: - name: gc image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }} imagePullPolicy: IfNotPresent env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName command: - "nfd-gc" args: - "-gc-interval={{ node_feature_discovery_gc_interval }}" securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsNonRoot: true ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2 ================================================ apiVersion: v1 kind: ConfigMap metadata: name: node-feature-discovery-master-conf namespace: {{ node_feature_discovery_namespace }} data: {% if node_feature_discovery_master_config %} nfd-master.conf: |- {{ node_feature_discovery_master_config | to_yaml(indent=2, width=1337) | indent(width=4) }} {% else %} nfd-master.conf: "null" {% endif %} ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2 ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: node-feature-discovery-master namespace: {{ node_feature_discovery_namespace }} labels: app.kubernetes.io/name: node-feature-discovery role: master spec: replicas: {{ node_feature_discovery_master_replicas }} selector: matchLabels: app.kubernetes.io/name: node-feature-discovery role: master template: metadata: labels: app.kubernetes.io/name: node-feature-discovery role: master spec: serviceAccountName: node-feature-discovery enableServiceLinks: false containers: - name: master securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsNonRoot: true image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }} imagePullPolicy: IfNotPresent livenessProbe: grpc: port: 8082 initialDelaySeconds: 10 periodSeconds: 10 readinessProbe: grpc: port: 8082 initialDelaySeconds: 5 periodSeconds: 10 failureThreshold: 10 ports: - containerPort: 8082 name: grpc - containerPort: 8081 name: metrics env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName command: - "nfd-master" args: - "-port=8080" {% if not node_feature_discovery_enable_nodefeature_api %} - "-enable-nodefeature-api=false" {% elif node_feature_discovery_master_replicas > 1 %} - "-enable-leader-election" {% endif %} {% if node_feature_discovery_master_crd_controller != none %} - "-crd-controller={{ node_feature_discovery_master_crd_controller }}" {% else %} {% if node_feature_discovery_master_instance %} ## By default, disable crd controller for other than the default instances - "-crd-controller=false" {% else %} ## By default, disable crd controller for other than the default instances - "-crd-controller=true" {% endif %} {% endif %} - "-metrics=8081" volumeMounts: - name: nfd-master-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true volumes: - name: nfd-master-conf configMap: name: node-feature-discovery-master-conf items: - key: nfd-master.conf path: nfd-master.conf affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - preference: matchExpressions: - key: node-role.kubernetes.io/master operator: In values: - "" weight: 1 - preference: matchExpressions: - key: node-role.kubernetes.io/control-plane operator: In values: - "" weight: 1 tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Equal - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Equal ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ node_feature_discovery_namespace }} labels: name: {{ node_feature_discovery_namespace }} ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: node-feature-discovery-worker namespace: {{ node_feature_discovery_namespace }} rules: - apiGroups: - nfd.k8s-sigs.io resources: - nodefeatures verbs: - create - get - update - apiGroups: - "" resources: - pods verbs: - get ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: node-feature-discovery-worker namespace: {{ node_feature_discovery_namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: node-feature-discovery-worker subjects: - kind: ServiceAccount name: {{ node_feature_discovery_worker_sa_name }} namespace: {{ node_feature_discovery_namespace }} ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-service.yaml.j2 ================================================ apiVersion: v1 kind: Service metadata: name: node-feature-discovery-master namespace: {{ node_feature_discovery_namespace }} labels: app.kubernetes.io/name: node-feature-discovery role: master spec: type: ClusterIP ports: - port: 8080 targetPort: grpc protocol: TCP name: grpc selector: app.kubernetes.io/name: node-feature-discovery role: master ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-serviceaccount.yaml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: node-feature-discovery namespace: {{ node_feature_discovery_namespace }} {% if node_feature_discovery_gc_sa_create %} --- apiVersion: v1 kind: ServiceAccount metadata: name: {{ node_feature_discovery_gc_sa_name }} namespace: {{ node_feature_discovery_namespace }} {% endif %} {% if node_feature_discovery_worker_sa_create %} --- apiVersion: v1 kind: ServiceAccount metadata: name: {{ node_feature_discovery_worker_sa_name }} namespace: {{ node_feature_discovery_namespace }} {% endif %} ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-topologyupdater-conf.yaml.j2 ================================================ apiVersion: v1 kind: ConfigMap metadata: name: node-feature-discovery-topology-updater-conf namespace: {{ node_feature_discovery_namespace }} data: nfd-topology-updater.conf: "null" ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker-conf.yaml.j2 ================================================ apiVersion: v1 kind: ConfigMap metadata: name: node-feature-discovery-worker-conf namespace: {{ node_feature_discovery_namespace }} data: {% if node_feature_discovery_worker_config %} nfd-worker.conf: |- {{ node_feature_discovery_worker_config | to_yaml(indent=2, width=1337) | indent(width=4) }} {% else %} nfd-worker.conf: "null" {% endif %} ================================================ FILE: roles/kubernetes-apps/node_feature_discovery/templates/nfd-worker.yaml.j2 ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: node-feature-discovery-worker namespace: {{ node_feature_discovery_namespace }} labels: app.kubernetes.io/name: node-feature-discovery role: worker spec: selector: matchLabels: app.kubernetes.io/name: node-feature-discovery role: worker template: metadata: labels: app.kubernetes.io/name: node-feature-discovery role: worker spec: dnsPolicy: ClusterFirstWithHostNet serviceAccountName: {{ node_feature_discovery_worker_sa_name }} containers: - name: worker securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsNonRoot: true image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }} imagePullPolicy: IfNotPresent env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName command: - "nfd-worker" args: - "-server=node-feature-discovery-master:8080" {% if not node_feature_discovery_enable_nodefeature_api %} - "-enable-nodefeature-api=false" {% endif %} - "-metrics=8081" ports: - name: metrics containerPort: 8081 volumeMounts: - name: host-boot mountPath: "/host-boot" readOnly: true - name: host-os-release mountPath: "/host-etc/os-release" readOnly: true - name: host-sys mountPath: "/host-sys" readOnly: true - name: host-usr-lib mountPath: "/host-usr/lib" readOnly: true - name: host-lib mountPath: "/host-lib" readOnly: true - name: source-d mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" readOnly: true - name: features-d mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" readOnly: true - name: nfd-worker-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true volumes: - name: host-boot hostPath: path: "/boot" - name: host-os-release hostPath: path: "/etc/os-release" - name: host-sys hostPath: path: "/sys" - name: host-usr-lib hostPath: path: "/usr/lib" - name: host-lib hostPath: path: "/lib" - name: source-d hostPath: path: "/etc/kubernetes/node-feature-discovery/source.d/" - name: features-d hostPath: path: "/etc/kubernetes/node-feature-discovery/features.d/" - name: nfd-worker-conf configMap: name: node-feature-discovery-worker-conf items: - key: nfd-worker.conf path: nfd-worker.conf {% if node_feature_discovery_worker_tolerations %} tolerations: {{ node_feature_discovery_worker_tolerations | to_yaml(indent=2, width=1337) | indent(width=8) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml ================================================ --- # To restrict which AZ the volume should be provisioned in # set this value to true and set the list of relevant AZs # For it to work, the flag aws_ebs_csi_enable_volume_scheduling # in AWS EBS Driver must be true restrict_az_provisioning: false aws_ebs_availability_zones: - eu-west-3c ================================================ FILE: roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml ================================================ --- - name: Kubernetes Persistent Volumes | Copy AWS EBS CSI Storage Class template template: src: "aws-ebs-csi-storage-class.yml.j2" dest: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml" mode: "0644" register: manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Persistent Volumes | Add AWS EBS CSI Storage Class kube: name: aws-ebs-csi kubectl: "{{ bin_dir }}/kubectl" resource: StorageClass filename: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] - manifests.changed ================================================ FILE: roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 ================================================ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: ebs-sc provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer parameters: csi.storage.k8s.io/fstype: xfs type: gp2 {% if restrict_az_provisioning %} allowedTopologies: - matchLabelExpressions: - key: topology.ebs.csi.aws.com/zone values: {% for value in aws_ebs_availability_zones %} - {{ value }} {% endfor %} {% endif %} ================================================ FILE: roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml ================================================ --- ## Available values: Standard_LRS, Premium_LRS, StandardSSD_LRS, UltraSSD_LRS storage_account_type: StandardSSD_LRS ================================================ FILE: roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml ================================================ --- - name: Kubernetes Persistent Volumes | Copy Azure CSI Storage Class template template: src: "azure-csi-storage-class.yml.j2" dest: "{{ kube_config_dir }}/azure-csi-storage-class.yml" mode: "0644" register: manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Persistent Volumes | Add Azure CSI Storage Class kube: name: azure-csi kubectl: "{{ bin_dir }}/kubectl" resource: StorageClass filename: "{{ kube_config_dir }}/azure-csi-storage-class.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] - manifests.changed ================================================ FILE: roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 ================================================ --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: disk.csi.azure.com provisioner: disk.csi.azure.com parameters: skuname: {{ storage_account_type }} {% if azure_csi_tags is defined %} tags: {{ azure_csi_tags }} {% endif %} reclaimPolicy: Delete volumeBindingMode: Immediate allowVolumeExpansion: true ================================================ FILE: roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml ================================================ --- storage_classes: - name: cinder-csi is_default: false parameters: availability: nova allowVolumeExpansion: false ================================================ FILE: roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml ================================================ --- - name: Kubernetes Persistent Volumes | Copy Cinder CSI Storage Class template template: src: "cinder-csi-storage-class.yml.j2" dest: "{{ kube_config_dir }}/cinder-csi-storage-class.yml" mode: "0644" register: manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Persistent Volumes | Add Cinder CSI Storage Class kube: name: cinder-csi kubectl: "{{ bin_dir }}/kubectl" resource: StorageClass filename: "{{ kube_config_dir }}/cinder-csi-storage-class.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] - manifests.changed ================================================ FILE: roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 ================================================ {% for class in storage_classes %} --- kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: "{{ class.name }}" annotations: storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" provisioner: cinder.csi.openstack.org volumeBindingMode: WaitForFirstConsumer parameters: {% for key, value in (class.parameters | default({})).items() %} "{{ key }}": "{{ value }}" {% endfor %} {% if cinder_topology is defined and cinder_topology is sameas true %} allowedTopologies: - matchLabelExpressions: - key: topology.cinder.csi.openstack.org/zone values: {% for zone in cinder_topology_zones %} - "{{ zone }}" {% endfor %} {% endif %} allowVolumeExpansion: {{ expand_persistent_volumes }} {% endfor %} ================================================ FILE: roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/defaults/main.yml ================================================ --- # Choose between pd-standard and pd-ssd gcp_pd_csi_volume_type: pd-standard gcp_pd_regional_replication_enabled: false gcp_pd_restrict_zone_replication: false gcp_pd_restricted_zones: - europe-west1-b - europe-west1-c ================================================ FILE: roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/tasks/main.yml ================================================ --- - name: Kubernetes Persistent Volumes | Copy GCP PD CSI Storage Class template template: src: "gcp-pd-csi-storage-class.yml.j2" dest: "{{ kube_config_dir }}/gcp-pd-csi-storage-class.yml" mode: "0644" register: manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Persistent Volumes | Add GCP PD CSI Storage Class kube: name: gcp-pd-csi kubectl: "{{ bin_dir }}/kubectl" resource: StorageClass filename: "{{ kube_config_dir }}/gcp-pd-csi-storage-class.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] - manifests.changed ================================================ FILE: roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/templates/gcp-pd-csi-storage-class.yml.j2 ================================================ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-gce-pd provisioner: pd.csi.storage.gke.io parameters: type: {{ gcp_pd_csi_volume_type }} {% if gcp_pd_regional_replication_enabled %} replication-type: regional-pd {% endif %} volumeBindingMode: WaitForFirstConsumer {% if gcp_pd_restrict_zone_replication %} allowedTopologies: - matchLabelExpressions: - key: topology.gke.io/zone values: {% for value in gcp_pd_restricted_zones %} - {{ value }} {% endfor %} {% endif %} ================================================ FILE: roles/kubernetes-apps/persistent_volumes/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/persistent_volumes/cinder-csi when: - cinder_csi_enabled tags: - persistent_volumes_cinder_csi - cinder-csi-driver - role: kubernetes-apps/persistent_volumes/aws-ebs-csi when: - aws_ebs_csi_enabled tags: - persistent_volumes_aws_ebs_csi - aws-ebs-csi-driver - role: kubernetes-apps/persistent_volumes/azuredisk-csi when: - azure_csi_enabled tags: - persistent_volumes_azure_csi - azure-csi-driver - role: kubernetes-apps/persistent_volumes/gcp-pd-csi when: - gcp_pd_csi_enabled tags: - persistent_volumes_gcp_pd_csi - gcp-pd-csi-driver - role: kubernetes-apps/persistent_volumes/upcloud-csi when: - upcloud_csi_enabled tags: - persistent_volumes_upcloud_csi - upcloud-csi-driver ================================================ FILE: roles/kubernetes-apps/persistent_volumes/upcloud-csi/defaults/main.yml ================================================ --- storage_classes: - name: standard is_default: true expand_persistent_volumes: true parameters: tier: maxiops - name: hdd is_default: false expand_persistent_volumes: true parameters: tier: hdd ================================================ FILE: roles/kubernetes-apps/persistent_volumes/upcloud-csi/tasks/main.yml ================================================ --- - name: Kubernetes Persistent Volumes | Copy UpCloud CSI Storage Class template template: src: "upcloud-csi-storage-class.yml.j2" dest: "{{ kube_config_dir }}/upcloud-csi-storage-class.yml" mode: "0644" register: manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Persistent Volumes | Add UpCloud CSI Storage Class kube: name: upcloud-csi kubectl: "{{ bin_dir }}/kubectl" resource: StorageClass filename: "{{ kube_config_dir }}/upcloud-csi-storage-class.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] - manifests.changed ================================================ FILE: roles/kubernetes-apps/persistent_volumes/upcloud-csi/templates/upcloud-csi-storage-class.yml.j2 ================================================ {% for class in storage_classes %} --- kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: "{{ class.name }}" annotations: storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" provisioner: storage.csi.upcloud.com reclaimPolicy: Delete parameters: {% for key, value in (class.parameters | default({})).items() %} "{{ key }}": "{{ value }}" {% endfor %} allowVolumeExpansion: {{ class.expand_persistent_volumes | default(true) | ternary("true","false") }} {% endfor %} ================================================ FILE: roles/kubernetes-apps/policy_controller/calico/defaults/main.yml ================================================ --- # Limits for calico apps calico_policy_controller_cpu_limit: 1000m calico_policy_controller_memory_limit: 256M calico_policy_controller_cpu_requests: 30m calico_policy_controller_memory_requests: 64M calico_policy_controller_deployment_nodeselector: "kubernetes.io/os: linux" calico_policy_controller_log_level: info # SSL calico_cert_dir: "/etc/calico/certs" ================================================ FILE: roles/kubernetes-apps/policy_controller/calico/tasks/main.yml ================================================ --- - name: Create calico-kube-controllers manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: calico-kube-controllers, file: calico-kube-controllers.yml, type: deployment} - {name: calico-kube-controllers, file: calico-kube-sa.yml, type: sa} - {name: calico-kube-controllers, file: calico-kube-cr.yml, type: clusterrole} - {name: calico-kube-controllers, file: calico-kube-crb.yml, type: clusterrolebinding} register: calico_kube_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - rbac_enabled or item.type not in rbac_resources - name: Start of Calico kube controllers kube: name: "{{ item.item.name }}" namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ calico_kube_manifests.results }}" register: calico_kube_controller_start until: calico_kube_controller_start is succeeded retries: 4 when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" ================================================ FILE: roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: replicas: 1 strategy: type: Recreate selector: matchLabels: k8s-app: calico-kube-controllers template: metadata: name: calico-kube-controllers namespace: kube-system labels: k8s-app: calico-kube-controllers spec: nodeSelector: {{ calico_policy_controller_deployment_nodeselector }} {% if calico_datastore == "etcd" %} hostNetwork: true {% endif %} serviceAccountName: calico-kube-controllers tolerations: - key: CriticalAddonsOnly operator: Exists - key: node-role.kubernetes.io/control-plane effect: NoSchedule - key: node-role.kubernetes.io/master effect: NoSchedule {% if policy_controller_extra_tolerations is defined %} {{ policy_controller_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} {% endif %} priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} resources: limits: cpu: {{ calico_policy_controller_cpu_limit }} memory: {{ calico_policy_controller_memory_limit }} requests: cpu: {{ calico_policy_controller_cpu_requests }} memory: {{ calico_policy_controller_memory_requests }} livenessProbe: exec: command: - /usr/bin/check-status - -l periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: exec: command: - /usr/bin/check-status - -r periodSeconds: 10 securityContext: runAsNonRoot: true env: - name: LOG_LEVEL value: {{ calico_policy_controller_log_level }} {% if calico_datastore == "kdd" %} - name: ENABLED_CONTROLLERS value: node - name: DATASTORE_TYPE value: kubernetes {% else %} - name: ENABLED_CONTROLLERS value: policy,namespace,serviceaccount,workloadendpoint,node - name: ETCD_ENDPOINTS value: "{{ etcd_access_addresses }}" - name: ETCD_CA_CERT_FILE value: "{{ calico_cert_dir }}/ca_cert.crt" - name: ETCD_CERT_FILE value: "{{ calico_cert_dir }}/cert.crt" - name: ETCD_KEY_FILE value: "{{ calico_cert_dir }}/key.pem" volumeMounts: - mountPath: {{ calico_cert_dir }} name: etcd-certs readOnly: true volumes: - hostPath: path: {{ calico_cert_dir }} name: etcd-certs {% endif %} ================================================ FILE: roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 ================================================ --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers namespace: kube-system rules: {% if calico_datastore == "etcd" %} # Pods are monitored for changing labels. # The node controller monitors Kubernetes nodes. # Namespace and serviceaccount labels are used for policy. - apiGroups: [""] resources: - pods - nodes - namespaces - serviceaccounts verbs: - watch - list - get # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list # Services are monitored for service LoadBalancer IP allocation - apiGroups: [""] resources: - services - services/status verbs: - get - list - update - watch {% elif calico_datastore == "kdd" %} # Nodes are watched to monitor for deletions. - apiGroups: [""] resources: - nodes verbs: - watch - list - get # Pods are queried to check for existence. - apiGroups: [""] resources: - pods verbs: - watch - list - get # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] resources: - ipreservations verbs: - list # Pools are watched to maintain a mapping of blocks to IP pools. - apiGroups: ["crd.projectcalico.org"] resources: - ippools verbs: - list - watch - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles - tiers verbs: - get - list - create - update - delete - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: - hostendpoints verbs: - get - list - create - update - delete - watch # Needs access to update clusterinformations. - apiGroups: ["crd.projectcalico.org"] resources: - clusterinformations verbs: - get - list - create - update - watch # KubeControllersConfiguration is where it gets its config - apiGroups: ["crd.projectcalico.org"] resources: - kubecontrollersconfigurations verbs: # read its own config - get - list # create a default if none exists - create # update status - update # watch for changes - watch # Services are monitored for service LoadBalancer IP allocation - apiGroups: [""] resources: - services - services/status verbs: - get - list - update - watch {% endif %} ================================================ FILE: roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-crb.yml.j2 ================================================ --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-kube-controllers roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-kube-controllers subjects: - kind: ServiceAccount name: calico-kube-controllers namespace: kube-system ================================================ FILE: roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-sa.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-kube-controllers namespace: kube-system ================================================ FILE: roles/kubernetes-apps/policy_controller/meta/main.yml ================================================ --- dependencies: - role: policy_controller/calico when: - kube_network_plugin in ['calico'] - enable_network_policy tags: - policy-controller ================================================ FILE: roles/kubernetes-apps/registry/defaults/main.yml ================================================ --- registry_namespace: "kube-system" registry_storage_class: "" registry_storage_access_mode: "ReadWriteOnce" registry_disk_size: "10Gi" registry_port: 5000 registry_replica_count: 1 # type of service: ClusterIP, LoadBalancer or NodePort registry_service_type: "ClusterIP" # you can specify your cluster IP address when registry_service_type is ClusterIP registry_service_cluster_ip: "" # you can specify your cloud provider assigned loadBalancerIP when registry_service_type is LoadBalancer registry_service_loadbalancer_ip: "" # annotations for managing Cloud Load Balancers registry_service_annotations: {} # you can specify the node port when registry_service_type is NodePort registry_service_nodeport: "" # name of kubernetes secret for registry TLS certs registry_tls_secret: "" registry_htpasswd: "" # registry configuration # see: https://docs.docker.com/registry/configuration/#list-of-configuration-options registry_config: version: 0.1 log: fields: service: registry storage: cache: blobdescriptor: inmemory http: addr: :{{ registry_port }} headers: X-Content-Type-Options: [nosniff] health: storagedriver: enabled: true interval: 10s threshold: 3 registry_ingress_annotations: {} registry_ingress_host: "" # name of kubernetes secret for registry ingress TLS certs registry_ingress_tls_secret: "" ================================================ FILE: roles/kubernetes-apps/registry/tasks/main.yml ================================================ --- - name: Registry | check registry_service_type value fail: msg: "registry_service_type can only be 'ClusterIP', 'LoadBalancer' or 'NodePort'" when: registry_service_type not in ['ClusterIP', 'LoadBalancer', 'NodePort'] - name: Registry | Stop if registry_service_cluster_ip is defined when registry_service_type is not 'ClusterIP' fail: msg: "registry_service_cluster_ip support only compatible with ClusterIP." when: - registry_service_cluster_ip is defined and registry_service_cluster_ip | length > 0 - registry_service_type != "ClusterIP" - name: Registry | Stop if registry_service_loadbalancer_ip is defined when registry_service_type is not 'LoadBalancer' fail: msg: "registry_service_loadbalancer_ip support only compatible with LoadBalancer." when: - registry_service_loadbalancer_ip is defined and registry_service_loadbalancer_ip | length > 0 - registry_service_type != "LoadBalancer" - name: Registry | Stop if registry_service_nodeport is defined when registry_service_type is not 'NodePort' fail: msg: "registry_service_nodeport support only compatible with NodePort." when: - registry_service_nodeport is defined and registry_service_nodeport | length > 0 - registry_service_type != "NodePort" - name: Registry | Create addon dir file: path: "{{ kube_config_dir }}/addons/registry" state: directory owner: root group: root mode: "0755" - name: Registry | Templates list set_fact: registry_templates: - { name: registry-ns, file: registry-ns.yml, type: ns } - { name: registry-sa, file: registry-sa.yml, type: sa } - { name: registry-svc, file: registry-svc.yml, type: svc } - { name: registry-secrets, file: registry-secrets.yml, type: secrets } - { name: registry-cm, file: registry-cm.yml, type: cm } - { name: registry-rs, file: registry-rs.yml, type: rs } - name: Registry | Append ingress templates to Registry Templates list when ALB ingress enabled set_fact: registry_templates: "{{ registry_templates + [item] }}" with_items: - [{ name: registry-ing, file: registry-ing.yml, type: ing }] when: ingress_alb_enabled - name: Registry | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/registry/{{ item.file }}" mode: "0644" with_items: "{{ registry_templates }}" register: registry_manifests when: inventory_hostname == groups['kube_control_plane'][0] - name: Registry | Apply manifests kube: name: "{{ item.item.name }}" namespace: "{{ registry_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/registry/{{ item.item.file }}" state: "latest" with_items: "{{ registry_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] - name: Registry | Create PVC manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/registry/{{ item.file }}" mode: "0644" with_items: - { name: registry-pvc, file: registry-pvc.yml, type: pvc } register: registry_manifests when: - registry_storage_class != none and registry_storage_class - registry_disk_size != none and registry_disk_size - inventory_hostname == groups['kube_control_plane'][0] - name: Registry | Apply PVC manifests kube: name: "{{ item.item.name }}" namespace: "{{ registry_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/registry/{{ item.item.file }}" state: "latest" with_items: "{{ registry_manifests.results }}" when: - registry_storage_class != none and registry_storage_class - registry_disk_size != none and registry_disk_size - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-cm.yml.j2 ================================================ apiVersion: v1 kind: ConfigMap metadata: name: registry-config namespace: {{ registry_namespace }} {% if registry_config %} data: config.yml: |- {{ registry_config | to_yaml(indent=2, width=1337) | indent(width=4) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-ing.yml.j2 ================================================ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: registry namespace: {{ registry_namespace }} {% if registry_ingress_annotations %} annotations: {{ registry_ingress_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }} {% endif %} spec: {% if registry_ingress_tls_secret %} tls: - hosts: - {{ registry_ingress_host }} secretName: {{ registry_ingress_tls_secret }} {% endif %} rules: - host: {{ registry_ingress_host }} http: paths: - path: / pathType: Prefix backend: service: name: registry port: number: {{ registry_port }} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ registry_namespace }} labels: name: {{ registry_namespace }} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 ================================================ --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: registry-pvc namespace: {{ registry_namespace }} labels: addonmanager.kubernetes.io/mode: Reconcile spec: accessModes: - {{ registry_storage_access_mode }} storageClassName: {{ registry_storage_class }} resources: requests: storage: {{ registry_disk_size }} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 ================================================ --- apiVersion: apps/v1 kind: ReplicaSet metadata: name: registry namespace: {{ registry_namespace }} labels: k8s-app: registry version: v{{ registry_image_tag }} addonmanager.kubernetes.io/mode: Reconcile spec: {% if registry_storage_class != "" and registry_storage_access_mode == "ReadWriteMany" %} replicas: {{ registry_replica_count }} {% else %} replicas: 1 {% endif %} selector: matchLabels: k8s-app: registry version: v{{ registry_image_tag }} template: metadata: labels: k8s-app: registry version: v{{ registry_image_tag }} spec: priorityClassName: {% if registry_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} serviceAccountName: registry securityContext: fsGroup: 1000 runAsUser: 1000 containers: - name: registry image: {{ registry_image_repo }}:{{ registry_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: - /bin/registry - serve - /etc/docker/registry/config.yml env: - name: REGISTRY_HTTP_ADDR value: :{{ registry_port }} - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY value: /var/lib/registry {% if registry_htpasswd != "" %} - name: REGISTRY_AUTH value: "htpasswd" - name: REGISTRY_AUTH_HTPASSWD_REALM value: "Registry Realm" - name: REGISTRY_AUTH_HTPASSWD_PATH value: "/auth/htpasswd" {% endif %} {% if registry_tls_secret != "" %} - name: REGISTRY_HTTP_TLS_CERTIFICATE value: /etc/ssl/docker/tls.crt - name: REGISTRY_HTTP_TLS_KEY value: /etc/ssl/docker/tls.key {% endif %} volumeMounts: - name: registry-pvc mountPath: /var/lib/registry - name: registry-config mountPath: /etc/docker/registry {% if registry_htpasswd != "" %} - name: auth mountPath: /auth readOnly: true {% endif %} {% if registry_tls_secret != "" %} - name: tls-cert mountPath: /etc/ssl/docker readOnly: true {% endif %} ports: - containerPort: {{ registry_port }} name: registry protocol: TCP livenessProbe: httpGet: {% if registry_tls_secret != "" %} scheme: HTTPS {% endif %} path: / port: {{ registry_port }} readinessProbe: httpGet: {% if registry_tls_secret != "" %} scheme: HTTPS {% endif %} path: / port: {{ registry_port }} volumes: - name: registry-pvc {% if registry_storage_class != "" %} persistentVolumeClaim: claimName: registry-pvc {% else %} emptyDir: {} {% endif %} - name: registry-config configMap: name: registry-config {% if registry_htpasswd != "" %} - name: auth secret: secretName: registry-secret items: - key: htpasswd path: htpasswd {% endif %} {% if registry_tls_secret != "" %} - name: tls-cert secret: secretName: {{ registry_tls_secret }} {% endif %} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-sa.yml.j2 ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: registry namespace: {{ registry_namespace }} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-secrets.yml.j2 ================================================ apiVersion: v1 kind: Secret metadata: name: registry-secret namespace: {{ registry_namespace }} type: Opaque data: {% if registry_htpasswd != "" %} htpasswd: {{ registry_htpasswd | b64encode }} {% endif %} ================================================ FILE: roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 ================================================ --- apiVersion: v1 kind: Service metadata: name: registry namespace: {{ registry_namespace }} labels: k8s-app: registry addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "KubeRegistry" {% if registry_service_annotations %} annotations: {{ registry_service_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }} {% endif %} spec: selector: k8s-app: registry type: {{ registry_service_type }} {% if registry_service_type == "ClusterIP" and registry_service_cluster_ip != "" %} clusterIP: {{ registry_service_cluster_ip }} {% endif %} {% if registry_service_type == "LoadBalancer" and registry_service_loadbalancer_ip != "" %} loadBalancerIP: {{ registry_service_loadbalancer_ip }} {% endif %} ports: - name: registry port: {{ registry_port }} protocol: TCP targetPort: {{ registry_port }} {% if registry_service_type == "NodePort" and registry_service_nodeport != "" %} nodePort: {{ registry_service_nodeport }} {% endif %} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/defaults/main.yml ================================================ --- scheduler_plugins_enabled: false scheduler_plugins_namespace: scheduler-plugins scheduler_plugins_controller_replicas: 1 scheduler_plugins_scheduler_replicas: 1 # The default is determined by the number of control plane nodes. scheduler_plugins_scheduler_leader_elect: "{{ ((groups['kube_control_plane'] | length) > 1) }}" # Plugins to enable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info. scheduler_plugins_enabled_plugins: - Coscheduling - CapacityScheduling - NodeResourceTopologyMatch - NodeResourcesAllocatable # Plugins to disable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info. scheduler_plugins_disabled_plugins: - PrioritySort # Customize the enabled plugins' config. # Refer to the "pluginConfig" section of https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests//scheduler-config.yaml. scheduler_plugins_plugin_config: - name: Coscheduling args: permitWaitingTimeSeconds: 10 # default is 60 ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/tasks/main.yml ================================================ --- - name: Scheduler Plugins | Ensure dir exists file: path: "{{ kube_config_dir }}/scheduler-plugins" state: directory owner: root group: root mode: "0755" when: inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins - name: Scheduler Plugins | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/scheduler-plugins/{{ item.file }}" mode: "0644" with_items: - { name: appgroup, file: appgroup.diktyo.x-k8s.io_appgroups.yaml, type: crd } - { name: networktopology, file: networktopology.diktyo.x-k8s.io_networktopologies.yaml, type: crd } - { name: elasticquotas, file: scheduling.x-k8s.io_elasticquotas.yaml, type: crd } - { name: podgroups, file: scheduling.x-k8s.io_podgroups.yaml, type: crd } - { name: noderesourcetopologies, file: topology.node.k8s.io_noderesourcetopologies.yaml, type: crd } - { name: namespace, file: namespace.yaml, type: namespace } - { name: sa, file: sa-scheduler-plugins.yaml, type: serviceaccount } - { name: rbac, file: rbac-scheduler-plugins.yaml, type: rbac } - { name: cm, file: cm-scheduler-plugins.yaml, type: configmap } - { name: deploy, file: deploy-scheduler-plugins.yaml, type: deployment } register: scheduler_plugins_manifests when: inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins - name: Scheduler Plugins | Apply manifests kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/scheduler-plugins/{{ item.item.file }}" state: "latest" with_items: "{{ scheduler_plugins_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins - name: Scheduler Plugins | Wait for controller pods to be ready command: "{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l app=scheduler-plugins-controller -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors register: controller_pods_not_ready until: controller_pods_not_ready.stdout.find("scheduler-plugins-controller")==-1 retries: 30 delay: 10 ignore_errors: true changed_when: false when: inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins - name: Scheduler Plugins | Wait for scheduler pods to be ready command: "{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l component=scheduler -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors register: scheduler_pods_not_ready until: scheduler_pods_not_ready.stdout.find("scheduler-plugins-scheduler")==-1 retries: 30 delay: 10 ignore_errors: true changed_when: false when: inventory_hostname == groups['kube_control_plane'][0] tags: - scheduler_plugins ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: appgroups.appgroup.diktyo.x-k8s.io spec: group: appgroup.diktyo.x-k8s.io names: kind: AppGroup listKind: AppGroupList plural: appgroups shortNames: - ag singular: appgroup scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: AppGroup is a collection of Pods belonging to the same application. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: AppGroupSpec defines the number of Pods and which Pods belong to the group. properties: numMembers: description: NumMembers defines the number of Pods belonging to the App Group format: int32 minimum: 1 type: integer topologySortingAlgorithm: description: The preferred Topology Sorting Algorithm type: string workloads: description: Workloads defines the workloads belonging to the group items: description: AppGroupWorkload represents the Workloads belonging to the App Group. properties: dependencies: description: Dependencies of the Workload. items: description: DependenciesInfo contains information about one dependency. properties: maxNetworkCost: description: Max Network Cost between workloads format: int64 maximum: 10000 minimum: 0 type: integer minBandwidth: anyOf: - type: integer - type: string description: MinBandwidth between workloads pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true workload: description: Workload reference Info. properties: apiVersion: description: ApiVersion defines the versioned schema of an object. type: string kind: description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string name: description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string namespace: description: Namespace of the workload type: string selector: description: Selector defines how to find Pods related to the Workload (key = workload). (e.g., workload=w1) type: string required: - kind - name - selector type: object required: - workload type: object type: array workload: description: Workload reference Info. properties: apiVersion: description: ApiVersion defines the versioned schema of an object. type: string kind: description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string name: description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string namespace: description: Namespace of the workload type: string selector: description: Selector defines how to find Pods related to the Workload (key = workload). (e.g., workload=w1) type: string required: - kind - name - selector type: object required: - workload type: object type: array required: - numMembers - topologySortingAlgorithm - workloads type: object status: description: AppGroupStatus defines the observed use. properties: runningWorkloads: description: The number of actively running workloads (e.g., number of pods). format: int32 minimum: 0 type: integer scheduleStartTime: description: ScheduleStartTime of the group format: date-time type: string topologyCalculationTime: description: TopologyCalculationTime of the group format: date-time type: string topologyOrder: description: Topology order for TopSort plugin (QueueSort) items: description: AppGroupTopologyInfo represents the calculated order for a given Workload. properties: index: description: Topology index. format: int32 type: integer workload: description: Workload reference Info. properties: apiVersion: description: ApiVersion defines the versioned schema of an object. type: string kind: description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string name: description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string namespace: description: Namespace of the workload type: string selector: description: Selector defines how to find Pods related to the Workload (key = workload). (e.g., workload=w1) type: string required: - kind - name - selector type: object type: object type: array type: object type: object served: true storage: true ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 ================================================ apiVersion: v1 kind: ConfigMap metadata: name: scheduler-config namespace: {{ scheduler_plugins_namespace }} data: scheduler-config.yaml: | apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration leaderElection: leaderElect: {{ scheduler_plugins_scheduler_leader_elect | bool | lower }} profiles: # Compose all plugins in one profile - schedulerName: scheduler-plugins-scheduler plugins: multiPoint: enabled: {% for enabeld_plugin in scheduler_plugins_enabled_plugins %} - name: {{ enabeld_plugin }} {% endfor %} disabled: {% for disabled_plugin in scheduler_plugins_disabled_plugins %} - name: {{ disabled_plugin }} {% endfor %} {% if scheduler_plugins_plugin_config is defined and scheduler_plugins_plugin_config | length != 0 %} pluginConfig: {{ scheduler_plugins_plugin_config | to_nice_yaml(indent=2, width=256) | indent(6, true) }} {% endif %} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 ================================================ kind: Deployment apiVersion: apps/v1 metadata: name: scheduler-plugins-controller namespace: {{ scheduler_plugins_namespace }} labels: app: scheduler-plugins-controller spec: replicas: {{ scheduler_plugins_controller_replicas }} selector: matchLabels: app: scheduler-plugins-controller template: metadata: labels: app: scheduler-plugins-controller spec: serviceAccountName: scheduler-plugins-controller containers: - name: scheduler-plugins-controller image: {{ scheduler_plugins_controller_image_repo }}:{{ scheduler_plugins_controller_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} --- apiVersion: apps/v1 kind: Deployment metadata: labels: component: scheduler name: scheduler-plugins-scheduler namespace: {{ scheduler_plugins_namespace }} spec: selector: matchLabels: component: scheduler replicas: {{ scheduler_plugins_scheduler_replicas }} template: metadata: labels: component: scheduler spec: serviceAccountName: scheduler-plugins-scheduler containers: - command: - /bin/kube-scheduler - --config=/etc/kubernetes/scheduler-config.yaml image: {{ scheduler_plugins_scheduler_image_repo }}:{{ scheduler_plugins_scheduler_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} livenessProbe: httpGet: path: /healthz port: 10259 scheme: HTTPS initialDelaySeconds: 15 name: scheduler-plugins-scheduler readinessProbe: httpGet: path: /healthz port: 10259 scheme: HTTPS resources: requests: cpu: '0.1' securityContext: privileged: false volumeMounts: - name: scheduler-config mountPath: /etc/kubernetes readOnly: true hostNetwork: false hostPID: false volumes: - name: scheduler-config configMap: name: scheduler-config ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ scheduler_plugins_namespace }} labels: name: {{ scheduler_plugins_namespace }} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: networktopologies.networktopology.diktyo.x-k8s.io spec: group: networktopology.diktyo.x-k8s.io names: kind: NetworkTopology listKind: NetworkTopologyList plural: networktopologies shortNames: - nt singular: networktopology scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: NetworkTopology defines network costs in the cluster between regions and zones properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: NetworkTopologySpec defines the zones and regions of the cluster. properties: configmapName: description: ConfigmapName to be used for cost calculation type: string weights: description: The manual defined weights of the cluster items: description: WeightInfo contains information about all network costs for a given algorithm. properties: name: description: Algorithm Name for network cost calculation (e.g., userDefined) type: string topologyList: description: TopologyList owns Costs between origins items: description: TopologyInfo contains information about network costs for a particular Topology Key. properties: originList: description: OriginList for a particular origin. items: description: OriginInfo contains information about network costs for a particular Origin. properties: costList: description: Costs for the particular origin. items: description: CostInfo contains information about networkCosts. properties: bandwidthAllocated: anyOf: - type: integer - type: string description: Bandwidth allocated between origin and destination. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true bandwidthCapacity: anyOf: - type: integer - type: string description: Bandwidth capacity between origin and destination. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true destination: description: Name of the destination (e.g., Region Name, Zone Name). type: string networkCost: description: Network Cost between origin and destination (e.g., Dijkstra shortest path, etc) format: int64 minimum: 0 type: integer required: - destination - networkCost type: object type: array origin: description: Name of the origin (e.g., Region Name, Zone Name). type: string required: - origin type: object type: array topologyKey: description: Topology key (e.g., "topology.kubernetes.io/region", "topology.kubernetes.io/zone"). type: string required: - originList - topologyKey type: object type: array required: - name - topologyList type: object type: array required: - configmapName - weights type: object status: description: NetworkTopologyStatus defines the observed use. properties: nodeCount: description: The total number of nodes in the cluster format: int64 minimum: 0 type: integer weightCalculationTime: description: The calculation time for the weights in the network topology CRD format: date-time type: string type: object type: object served: true storage: true ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: scheduler-plugins-scheduler rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list", "watch"] - apiGroups: ["", "events.k8s.io"] resources: ["events"] verbs: ["create", "patch", "update"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create"] - apiGroups: ["coordination.k8s.io"] resourceNames: ["kube-scheduler"] resources: ["leases"] verbs: ["get", "update"] - apiGroups: [""] resources: ["endpoints"] verbs: ["create"] - apiGroups: [""] resourceNames: ["kube-scheduler"] resources: ["endpoints"] verbs: ["get", "update"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch", "patch"] - apiGroups: [""] resources: ["pods"] verbs: ["delete", "get", "list", "watch", "update"] - apiGroups: [""] resources: ["bindings", "pods/binding"] verbs: ["create"] - apiGroups: [""] resources: ["pods/status"] verbs: ["patch", "update"] - apiGroups: [""] resources: ["replicationcontrollers", "services"] verbs: ["get", "list", "watch"] - apiGroups: ["apps", "extensions"] resources: ["replicasets"] verbs: ["get", "list", "watch"] - apiGroups: ["apps"] resources: ["statefulsets"] verbs: ["get", "list", "watch"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims", "persistentvolumes"] verbs: ["get", "list", "watch", "patch", "update"] - apiGroups: ["authentication.k8s.io"] resources: ["tokenreviews"] verbs: ["create"] - apiGroups: ["authorization.k8s.io"] resources: ["subjectaccessreviews"] verbs: ["create"] - apiGroups: ["storage.k8s.io"] resources: ["csinodes", "storageclasses" , "csidrivers" , "csistoragecapacities"] verbs: ["get", "list", "watch"] - apiGroups: ["topology.node.k8s.io"] resources: ["noderesourcetopologies"] verbs: ["get", "list", "watch"] # resources need to be updated with the scheduler plugins used - apiGroups: ["scheduling.x-k8s.io"] resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] # for network-aware plugins add the following lines (scheduler-plugins v0.27.8) #- apiGroups: [ "appgroup.diktyo.x-k8s.io" ] # resources: [ "appgroups" ] # verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ] #- apiGroups: [ "networktopology.diktyo.x-k8s.io" ] # resources: [ "networktopologies" ] # verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: scheduler-plugins-scheduler roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: scheduler-plugins-scheduler subjects: - kind: ServiceAccount name: scheduler-plugins-scheduler namespace: {{ scheduler_plugins_namespace }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: scheduler-plugins-controller rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch", "update"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch", "patch"] - apiGroups: ["topology.node.k8s.io"] resources: ["noderesourcetopologies"] verbs: ["get", "list", "watch"] # resources need to be updated with the scheduler plugins used - apiGroups: ["scheduling.x-k8s.io"] resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: scheduler-plugins-controller subjects: - kind: ServiceAccount name: scheduler-plugins-controller namespace: {{ scheduler_plugins_namespace }} roleRef: kind: ClusterRole name: scheduler-plugins-controller apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: sched-plugins::extension-apiserver-authentication-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: scheduler-plugins-scheduler namespace: {{ scheduler_plugins_namespace }} - kind: ServiceAccount name: scheduler-plugins-controller namespace: {{ scheduler_plugins_namespace }} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: scheduler-plugins-scheduler namespace: {{ scheduler_plugins_namespace }} --- apiVersion: v1 kind: ServiceAccount metadata: name: scheduler-plugins-controller namespace: {{ scheduler_plugins_namespace }} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/52 controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: elasticquotas.scheduling.x-k8s.io spec: group: scheduling.x-k8s.io names: kind: ElasticQuota listKind: ElasticQuotaList plural: elasticquotas shortNames: - eq - eqs singular: elasticquota scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: ElasticQuota sets elastic quota restrictions per namespace properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: ElasticQuotaSpec defines the Min and Max for Quota. properties: max: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Max is the set of desired max limits for each named resource. The usage of max is based on the resource configurations of successfully scheduled pods. type: object min: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Min is the set of desired guaranteed limits for each named resource. type: object type: object status: description: ElasticQuotaStatus defines the observed use. properties: used: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Used is the current observed total usage of the resource in the namespace. type: object type: object type: object served: true storage: true subresources: status: {} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/50 controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: podgroups.scheduling.x-k8s.io spec: group: scheduling.x-k8s.io names: kind: PodGroup listKind: PodGroupList plural: podgroups shortNames: - pg - pgs singular: podgroup scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: PodGroup is a collection of Pod; used for batch workload. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: Specification of the desired behavior of the pod group. properties: minMember: description: MinMember defines the minimal number of members/tasks to run the pod group; if there's not enough resources to start all tasks, the scheduler will not start anyone. format: int32 type: integer minResources: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: MinResources defines the minimal resource of members/tasks to run the pod group; if there's not enough resources to start all tasks, the scheduler will not start anyone. type: object scheduleTimeoutSeconds: description: ScheduleTimeoutSeconds defines the maximal time of members/tasks to wait before run the pod group; format: int32 type: integer type: object status: description: Status represents the current information about a pod group. This data may not be up to date. properties: failed: description: The number of pods which reached phase Failed. format: int32 type: integer occupiedBy: description: OccupiedBy marks the workload (e.g., deployment, statefulset) UID that occupy the podgroup. It is empty if not initialized. type: string phase: description: Current phase of PodGroup. type: string running: description: The number of actively running pods. format: int32 type: integer scheduleStartTime: description: ScheduleStartTime of the group format: date-time type: string succeeded: description: The number of pods which reached phase Succeeded. format: int32 type: integer type: object type: object served: true storage: true subresources: status: {} ================================================ FILE: roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: noderesourcetopologies.topology.node.k8s.io spec: group: topology.node.k8s.io names: kind: NodeResourceTopology listKind: NodeResourceTopologyList plural: noderesourcetopologies shortNames: - node-res-topo singular: noderesourcetopology scope: Cluster versions: - name: v1alpha2 schema: openAPIV3Schema: description: NodeResourceTopology describes node resources and their topology. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string attributes: description: AttributeList contains an array of AttributeInfo objects. items: description: AttributeInfo contains one attribute of a Zone. properties: name: type: string value: type: string required: - name - value type: object type: array kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object topologyPolicies: description: 'DEPRECATED (to be removed in v1beta1): use top level attributes if needed' items: type: string type: array zones: description: ZoneList contains an array of Zone objects. items: description: Zone represents a resource topology zone, e.g. socket, node, die or core. properties: attributes: description: AttributeList contains an array of AttributeInfo objects. items: description: AttributeInfo contains one attribute of a Zone. properties: name: type: string value: type: string required: - name - value type: object type: array costs: description: CostList contains an array of CostInfo objects. items: description: CostInfo describes the cost (or distance) between two Zones. properties: name: type: string value: format: int64 type: integer required: - name - value type: object type: array name: type: string parent: type: string resources: description: ResourceInfoList contains an array of ResourceInfo objects. items: description: ResourceInfo contains information about one resource type. properties: allocatable: anyOf: - type: integer - type: string description: Allocatable quantity of the resource, corresponding to allocatable in node status, i.e. total amount of this resource available to be used by pods. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true available: anyOf: - type: integer - type: string description: Available is the amount of this resource currently available for new (to be scheduled) pods, i.e. Allocatable minus the resources reserved by currently running pods. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true capacity: anyOf: - type: integer - type: string description: Capacity of the resource, corresponding to capacity in node status, i.e. total amount of this resource that the node has. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true name: description: Name of the resource. type: string required: - allocatable - available - capacity - name type: object type: array type: type: string required: - name - type type: object type: array required: - zones type: object served: true storage: true ================================================ FILE: roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml ================================================ --- snapshot_classes: - name: cinder-csi-snapshot is_default: false force_create: true deletionPolicy: Delete ================================================ FILE: roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml ================================================ --- - name: Kubernetes Snapshots | Copy Cinder CSI Snapshot Class template template: src: "cinder-csi-snapshot-class.yml.j2" dest: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml" mode: "0644" register: manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Snapshots | Add Cinder CSI Snapshot Class kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml" state: "latest" when: - inventory_hostname == groups['kube_control_plane'][0] - manifests.changed ================================================ FILE: roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 ================================================ {% for class in snapshot_classes %} --- kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1 metadata: name: "{{ class.name }}" annotations: storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" driver: cinder.csi.openstack.org deletionPolicy: "{{ class.deletionPolicy | default("Delete") }}" parameters: force-create: "{{ class.force_create }}" {% endfor %} ================================================ FILE: roles/kubernetes-apps/snapshots/meta/main.yml ================================================ --- dependencies: - role: kubernetes-apps/snapshots/snapshot-controller when: - cinder_csi_enabled or csi_snapshot_controller_enabled tags: - snapshot-controller - role: kubernetes-apps/snapshots/cinder-csi when: - cinder_csi_enabled tags: - snapshot - cinder-csi-driver ================================================ FILE: roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml ================================================ --- snapshot_controller_replicas: 1 snapshot_controller_namespace: kube-system ================================================ FILE: roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml ================================================ --- - name: Check if snapshot namespace exists register: snapshot_namespace_exists kube: kubectl: "{{ bin_dir }}/kubectl" name: "{{ snapshot_controller_namespace }}" resource: "namespace" state: "exists" when: inventory_hostname == groups['kube_control_plane'][0] tags: snapshot-controller - name: Snapshot Controller | Generate Manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: snapshot-ns, file: snapshot-ns.yml, apply: not snapshot_namespace_exists} - {name: rbac-snapshot-controller, file: rbac-snapshot-controller.yml} - {name: snapshot-controller, file: snapshot-controller.yml} register: snapshot_controller_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - item.apply | default(True) | bool tags: snapshot-controller - name: Snapshot Controller | Apply Manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ snapshot_controller_manifests.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" tags: snapshot-controller ================================================ FILE: roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 ================================================ # RBAC file for the snapshot controller. # # The snapshot controller implements the control loop for CSI snapshot functionality. # It should be installed as part of the base Kubernetes distribution in an appropriate # namespace for components implementing base system functionality. For installing with # Vanilla Kubernetes, kube-system makes sense for the namespace. apiVersion: v1 kind: ServiceAccount metadata: name: snapshot-controller namespace: {{ snapshot_controller_namespace }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: snapshot-controller-runner rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update", "patch", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update", "patch"] - apiGroups: ["groupsnapshot.storage.k8s.io"] resources: ["volumegroupsnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["groupsnapshot.storage.k8s.io"] resources: ["volumegroupsnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - apiGroups: ["groupsnapshot.storage.k8s.io"] resources: ["volumegroupsnapshotcontents/status"] verbs: ["patch"] - apiGroups: ["groupsnapshot.storage.k8s.io"] resources: ["volumegroupsnapshots"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["groupsnapshot.storage.k8s.io"] resources: ["volumegroupsnapshots/status"] verbs: ["update", "patch"] # Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true # - apiGroups: [""] # resources: ["nodes"] # verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: snapshot-controller-role subjects: - kind: ServiceAccount name: snapshot-controller namespace: {{ snapshot_controller_namespace }} roleRef: kind: ClusterRole name: snapshot-controller-runner apiGroup: rbac.authorization.k8s.io --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: snapshot-controller-leaderelection namespace: {{ snapshot_controller_namespace }} rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: snapshot-controller-leaderelection namespace: {{ snapshot_controller_namespace }} subjects: - kind: ServiceAccount name: snapshot-controller namespace: {{ snapshot_controller_namespace }} roleRef: kind: Role name: snapshot-controller-leaderelection apiGroup: rbac.authorization.k8s.io ================================================ FILE: roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 ================================================ # This YAML file shows how to deploy the snapshot controller # The snapshot controller implements the control loop for CSI snapshot functionality. # It should be installed as part of the base Kubernetes distribution in an appropriate # namespace for components implementing base system functionality. For installing with # Vanilla Kubernetes, kube-system makes sense for the namespace. --- kind: Deployment apiVersion: apps/v1 metadata: name: snapshot-controller namespace: {{ snapshot_controller_namespace }} spec: replicas: {{ snapshot_controller_replicas }} selector: matchLabels: app.kubernetes.io/name: snapshot-controller # The snapshot controller won't be marked as ready if the v1 CRDs are unavailable. # The flag --retry-crd-interval-max is used to determine how long the controller # will wait for the CRDs to become available before exiting. The default is 30 seconds # so minReadySeconds should be set slightly higher than the flag value. minReadySeconds: 35 strategy: rollingUpdate: maxSurge: 0 maxUnavailable: 1 type: RollingUpdate template: metadata: labels: app.kubernetes.io/name: snapshot-controller spec: serviceAccountName: snapshot-controller containers: - name: snapshot-controller image: {{ snapshot_controller_image_repo }}:{{ snapshot_controller_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--v=5" - "--leader-election={{ 'true' if snapshot_controller_replicas > 1 else 'false' }}" ================================================ FILE: roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-ns.yml.j2 ================================================ --- apiVersion: v1 kind: Namespace metadata: name: {{ snapshot_controller_namespace }} labels: name: {{ snapshot_controller_namespace }} ================================================ FILE: roles/kubernetes-apps/utils/defaults/main.yml ================================================ --- k8s_namespace: kube-system ================================================ FILE: roles/kubernetes-apps/utils/vars/main.yml ================================================ --- _kubectl_apply_stdin: - "{{ kubectl }}" - apply - -f - "-" - -n - "{{ k8s_namespace }}" - --server-side="{{ server_side_apply | lower }}" # TODO: switch to default SSA server_side_apply: false kubectl_apply_stdin: "{{ _kubectl_apply_stdin | join(' ') }}" ================================================ FILE: roles/kubespray-defaults/tasks/main.yml ================================================ --- - name: Warn for usage of deprecated role fail: msg: kubespray-defaults is deprecated, switch to kubespray_defaults ignore_errors: true # noqa ignore-errors run_once: true - name: Compat for direct role import import_role: name: kubespray_defaults ================================================ FILE: roles/kubespray_defaults/defaults/main/download.yml ================================================ --- local_release_dir: /tmp/releases download_cache_dir: /tmp/kubespray_cache # If this is true, debug information will be displayed but # may contain some private data, so it is recommended to set it to false # in the production environment. # false by default, unless we're running in CI. (CI_PROJECT_URL should be globally unique even if kubespray happens to run # in gitlab-ci in other contexts unsafe_show_logs: "{{ lookup('env', 'CI_PROJECT_URL') == 'https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray' }}" # do not delete remote cache files after using them # NOTE: Setting this parameter to TRUE is only really useful when developing kubespray download_keep_remote_cache: false # Only useful when download_run_once is false: Localy cached files and images are # uploaded to kubernetes nodes. Also, images downloaded on those nodes are copied # back to the ansible runner's cache, if they are not yet preset. download_force_cache: false # Used to only evaluate vars from download role skip_downloads: false # Optionally skip kubeadm images download skip_kubeadm_images: false kubeadm_images: {} # if this is set to true will only download files once. Doesn't work # on Flatcar Container Linux by Kinvolk unless the download_localhost is true and localhost # is running another OS type. Default compress level is 1 (fastest). download_run_once: false download_compress: 1 # if this is set to true will download container download_container: true # if this is set to true, uses the localhost for download_run_once mode # (requires docker and sudo to access docker). You may want this option for # local caching of docker images or for Flatcar Container Linux by Kinvolk cluster nodes. # Otherwise, uses the first node in the kube_control_plane group to store images # in the download_run_once mode. download_localhost: false # Always pull images if set to True. Otherwise check by the repo's tag/digest. download_always_pull: false # Some problems may occur when downloading files over https proxy due to ansible bug # https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable # SSL validation of get_url module. Note that kubespray will still be performing checksum validation. download_validate_certs: true # Use the first kube_control_plane if download_localhost is not set download_delegate: "{% if download_localhost %}localhost{% else %}{{ groups['kube_control_plane'][0] }}{% endif %}" # Allow control the times of download retries for files and containers download_retries: 4 # The docker_image_info_command might seems weird but we are using raw/endraw and `{{ `{{` }}` to manage the double jinja2 processing docker_image_pull_command: "{{ docker_bin_dir }}/docker pull" docker_image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs -i {{ '{{' }} docker_bin_dir }}/docker inspect -f {% raw %}'{{ '{{' }} if .RepoTags }}{{ '{{' }} join .RepoTags \",\" }}{{ '{{' }} end }}{{ '{{' }} if .RepoDigests }},{{ '{{' }} join .RepoDigests \",\" }}{{ '{{' }} end }}' {% endraw %} {} | tr '\n' ','" nerdctl_image_info_command: "{{ bin_dir }}/nerdctl -n k8s.io images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' 2>/dev/null | grep -v ^:$ | tr '\n' ','" nerdctl_image_pull_command: "{{ bin_dir }}/nerdctl -n k8s.io pull --quiet" crictl_image_info_command: "{{ bin_dir }}/crictl images --verbose | awk -F ': ' '/RepoTags|RepoDigests/ {print $2}' | tr '\n' ','" crictl_image_pull_command: "{{ bin_dir }}/crictl pull" image_command_tool: "{%- if container_manager == 'containerd' -%}nerdctl{%- elif container_manager == 'crio' -%}crictl{%- else -%}{{ container_manager }}{%- endif -%}" image_command_tool_on_localhost: "{{ image_command_tool }}" image_pull_command: "{{ lookup('vars', image_command_tool + '_image_pull_command') }}" image_info_command: "{{ lookup('vars', image_command_tool + '_image_info_command') }}" image_pull_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localhost + '_image_pull_command') }}" image_info_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localhost + '_image_info_command') }}" # Arch of Docker images and needed packages image_arch: "{{ host_architecture | default('amd64') }}" # Versions crun_version: "{{ (crun_checksums['amd64'] | dict2items)[0].key }}" runc_version: "{{ (runc_checksums['amd64'] | dict2items)[0].key }}" kata_containers_version: "{{ (kata_containers_binary_checksums['amd64'] | dict2items)[0].key }}" youki_version: "{{ (youki_checksums['amd64'] | dict2items)[0].key }}" gvisor_version: "{{ (gvisor_runsc_binary_checksums['amd64'] | dict2items)[0].key }}" containerd_version: "{{ (containerd_archive_checksums['amd64'] | dict2items)[0].key }}" cri_dockerd_version: "{{ (cri_dockerd_archive_checksums['amd64'] | dict2items)[0].key }}" argocd_version: "{{ (argocd_install_checksums.no_arch | dict2items)[0].key }}" # this is relevant when container_manager == 'docker' docker_containerd_version: 1.6.32 # gcr and kubernetes image repo define gcr_image_repo: "gcr.io" kube_image_repo: "registry.k8s.io" kubeadm_image_repo: "{{ kube_image_repo }}" # docker image repo define docker_image_repo: "docker.io" # quay image repo define quay_image_repo: "quay.io" # github image repo define (ex multus only use that) github_image_repo: "ghcr.io" # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download calico_version: "{{ (calicoctl_binary_checksums['amd64'] | dict2items)[0].key }}" calico_ctl_version: "{{ calico_version }}" calico_cni_version: "{{ calico_version }}" calico_policy_version: "{{ calico_version }}" calico_typha_version: "{{ calico_version }}" calico_apiserver_version: "{{ calico_version }}" typha_enabled: false calico_apiserver_enabled: false flannel_version: 0.27.3 flannel_cni_version: 1.7.1-flannel1 cni_version: "{{ (cni_binary_checksums['amd64'] | dict2items)[0].key }}" cilium_version: "1.19.1" cilium_cli_version: "{{ (ciliumcli_binary_checksums['amd64'] | dict2items)[0].key }}" cilium_enable_hubble: false kube_ovn_version: "1.12.21" kube_ovn_dpdk_version: "19.11-v{{ kube_ovn_version }}" kube_router_version: "2.1.1" multus_version: "4.2.2" helm_version: "{{ (helm_archive_checksums['amd64'] | dict2items)[0].key }}" nerdctl_version: "{{ (nerdctl_archive_checksums['amd64'] | dict2items)[0].key }}" skopeo_version: "{{ (skopeo_binary_checksums['amd64'] | dict2items)[0].key }}" pod_infra_version: "{{ pod_infra_supported_versions[kube_major_version] }}" etcd_version: "{{ etcd_supported_versions[kube_major_version] }}" crictl_version: "{{ (crictl_checksums['amd64'].keys() | select('version', kube_major_next_version, '<'))[0] }}" crio_version: "{{ (crio_archive_checksums['amd64'].keys() | select('version', kube_major_next_version, '<'))[0] }}" # Scheduler plugins doesn't build for K8s 1.29 yet scheduler_plugins_supported_versions: '1.31': 0 '1.30': 0 '1.29': 0 scheduler_plugins_version: "{{ scheduler_plugins_supported_versions[kube_major_version] }}" yq_version: "{{ (yq_checksums['amd64'] | dict2items)[0].key }}" gateway_api_version: "{{ (gateway_api_standard_crds_checksums.no_arch | dict2items)[0].key }}" gateway_api_channel: "standard" prometheus_operator_crds_version: "{{ (prometheus_operator_crds_checksums.no_arch | dict2items)[0].key }}" github_url: https://github.com dl_k8s_io_url: https://dl.k8s.io storage_googleapis_url: https://storage.googleapis.com get_helm_url: https://get.helm.sh # Download URLs kubelet_download_url: "{{ dl_k8s_io_url }}/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet" kubectl_download_url: "{{ dl_k8s_io_url }}/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl" kubeadm_download_url: "{{ dl_k8s_io_url }}/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm" etcd_download_url: "{{ github_url }}/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" cni_download_url: "{{ github_url }}/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz" calicoctl_download_url: "{{ github_url }}/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" calico_crds_download_url: "{{ github_url }}/projectcalico/calico/raw/v{{ calico_version }}/manifests/crds.yaml" ciliumcli_download_url: "{{ github_url }}/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz" crictl_download_url: "{{ github_url }}/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" crio_download_url: "{{ storage_googleapis_url }}/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz" helm_download_url: "{{ get_helm_url }}/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz" runc_download_url: "{{ github_url }}/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}" crun_download_url: "{{ github_url }}/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}" youki_download_url: "{{ github_url }}/youki-dev/youki/releases/download/v{{ youki_version }}/youki-{{ youki_version }}-{{ ansible_architecture }}-gnu.tar.gz" kata_containers_download_url: "{{ github_url }}/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz" # gVisor only supports amd64 and uses x86_64 to in the download link gvisor_runsc_download_url: "{{ storage_googleapis_url }}/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc" gvisor_containerd_shim_runsc_download_url: "{{ storage_googleapis_url }}/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1" nerdctl_download_url: "{{ github_url }}/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" containerd_download_url: "{{ github_url }}/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ 'static-' if containerd_static_binary }}{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" cri_dockerd_download_url: "{{ github_url }}/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz" skopeo_download_url: "{{ github_url }}/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}" yq_download_url: "{{ github_url }}/mikefarah/yq/releases/download/v{{ yq_version }}/yq_linux_{{ image_arch }}" argocd_install_url: "https://raw.githubusercontent.com/argoproj/argo-cd/v{{ argocd_version }}/manifests/install.yaml" gateway_api_crds_download_url: "{{ github_url }}/kubernetes-sigs/gateway-api/releases/download/v{{ gateway_api_version }}/{{ gateway_api_channel }}-install.yaml" prometheus_operator_crds_download_url: "{{ github_url }}/prometheus-operator/prometheus-operator/releases/download/v{{ prometheus_operator_crds_version }}/stripped-down-crds.yaml" etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch][etcd_version] }}" cni_binary_checksum: "{{ cni_binary_checksums[image_arch][cni_version] }}" kubelet_binary_checksum: "{{ kubelet_checksums[image_arch][kube_version] }}" kubectl_binary_checksum: "{{ kubectl_checksums[image_arch][kube_version] }}" kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kube_version] }}" yq_binary_checksum: "{{ yq_checksums[image_arch][yq_version] }}" argocd_install_checksum: "{{ argocd_install_checksums.no_arch[argocd_version] }}" calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" ciliumcli_binary_checksum: "{{ ciliumcli_binary_checksums[image_arch][cilium_cli_version] }}" crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" crio_archive_checksum: "{{ crio_archive_checksums[image_arch][crio_version] }}" cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}" crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" youki_archive_checksum: "{{ youki_checksums[image_arch][youki_version] }}" kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}" gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}" gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}" nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}" containerd_archive_checksum: "{{ containerd_archive_checksums[image_arch][containerd_version] }}" containerd_static_archive_checksum: "{{ containerd_static_archive_checksums[image_arch][containerd_version] }}" containerd_checksum: "{{ containerd_static_archive_checksum if containerd_static_binary else containerd_archive_checksum }}" skopeo_binary_checksum: "{{ skopeo_binary_checksums[image_arch][skopeo_version] }}" # Containers # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, # it helps a lot for local private development or bare metal environment. # So you need define --registry-mirror or --insecure-registry, and modify the following url address. # example: # You need to deploy kubernetes cluster on local private development. # Also provide the address of your own private registry. # And use --insecure-registry options for docker kube_proxy_image_repo: "{{ kube_image_repo }}/kube-proxy" etcd_image_repo: "{{ quay_image_repo }}/coreos/etcd" etcd_image_tag: "v{{ etcd_version }}" flannel_image_repo: "{{ docker_image_repo }}/flannel/flannel" flannel_image_tag: "v{{ flannel_version }}" flannel_init_image_repo: "{{ docker_image_repo }}/flannel/flannel-cni-plugin" flannel_init_image_tag: "v{{ flannel_cni_version }}" calico_node_image_repo: "{{ quay_image_repo }}/calico/node" calico_node_image_tag: "v{{ calico_version }}" calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni" calico_cni_image_tag: "v{{ calico_cni_version }}" calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers" calico_policy_image_tag: "v{{ calico_policy_version }}" calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha" calico_typha_image_tag: "v{{ calico_typha_version }}" calico_apiserver_image_repo: "{{ quay_image_repo }}/calico/apiserver" calico_apiserver_image_tag: "v{{ calico_apiserver_version }}" pod_infra_image_repo: "{{ kube_image_repo }}/pause" pod_infra_image_tag: "{{ pod_infra_version }}" cilium_image_repo: "{{ quay_image_repo }}/cilium/cilium" cilium_image_tag: "v{{ cilium_version }}" cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator" cilium_operator_image_tag: "v{{ cilium_version }}" cilium_hubble_relay_image_repo: "{{ quay_image_repo }}/cilium/hubble-relay" cilium_hubble_relay_image_tag: "v{{ cilium_version }}" cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen" cilium_hubble_certgen_image_tag: "v0.2.4" cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui" cilium_hubble_ui_image_tag: "v0.13.3" cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend" cilium_hubble_ui_backend_image_tag: "v0.13.3" cilium_hubble_envoy_image_repo: "{{ quay_image_repo }}/cilium/cilium-envoy" cilium_hubble_envoy_image_tag: "v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5" kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" kube_ovn_container_image_tag: "v{{ kube_ovn_version }}" kube_ovn_vpc_container_image_repo: "{{ docker_image_repo }}/kubeovn/vpc-nat-gateway" kube_ovn_vpc_container_image_tag: "v{{ kube_ovn_version }}" kube_ovn_dpdk_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-dpdk" kube_ovn_dpdk_container_image_tag: "{{ kube_ovn_dpdk_version }}" kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router" kube_router_image_tag: "v{{ kube_router_version }}" multus_image_repo: "{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni" multus_image_tag: "v{{ multus_version }}" external_openstack_cloud_controller_image_repo: "{{ kube_image_repo }}/provider-os/openstack-cloud-controller-manager" external_openstack_cloud_controller_image_tag: "v1.35.0" kube_vip_version: 1.0.3 kube_vip_image_repo: "{{ github_image_repo }}/kube-vip/kube-vip{{ '-iptables' if kube_vip_lb_fwdmethod == 'masquerade' else '' }}" kube_vip_image_tag: "v{{ kube_vip_version }}" nginx_image_repo: "{{ docker_image_repo }}/library/nginx" nginx_image_tag: 1.28.2-alpine haproxy_image_repo: "{{ docker_image_repo }}/library/haproxy" haproxy_image_tag: 3.2.13-alpine # Coredns version should be supported by corefile-migration (or at least work with) # bundle with kubeadm; if not 'basic' upgrade can sometimes fail coredns_supported_versions: '1.35': 1.12.4 '1.34': 1.12.1 '1.33': 1.12.0 coredns_version: "{{ coredns_supported_versions[kube_major_version] }}" coredns_image_repo: "{{ kube_image_repo }}{{ '/coredns' if coredns_version is version('1.7.1', '>=') else '' }}/coredns" coredns_image_tag: "{{ 'v' if coredns_version is version('1.7.1', '>=') else '' }}{{ coredns_version }}" nodelocaldns_version: "1.25.0" nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache" nodelocaldns_image_tag: "{{ nodelocaldns_version }}" dnsautoscaler_version: 1.8.8 dnsautoscaler_image_repo: "{{ kube_image_repo }}/cpa/cluster-proportional-autoscaler" dnsautoscaler_image_tag: "v{{ dnsautoscaler_version }}" scheduler_plugins_controller_image_repo: "{{ kube_image_repo }}/scheduler-plugins/controller" scheduler_plugins_controller_image_tag: "v{{ scheduler_plugins_version }}" scheduler_plugins_scheduler_image_repo: "{{ kube_image_repo }}/scheduler-plugins/kube-scheduler" scheduler_plugins_scheduler_image_tag: "v{{ scheduler_plugins_version }}" registry_version: "2.8.1" registry_image_repo: "{{ docker_image_repo }}/library/registry" registry_image_tag: "{{ registry_version }}" metrics_server_version: "0.8.0" metrics_server_image_repo: "{{ kube_image_repo }}/metrics-server/metrics-server" metrics_server_image_tag: "v{{ metrics_server_version }}" local_volume_provisioner_version: "2.5.0" local_volume_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/local-volume-provisioner" local_volume_provisioner_image_tag: "v{{ local_volume_provisioner_version }}" local_path_provisioner_version: "0.0.32" local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner" local_path_provisioner_image_tag: "v{{ local_path_provisioner_version }}" alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller" alb_ingress_image_tag: "v1.1.9" cert_manager_version: "1.15.3" cert_manager_controller_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-controller" cert_manager_controller_image_tag: "v{{ cert_manager_version }}" cert_manager_cainjector_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-cainjector" cert_manager_cainjector_image_tag: "v{{ cert_manager_version }}" cert_manager_webhook_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-webhook" cert_manager_webhook_image_tag: "v{{ cert_manager_version }}" csi_attacher_image_repo: "{{ kube_image_repo }}/sig-storage/csi-attacher" csi_attacher_image_tag: "v4.4.2" csi_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/csi-provisioner" csi_provisioner_image_tag: "v3.6.2" csi_snapshotter_image_repo: "{{ kube_image_repo }}/sig-storage/csi-snapshotter" csi_snapshotter_image_tag: "v6.3.2" csi_resizer_image_repo: "{{ kube_image_repo }}/sig-storage/csi-resizer" csi_resizer_image_tag: "v1.9.2" csi_node_driver_registrar_image_repo: "{{ kube_image_repo }}/sig-storage/csi-node-driver-registrar" csi_node_driver_registrar_image_tag: "v2.4.0" csi_livenessprobe_image_repo: "{{ kube_image_repo }}/sig-storage/livenessprobe" csi_livenessprobe_image_tag: "v2.11.0" snapshot_controller_supported_versions: '1.35': "v7.0.2" '1.34': "v7.0.2" '1.33': "v7.0.2" snapshot_controller_image_repo: "{{ kube_image_repo }}/sig-storage/snapshot-controller" snapshot_controller_image_tag: "{{ snapshot_controller_supported_versions[kube_major_version] }}" cinder_csi_plugin_version: "1.30.0" cinder_csi_plugin_image_repo: "{{ kube_image_repo }}/provider-os/cinder-csi-plugin" cinder_csi_plugin_image_tag: "v{{ cinder_csi_plugin_version }}" aws_ebs_csi_plugin_version: "0.5.0" aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver" aws_ebs_csi_plugin_image_tag: "v{{ aws_ebs_csi_plugin_version }}" gcp_pd_csi_plugin_version: "1.9.2" gcp_pd_csi_plugin_image_repo: "{{ kube_image_repo }}/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver" gcp_pd_csi_plugin_image_tag: "v{{ gcp_pd_csi_plugin_version }}" azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi" azure_csi_provisioner_image_tag: "v2.2.2" azure_csi_attacher_image_tag: "v3.3.0" azure_csi_resizer_image_tag: "v1.3.0" azure_csi_livenessprobe_image_tag: "v2.5.0" azure_csi_node_registrar_image_tag: "v2.4.0" azure_csi_snapshotter_image_tag: "v3.0.3" azure_csi_plugin_version: "1.10.0" azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi" azure_csi_plugin_image_tag: "v{{ azure_csi_plugin_version }}" gcp_pd_csi_image_repo: "gke.gcr.io" gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0" gcp_pd_csi_provisioner_image_tag: "v1.5.0-gke.0" gcp_pd_csi_attacher_image_tag: "v2.1.1-gke.0" gcp_pd_csi_resizer_image_tag: "v0.4.0-gke.0" gcp_pd_csi_registrar_image_tag: "v1.2.0-gke.0" metallb_speaker_image_repo: "{{ quay_image_repo }}/metallb/speaker" metallb_controller_image_repo: "{{ quay_image_repo }}/metallb/controller" metallb_version: 0.13.9 metallb_image_tag: "v{{ metallb_version }}" node_feature_discovery_version: 0.16.4 node_feature_discovery_image_repo: "{{ kube_image_repo }}/nfd/node-feature-discovery" node_feature_discovery_image_tag: "v{{ node_feature_discovery_version }}" downloads: etcd: container: "{{ etcd_deployment_type != 'host' }}" file: "{{ etcd_deployment_type == 'host' }}" enabled: true dest: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" repo: "{{ etcd_image_repo }}" tag: "{{ etcd_image_tag }}" checksum: >- {{ etcd_binary_checksum if (etcd_deployment_type == 'host') else etcd_digest_checksum | d(None) }} url: "{{ etcd_download_url }}" unarchive: "{{ etcd_deployment_type == 'host' }}" owner: "root" mode: "0755" groups: - etcd cni: enabled: true file: true dest: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" checksum: "{{ cni_binary_checksum }}" url: "{{ cni_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster kubeadm: enabled: true file: true dest: "{{ local_release_dir }}/kubeadm-{{ kube_version }}-{{ image_arch }}" checksum: "{{ kubeadm_binary_checksum }}" url: "{{ kubeadm_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster kubelet: enabled: true file: true dest: "{{ local_release_dir }}/kubelet-{{ kube_version }}-{{ image_arch }}" checksum: "{{ kubelet_binary_checksum }}" url: "{{ kubelet_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster kubectl: enabled: true file: true dest: "{{ local_release_dir }}/kubectl-{{ kube_version }}-{{ image_arch }}" checksum: "{{ kubectl_binary_checksum }}" url: "{{ kubectl_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - kube_control_plane crictl: file: true enabled: true dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" checksum: "{{ crictl_binary_checksum }}" url: "{{ crictl_download_url }}" unarchive: true owner: "root" mode: "0755" groups: - k8s_cluster crio: file: true enabled: "{{ container_manager == 'crio' }}" dest: "{{ local_release_dir }}/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz" checksum: "{{ crio_archive_checksum }}" url: "{{ crio_download_url }}" unarchive: true owner: "root" mode: "0755" groups: - k8s_cluster cri_dockerd: file: true enabled: "{{ container_manager == 'docker' }}" dest: "{{ local_release_dir }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tar.gz" checksum: "{{ cri_dockerd_archive_checksum }}" url: "{{ cri_dockerd_download_url }}" unarchive: true unarchive_extra_opts: - --strip=1 owner: "root" mode: "0755" groups: - k8s_cluster crun: file: true enabled: "{{ crun_enabled }}" dest: "{{ local_release_dir }}/crun-{{ crun_version }}-{{ image_arch }}" checksum: "{{ crun_binary_checksum }}" url: "{{ crun_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster youki: file: true enabled: "{{ youki_enabled }}" dest: "{{ local_release_dir }}/youki-{{ youki_version }}-{{ ansible_architecture }}.tar.gz" checksum: "{{ youki_archive_checksum }}" url: "{{ youki_download_url }}" unarchive: true owner: "root" mode: "0755" groups: - k8s_cluster runc: file: true enabled: "{{ container_manager == 'containerd' }}" dest: "{{ local_release_dir }}/runc-{{ runc_version }}.{{ image_arch }}" checksum: "{{ runc_binary_checksum }}" url: "{{ runc_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster kata_containers: enabled: "{{ kata_containers_enabled }}" file: true dest: "{{ local_release_dir }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz" checksum: "{{ kata_containers_binary_checksum }}" url: "{{ kata_containers_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster containerd: enabled: "{{ container_manager == 'containerd' }}" file: true dest: "{{ local_release_dir }}/containerd-{{ 'static-' if containerd_static_binary }}{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" checksum: "{{ containerd_checksum }}" url: "{{ containerd_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster gvisor_runsc: enabled: "{{ gvisor_enabled }}" file: true dest: "{{ local_release_dir }}/gvisor-runsc-{{ gvisor_version }}-{{ ansible_architecture }}" checksum: "{{ gvisor_runsc_binary_checksum }}" url: "{{ gvisor_runsc_download_url }}" unarchive: false owner: "root" mode: 755 groups: - k8s_cluster gvisor_containerd_shim: enabled: "{{ gvisor_enabled }}" file: true dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1-{{ gvisor_version }}-{{ ansible_architecture }}" checksum: "{{ gvisor_containerd_shim_binary_checksum }}" url: "{{ gvisor_containerd_shim_runsc_download_url }}" unarchive: false owner: "root" mode: 755 groups: - k8s_cluster nerdctl: file: true enabled: "{{ container_manager == 'containerd' }}" dest: "{{ local_release_dir }}/nerdctl-{{ nerdctl_version }}-linux-{{ image_arch }}.tar.gz" checksum: "{{ nerdctl_archive_checksum }}" url: "{{ nerdctl_download_url }}" unarchive: true owner: "root" mode: "0755" groups: - k8s_cluster skopeo: file: true enabled: "{{ container_manager == 'crio' }}" dest: "{{ local_release_dir }}/skopeo-{{ skopeo_version }}-{{ image_arch }}" checksum: "{{ skopeo_binary_checksum }}" url: "{{ skopeo_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - kube_control_plane cilium: enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally }}" container: true repo: "{{ cilium_image_repo }}" tag: "{{ cilium_image_tag }}" checksum: "{{ cilium_digest_checksum | default(None) }}" groups: - k8s_cluster cilium_operator: enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally }}" container: true repo: "{{ cilium_operator_image_repo }}" tag: "{{ cilium_operator_image_tag }}" checksum: "{{ cilium_operator_digest_checksum | default(None) }}" groups: - k8s_cluster cilium_hubble_relay: enabled: "{{ cilium_enable_hubble }}" container: true repo: "{{ cilium_hubble_relay_image_repo }}" tag: "{{ cilium_hubble_relay_image_tag }}" checksum: "{{ cilium_hubble_relay_digest_checksum | default(None) }}" groups: - k8s_cluster cilium_hubble_certgen: enabled: "{{ cilium_enable_hubble }}" container: true repo: "{{ cilium_hubble_certgen_image_repo }}" tag: "{{ cilium_hubble_certgen_image_tag }}" checksum: "{{ cilium_hubble_certgen_digest_checksum | default(None) }}" groups: - k8s_cluster cilium_hubble_ui: enabled: "{{ cilium_enable_hubble }}" container: true repo: "{{ cilium_hubble_ui_image_repo }}" tag: "{{ cilium_hubble_ui_image_tag }}" checksum: "{{ cilium_hubble_ui_digest_checksum | default(None) }}" groups: - k8s_cluster cilium_hubble_ui_backend: enabled: "{{ cilium_enable_hubble }}" container: true repo: "{{ cilium_hubble_ui_backend_image_repo }}" tag: "{{ cilium_hubble_ui_backend_image_tag }}" checksum: "{{ cilium_hubble_ui_backend_digest_checksum | default(None) }}" groups: - k8s_cluster cilium_hubble_envoy: enabled: "{{ cilium_enable_hubble }}" container: true repo: "{{ cilium_hubble_envoy_image_repo }}" tag: "{{ cilium_hubble_envoy_image_tag }}" checksum: "{{ cilium_hubble_envoy_digest_checksum | default(None) }}" groups: - k8s_cluster ciliumcli: enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally }}" file: true dest: "{{ local_release_dir }}/cilium-{{ cilium_cli_version }}-{{ image_arch }}.tar.gz" checksum: "{{ ciliumcli_binary_checksum }}" url: "{{ ciliumcli_download_url }}" unarchive: true owner: "root" mode: "0755" groups: - k8s_cluster multus: enabled: "{{ kube_network_plugin_multus }}" container: true repo: "{{ multus_image_repo }}" tag: "{{ multus_image_tag }}" checksum: "{{ multus_digest_checksum | default(None) }}" groups: - k8s_cluster flannel: enabled: "{{ kube_network_plugin == 'flannel' }}" container: true repo: "{{ flannel_image_repo }}" tag: "{{ flannel_image_tag }}" checksum: "{{ flannel_digest_checksum | default(None) }}" groups: - k8s_cluster flannel_init: enabled: "{{ kube_network_plugin == 'flannel' }}" container: true repo: "{{ flannel_init_image_repo }}" tag: "{{ flannel_init_image_tag }}" checksum: "{{ flannel_init_digest_checksum | default(None) }}" groups: - k8s_cluster calicoctl: enabled: "{{ kube_network_plugin == 'calico' }}" file: true dest: "{{ local_release_dir }}/calicoctl-{{ calico_ctl_version }}-{{ image_arch }}" checksum: "{{ calicoctl_binary_checksum }}" url: "{{ calicoctl_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - k8s_cluster calico_node: enabled: "{{ kube_network_plugin == 'calico' }}" container: true repo: "{{ calico_node_image_repo }}" tag: "{{ calico_node_image_tag }}" checksum: "{{ calico_node_digest_checksum | default(None) }}" groups: - k8s_cluster calico_cni: enabled: "{{ kube_network_plugin == 'calico' }}" container: true repo: "{{ calico_cni_image_repo }}" tag: "{{ calico_cni_image_tag }}" checksum: "{{ calico_cni_digest_checksum | default(None) }}" groups: - k8s_cluster calico_policy: enabled: "{{ enable_network_policy and kube_network_plugin in ['calico'] }}" container: true repo: "{{ calico_policy_image_repo }}" tag: "{{ calico_policy_image_tag }}" checksum: "{{ calico_policy_digest_checksum | default(None) }}" groups: - k8s_cluster calico_typha: enabled: "{{ typha_enabled }}" container: true repo: "{{ calico_typha_image_repo }}" tag: "{{ calico_typha_image_tag }}" checksum: "{{ calico_typha_digest_checksum | default(None) }}" groups: - k8s_cluster calico_apiserver: enabled: "{{ calico_apiserver_enabled }}" container: true repo: "{{ calico_apiserver_image_repo }}" tag: "{{ calico_apiserver_image_tag }}" checksum: "{{ calico_apiserver_digest_checksum | default(None) }}" groups: - k8s_cluster calico_crds: file: true enabled: "{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}" dest: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/crds.yaml" checksum: "{{ calico_crds_checksums.no_arch[calico_version] }}" url: "{{ calico_crds_download_url }}" owner: "root" mode: "0755" groups: - kube_control_plane kube_ovn: enabled: "{{ kube_network_plugin == 'kube-ovn' }}" container: true repo: "{{ kube_ovn_container_image_repo }}" tag: "{{ kube_ovn_container_image_tag }}" checksum: "{{ kube_ovn_digest_checksum | default(None) }}" groups: - k8s_cluster kube_router: enabled: "{{ kube_network_plugin == 'kube-router' }}" container: true repo: "{{ kube_router_image_repo }}" tag: "{{ kube_router_image_tag }}" checksum: "{{ kube_router_digest_checksum | default(None) }}" groups: - k8s_cluster pod_infra: enabled: true container: true repo: "{{ pod_infra_image_repo }}" tag: "{{ pod_infra_image_tag }}" checksum: "{{ pod_infra_digest_checksum | default(None) }}" groups: - k8s_cluster kube-vip: enabled: "{{ kube_vip_enabled }}" container: true repo: "{{ kube_vip_image_repo }}" tag: "{{ kube_vip_image_tag }}" checksum: "{{ kube_vip_digest_checksum | default(None) }}" groups: - kube_control_plane nginx: enabled: "{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'nginx' }}" container: true repo: "{{ nginx_image_repo }}" tag: "{{ nginx_image_tag }}" checksum: "{{ nginx_digest_checksum | default(None) }}" groups: - kube_node haproxy: enabled: "{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'haproxy' }}" container: true repo: "{{ haproxy_image_repo }}" tag: "{{ haproxy_image_tag }}" checksum: "{{ haproxy_digest_checksum | default(None) }}" groups: - kube_node coredns: enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}" container: true repo: "{{ coredns_image_repo }}" tag: "{{ coredns_image_tag }}" checksum: "{{ coredns_digest_checksum | default(None) }}" groups: - k8s_cluster nodelocaldns: enabled: "{{ enable_nodelocaldns }}" container: true repo: "{{ nodelocaldns_image_repo }}" tag: "{{ nodelocaldns_image_tag }}" checksum: "{{ nodelocaldns_digest_checksum | default(None) }}" groups: - k8s_cluster dnsautoscaler: enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] and enable_dns_autoscaler }}" container: true repo: "{{ dnsautoscaler_image_repo }}" tag: "{{ dnsautoscaler_image_tag }}" checksum: "{{ dnsautoscaler_digest_checksum | default(None) }}" groups: - kube_control_plane helm: enabled: "{{ helm_enabled }}" file: true dest: "{{ local_release_dir }}/helm-{{ helm_version }}/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" checksum: "{{ helm_archive_checksum }}" url: "{{ helm_download_url }}" unarchive: true owner: "root" mode: "0755" groups: - kube_control_plane registry: enabled: "{{ registry_enabled }}" container: true repo: "{{ registry_image_repo }}" tag: "{{ registry_image_tag }}" checksum: "{{ registry_digest_checksum | default(None) }}" groups: - kube_node metrics_server: enabled: "{{ metrics_server_enabled }}" container: true repo: "{{ metrics_server_image_repo }}" tag: "{{ metrics_server_image_tag }}" checksum: "{{ metrics_server_digest_checksum | default(None) }}" groups: - kube_control_plane local_volume_provisioner: enabled: "{{ local_volume_provisioner_enabled }}" container: true repo: "{{ local_volume_provisioner_image_repo }}" tag: "{{ local_volume_provisioner_image_tag }}" checksum: "{{ local_volume_provisioner_digest_checksum | default(None) }}" groups: - kube_node local_path_provisioner: enabled: "{{ local_path_provisioner_enabled }}" container: true repo: "{{ local_path_provisioner_image_repo }}" tag: "{{ local_path_provisioner_image_tag }}" checksum: "{{ local_path_provisioner_digest_checksum | default(None) }}" groups: - kube_node ingress_alb_controller: enabled: "{{ ingress_alb_enabled }}" container: true repo: "{{ alb_ingress_image_repo }}" tag: "{{ alb_ingress_image_tag }}" checksum: "{{ ingress_alb_controller_digest_checksum | default(None) }}" groups: - kube_node cert_manager_controller: enabled: "{{ cert_manager_enabled }}" container: true repo: "{{ cert_manager_controller_image_repo }}" tag: "{{ cert_manager_controller_image_tag }}" checksum: "{{ cert_manager_controller_digest_checksum | default(None) }}" groups: - kube_node cert_manager_cainjector: enabled: "{{ cert_manager_enabled }}" container: true repo: "{{ cert_manager_cainjector_image_repo }}" tag: "{{ cert_manager_cainjector_image_tag }}" checksum: "{{ cert_manager_cainjector_digest_checksum | default(None) }}" groups: - kube_node cert_manager_webhook: enabled: "{{ cert_manager_enabled }}" container: true repo: "{{ cert_manager_webhook_image_repo }}" tag: "{{ cert_manager_webhook_image_tag }}" checksum: "{{ cert_manager_webhook_digest_checksum | default(None) }}" groups: - kube_node gateway_api_crds: enabled: "{{ gateway_api_enabled }}" file: true version: "{{ gateway_api_version }}" dest: "{{ local_release_dir }}/gateway-api-{{ gateway_api_channel }}-install.yaml" checksum: "{{ lookup('vars', 'gateway_api_' + gateway_api_channel + '_crds_checksums').no_arch[gateway_api_version] }}" url: "{{ gateway_api_crds_download_url }}" owner: "root" mode: "0755" groups: - kube_control_plane prometheus_operator_crds: enabled: "{{ prometheus_operator_crds_enabled }}" file: true version: "{{ prometheus_operator_crds_version }}" dest: "{{ local_release_dir }}/prometheus-operator-crds.yaml" checksum: "{{ prometheus_operator_crds_checksums.no_arch[prometheus_operator_crds_version] }}" url: "{{ prometheus_operator_crds_download_url }}" owner: "root" mode: "0755" groups: - kube_control_plane csi_attacher: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true repo: "{{ csi_attacher_image_repo }}" tag: "{{ csi_attacher_image_tag }}" checksum: "{{ csi_attacher_digest_checksum | default(None) }}" groups: - kube_node csi_provisioner: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true repo: "{{ csi_provisioner_image_repo }}" tag: "{{ csi_provisioner_image_tag }}" checksum: "{{ csi_provisioner_digest_checksum | default(None) }}" groups: - kube_node csi_snapshotter: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true repo: "{{ csi_snapshotter_image_repo }}" tag: "{{ csi_snapshotter_image_tag }}" checksum: "{{ csi_snapshotter_digest_checksum | default(None) }}" groups: - kube_node snapshot_controller: enabled: "{{ csi_snapshot_controller_enabled }}" container: true repo: "{{ snapshot_controller_image_repo }}" tag: "{{ snapshot_controller_image_tag }}" checksum: "{{ snapshot_controller_digest_checksum | default(None) }}" groups: - kube_node csi_resizer: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true repo: "{{ csi_resizer_image_repo }}" tag: "{{ csi_resizer_image_tag }}" checksum: "{{ csi_resizer_digest_checksum | default(None) }}" groups: - kube_node csi_livenessprobe: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true repo: "{{ csi_livenessprobe_image_repo }}" tag: "{{ csi_livenessprobe_image_tag }}" checksum: "{{ csi_livenessprobe_digest_checksum | default(None) }}" groups: - kube_node csi_node_driver_registrar: enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true repo: "{{ csi_node_driver_registrar_image_repo }}" tag: "{{ csi_node_driver_registrar_image_tag }}" checksum: "{{ csi_node_driver_registrar_digest_checksum | default(None) }}" groups: - kube_node cinder_csi_plugin: enabled: "{{ cinder_csi_enabled }}" container: true repo: "{{ cinder_csi_plugin_image_repo }}" tag: "{{ cinder_csi_plugin_image_tag }}" checksum: "{{ cinder_csi_plugin_digest_checksum | default(None) }}" groups: - kube_node aws_ebs_csi_plugin: enabled: "{{ aws_ebs_csi_enabled }}" container: true repo: "{{ aws_ebs_csi_plugin_image_repo }}" tag: "{{ aws_ebs_csi_plugin_image_tag }}" checksum: "{{ aws_ebs_csi_plugin_digest_checksum | default(None) }}" groups: - kube_node metallb_speaker: enabled: "{{ metallb_speaker_enabled }}" container: true repo: "{{ metallb_speaker_image_repo }}" tag: "{{ metallb_image_tag }}" checksum: "{{ metallb_speaker_digest_checksum | default(None) }}" groups: - kube_control_plane metallb_controller: enabled: "{{ metallb_enabled }}" container: true repo: "{{ metallb_controller_image_repo }}" tag: "{{ metallb_image_tag }}" checksum: "{{ metallb_controller_digest_checksum | default(None) }}" groups: - kube_control_plane yq: enabled: "{{ argocd_enabled }}" file: true version: "{{ yq_version }}" dest: "{{ local_release_dir }}/yq-{{ yq_version }}-{{ image_arch }}" checksum: "{{ yq_binary_checksum }}" url: "{{ yq_download_url }}" unarchive: false owner: "root" mode: "0755" groups: - kube_control_plane argocd_install: enabled: "{{ argocd_enabled }}" file: true version: "{{ argocd_version }}" dest: "{{ local_release_dir }}/argocd-install.yml" checksum: "{{ argocd_install_checksum }}" url: "{{ argocd_install_url }}" unarchive: false owner: "root" mode: "0644" groups: - kube_control_plane download_defaults: container: false file: false repo: None tag: None enabled: false dest: None url: None unarchive: false owner: "{{ kube_owner }}" mode: None ================================================ FILE: roles/kubespray_defaults/defaults/main/main.yml ================================================ --- # Use proxycommand if bastion host is in group all # This change obseletes editing ansible.cfg file depending on bastion existence ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p -p {{ hostvars['bastion']['ansible_port'] | default(22) }} {{ hostvars['bastion']['ansible_user'] }}@{{ hostvars['bastion']['ansible_host'] }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} ' {% endif %}" # selinux state preinstall_selinux_state: permissive # Check if access_ip responds to ping. Set false if your firewall blocks ICMP. ping_access_ip: true # Setting this value to false will fail # For details, read this comment https://github.com/kubernetes-sigs/kubespray/pull/11016#issuecomment-2004985001 # if kube_api_anonymous_auth: "{{ undef() }}", remove --anonymous-auth argument kube_api_anonymous_auth: true # Default value, but will be set to true automatically if detected is_fedora_coreos: false # Kubernetes 1.35+: fail on cgroup v1 by default kubelet_fail_cgroup_v1: true # Swap settings kubelet_fail_swap_on: true kubelet_swap_behavior: LimitedSwap ## Change this to use another Kubernetes version, e.g. a current beta release kube_version: "{{ (kubelet_checksums['amd64'] | dict2items)[0].key }}" ## The minimum version working kube_version_min_required: "{{ (kubelet_checksums['amd64'] | dict2items)[-1].key }}" ## Kube Proxy mode One of ['ipvs', 'iptables', 'nftables'] kube_proxy_mode: ipvs # Debugging option for the kubeadm config validate command # Set to false only for development and testing scenarios where validation is expected to fail (pre-release Kubernetes versions, etc.) kubeadm_config_validate_enabled: true ## The timeout for init first control-plane kubeadm_init_timeout: 300s # TODO: remove this kube_reserved_cgroups_for_service_slice: kube.slice ## List of kubeadm init phases that should be skipped during control plane setup ## By default 'addon/coredns' is skipped ## 'addon/kube-proxy' gets skipped for some network plugins kubeadm_init_phases_skip_default: [ "addon/coredns" ] kubeadm_init_phases_skip: >- {%- if kube_network_plugin == 'kube-router' and (kube_router_run_service_proxy is defined and kube_router_run_service_proxy) -%} {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} {%- elif kube_network_plugin == 'cilium' and (cilium_kube_proxy_replacement is defined and (cilium_kube_proxy_replacement == 'strict' or (cilium_kube_proxy_replacement | bool) or (cilium_kube_proxy_replacement | string | lower == 'true') )) -%} {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} {%- elif kube_network_plugin == 'calico' and (calico_bpf_enabled is defined and calico_bpf_enabled) -%} {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} {%- elif kube_proxy_remove is defined and kube_proxy_remove -%} {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} {%- else -%} {{ kubeadm_init_phases_skip_default }} {%- endif -%} # List of kubeadm phases that should be skipped when joining a new node # You may need to set this to ['preflight'] for air-gaped deployments to avoid failing connectivity tests. kubeadm_join_phases_skip_default: [] kubeadm_join_phases_skip: >- {{ kubeadm_join_phases_skip_default }} # List of kubeadm upgrade node phases that should be skipped when upgrading a secondary control plane node (supports different phases than kubeadm init and kubeadm upgrade apply) kubeadm_upgrade_node_phases_skip_default: [] kubeadm_upgrade_node_phases_skip: >- {%- if kube_version is version('1.32.0', '>=') -%} {{ kubeadm_upgrade_node_phases_skip_default + kubeadm_init_phases_skip }} {%- else -%} {{ kubeadm_upgrade_node_phases_skip_default }} {%- endif -%} # Set to true to remove the role binding to anonymous users created by kubeadm remove_anonymous_access: false # A string slice of values which specify the addresses to use for NodePorts. # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). # The default empty string slice ([]) means to use all local addresses. # kube_proxy_nodeport_addresses_cidr is retained for legacy config kube_proxy_nodeport_addresses: >- {%- if kube_proxy_nodeport_addresses_cidr is defined -%} [{{ kube_proxy_nodeport_addresses_cidr }}] {%- else -%} [] {%- endif -%} # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false # kube-vip kube_vip_enabled: false kube_vip_lb_fwdmethod: local kube_vip_address: # nginx-proxy configure nginx_config_dir: "/etc/nginx" # haproxy configure haproxy_config_dir: "/etc/haproxy" # Directory where the binaries will be installed bin_dir: /usr/local/bin docker_bin_dir: /usr/bin containerd_bin_dir: "{{ bin_dir }}" etcd_data_dir: /var/lib/etcd # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) local_release_dir: "/tmp/releases" # Random shifts for retrying failed ops like pushing/downloading retry_stagger: 5 # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 # Default resolv.conf options docker_dns_options: - ndots:{{ ndots }} - timeout:2 - attempts:2 # Can be coredns, coredns_dual, manual, or none dns_mode: coredns # Enable dns autoscaler enable_dns_autoscaler: true # DNS servers added after the cluster DNS # These will also be used as upstream by Coredns for out-cluster queries upstream_dns_servers: [] # Enable nodelocal dns cache enable_nodelocaldns: true enable_nodelocaldns_secondary: false nodelocaldns_ip: 169.254.25.10 nodelocaldns_health_port: 9254 nodelocaldns_second_health_port: 9256 nodelocaldns_bind_metrics_host_ip: false nodelocaldns_secondary_skew_seconds: 5 # Should be set to a cluster IP if using a custom cluster DNS manual_dns_server: "" # Can be host_resolvconf, docker_dns or none resolvconf_mode: host_resolvconf # Ip address of the kubernetes DNS service (called skydns for historical reasons) skydns_server: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" skydns_server_secondary: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" dns_domain: "{{ cluster_name }}" docker_dns_search_domains: - 'default.svc.{{ dns_domain }}' - 'svc.{{ dns_domain }}' searchdomains: [] kube_dns_servers: coredns: ["{{ skydns_server }}"] coredns_dual: "{{ [skydns_server] + [skydns_server_secondary] }}" manual: ["{{ manual_dns_server }}"] dns_servers: "{{ kube_dns_servers[dns_mode] }}" enable_coredns_k8s_external: false coredns_k8s_external_zone: k8s_external.local enable_coredns_k8s_endpoint_pod_names: false # Kubernetes configuration dirs and system namespace. # Those are where all the additional config stuff goes # the kubernetes normally puts in /srv/kubernetes. # This puts them in a sane location and namespace. # Editing those values will almost surely break something. kube_config_dir: /etc/kubernetes kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" kube_manifest_dir: "{{ kube_config_dir }}/manifests" # Kubectl command # This is for consistency when using kubectl command in roles, and ensure kubectl: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf" # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/ssl" # compatibility directory for kubeadm kube_cert_compat_dir: "/etc/kubernetes/pki" # This is where all of the bearer tokens will be stored kube_token_dir: "{{ kube_config_dir }}/tokens" # This is the user that owns the cluster installation. kube_owner: kube # This is the group that the cert creation scripts chgrp the # cert files to. Not really changeable... kube_cert_group: kube-cert # Set to true when the CAs are managed externally. # When true, disables all tasks manipulating certificates. Ensure before the kubespray run that: # - Certificates and CAs are present in kube_cert_dir # - Kubeconfig files are present in kube_config_dir kube_external_ca_mode: false # Cluster Loglevel configuration kube_log_level: 2 # Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico kube_network_plugin_multus: false ## Network plugin options with dependencies across the whole playbook # Deploy cilium even if kube_network_plugin is not cilium. # This enables to deploy cilium alongside another CNI to replace kube-proxy. cilium_deploy_additionally: false # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). # These can be queried with: # `kubectl get ciliumid` # - "kvstore" stores identities in an etcd kvstore. # - In order to support External Workloads, "crd" is required # - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta # - KVStore operations are only required when cilium-operator is running with any of the below options: # - --synchronize-k8s-services # - --synchronize-k8s-nodes # - --identity-allocation-mode=kvstore # - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations cilium_identity_allocation_mode: crd # Determines if calico_rr group exists peer_with_calico_rr: "{{ 'calico_rr' in groups and groups['calico_rr'] | length > 0 }}" # Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) calico_datastore: "kdd" # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 # internal network. When used, it will assign IP # addresses from this range to individual pods. # This network must be unused in your network infrastructure! kube_pods_subnet: 10.233.64.0/18 # internal network node size allocation (optional). This is the size allocated # to each node for pod IP address allocation. Note that the number of pods per node is # also limited by the kubelet_max_pods variable which defaults to 110. # # Example: # Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node: # - kube_pods_subnet: 10.233.64.0/18 # - kube_network_node_prefix: 24 # - kubelet_max_pods: 110 # # Example: # Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node: # - kube_pods_subnet: 10.233.64.0/18 # - kube_network_node_prefix: 25 # - kubelet_max_pods: 110 kube_network_node_prefix: 24 # Configure Dual Stack networking (i.e. both IPv4 and IPv6) # enable_dual_stack_networks: false # deprecated # Configure IPv4 Stack networking ipv4_stack: true # Configure IPv6 Stack networking ipv6_stack: "{{ enable_dual_stack_networks | default(false) }}" # Kubernetes internal network for IPv6 services, unused block of space. # This is only used if ipv6_stack is set to true # This provides 4096 IPv6 IPs kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 # Internal network. When used, it will assign IPv6 addresses from this range to individual pods. # This network must not already be in your network infrastructure! # This is only used if ipv6_stack is set to true. # This provides room for 256 nodes with 254 pods per node. kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # IPv6 subnet size allocated to each for pods. # This is only used if ipv6_stack is set to true # This provides room for 254 pods per node. kube_network_node_prefix_ipv6: 120 # The virtual cluster IP, real host IPs and ports the API Server will be # listening on. # NOTE: loadbalancer_apiserver_localhost somewhat alters the final API enpdoint # access IP value (automatically evaluated below) kube_apiserver_ip: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" # NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost # loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too. kube_apiserver_bind_address: "::" # https kube_apiserver_port: 6443 # If non-empty, will use this string as identification instead of the actual hostname kube_override_hostname: "{{ inventory_hostname }}" # define kubelet config dir for dynamic kubelet # kubelet_config_dir: default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" # Aggregator kube_api_aggregator_routing: false # Profiling kube_profiling: false # Graceful Node Shutdown kubelet_shutdown_grace_period: 60s # kubelet_shutdown_grace_period_critical_pods should be less than kubelet_shutdown_grace_period # to give normal pods time to be gracefully evacuated kubelet_shutdown_grace_period_critical_pods: 20s # Cloud Provider # This variable can only be set to "external" or empty string, otherwise the check will fail. cloud_provider: "" # External Cloud Controller Manager (Formerly known as cloud provider) # cloud_provider must be "external", otherwise this setting is invalid. # Supported external cloud controllers are: 'openstack', 'vsphere', 'oci', 'huaweicloud', 'hcloud' and 'manual' # 'manual' does not install the cloud controller manager used by Kubespray. # If you fill in a value other than the above, the check will fail. external_cloud_provider: "" # Whether to deploy the container engine deploy_container_engine: "{{ 'k8s_cluster' in group_names or etcd_deployment_type == 'docker' }}" # Container for runtime container_manager: containerd # Enable Node Resource Interface plugin for containerd nri_enabled: "{{ container_manager == 'containerd' }}" # Enable Kata Containers as additional container runtime # When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false # Enable gVisor as an additional container runtime # gVisor is only supported with container_manager Docker or containerd gvisor_enabled: false # Enable runc as additional container runtime # When enabled, it requires container_manager=crio runc_enabled: false # Enable crun as additional container runtime # When enabled, it requires container_manager=crio crun_enabled: false # Enable youki as additional container runtime # When enabled, it requires container_manager=crio youki_enabled: false # Container on localhost (download images when download_localhost is true) container_manager_on_localhost: "{{ container_manager }}" # CRI socket path cri_socket: >- {%- if container_manager == 'crio' -%} unix:///var/run/crio/crio.sock {%- elif container_manager == 'containerd' -%} unix:///var/run/containerd/containerd.sock {%- elif container_manager == 'docker' -%} unix:///var/run/cri-dockerd.sock {%- endif -%} crio_insecure_registries: [] ## Uncomment this if you want to force overlay/overlay2 as docker storage driver ## Please note that overlay2 is only supported on newer kernels # docker_storage_options: -s overlay2 ## Only set this if you have more than 3 nameservers: ## If true Kubespray will only use the first 3, otherwise it will fail docker_dns_servers_strict: false # Path used to store Docker data docker_daemon_graph: "/var/lib/docker" ## Used to set docker daemon iptables options to true docker_iptables_enabled: "false" # Docker log options # Rotate container stderr/stdout logs at 50m and keep last 5 docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" ## A list of insecure docker registries (IP address or domain name), for example ## to allow insecure-registry access to self-hosted registries. Empty by default. # docker_insecure_registries: # - mirror.registry.io # - 172.19.16.11 docker_insecure_registries: [] ## A list of additional registry mirrors, for example China registry mirror. Empty by default. # docker_registry_mirrors: # - https://registry.docker-cn.com # - https://mirror.aliyuncs.com docker_registry_mirrors: [] ## If non-empty will override default system MounFlags value. ## This option takes a mount propagation flag: shared, slave ## or private, which control whether mounts in the file system ## namespace set up for docker will receive or propagate mounts ## and unmounts. Leave empty for system default # docker_mount_flags: ## A string of extra options to pass to the docker daemon. # docker_options: "" ## A list of plugins to install using 'docker plugin install --grant-all-permissions' ## Empty by default so no plugins will be installed. docker_plugins: [] # Containerd options - thse are relevant when container_manager == 'containerd' containerd_use_systemd_cgroup: true # Use static containerd binary for compatibility with older distributions (e.g., Debian 11). containerd_static_binary: false # Containerd conf default dir containerd_storage_dir: "/var/lib/containerd" containerd_state_dir: "/run/containerd" containerd_systemd_dir: "/etc/systemd/system/containerd.service.d" containerd_cfg_dir: "/etc/containerd" # Settings for containerized control plane (etcd/kubelet/secrets) # deployment type for legacy etcd mode etcd_deployment_type: host cert_management: script # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts kubeconfig_localhost: false # Download kubectl onto the host that runs Ansible in {{ bin_dir }} kubectl_localhost: false # Define credentials_dir here so it can be overridden credentials_dir: "{{ inventory_dir }}/credentials" # K8s image pull policy (imagePullPolicy) k8s_image_pull_policy: IfNotPresent # Addons which can be enabled helm_enabled: false registry_enabled: false metrics_server_enabled: false enable_network_policy: true local_path_provisioner_enabled: false local_volume_provisioner_enabled: false local_volume_provisioner_directory_mode: "0700" cinder_csi_enabled: false aws_ebs_csi_enabled: false azure_csi_enabled: false gcp_pd_csi_enabled: false vsphere_csi_enabled: false upcloud_csi_enabled: false csi_snapshot_controller_enabled: false persistent_volumes_enabled: false ingress_alb_enabled: false cert_manager_enabled: false expand_persistent_volumes: false metallb_enabled: false metallb_speaker_enabled: "{{ metallb_enabled }}" argocd_enabled: false gateway_api_enabled: false prometheus_operator_crds_enabled: false ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) # openstack_blockstorage_version: "v1/v2/auto (default)" openstack_blockstorage_ignore_volume_az: "{{ volume_cross_zone_attachment | default('false') }}" # set max volumes per node (cinder-csi), default not set # node_volume_attach_limit: 25 # Cinder CSI topology, when false volumes can be cross-mounted between availability zones # cinder_topology: false # Set Cinder topology zones (can be multiple zones, default not set) # cinder_topology_zones: # - nova cinder_csi_ignore_volume_az: "{{ volume_cross_zone_attachment | default('false') }}" ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. openstack_lbaas_enabled: false # openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" ## To enable automatic floating ip provisioning, specify a subnet. # openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default" ## Override default LBaaS behavior # openstack_lbaas_use_octavia: False # openstack_lbaas_method: "ROUND_ROBIN" # openstack_lbaas_provider: "haproxy" openstack_lbaas_create_monitor: "yes" openstack_lbaas_monitor_delay: "1m" openstack_lbaas_monitor_timeout: "30s" openstack_lbaas_monitor_max_retries: "3" openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}" # Default values for the external OpenStack Cloud Controller external_openstack_lbaas_enabled: true external_openstack_network_ipv6_disabled: false external_openstack_network_internal_networks: [] external_openstack_network_public_networks: [] # Default values for the external Hcloud Cloud Controller external_hcloud_cloud: hcloud_api_token: "" token_secret_name: hcloud service_account_name: cloud-controller-manager controller_image_tag: "latest" ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset ## Format: ## external_hcloud_cloud.controller_extra_args: ## arg1: "value1" ## arg2: "value2" controller_extra_args: {} ## List of authorization modes that must be configured for ## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and ## 'RBAC' modes are tested. Order is important. authorization_modes: ['Node', 'RBAC'] ## Structured authorization config ## Structured AuthorizationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authorization modes with a structured configuration file. ## AuthorizationConfiguration files offer features not available with the `--authorization-mode` flag, although Kubespray supports both methods and authorization-mode remains the default for now. ## Note: Because the `--authorization-config` and `--authorization-mode` flags are mutually exclusive, the `authorization_modes` ansible variable is ignored when `kube_apiserver_use_authorization_config_file` is set to true. The two features cannot be used at the same time. ## Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/#configuring-the-api-server-using-an-authorization-config-file ## Examples: https://kubernetes.io/blog/2024/04/26/multi-webhook-and-modular-authorization-made-much-easier/ ## KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3221-structured-authorization-configuration kube_apiserver_use_authorization_config_file: false kube_apiserver_authorization_config_api_version: "{{ 'v1alpha1' if kube_version is version('1.30.0', '<') else 'v1beta1' if kube_version is version('1.32.0', '<') else 'v1' }}" kube_apiserver_authorization_config_authorizers: - type: Node name: node - type: RBAC name: rbac ## Example for use with kube_webhook_authorization: true # - type: Webhook # name: webhook # webhook: # connectionInfo: # type: KubeConfigFile # kubeConfigFile: "{{ kube_config_dir }}/webhook-authorization-config.yaml" # subjectAccessReviewVersion: v1beta1 # matchConditionSubjectAccessReviewVersion: v1 # timeout: 3s # failurePolicy: NoOpinion # matchConditions: # # Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/ # # only send resource requests to the webhook # - expression: has(request.resourceAttributes) # # Don't intercept requests from kube-system service accounts # - expression: "!('system:serviceaccounts:kube-system' in request.groups)" # ## Below expressions avoid issues with kubeadm init and other system components that should be authorized by Node and RBAC # # Don't process node and bootstrap token requests with the webhook # - expression: "!('system:nodes' in request.groups)" # - expression: "!('system:bootstrappers' in request.groups)" # - expression: "!('system:bootstrappers:kubeadm:default-node-token' in request.groups)" # # Don't process kubeadm requests with the webhook # - expression: "!('kubeadm:cluster-admins' in request.groups)" # - expression: "!('system:masters' in request.groups)" ## Two workarounds are required to use AuthorizationConfiguration with kubeadm v1.29.x: ## 1. Enable the StructuredAuthorizationConfiguration feature gate: # kube_apiserver_feature_gates: # - StructuredAuthorizationConfiguration=true ## 2. Use the following kubeadm_patches to remove defaulted authorization-mode flags (Workaround for a kubeadm defaulting bug on v1.29.x. fixed in 1.30+ via: https://github.com/kubernetes/kubernetes/pull/123654) # kubeadm_patches: # - target: kube-apiserver # type: strategic # patch: # spec: # containers: # - name: kube-apiserver # $deleteFromPrimitiveList/command: # - --authorization-mode=Node,RBAC rbac_enabled: "{{ ('RBAC' in authorization_modes and not kube_apiserver_use_authorization_config_file) or (kube_apiserver_use_authorization_config_file and kube_apiserver_authorization_config_authorizers | selectattr('type', 'equalto', 'RBAC') | list | length > 0) }}" # When enabled, API bearer tokens (including service account tokens) can be used to authenticate to the kubelet's HTTPS endpoint kubelet_authentication_token_webhook: true # When enabled, access to the kubelet API requires authorization by delegation to the API server kubelet_authorization_mode_webhook: true # kubelet uses certificates for authenticating to the Kubernetes API # Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration kubelet_rotate_certificates: true # kubelet can also request a new server certificate from the Kubernetes API kubelet_rotate_server_certificates: false # If set to true, kubelet errors if any of kernel tunables is different than kubelet defaults kubelet_protect_kernel_defaults: true # Set additional sysctl variables to modify Linux kernel variables, for example: # additional_sysctl: # - { name: kernel.pid_max, value: 131072 } # additional_sysctl: [] ## List of key=value pairs that describe feature gates for ## the k8s cluster. kube_feature_gates: [] kube_apiserver_feature_gates: [] kube_controller_feature_gates: [] kube_scheduler_feature_gates: [] kube_proxy_feature_gates: [] kubelet_feature_gates: [] kubeadm_feature_gates: [] # Local volume provisioner storage classes # Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted # see https://github.com/ansible/ansible/issues/17324 local_volume_provisioner_storage_classes: | { "{{ local_volume_provisioner_storage_class | default('local-storage') }}": { "host_dir": "{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}", "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}", "volume_mode": "Filesystem", "fs_type": "ext4" } } ssl_ca_dirs: |- [ {% if ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] -%} '/usr/share/ca-certificates', {% elif ansible_os_family == 'RedHat' -%} '/etc/pki/tls', '/etc/pki/ca-trust', {% elif ansible_os_family == 'Debian' -%} '/usr/share/ca-certificates', {% endif -%} ] # used for delegating tasks on a working control plane node first_kube_control_plane: "{{ groups['kube_control_plane'] | first }}" # Vars for pointing to kubernetes api endpoints kube_apiserver_count: "{{ groups['kube_control_plane'] | length }}" kube_apiserver_address: "{{ hostvars[inventory_hostname]['main_ip'] }}" kube_apiserver_access_address: "{{ hostvars[inventory_hostname]['main_access_ip'] }}" first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['main_access_ip'] }}" loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" loadbalancer_apiserver_type: "nginx" # applied if only external loadbalancer_apiserver is defined, otherwise ignored apiserver_loadbalancer_domain_name: "{{ 'localhost' if loadbalancer_apiserver_localhost else (loadbalancer_apiserver.address | d(undef())) }}" kube_apiserver_global_endpoint: |- {% if loadbalancer_apiserver is defined -%} https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} {%- elif loadbalancer_apiserver_localhost -%} https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} {%- else -%} https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} {%- endif %} kube_apiserver_endpoint: |- {% if loadbalancer_apiserver is defined -%} https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} {%- elif ('kube_control_plane' not in group_names) and loadbalancer_apiserver_localhost -%} https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} {%- elif 'kube_control_plane' in group_names -%} https://{{ kube_apiserver_bind_address | regex_replace('::', '127.0.0.1') | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} {%- else -%} https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} {%- endif %} kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt" kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key" # Set to true to deploy etcd-events cluster etcd_events_cluster_enabled: false # etcd group can be empty when kubeadm manages etcd etcd_hosts: "{{ groups['etcd'] | default(groups['kube_control_plane']) }}" # Vars for pointing to etcd endpoints etcd_address: "{{ hostvars[inventory_hostname]['main_ip'] }}" etcd_access_address: "{{ hostvars[inventory_hostname]['main_access_ip'] }}" etcd_events_access_address: "{{ hostvars[inventory_hostname]['main_access_ip'] }}" etcd_peer_url: "https://{{ etcd_access_address | ansible.utils.ipwrap }}:2380" etcd_client_url: "https://{{ etcd_access_address | ansible.utils.ipwrap }}:2379" etcd_events_peer_url: "https://{{ etcd_events_access_address | ansible.utils.ipwrap }}:2382" etcd_events_client_url: "https://{{ etcd_events_access_address | ansible.utils.ipwrap }}:2383" etcd_access_addresses: |- {% for item in etcd_hosts -%} https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:2379{% if not loop.last %},{% endif %} {%- endfor %} etcd_events_access_addresses_list: |- [ {% for item in etcd_hosts -%} 'https://{{ hostvars[item].main_access_ip | ansible.utils.ipwrap }}:2383'{% if not loop.last %},{% endif %} {%- endfor %} ] etcd_metrics_addresses: |- {% for item in etcd_hosts -%} https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:{{ etcd_metrics_port | default(2381) }}{% if not loop.last %},{% endif %} {%- endfor %} etcd_events_access_addresses: "{{ etcd_events_access_addresses_list | join(',') }}" etcd_events_access_addresses_semicolon: "{{ etcd_events_access_addresses_list | join(';') }}" # user should set etcd_member_name in inventory/mycluster/hosts.ini etcd_member_name: |- {% for host in groups['etcd'] %} {% if inventory_hostname == host %}{{ hostvars[host].etcd_member_name | default("etcd" + loop.index | string) }}{% endif %} {% endfor %} etcd_peer_addresses: |- {% for item in groups['etcd'] -%} {{ hostvars[item].etcd_member_name | default("etcd" + loop.index | string) }}=https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:2380{% if not loop.last %},{% endif %} {%- endfor %} etcd_events_peer_addresses: |- {% for item in groups['etcd'] -%} {{ hostvars[item].etcd_member_name | default("etcd" + loop.index | string) }}-events=https://{{ hostvars[item]['main_access_ip'] | ansible.utils.ipwrap }}:2382{% if not loop.last %},{% endif %} {%- endfor %} etcd_heartbeat_interval: "250" etcd_election_timeout: "5000" etcd_snapshot_count: "100000" certificates_key_size: 2048 certificates_duration: 36500 etcd_config_dir: /etc/ssl/etcd etcd_events_data_dir: "/var/lib/etcd-events" etcd_cert_dir: "{{ etcd_config_dir }}/ssl" typha_enabled: false calico_apiserver_enabled: false _host_architecture_groups: x86_64: amd64 aarch64: arm64 armv7l: arm host_architecture: >- {%- if ansible_architecture in _host_architecture_groups -%} {{ _host_architecture_groups[ansible_architecture] }} {%- else -%} {{ ansible_architecture }} {%- endif -%} _host_os_groups: Linux: linux Darwin: darwin Win32NT: windows host_os: >- {%- if ansible_system in _host_os_groups -%} {{ _host_os_groups[ansible_system] }} {%- else -%} {{ ansible_system }} {%- endif -%} # Sets the eventRecordQPS parameter in kubelet-config.yaml. # Setting it to 0 allows unlimited requests per second. kubelet_event_record_qps: 50 proxy_env_defaults: http_proxy: "{{ http_proxy | default('') }}" HTTP_PROXY: "{{ http_proxy | default('') }}" https_proxy: "{{ https_proxy | default('') }}" HTTPS_PROXY: "{{ https_proxy | default('') }}" no_proxy: "{{ no_proxy | default('') }}" NO_PROXY: "{{ no_proxy | default('') }}" # If we use SSL_CERT_FILE: {{ omit }} it cause in value __omit_place_holder__ and break environments # Combine dict is avoiding the problem with omit placeholder. Maybe it can be better solution? proxy_env: "{{ proxy_env_defaults | combine({'SSL_CERT_FILE': https_proxy_cert_file}) if https_proxy_cert_file is defined else proxy_env_defaults }}" proxy_disable_env: ALL_PROXY: '' FTP_PROXY: '' HTTPS_PROXY: '' HTTP_PROXY: '' NO_PROXY: '' all_proxy: '' ftp_proxy: '' http_proxy: '' https_proxy: '' no_proxy: '' # sysctl_file_path to add sysctl conf to sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" # ignore sysctl errors about unknown keys sysctl_ignore_unknown_keys: false system_upgrade: false system_upgrade_reboot: on-upgrade # never, always # Enables or disables the scheduler plugins. scheduler_plugins_enabled: false ## NTP Settings # Start the ntpd or chrony service and enable it at system boot. ntp_enabled: false # TODO: Refactor NTP package selection to integrate with the general package installation system # instead of using a separate variable approach # The package to install which provides NTP functionality. # The default is ntp for most platforms, or chrony on RHEL/CentOS 7 and later. # The ntp_package can be one of ['ntp', 'ntpsec', 'chrony'] ntp_package: >- {% if ansible_os_family == "RedHat" -%} chrony {%- else -%} ntp {%- endif -%} ================================================ FILE: roles/kubespray_defaults/vars/main/checksums.yml ================================================ --- crictl_checksums: arm64: 1.35.0: sha256:519071de89b64c43e2a1661bb5489c6c3fd5e9e5fcef75e50e542b0c891f1118 1.34.0: sha256:c31d252e203df5f4cf37f314bd3092eb79087e791631c1e607087c74b6d0423f 1.33.0: sha256:e1f34918d77d5b4be85d48f5d713ca617698a371b049ea1486000a5e86ab1ff3 amd64: 1.35.0: sha256:2e141e5b22cb189c40365a11807d69b76b9b3caced89fac2f4ec879408ce2177 1.34.0: sha256:a8ff2a3edb37a98daf3aba7c3b284fe0aa5bff24166d896ab9ef64c8913c9f51 1.33.0: sha256:8307399e714626e69d1213a4cd18c8dec3d0201ecdac009b1802115df8973f0f ppc64le: 1.35.0: sha256:786522b14d684604c8b435312a310972bc1b460cddb1bb216a298098cd86b22e 1.34.0: sha256:1da50181f2f6f6f6332b9dbc7d7cc020457ccd542620167953c0e288535acc93 1.33.0: sha256:4224acfef4d1deba2ba456b7d93fa98feb0a96063ef66024375294f1de2b064f crio_archive_checksums: arm64: 1.35.0: sha256:e57175a4d00387b78adfbe248d087d8127bed625afb529e34b2c90d08cfdaf87 1.34.5: sha256:999a5dc2dc9854222aeff8a20897e0b34f0ba02c9b260b611d66c62e00e279e0 1.34.4: sha256:d176f6256d606a3fc279f9f2994ef4a4c4cbaaa0601f4d1bba1a19bec5674ce9 1.34.3: sha256:314595247054b53767a736e24bc3030a5f7c17552944c62b2e190c9e95fe4ca6 1.34.2: sha256:ac7530f7fc9d531a87bfdfcae9cf8bf81a8bbdb75e63a046ed96911aa7b68ebd 1.34.1: sha256:41a71cab6a61ae429ec447d572fd1cdea0a7e33d62aaa58c3b07467665b50b9f 1.34.0: sha256:3006658270477c5fb1e88e9124e40982d2ba7b34495fcc12f0fecd33bbab9a5a 1.33.9: sha256:bfcd534db3d1a9380dd7007d623e1eb3250ba64f7c4657e79e9e99b1d874f8f1 1.33.8: sha256:59c91726535dcadd0372df0c6aa8595e4d59590994b598b2d97ea2510b216359 1.33.7: sha256:af3ea22d3d6944c9a907c6c13d77e9fc4dbcf3972ffbde18dd6f37f1c2ffbd0d 1.33.6: sha256:6ee49e746d1a5be1a664a6f801c68b169cb181a9aaf12218eed121e2b151bfdb 1.33.5: sha256:ef1b5e2162b0f55722e0966db0cfe387f3ba7cb91d6a803f627121733132792d 1.33.4: sha256:6a04cb1ab2020508927d7237ff1174bb330211a1076683417b30642a9c8e4996 1.33.3: sha256:39cfbb196326952e554e0fb5f95ebcb6cc1735cf6d56a88b8ecd17d89fbc6c26 1.33.2: sha256:0a161cb1437a50fbdb04bf5ca11dbec8bfc567871d0597a5676737278a945a36 1.33.1: sha256:6bf135db438937f0ab7a533af64564a0fb1d2079a43723ce9255ecbf9556ae05 1.33.0: sha256:8a0dbee2879495d5b33e6fdeac32e5d86c356897bdcf3a94cd602851620ce8b5 amd64: 1.35.0: sha256:55b6d3e9fc9a5864ab5cdf0b24d54b1dcbaf6d4919274b3b9eb37bfc4b0b8cb5 1.34.5: sha256:d6606fb6d686b8f814dfec801f0f3cf2ded974c194fa90facefda36075b6fab2 1.34.4: sha256:f6348a781c34b433fe1c5150da3408e51e828b610eacbe734405e9c31136d810 1.34.3: sha256:e269914f3bc4f36ac87cd593d74daaa43c390571994062180019248be32cc6f7 1.34.2: sha256:3a0012938ed389e9270a208bb73b250062d5f1be5798472b1728403d55ddc1da 1.34.1: sha256:22c1e4d68d9339aa58a1b0f1b40a8944102934a7505105abe461dc8a7e3de540 1.34.0: sha256:5a8bc5c3b8072cb9bde1cf025d5597f75bf21018712c5b72d5cb0657948595c8 1.33.9: sha256:81c20a12866d9a7c08c6e381ed326141c917454b696a05b46ae27665fe3c5cfa 1.33.8: sha256:537adda39074377893f1f650a71b576ba487b3c4d2ee55e9b22f4e95fc188594 1.33.7: sha256:e2999436a272c77370241a4f962c80737698dd8c2400fe75e5c7cf2142c96001 1.33.6: sha256:4d0d446f73d9db6d5bf2c03ecdc39d9d702836886f4715886c15dc2f461cc810 1.33.5: sha256:b8883e51837ee7fd45c88c762f37ca4b96d80ec6a7b46ec989381089e762aa7f 1.33.4: sha256:8f6d14828659b85da7c83bad798d50c2f7e0311742615fb7ed305f77bab54e50 1.33.3: sha256:2ee843fd1bbdf32607015771a2e1320b46829f22516e559a49dc7c4e29bb756e 1.33.2: sha256:6e82739bbbeae12d571a277a88d85e8a0e23dbc87529414a91ee5f2e23792dcf 1.33.1: sha256:036063194028d24c75b9ce080e475ad97bacc955de796b7c895845294db8edbf 1.33.0: sha256:dad0cec9e09368b37b35ce824b0ef517a1b33365c4bb164fe82310c73c886f7e ppc64le: 1.35.0: sha256:081ab73a6970ac3c68893dea9a03b0732ca22ab44a2aa8794fddac0bd4dfa749 1.34.5: sha256:3a10d4c1406df01bd9ab88750eabc1273964e9c5f24c7d4a0b719ae77e6cfec2 1.34.4: sha256:dca59a28fe9b0b9163418eca1545c9ed01cf514179f108d14e462c6074fd103c 1.34.3: sha256:4dd782484eeb460b9a95e6e2e07474216fc02ad45a27ba871799d18f2b6ee0ae 1.34.2: sha256:d4c3c9ba24b1b0eabf3c11ddec98801dda7a87b0529706e9ede18b8cc9e4182a 1.34.1: sha256:cba0ac74e7202fe28cf8aa895b83f7a30d78b148666add78e19215259f629bb0 1.34.0: sha256:e9e41d14439db0ca88cf2cd8533038203f379c25cd612f37635c17908e050ebf 1.33.9: sha256:c0a9e60800f66f85c70615128fec5a8358ffde0f715a4058163707dbcca8eb94 1.33.8: sha256:1d69c01512e8ebdd51fc70fc64473a31d492e8db095c0ee5d3ee58722048150c 1.33.7: sha256:076e7519bfff72a43fb1121ce836eee3cc1fec5bb5a59a11747c514e9d162d26 1.33.6: sha256:3643eefe295604288f5b652fb9c672a60f96dc803e63edaf9ee64ed4047a50dd 1.33.5: sha256:cf85062f39d755418da0ee4f869c7a4817bf95daee6e35df53010ad29be37c88 1.33.4: sha256:2b1594dad9af944e29ee74e788a8d28e1304e3f435f2efb61e5c38f20c2106f7 1.33.3: sha256:4293bc74f348db58adb0b0dd6affb918abee999cbaf0e42ea8a33427b8d278a5 1.33.2: sha256:8ed65404a57262a9f8eb75b61afa37fcec134472eb1a6d81f1889a74ff32c651 1.33.1: sha256:12646aca33f65fe335c27d3af582c599584d3f51185f01044e7ddd0668bb2b4c 1.33.0: sha256:b4fa46b25538d8145197f8bf2e935486392c0ca2a9fa609aedd02b9f106d37a6 kubelet_checksums: arm64: 1.35.1: sha256:73475c6db8fd8a9780b1b378fa2f917875e6146166c24603c1abc6eafd4493a8 1.35.0: sha256:aa658d077348b43d238f50966a583f4244b2a7d45590c77b3b165b7d44983ab8 1.34.4: sha256:c78845473c434ee85a2444eeab87f8b20f524e3ab6854a078f79468f44aad8f5 1.34.3: sha256:765b740e3ad9c590852652a2623424ec60e2dddce2c6280d7f042f56c8c98619 1.34.2: sha256:3e31b1bee9ab32264a67af8a19679777cd372b1c3a04b5d7621289cf137b357c 1.34.1: sha256:6a66bc08d6c637fcea50c19063cf49e708fde1630a7f1d4ceca069a45a87e6f1 1.34.0: sha256:e45a7795391cd62ee226666039153832d3096c0f892266cd968936e18b2b40b0 1.33.8: sha256:e835f15be6d8b7b27b963a46c4a054f7663c26741f17e003bfcb8271350cf882 1.33.7: sha256:3035c44e0d429946d6b4b66c593d371cf5bbbfc85df39d7e2a03c422e4fe404a 1.33.6: sha256:7d8b7c63309cfe2da2331a1ae13cce070b9ba01e487099e7881a4281667c131d 1.33.5: sha256:c6ad0510c089d49244eede2638b4a4ff125258fd29a0649e7eef05c7f79c737f 1.33.4: sha256:623329b1a5f4858e3a5406d3947807b75144f4e71dde11ef1a71362c3a8619cc 1.33.3: sha256:3f69bb32debfaf25fce91aa5e7181e1e32f3550f3257b93c17dfb37bed621a9c 1.33.2: sha256:0fa15aca9b90fe7aef1ed3aad31edd1d9944a8c7aae34162963a6aaaf726e065 1.33.1: sha256:10540261c311ae005b9af514d83c02694e12614406a8524fd2d0bad75296f70d 1.33.0: sha256:ae5a4fc6d733fc28ff198e2d80334e21fcb5c34e76b411c50fff9cb25accf05a amd64: 1.35.1: sha256:e7343310e03ff0d424df4397bdfa4468947d6d1f0f93dac586c1e8d6e4086d5d 1.35.0: sha256:2f4ed7778681649b81244426c29c5d98df60ccabf83d561d69e61c1cbb943ddf 1.34.4: sha256:03b8fea715a7ef82eeaf518dee34c72670c57cc7bc40dc1320c04fbf4f15172f 1.34.3: sha256:0e759f40bbc717c05227ae3994b77786f58f59ffa0137a34958c6b26fa5bcbbd 1.34.2: sha256:9c5e717b774ee9b9285ce47e7d2150c29e84837eb19a7eaa24b60b1543c9d58f 1.34.1: sha256:5a72c596c253ea0b0e5bcc6f29903fd41d1d542a7cadf3700c165a2a041a8d82 1.34.0: sha256:5c0d28cea2a3a5c91861dda088a29d56c1b027e184dae1d792686f0710750076 1.33.8: sha256:1caa69c5328cfa774218f75f0621a6f10a1b97e095af85015f468aeb8fdf956a 1.33.7: sha256:2cea40c8c6929330e799f8fc73233a4b61e63f208739669865e2a23a39c3a007 1.33.6: sha256:10cd08fe1f9169fd7520123bcdfff87e37b8a4e21c39481faa382f00355b6973 1.33.5: sha256:8f6106b970259486c5af5cbee404d4f23406d96d99dfb92a6965b299c2a4db0e 1.33.4: sha256:109bd2607b054a477ede31c55ae814eae8e75543126dc4cea40b04424d843489 1.33.3: sha256:37f9093ed2b4669cccf5474718e43ec412833e1267c84b01e662df2c4e5d7aaa 1.33.2: sha256:77fa5d29995653fe7e2855759a909caf6869c88092e2f147f0b84cbdba98c8f3 1.33.1: sha256:f7224648451dd4f9f2c4f79416f9874223c286ce41727788965fd0341ddb59c4 1.33.0: sha256:dd416d94850c342226d3dcdce838518b040ccea16548bfeaf2595934af88ef60 ppc64le: 1.35.1: sha256:ec8b7f870043f711b5d73e528342af1705d6ad7f8308d7f31d74d967986b54f6 1.35.0: sha256:f24eb1244878a3876fe180e6052822cc9998033850478b2f4776e5c3b09baecd 1.34.4: sha256:fab75e3eb1e0edf15aef7e8ba219256b44f047544ac421737d1778784fa46676 1.34.3: sha256:67dcceb6d91710e4da7af720eda7b20fd4e8c24237fc345602bb54439ad8ccca 1.34.2: sha256:a195f278b9bac26803f1e26b0f608e0dce66aad033e8c043e8555775612530c9 1.34.1: sha256:c4782dbf1987680e9b2baa3ecf5db9e66395772e82b251eb73a150fbfbe0b906 1.34.0: sha256:ed663fa4ff3e305276dd889885303e07989dfab073e95ef2da931b975f6686e8 1.33.8: sha256:392ed39b6c037bc5c510412c9b5cfd29238d31dd67d1a3cbae7ef4a274304c63 1.33.7: sha256:f96dd4272ca8eccf1f93fb5162323840b9286c5a42a5305fcc1b4d47889534d3 1.33.6: sha256:00ae91297503518efd237d40900af4de0067597ae4f2ab8250ddb629ffb6df05 1.33.5: sha256:1d785ead3f6709f66a105c629a020b9dfe6dff775fae42f7d147edec2d178351 1.33.4: sha256:5133077024e5a59ece48d2e6d0fdaeed5c4f90c5e781f25c89c984ee4da396a6 1.33.3: sha256:bb4123e09734348d4b553c031bfe7710adcffffe79ed9973a526e36d87aa19fe 1.33.2: sha256:be8412cb9bf30125e3a88ecb9bfca4df1ff5d4e650947c46222683071f1a17d7 1.33.1: sha256:c1bc01115a513eaec76d56dc52a52aeb05f866a6d07c55335c1fff56c868543d 1.33.0: sha256:6fa5abbc14d65b943b00fcfc8a6ac7eb39fd7e924271738c6f17e0b7e74c665b kubectl_checksums: arm: 1.35.1: sha256:dbe14e5b12184d72978b17b167aedc3f42f4a1faf249180025d6359eebcd983e 1.35.0: sha256:dca28f6af03b31ca6043baa1da7332472c7a3df743606a758534b9ac3ed7ecce 1.34.4: sha256:3a6e631bdbb79e633d23055dadc97b45f45d325105ddf40e696de2a324a254c0 1.34.3: sha256:e0cf1eddede6abfd539e30ccbb4e50f65b2d6ff44b3bb9d9107ea8775a90a7e4 1.34.2: sha256:18e03c1c6ab1dbff6d2a648bf944213f627369d1daeea5b43a7890181ab33abf 1.34.1: sha256:ca6218ae8bf366bd8ccdcb440b756c67422a4e04936163845f74d8c056e786ee 1.34.0: sha256:69d2ce88274caf9d9117b359cc27656fb6f9dd6517c266cfd93c6513043968b8 1.33.8: sha256:734dea07663751c8b45926c843e2c250f13473d65f396555a1ecfe0c9c502fa8 1.33.7: sha256:f6b9ac99f4efb406c5184d0a51d9ed896690c80155387007291309cbb8cdd847 1.33.6: sha256:89bcef827ac8662781740d092cff410744c0653d828b68cc14051294fcd717e6 1.33.5: sha256:5a3a416a85cfc9f7a348c0c0e6334b7449e00a57288ab5a57286ccf68a4d06af 1.33.4: sha256:eefd3864ce5440e0ba648b12d53ccffaad97f1c049781b1aa21af6a5278f035f 1.33.3: sha256:0124dba9e9091b872591cabcbaea7df07069cb132d38d95f3c7bc8d5b8b621a9 1.33.2: sha256:f3992382aa0ea21f71a976b6fd6a213781c9b58be60c42013950110cf2184f2a 1.33.1: sha256:6b1cd6e2bf05c6adaa76b952f9c4ea775f5255913974ccdb12145175d4809e93 1.33.0: sha256:bbb4b4906d483f62b0fc3a0aea3ddac942820984679ad11635b81ee881d69ab3 arm64: 1.35.1: sha256:706256e21a4e9192ee62d1a007ac0bfcff2b0b26e92cc7baad487a6a5d08ff82 1.35.0: sha256:58f82f9fe796c375c5c4b8439850b0f3f4d401a52434052f2df46035a8789e25 1.34.4: sha256:5b982c0644ab1e27780246b9085a5886651b4a7ed86243acbb2bacc1bea01dda 1.34.3: sha256:46913a7aa0327f6cc2e1cc2775d53c4a2af5e52f7fd8dacbfbfd098e757f19e9 1.34.2: sha256:95df604e914941f3172a93fa8feeb1a1a50f4011dfbe0c01e01b660afc8f9b85 1.34.1: sha256:420e6110e3ba7ee5a3927b5af868d18df17aae36b720529ffa4e9e945aa95450 1.34.0: sha256:00b182d103a8a73da7a4d11e7526d0543dcf352f06cc63a1fde25ce9243f49a0 1.33.8: sha256:76e284669f1f6343bd9fe2a011757809c8c01cf51da9f85ee6ef4eb93c8393a8 1.33.7: sha256:fa7ee98fdb6fba92ae05b5e0cde0abd5972b2d9a4a084f7052a1fd0dce6bc1de 1.33.6: sha256:3ab32d945a67a6000ba332bf16382fc3646271da6b7d751608b320819e5b8f38 1.33.5: sha256:6db7c5d846c3b3ddfd39f3137a93fe96af3938860eefdbf2429805ee1656e381 1.33.4: sha256:76cd7a2aa59571519b68c3943521404cbce55dafb7d8866f8d0ea2995b396eef 1.33.3: sha256:3d514dbae5dc8c09f773df0ef0f5d449dfad05b3aca5c96b13565f886df345fd 1.33.2: sha256:54dc02c8365596eaa2b576fae4e3ac521db9130e26912385e1e431d156f8344d 1.33.1: sha256:d595d1a26b7444e0beb122e25750ee4524e74414bbde070b672b423139295ce6 1.33.0: sha256:48541d119455ac5bcc5043275ccda792371e0b112483aa0b29378439cf6322b9 amd64: 1.35.1: sha256:36e2f4ac66259232341dd7866952d64a958846470f6a9a6a813b9117bd965207 1.35.0: sha256:a2e984a18a0c063279d692533031c1eff93a262afcc0afdc517375432d060989 1.34.4: sha256:d50c359d95e0841eaad08ddc27c7be37cba8fdccfba5c8e2ded65e121ff112db 1.34.3: sha256:ab60ca5f0fd60c1eb81b52909e67060e3ba0bd27e55a8ac147cbc2172ff14212 1.34.2: sha256:9591f3d75e1581f3f7392e6ad119aab2f28ae7d6c6e083dc5d22469667f27253 1.34.1: sha256:7721f265e18709862655affba5343e85e1980639395d5754473dafaadcaa69e3 1.34.0: sha256:cfda68cba5848bc3b6c6135ae2f20ba2c78de20059f68789c090166d6abc3e2c 1.33.8: sha256:7f9c3faab7c9f9cc3f318d49eb88efc60eb3b3a7ce9eee5feb39b1280e108a29 1.33.7: sha256:471d94e208a89be62eb776700fc8206cbef11116a8de2dc06fc0086b0015375b 1.33.6: sha256:d25d9b63335c038333bed785e9c6c4b0e41d791a09cac5f3e8df9862c684afbe 1.33.5: sha256:6a12d6c39e4a611a3687ee24d8c733961bb4bae1ae975f5204400c0a6930c6fc 1.33.4: sha256:c2ba72c115d524b72aaee9aab8df8b876e1596889d2f3f27d68405262ce86ca1 1.33.3: sha256:2fcf65c64f352742dc253a25a7c95617c2aba79843d1b74e585c69fe4884afb0 1.33.2: sha256:33d0cdec6967817468f0a4a90f537dfef394dcf815d91966ca651cc118393eea 1.33.1: sha256:5de4e9f2266738fd112b721265a0c1cd7f4e5208b670f811861f699474a100a3 1.33.0: sha256:9efe8d3facb23e1618cba36fb1c4e15ac9dc3ed5a2c2e18109e4a66b2bac12dc ppc64le: 1.35.1: sha256:bced44e491ce52cce11e2b4bd4bd9181f4f963ffe868438778d028d56485c5d9 1.35.0: sha256:8989809d0ac771244dabe50ed742249ac60eeb6d385cd234ee151eb40b7c32c4 1.34.4: sha256:b083c39879483816f34d1f7e2e31e70ec48984fcc1753c79f4b846cfedbf41ac 1.34.3: sha256:ae239b7f6f071e47014e1b5b20aa60626e06b32922a6b5054562ae2c5fa82c18 1.34.2: sha256:49a985986a9add6c229c628bf2a83addebbdeeef40469fce2a54e51b6f1bb05b 1.34.1: sha256:45499f0728b4a3428400db289edb444609d41787061f09b66f18028c0a73652f 1.34.0: sha256:1773805a0c128f4d267b2e11f4c74cac287e9a07fffaecc3f7af6df9c8aaf82c 1.33.8: sha256:aa079f403c80ba6017449c230733fed4e5d7b0a8700bd6590ee202161b8b12af 1.33.7: sha256:0807c38a1342ab8dea6435f33d5897a01527d348a968a5c4ca2929769f3d54f2 1.33.6: sha256:4b056b1749c619fab6a855247c3bd04123f2b61cf136ca6bddf69ff97a727e32 1.33.5: sha256:37e2204d371bbbb90fd693049a7a45b81991ca8bcc9b8baf041a7c9f23e9035c 1.33.4: sha256:fa61404b9c3d76f342f2ad05616753475739ab488e0beffd22942e0cb266cfa9 1.33.3: sha256:e0261727822c685f63902f7d78ddfcd112bfde4619692b6c1aae68d162245f67 1.33.2: sha256:d1cdf13cb786c1ee6d5bf6d85034f496aa2fee97b287028043eb14c5dc74993f 1.33.1: sha256:f922dd8f558dc616ebaa34908ceb7964ebb8caadd7c48699d0b791ffff2be1aa 1.33.0: sha256:580d076c891711ec37afaf5994f72a8aad9d45c25413e6e94648e988a5a9933a kubeadm_checksums: arm64: 1.35.1: sha256:80097a3c4ef824f4edfe131d2bd429772c4be3a460c42a44f2320164a917de32 1.35.0: sha256:1dac7dc2c6a56548bbc6bf8a7ecf4734f2e733fb336d7293d84541ebe52d0e50 1.34.4: sha256:d8028b7e8c8d6c9b3fc3da6bc88d4d0cfb33df1b4b026a7d6e8c35d1471c9f6e 1.34.3: sha256:697cf3aa54f1a5740b883a3b18a5d051b4032fd68ba89af626781a43ec9bccc3 1.34.2: sha256:065f7de266c59831676cc48b50f404fd18d1f6464502d53980957158e4cab3a7 1.34.1: sha256:b0dc5cf091373caf87d069dc3678e661464837e4f10156f1436bd35a9a7db06b 1.34.0: sha256:6b7108016bb2b74132f7494e200501d6522682c01759db91892051a052079c77 1.33.8: sha256:b5248b51e66e4716261f2c926fe2f08a293795e6863099e7792b4d57dbb9109e 1.33.7: sha256:b24eeeff288f9565e11a2527e5aed42c21386596110537adb805a5a2a7b3e9ce 1.33.6: sha256:ef80c198ca15a0850660323655ebf5c32cc4ab00da7a5a59efe95e4bcf8503ab 1.33.5: sha256:b1c00657649e35771569d095e531d826bd19baf57bcb53cccf3f91d7d60b7808 1.33.4: sha256:ef471b454d68ee211e279ddeaebde6ee7a8e14b66ae58e0d0184e967c3595892 1.33.3: sha256:bf8ed3bc3952e04f29863c6910ae84b359fe7ac1e642ed4d742ceb396e62c6f2 1.33.2: sha256:21efc1ba54a1cf25ac68208b7dde2e67f6d0331259f432947d83e70b975ad4cc 1.33.1: sha256:5b3e3a1e18d43522fdee0e15be13a42cee316e07ddcf47ef718104836edebb3e 1.33.0: sha256:746c0ee45f4d32ec5046fb10d4354f145ba1ff0c997f9712d46036650ad26340 amd64: 1.35.1: sha256:8a7ff344eef1bfba88f9a74b3fdc9ea4448c94f1b3cefb8c0aeeaf1f96e05053 1.35.0: sha256:729e7fb34e4f1bfcf2bdaf2a14891ed64bd18c47aaab42f8cc5030875276cfed 1.34.4: sha256:b967f1fa0e36621c402d38bb560eb4a943954d5cf5a00e5150842f6f5da73455 1.34.3: sha256:f9ce265434d306e59d800b26f3049b8430ba71f815947f4bacdcdc33359417fb 1.34.2: sha256:6a2346006132f6e1ed0b5248e518098cf5abbce25bf11b8926fb1073091b83f4 1.34.1: sha256:20654fd7c5155057af5c30b86c52c9ba169db6229eee6ac7abab4309df4172e7 1.34.0: sha256:aecc23726768d1753fd417f6e7395cb1a350373295e8e9d9f80e95ed3618e38e 1.33.8: sha256:8259af514dc3655e8abec1a69b637f31cce2ecb940a80ae4a268e5287890f009 1.33.7: sha256:c10813d54f58ef33bbe6675f3d39c8bd401867743ebc729afdd043265040c31d 1.33.6: sha256:c1b84cb3482dd79e26629012f432541ccb505c17f5073aa1fdbca26b1e4909fd 1.33.5: sha256:6761219749c6c67a56a5668dfe65d669e0c1f34d4b280b72de6d74d47c601f1e 1.33.4: sha256:a109ebcb68e52d3dd605d92f92460c884dcc8b68aebe442404af19b6d9d778ec 1.33.3: sha256:baaa1f7621c9c239cd4ac3be5b7e427df329d7e1e15430db5f6ea5bb7a15a02b 1.33.2: sha256:5c623ec9a9b8584beba510da5c2b775c41cf51c0accdfb43af093bc084563845 1.33.1: sha256:9a481b0a5f1cee1e071bc9a0867ca0aad5524408c2580596c00767ba1a7df0bd 1.33.0: sha256:5a65cfec0648cabec124c41be8c61040baf2ba27a99f047db9ca08cac9344987 ppc64le: 1.35.1: sha256:eec12948cfabc18115636c44aca894bf9abef3b2ea73cba180314ee3c218dcca 1.35.0: sha256:77a466e1b6a8e28362a729541269de0a7c4a6b9e7770cccefcd745502e656b90 1.34.4: sha256:69f1065e718ef2aa5f0287444ef97bd4a5fb8841fc0662f54ca8992a39865391 1.34.3: sha256:2b8b48b3b0eb657e04122a158cb7fcad964fba5bd2d8e07f8eeec6f856a63ecf 1.34.2: sha256:bea4ed6d971523da794a802de15910b08c09e23bc4c850ee3b953c4bdb0b7976 1.34.1: sha256:ddb6bd80bee0719924ae901672b99205226badab74fb13a9e1bb6d3de49fbb21 1.34.0: sha256:7201ba36f44187f408a036c4a545e2a3cd12943b1297092687bb66c9a1a9fed6 1.33.8: sha256:d618fa97b5782b57512e0a8ab9ed17af190236907af7bd3c9c0776d81c78273f 1.33.7: sha256:db2e20d0c20928ae7d68d7603020f8ffd89dcdac4fdc160ef83f1da663868bed 1.33.6: sha256:58aaec7b5066b6e3705e0493a2f51c7f101b17165ce714c4d52a2b53861c078b 1.33.5: sha256:b1e261109a4e22e0a417d10724bed7f71ba12c2acc167a55d89211e49c2e5eee 1.33.4: sha256:eb4f3b7a875ffe06aadd5b5ff7b3dccec125933b7ba6fcb5baed39c9c01220c4 1.33.3: sha256:d9f30f0eb538be98cd07603b945611b056be5e5871369b16e23090545ef8cdfa 1.33.2: sha256:1b818900ac7af72a14f50300d6c6ad600eecdc578c37b75fa488cc654ca08c25 1.33.1: sha256:a772834ba22478c9119f03ecca2a27a70234623d74ff1d7671ee85675a4e830b 1.33.0: sha256:26cb7ac57d522a59c84c4784b176097d23c7b4e61874fab84ae719d0e43ac0bc etcd_binary_checksums: arm64: 3.6.8: sha256:438f56a700d17ce761510a3e63e6fa5c1d587b2dd4d7a22c179c09a649366760 3.6.7: sha256:ef5fc443cf7cc5b82738f3c28363704896551900af90a6d622cae740b5644270 3.6.6: sha256:8a15f5427c111ff4692753682374970fb68878401d946c2c28bdad6857db652f 3.6.5: sha256:7010161787077b07de29b15b76825ceacbbcedcb77fe2e6832f509be102cab6b 3.6.4: sha256:323421fa279f4f3d7da4c7f2dfa17d9e49529cb2b4cdf40899a7416bccdde42d 3.6.3: sha256:4b39989093699da7502d1cdd649c412055a2bddd26b3d80ed87d0db31957075c 3.6.2: sha256:79d0a2488967aa07ecfde79158b1dab458158522f834810c2827eecac4695a31 3.6.1: sha256:5f8ed6e314df44128c218decbf0d146cf882583d05c6f6d9023ce905d232aaec 3.6.0: sha256:81477b120ef66ff338fe7de63d894e5feec17e6e1f1d98507676832e089d9b58 3.5.27: sha256:1277309f540c5a0329c428f95455c9f76d24f768c8d28fd2753e891c379053fa 3.5.26: sha256:93ac1667df0e178ea6d152476ce4088df4075604fe4bc7f85f4719e863cd030b 3.5.25: sha256:419dce0b679df31cc45201ef2449b7a6a48e9d241af01741957c9ac86a35badc 3.5.24: sha256:efc01f6b3fbef0f000cb53bcad4845c116d7fdd8769ca39d9c40d2fe4d2e509f 3.5.23: sha256:d95118595a9556a29775f99c1ef2ee16f6be113df76b7178643a2bfd6a6f37c3 3.5.22: sha256:22aca5a253c4a9f2850300b0a1dd209587586ff0e985f5cb1c34e9e5edc07848 3.5.21: sha256:95bf6918623a097c0385b96f139d90248614485e781ec9bee4768dbb6c79c53f 3.5.20: sha256:f034232e6fb64b0d89c45fd78b8b4c3e9fb8d69605dddddcdebf5d7cd96a1531 3.5.19: sha256:a786fd2c92c3c0404586ffedf1b318e4944a17aefed1fa6566f5712ddb8359ad 3.5.18: sha256:c2bcaf465537d6d47c8bb82a69e31f786f32257050e3bca445bc4e63479ec714 3.5.17: sha256:7d717a62520bf39fa1115dfbb1df79479ff74b5eda0914f4132bfa60a48b9549 3.5.16: sha256:8e68c55e6d72b791a9e98591c755af36f6f55aa9eca63767822cd8a3817fdb23 3.5.15: sha256:5aa28b435edb1f22bf6455f754e16a13e3e4eb1ad8fc7c22ad47aa8e722febf2 3.5.14: sha256:90510c79c4aae3c3313691f5770fc53b3ac883338fc0254bf8d22460acd3c19d 3.5.13: sha256:2854993bf622764ccdeb7a146fae2965cb0fcba93c5c8f391e0d5f153c8a7a02 3.5.12: sha256:31f30c01918771ece28d6e553e0f33be9483ced989896ecf6bbe1edb07786141 3.5.11: sha256:6edf0cddc8fa2d7674129abe2e44d5a37cc3a6e3b500c13c6cbc2ed2ecf08bf4 3.5.10: sha256:ff74a6018d9b2a1320bff30e5a11b4f2f5c2a3d147df8a8bad53c01b9f800ee1 3.5.9: sha256:bb201c106a61bbab59e2d9f37f4bdff99d50201f513c66b4578741eab581fb28 3.5.8: sha256:3f4441b293a2d0d4d2f8b2cd9504376e15818f7b865ef4b436e8e6f865f895ff 3.5.7: sha256:1a35314900da7db006b198dd917e923459b462128101736c63a3cda57ecdbf51 3.5.6: sha256:888e25c9c94702ac1254c7655709b44bb3711ebaabd3cb05439f3dd1f2b51a87 amd64: 3.6.8: sha256:cf9cfe91a4856cb90eed9c99e6aee4b708db2c7888b88a6f116281f04b0ea693 3.6.7: sha256:cf8af880c5a01ee5363cefa14a3e0cb7e5308dcf4ed17a6973099c9a7aee5a9a 3.6.6: sha256:887afaa4a99f22d802ccdfbe65730a5e79aa5c9ce2c8799c67e9d804c50ecedb 3.6.5: sha256:66bad39ed920f6fc15fd74adcb8bfd38ba9a6412f8c7852d09eb11670e88cac3 3.6.4: sha256:4d5f3101daa534e45ccaf3eec8d21c19b7222db377bcfd5e5a9144155238c105 3.6.3: sha256:3f3b4aa9785d86322c50b296eebdc7a0a57b27065190154b5858bf6a7512ac10 3.6.2: sha256:4b5d55d61e2218fab7c1cc1c00b341c469159ecde8cedd575fa858683f67e9f4 3.6.1: sha256:1324664bfe56d178d1362a57462ca5a7b26a6d2cbe9e1c94b6820e32cb82d673 3.6.0: sha256:42305b0dcbba7b6fdff0382d0c7b99c42026c88c44847a619ab58cde216725d9 3.5.27: sha256:0aad9a9e4e0817a021e933f9806a2b2960a62f949ad5a3d6436d8886945cb1bc 3.5.26: sha256:0a682a91201dc8351d507210bc30b021a11e254eab806f03224b51e8fad29abb 3.5.25: sha256:168af82b59772e1811a9af7b358d42f5c6df44e0d9767afb006ecf12c4bbd607 3.5.24: sha256:042497e2ddcee06f22e5d486d81f58affa26b53ee423e2a6aaca3d3ea98c8191 3.5.23: sha256:8b50d62d38cb2de005b42227dd14b33f6e01758970f248f9257789ecfaf634c9 3.5.22: sha256:20174ab70a6f3df94da13ecac4610f42c47d25af82426f21c112c2c841ec499a 3.5.21: sha256:adddda4b06718e68671ffabff2f8cee48488ba61ad82900e639d108f2148501c 3.5.20: sha256:9ac85616fb8c0e45f485074dde0258ca2b7b42f1dd5320821af5a8b66daf7072 3.5.19: sha256:16ae742def5f330800590e8d505d72830a3b0b7012e559e6bd76f0bc9864bf42 3.5.18: sha256:6ddde039a7a506badf34e7edfb38e1ea90e36f05c8cfceba602045df623d86fa 3.5.17: sha256:eff6ac621d41711085d0f38fab17d8fa3705f6326c3ff11301a1f5a71fc94edd 3.5.16: sha256:b414b27a5ad05f7cb01395c447c85d3227e3fb1c176e51757a283b817f645ccc 3.5.15: sha256:3f6b48d8c2844699f2b19c1880508ecf63e1489769ed37ebb97495d5cd848a89 3.5.14: sha256:b0b34298f53f6830f08e7ddc57fc74dc45563216a66e94d9e6b0b9e0b0281b34 3.5.13: sha256:31e6fcbee0e8c3df27cf1ba69b522e338377f5ed6447f5d05700aee367f3b7e7 3.5.12: sha256:f2ff0cb43ce119f55a85012255609b61c64263baea83aa7c8e6846c0938adca5 3.5.11: sha256:e256885e753dc99001335e099d3c2eb8cf21a865a087ee4d7e3665752ae5929a 3.5.10: sha256:26e90d024fa2310bc52bb40e7f2132e81640b55f8fc446c00ae07e30af2a44fd 3.5.9: sha256:d59017044eb776597eca480432081c5bb26f318ad292967029af1f62b588b042 3.5.8: sha256:d4c1b8d90ad53658f12ffc293afc5694b7bc6cb093af609188649a799e1cc8dc 3.5.7: sha256:a43119af79c592a874e8f59c4f23832297849d0c479338f9df36e196b86bc396 3.5.6: sha256:4db32e3bc06dd0999e2171f76a87c1cffed8369475ec7aa7abee9023635670fb ppc64le: 3.6.8: sha256:3b9bb486b0eb8d79b30410749ec26e174db075956c9ecb533b313b9263e7ba78 3.6.7: sha256:de3b1ed50fc8868cdd56b12b0cd81d6740bf53edbca570400a78e530e4829b7b 3.6.6: sha256:e4f528b63a731e9b96f5d10f55ce096223fb4e1bc1778aa2535a3d47e9a129e5 3.6.5: sha256:3cf99879c7c5b8678a0ec2edf9102b268ea934584db2850f049d89ed8e36b61c 3.6.4: sha256:2910fc73e42e1eeb9cc7da8080b821c7649558465e0e6122e49afce832e4b9da 3.6.3: sha256:de8ee412ee2669483fd9c730e915c5bd4fe113ba33be4a70305d13ff35e1f919 3.6.2: sha256:bf79b9d4c7e9f86e611e73de9fe54a195bc0ad54aeb17200b1c8bda3c4119705 3.6.1: sha256:bb87fcd0ea4b9fabf502703512c416ca1d9f4082679cb7f6dbc34bed3dfc13f6 3.6.0: sha256:1180d06e3a3787ab65078d9a488f778a4712c59cc82d614abde80c5d06efe38f 3.5.27: sha256:b41d488dcd579e780f49f5bd747e9386e17e1376ffb77bfff061f7944818a678 3.5.26: sha256:9678ddaced9fcd4878b76b0b76c9c2a3638a70bdc362c9f4cb25ecc48de2c6d3 3.5.25: sha256:0dee64e99a43a06dd9541a40a18b52c7309eb1682a2a32740d4bdf358296c007 3.5.24: sha256:4b252266a59a00c0f608f481c836fb469d2cd0f60ecbc119c4f1fe0611910ab1 3.5.23: sha256:9b9caa29715f387633c6f3639efd3b85284665f7e8632551c904c9859edbb7a8 3.5.22: sha256:4dbe98bf5fd82a0f9295bd7ab47429381ab0a80f1e83861c50b01452fa515353 3.5.21: sha256:6fb6ecb3d1b331eb177dc610a8efad3aceb1f836d6aeb439ba0bfac5d5c2a38c 3.5.20: sha256:563bdac64fc92442cf366c02294dff1cbbd3885a86dbcf7f2e87d9388c3b3223 3.5.19: sha256:cc8651929f4d5794892eeeabf612a243ea6233125bc5f8b0f711118736e2710f 3.5.18: sha256:ad90260978a9a94572c8aedfa3c4ab225a451e84ab01a1df35e4124863672999 3.5.17: sha256:5c737b586a1ebcc12bf0d68a2b56583764f4aba82ab4934629626da93d4a9ecc 3.5.16: sha256:33322806f4a2aa3d4947a4d42ec6a120296535e7f00e2f9d74e515e9386333e0 3.5.15: sha256:6512f7308a0d0af7c9bb46bca1c4f4e816304304d838d1045ae21d9772c5490e 3.5.14: sha256:01681d4d33bba5130c9cffca42c35b0f68e0d991b0b4ee65dab6fd36568d4fee 3.5.13: sha256:f372b524e2c118dbb0dbe1097474a072fc93bc30da65efa92999137303bcd9a7 3.5.12: sha256:ebd8060508d572678d8d1e4f90f87863e3a6cfcba856ceca32379b03251c0597 3.5.11: sha256:a2e70b291811fa8ccc34cc7d297bf7d31e3af790bc31e54cad034a49e9db2eb7 3.5.10: sha256:10cd8e4ecf6718b9712bf2edfac2e4924d7f21dbe58d368e6e10578c85bd8c01 3.5.9: sha256:551539ebb344ebdc77f170ea51512a6cda35877ffdcbd8b3316b2495a8b2bd87 3.5.8: sha256:20e28302c1424b1a3daf7d817f2662e4c64e395a82765d1696cb53cb6bc37a4e 3.5.7: sha256:e861aa6acd4d326ec01bfa06fffb80d33f3f8c26e0eb8b73e4424578d149bd04 3.5.6: sha256:e235cb885996b8aac133975e0077eaf0a2f8dc7062ad052fa7395668a365906b cni_binary_checksums: arm: 1.8.0: sha256:7ed51af2ee5c7784c8b978b293ad1cf6c3022d7792a2e79197fb9cdc98f3f752 1.7.1: sha256:1df4fa20a0fe279bda5a671d172911de2c1a81813bfe8fb0398b46fa9e49d0fb 1.6.2: sha256:ee31b0117206dc6242a0349443020025a71752d20d683c08caa70b32de179c4b 1.6.0: sha256:10cd1b6b0f7c1e6faf18b2e46ee338beb1e1cce253efc7086f8bc1f4e1061d1a arm64: 1.8.0: sha256:57ce466fc3b79db1f19b8f4c63e07a1112306efa53c94fe810a2150dd9e07ddb 1.7.1: sha256:119fcb508d1ac2149e49a550752f9cd64d023a1d70e189b59c476e4d2bf7c497 1.6.2: sha256:01e0e22acc7f7004e4588c1fe1871cc86d7ab562cd858e1761c4641d89ebfaa4 1.6.0: sha256:db09ab057ecf60b05ba05cbec38d55b95cc139c7f1078e2e4857cc13af158cee amd64: 1.8.0: sha256:ab3bda535f9d90766cccc90d3dddb5482003dd744d7f22bcf98186bf8eea8be6 1.7.1: sha256:1a28a0506bfe5bcdc981caf1a49eeab7e72da8321f1119b7be85f22621013098 1.6.2: sha256:b8e811578fb66023f90d2e238d80cec3bdfca4b44049af74c374d4fae0f9c090 1.6.0: sha256:682b49ff8933a997a52107161f1745f8312364b4c7f605ccdf7a77499130d89d ppc64le: 1.8.0: sha256:63f762df723eb7dbee83e3751167ff1e18cf7b86ef5b48eb620c91af2def434a 1.7.1: sha256:15a4070b20c4d6c8bc9b3db52d8303d0520ff7a89cd2e87a2fdf1e9b8dd69373 1.6.2: sha256:66dcb90886a039e919904f2fb761b88e03dedd25fb718196855432771eaaa325 1.6.0: sha256:d8d4bd74247407c8c73de057bc00adac28bb1ed2d2ee60a9dda278e3b398bcc2 calicoctl_binary_checksums: arm64: 3.30.6: sha256:47ecc00bdd797f82e4bac0ff3904c3a5143ba2d61e8ae1cbbce286ca76d3790a 3.30.5: sha256:7611343e7a56e770b95e2bb882dda787efbbd4331b1dd6316ff8ea189238dfaa 3.30.4: sha256:b21fbbc55b6f5d50c1c0faae714242cae3e013185cb8e26ce56981bd10da260d 3.30.3: sha256:2ae0474b88a6042e5489d7410d2669a9d443c9d5c51e2bdc8ebe4d6dd98f2475 3.30.2: sha256:b7aaa3ef1d3f280f835f3e339cbc32d4c6cd6493c7adf46b1ecae29fca848e7f 3.30.1: sha256:e3ae853fecfbf09bb2cb8d5ecded553b2d8d1a7229cfbf911fb3ff13c3ab5412 3.30.0: sha256:3b97d2e15a0ffc42755571ca0749a1722c7d87d045d2345606eec8e18e95ccc6 3.29.7: sha256:484479aa441401adf0db2effb911562d25a4c9a3f022aa6a3b082d027108edec 3.29.6: sha256:db3da30af65b3610ee87c8d1bbefb54ca6c1982ed2a0a950215f5eb812bf217f 3.29.5: sha256:b121a5fb297271c72ba252f6650d4752c67772f16dd57022d69439866e947957 3.29.4: sha256:3139c83cdd3e648d9605c2cebb4657871e41310aad0c7b4bf198e3a8393c5cec 3.29.3: sha256:d6cba570af9162dff56714ac5e22dfdd170742bc58a51211f587875a3de79fc4 3.29.2: sha256:3a9b80335338b7f4af762d4a7cf68e67b40839e50711fbe6e67f9a62b69bafdd 3.29.1: sha256:6f662d316a267854dc5487242ca7ec8ca70c35b52bed258aafb76c2d113643c2 3.29.0: sha256:ab23afb283fcdffcf0e1156cdced68d05b6c2b70fd4ea2cbc3189d0ecd43bdfd 3.28.5: sha256:691baba17e6a50d0ceee7f95569b864729e436f474fce1e9842041b33cc14316 3.28.4: sha256:48887a6dd715f7340511788c3f311810326e61dcce5a6c1554e365cd372ffab1 3.28.3: sha256:b61b5206bc7795793edf792040acf5c52d48ff5de701001d0dbbd850edd0c077 3.28.2: sha256:8ebe965424ac94084499182b2853de62e5d18cdc346a3b8974e991d8b7a9592d 3.28.1: sha256:c062d13534498a427c793a4a9190be4df3cf796a3feb29e4a501e1d6f48daa7c 3.28.0: sha256:c4ca8563d2a920729116a3a30171c481580c8c447938ce974ce14d7ce25a31bf amd64: 3.30.6: sha256:2017e19727dca689d8bb73a9d8dff3c6a8ba7d8c75049f99ee207272161b5749 3.30.5: sha256:6cdfb17b0276f648f4fdb051a5d75617a50b3c328d4cccfc40d087b96c361d80 3.30.4: sha256:7e2e5e75b25c55683b68eabeb9b00390b1d359e72bf57f7ec2b76bb006fd175f 3.30.3: sha256:a7d017d1abf6ef5d6e03267187c0dd68c32f5e937b64decd29d003be44fa6b94 3.30.2: sha256:a89e03de84194ab445a998940768959aaabaa5d92be31fe00814b21f86f48c76 3.30.1: sha256:85208c36a4fb4e251322a4e544a7b8fc9bd6b9ba2922685a2a2b0aa7dc75d743 3.30.0: sha256:a8d385b40526e117a622eb2134f7ed0ce77b9048dbd560dfa2be49fbb4ee6f25 3.29.7: sha256:c4e281a404d14da561c65838195b2a4bcf18aebca1bede5ef6b99e0bf78cb8a0 3.29.6: sha256:42ee55f279a9df0ef205d001bdde1a4cd4ad3f4dce4d0a3cb7d04ef10f1fed6b 3.29.5: sha256:0b118c7efa08e84751a6c7f6647d61ecac58793343d949ad75b71a1d55480a9c 3.29.4: sha256:f2a6da6e97052da3b8b787aaea61fa83298586e822af8b9ec5f3858859de759c 3.29.3: sha256:8101eef6d31ca80db0c64c7ab8930f657dafc1f8696f145ef5d5f162026eedda 3.29.2: sha256:6076d6745c4d60c0c4322961cbb256a0ffa8476cf7f8dbe5de4ae82c55bca020 3.29.1: sha256:2ac849181cb1fb40c61c06d075711025cdb909d80562d078cc548d50a0edcd3d 3.29.0: sha256:df5048549d72a1f7ea4f61c655699d3b16d8a45873f28c3855c39597b73e8a3d 3.28.5: sha256:fe4702ce171045728b6c37b2c01e6f903780997aea7e695b35735754eeeeaf64 3.28.4: sha256:ff07f5ac4dbf9a849adb12db20e7b35857869fb98b23e802404dbb4a8a98e013 3.28.3: sha256:b7dc6d01407ea04c110b8d50312591d7a7c3aa5239c875354ced83ac6b924137 3.28.2: sha256:d7f30447f0f59262051b95bdc656407442c4f71066dc37ddd3b676108fab569d 3.28.1: sha256:22ec5727c38dbe19001792b4ca64ac760a6e2985d5c1a231d919dbebe5bca171 3.28.0: sha256:4ea270699e67ca29e5533ddb0a68d370cb0005475796c7e841f83047da6297b6 ppc64le: 3.30.6: sha256:9a9c368499b1e3d08418dfbb566379483e15c50d08dd1bcaf6148c115d82ed36 3.30.5: sha256:5b6de49da1af2633549bff5e8f4d8a573a175b65c47c29d327ef6a0760d39a93 3.30.4: sha256:8fc8ef492d463e184e714bc6d31b05f9066c8af3445928efef233850f036bb92 3.30.3: sha256:ccd13ced62baf633fb4347fbe6c9fdc0d3b1b7deb1794c83c015507a0cb8238e 3.30.2: sha256:6364bbadfcaf5935427a5f6ef431435b39ccacc22c8a1b5f9071fda00bdfd8ec 3.30.1: sha256:d35943a9eccf7def1f3f92b782b221331a2e9163625233b98a72189e2f1b8a6b 3.30.0: sha256:5694651bbc6a50dc5537f1f0acf4d4c3897eec5f3b114bdc7cb6bce21df7ba83 3.29.7: sha256:b76f1bd1feb3a83a6591196d403b35e1ce97dfdc88babcf1f52f905c0c3cc94e 3.29.6: sha256:f1369d48577ff441f893ed08197987e17851cd19b99911d9999ecda667567a69 3.29.5: sha256:ec2ab56be3ffdc7ee41f14f3a5fe17564c13a8103939113437c124e2a9900a0e 3.29.4: sha256:c7798ef7817bb67f84d54aef661066e64f957977fc80f88b49be2b13a3492d06 3.29.3: sha256:edb98d2a0d3f8afbf98eb000f0d535d4678af39dd6e10a09ea5615a4824f692f 3.29.2: sha256:6f3fd72be26fcf52605d9ece716363a73bb194ca59ee34a257156d30fa5c1542 3.29.1: sha256:ef6064f2ec1a09b5eb8c43ab0c64bd42785c24f5b22b950583fb5074f472c2b7 3.29.0: sha256:c9c2a29a349c6f681aa79b5f5d6aee738305d95aa7f158b6217f487808758e53 3.28.5: sha256:1aa3b36f198aecdad312664c7b2dd2b15daced54fd4e2db56563d57431fb10d3 3.28.4: sha256:9646b8b66981ed68017d30291f44e3e4ff1f6ce318c88c1e837097c061e2bb79 3.28.3: sha256:08bfe47df894ae22f2a1256f28b46345cc1718cd9c936ca8248ae5b761c33dab 3.28.2: sha256:9889a2f9c26ae82a501b33440b3a0772f552a4ece128cd57a21e395452b4238f 3.28.1: sha256:985caad36fed7b883a2cd4cf91e556974bcca95fe4e6b7ff4cb64d8d8fbe9223 3.28.0: sha256:0789cb0d1478ec3f0a44db265b19042be9dfc18bc1776343c7ea8d246561d12b ciliumcli_binary_checksums: arm64: 0.18.9: sha256:eaa2b3570d3737592ec912505a247173e25fc7bca92d16b32d72b3aca94a743f 0.18.8: sha256:44e6dd188336b9168986945c99f8e0882ec4e54a4b6422d44d8e36ec449ba929 0.18.7: sha256:dbaa2ab4b1969f4402adf430d6a1bd914c5ab52475ec68f50b3af6fa7fe2fecc 0.18.6: sha256:7639c69b410c26d0276fe1297d53e9573f094b56822bd01e85153acb3ca7dd43 0.18.5: sha256:54a517c05e1f60c4d3230191edcdc80647dc666173896a8e724817694aae4194 0.18.4: sha256:ee438745fedf3ffeb289a12e5d1a8b1a4e8f931a66835ad705bd491461051087 0.18.3: sha256:e0588268fc9ab6e0b7a363c4e15ecf69ed2a4cade956ab272745262e456f0e54 0.18.2: sha256:db3fae09ba005d6d345858655777bb5c972c9c841f98dc3fad3455d3084dba61 0.18.1: sha256:e6556fc7ccd071d7612446945d361c869dfeb423e0738147e0b46b2550bc2bf9 0.18.0: sha256:fd20a79875c8089694fb9b5dc3a0bf89d51711f9239637931ff0ace76ce78816 0.17.0: sha256:dee29ad27f3958882b450019e2021698282e8fcf8b136c27397798102cc1ad13 0.16.24: sha256:cf7f1276bbcf4aa5e6347d5619efe990cf1340d5898f8405931e277a1f76c670 0.16.23: sha256:7973302bead01c3f2e1d0f03e2766a0d6e76d3c52c666c750b9871a28b9afb32 0.16.22: sha256:b70c15e40b36ac34d59597f2448c5b4e0033964c517f926dbb9654aa07fb1e5b 0.16.21: sha256:7b710ddc0d3250f8c5d49a4cefb459b98c3c6651da9389d65cc727451fe3c4e6 0.16.20: sha256:4e3f50a6021fa8985232cebb0d9d33ba63e51f43f8fe5d657fc3e91179f8b96a 0.16.19: sha256:4a7a401d671054efb98dc9a8028aea82d66bed0bfc86ff008d823d1200d20030 0.16.18: sha256:8cd07955edf821ab72f7b765d9b23c196c0e69189e38864ec2470987c2b234fc 0.16.17: sha256:3a2ce18f06b4e88944b232fe1ef30678ab6157846628303da56c15e423d609fe 0.16.16: sha256:934232e6524f0236edc497664c491a84c7599eb909b7806c789e96b72bb92238 0.16.15: sha256:af297814428dbcf2da4c6c5c03a359f0e5592a01f882ddf39aef0dcd0542223f 0.16.14: sha256:f335a6af685d6cc47fe4b53bb3fe1eef4bb567d02eff11dc44f4c33fb058e131 0.16.13: sha256:2465d104b4e463cdd01021f04e064cef075dd93918eaf1ef4c6b02273de772a6 0.16.12: sha256:a093cd04332be79510a78ee6d7a35f61cb8dc068b7d77441fccef7e1b2dfe36f 0.16.11: sha256:b118df2e75b950335385c9ead4c59dff977c1e36c845320fe5308d4a2c8ce39d 0.16.10: sha256:625e880a9a5a9f38931a3e129fe2c811bd4883105b6f4dbe562734d48a082f7a 0.16.9: sha256:3417584bd63f36617b0556e0f327d45ad35aa462b4f3dd2907603f5c9040fb80 0.16.8: sha256:a88d984cf962a07a60bbe3da75494661cf9acd4166575513d7875c960349174a 0.16.7: sha256:9b6ef0f1097e3937127a05e44c9506fb476ef2cad88ecacd1b2fd9c70de680cd 0.16.6: sha256:3c36f52012fa3ebfed5c3959cc7e429ed6c3da640a5ffdd28bc0d9818db383dc 0.16.5: sha256:5d5692970af1abcdc22fcb38aec5fec770483ed4b5c1e17d26ffd4923f07a851 0.16.4: sha256:d83461431b5c853888c4213f9276991a95806960a87af64f2131b7e445ee9ce0 0.16.3: sha256:c6d5f4dec27a5937814afb2750194d15c6caaed130aabb539793fce42698f860 0.16.2: sha256:b5c48449c7d7d80bc537d9d231e1242e92fa0db9c7fef3c5e4198e21b3965e8b 0.16.1: sha256:07f3f2ba4d772140e46004ee7fa239134acc27afc4f79fc301ee7037617babc1 0.16.0: sha256:fe16bcd447fc6fe764ca75712f5832d7504845e9f782684ff09c9f52548237fe amd64: 0.18.9: sha256:15978aaf82373b0682aa87ab217848b3fb6e3cd80adad365d34696fe92543923 0.18.8: sha256:422940f0b7eb6eabc1e126945d1772e3f824c3f4f9fbb0df0dbbf00a271311ba 0.18.7: sha256:7f5ff96f1793ee389ef77435a72debf122f5ec253d41418fca99d2c21472016a 0.18.6: sha256:075bcc605308ff40a488d75a9a8555713dc0139d36536e032d3ebc2a1e7a9f4d 0.18.5: sha256:e63893745b67f58032d9b4f142ae7d6e97286df66af27ff24cd72dc81efc9ff9 0.18.4: sha256:6d7d2831380e8741cd46ef92e8f5132c66864311432fae5ab572608ca2d69353 0.18.3: sha256:5fe565f3b98b5846b867319aa76bc057fca37894d80db56edc20e4e809d10b25 0.18.2: sha256:1b4bd5fd5c96ab1195cd4eb56841c983a21149c62ee39922b7955f1cd0eda23a 0.18.1: sha256:c472639d460173e8d807a3f57048f9d1bcdb325e9edba320550d7ec62b72f956 0.18.0: sha256:3ac8bd270763e40a7853c73f8c7ec9e49707e1723801884a083dc25469b6b4ba 0.17.0: sha256:4ba0687ff7d47e182a7328409fb0eae123e64fa6099cd6f8b9bf240c0012ecf4 0.16.24: sha256:019c9c765222b3db5786f7b3a0bff2cd62944a8ce32681acfb47808330f405a7 0.16.23: sha256:e7cd3b982eca9b6214226536a147490ebb6ea3caad40d5a724daeea0bec5e3be 0.16.22: sha256:8bd9faae272aef2e75c686a55de782018013098b66439a1ee0c8ff1e05c5d32c 0.16.21: sha256:f5c4fed6f78b340a6678793c299cc5a8907e55bde9c9e0cc1759c26f6f37db05 0.16.20: sha256:8d0e75bf3a867880cb82bbec57027864879775997e3824737e36cf16467d9d04 0.16.19: sha256:211c994c9cabe1cb194f308510b78d8498681ee88771fd7c503e259e74eeb541 0.16.18: sha256:aa77d11150fac1e59431e5a50d5977219aae6c4bd9ccb051899c2ab99256f30e 0.16.17: sha256:77391d4c157e26685e030ec97c075e8cf82dda7fb11875ea9ee4df3bcb0e0070 0.16.16: sha256:2d493a70e08d0df947a19d6065f40bd3886554a7fad30b02a7839aeee7400c02 0.16.15: sha256:20a5296a9eca3aa50ad9bfa061805b5f332df9d25829f2fd34ecd42d0bf26caf 0.16.14: sha256:65c6fc67cf7cff7d958266642032386b4a9a715c502f016aff1761ec2c736b85 0.16.13: sha256:9e7597f97192f3d2cbe61a25d1a4aec33a43ef9c307bff27c3c1e81af0011891 0.16.12: sha256:535d3a72771f4b1c732701289dbde74ec9efae2e6b56aafc2b448773bd6d815b 0.16.11: sha256:f3afc4f32aebeedbdf082c0aa69e243809bfcfba8f60cb1bbf19abc56b68a750 0.16.10: sha256:aeb9d7c56108283a9fb9b370ec36b33f28f3126f6c4e6b4176a15cc6b2d3fc70 0.16.9: sha256:c6c381044025167a2dec943dfae26887cfa4ade0462f463466fcf82da5df8446 0.16.8: sha256:1903abd3055d2ddfccd5a4f6a2a45a00943d6fe9a14c889bf22714d46601f318 0.16.7: sha256:7acd60b565428203c3015a3273fae431bad0e833bf465a7460a5d4b432a2badd 0.16.6: sha256:3172f3ca66df44233f39bcc2c43c4c149ed90da29acd1da44e8efd498828cf93 0.16.5: sha256:699ac1b5f1793145477005cb6b7bd0205458057120e660bed4bc617d3457e261 0.16.4: sha256:cbc30655a886ee05c8ffb5eff3ad6f92d37f129c49f4515ce258658f9078fb10 0.16.3: sha256:0ff241d8c31d9f4a10d570dc2e9c8d35648bd9e5e21242d07e40a0c55c9485f1 0.16.2: sha256:9cfb7ee5b1d6bd125174675e8aab6f9a09283cde699b5ac201475784ecb864f8 0.16.1: sha256:9c3bb746e554efc788631629932ab00d13979c75799d957f6d3c17219deb6d45 0.16.0: sha256:da98675f961833d4ffd68b1046d907b228a7d394ded2abd70a50b20eaca171c4 calico_crds_checksums: no_arch: 3.30.6: sha256:b0eb83f6d70afac27e8830f22642cd12b0692e4d1a1b5060caa9231a951e736a 3.30.5: sha256:68bbe7f44693374f1379aa3fa55f254e9a689d070c26d0de26b2c9fb8d1166ab 3.30.4: sha256:be1d346a966a0be79cad7c9856ee5fc0eef8d88b70eba8d4d0bc4be057138bd3 3.30.3: sha256:f813232c182229da17658f59db511acb3997e51973eeb293b57ac0dbb5ccf791 3.30.2: sha256:57ccedd965f3dcbfd2a38a53b6b9e84e07232205fb83d23ed2afcee94590eed2 3.30.1: sha256:af066bd48e68c391dec3645b94d11a1ca513398ee6c56b5a67f0eb13f13fe21e 3.30.0: sha256:ee795478b6ab659ee172de74e1cf974871b37f10290dfc75f4d5eae887ba4ce0 3.29.7: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5 3.29.6: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5 3.29.5: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5 3.29.4: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5 3.29.3: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5 3.29.2: sha256:1620ee6f539de44bbb3ec4aa3c2687b5023d4ee30795b30663ab3423b0c5f5d5 3.29.1: sha256:aaa336bf0ef87495eccecae7eb65acaf59508a7f0a44dbeec933e05d73bbe0a0 3.29.0: sha256:ed35a2bd383674f4d61b013f2588be1ee08b5e7a26eb3208ba6a5565ebf0175c 3.28.5: sha256:541635bf3e0cd409ff2f5b9b78363ac8901da4565fffaeb4c1507e19461bf4c7 3.28.4: sha256:541635bf3e0cd409ff2f5b9b78363ac8901da4565fffaeb4c1507e19461bf4c7 3.28.3: sha256:541635bf3e0cd409ff2f5b9b78363ac8901da4565fffaeb4c1507e19461bf4c7 3.28.2: sha256:f09dbaf5b25419659af654f3b50edb3a2b1ebcfeab80b0e56f7fbc79721e8ec3 3.28.1: sha256:f09dbaf5b25419659af654f3b50edb3a2b1ebcfeab80b0e56f7fbc79721e8ec3 3.28.0: sha256:f09dbaf5b25419659af654f3b50edb3a2b1ebcfeab80b0e56f7fbc79721e8ec3 helm_archive_checksums: arm: 3.18.4: sha256:34ea88aef15fd822e839da262176a36e865bb9cfdb89b1f723811c0cc527f981 3.18.3: sha256:5ec62879f57d6acc0436440c88459d2a5c8de233273e73ff6498d79fd2d92653 3.18.2: sha256:a848c9db5e51f7cc4975bcfbba415c30cdfb67e141b6efc0e8b3a66cd89e8607 3.18.1: sha256:6a58bf4091f798ac1a6fa7e21ea142fe138f5b688f2e3af0c49147f99210fff6 3.18.0: sha256:88f6264801fd9c5bb3d2d24c7b3da4e239d137b39bacd18d25b22823e6bd31f7 3.17.3: sha256:60d76d1e12d3e058a9e9a8209eff748a6fab5948028a1f0860f48e141243d33d 3.17.2: sha256:0b13ec8580dd5498b5a2d7cb34146e098049f59500a266db1bb98f59649eb90a 3.17.1: sha256:1dc5ed54350f4f7ae87441e878be4f4fd9b727a86b11b1d20b1001358c83bed3 3.16.4: sha256:432e774d1087d3773737888d384c62477b399227662b42cbf0c32e95e6e72556 3.16.3: sha256:02ba2f3b1524113f49be6df25a0b4b3190010d6e218c8e2b2fde4578a8439a9c 3.16.2: sha256:f0f606d0806a518b749bd82e8dbfe6a803aa33340215590ef3977c60e366ba82 3.16.1: sha256:a15a8ddfc373628b13cd2a987206756004091a1f6a91c3b9ee8de6f0b1e2ce90 3.16.0: sha256:73efcd63d1b7f1d9db6afc2e039e03a638fe43d3633735363184692ebc8eef02 arm64: 3.18.4: sha256:c0a45e67eef0c7416a8a8c9e9d5d2d30d70e4f4d3f7bea5de28241fffa8f3b89 3.18.3: sha256:3382ebdc6d6e027371551a63fc6e0a3073a1aec1061e346692932da61cfd8d24 3.18.2: sha256:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d 3.18.1: sha256:5ddc8fbd4b17857754a95be799543ceafa5aa9532b05f738ee590a76bb049988 3.18.0: sha256:489c9d2d3ea4e095331249d74b4407fb5ac1d338c28429d70cdedccfe6e2b029 3.17.3: sha256:7944e3defd386c76fd92d9e6fec5c2d65a323f6fadc19bfb5e704e3eee10348e 3.17.2: sha256:d78d76ec7625a94991e887ac049d93f44bd70e4876200b945f813c9e1ed1df7c 3.17.1: sha256:c86c9b23602d4abbfae39d9634e25ab1d0ea6c4c16c5b154113efe316a402547 3.16.4: sha256:d3f8f15b3d9ec8c8678fbf3280c3e5902efabe5912e2f9fcf29107efbc8ead69 3.16.3: sha256:5bd34ed774df6914b323ff84a0a156ea6ff2ba1eaf0113962fa773f3f9def798 3.16.2: sha256:1888301aeb7d08a03b6d9f4d2b73dcd09b89c41577e80e3455c113629fc657a4 3.16.1: sha256:780b5b86f0db5546769b3e9f0204713bbdd2f6696dfdaac122fbe7f2f31541d2 3.16.0: sha256:fc121590b532d7f2037ae5cdd39d2b56dec96069d8bc613a08965f29a156e84f amd64: 3.18.4: sha256:f8180838c23d7c7d797b208861fecb591d9ce1690d8704ed1e4cb8e2add966c1 3.18.3: sha256:6ec85f306dd8fe9eb05c61ba4593182b2afcfefb52f21add3fe043ebbdc48e39 3.18.2: sha256:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce 3.18.1: sha256:b1c7e8e261fd30f34c617282813ecafc63628fcd59a255a9fc51b1fe43394c05 3.18.0: sha256:961e587fc2c03807f8a99ac25ef063fa9e6915f1894729399cbb95d2a79af931 3.17.3: sha256:ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd 3.17.2: sha256:90c28792a1eb5fb0b50028e39ebf826531ebfcf73f599050dbd79bab2f277241 3.17.1: sha256:3b66f3cd28409f29832b1b35b43d9922959a32d795003149707fea84cbcd4469 3.16.4: sha256:fc307327959aa38ed8f9f7e66d45492bb022a66c3e5da6063958254b9767d179 3.16.3: sha256:f5355c79190951eed23c5432a3b920e071f4c00a64f75e077de0dd4cb7b294ea 3.16.2: sha256:9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb 3.16.1: sha256:e57e826410269d72be3113333dbfaac0d8dfdd1b0cc4e9cb08bdf97722731ca9 3.16.0: sha256:327cfbc7ddc5a3eb644039ceb0cff66394628654c4f5a76bf715ed15b893983b ppc64le: 3.18.4: sha256:dbd74c59e7710f26e058596723abbf73662b553e01f40dfb08508ffffaeb7b81 3.18.3: sha256:ca5ab0bb205488276095881f04b72bfed5c0ddb92f715940dde6a7ccae72818c 3.18.2: sha256:1d21b2acdd79a13d20585b61fe90bababdd3f7047530d66aad650869c772b5c3 3.18.1: sha256:4d03617f742e4774ddf2170840ec385c67abf0db93e5df1aa9a036bb1275988e 3.18.0: sha256:559036fe183593488275a19796ca0b13f56e9d586b697a0d968e8b1e24472d7c 3.17.3: sha256:b821885a502b2fa159e3ef3afe9cde6e6c9876d4a623f18868829c3ee4a3c64c 3.17.2: sha256:6bb1c83078bdd5e9acad5793dfc9ab3b5b56d410723a660ff1da61dbdff3207b 3.17.1: sha256:4223394f3fca82a7f8e8d083caf6faf0ee0639d8f235071334579237078a2c2e 3.16.4: sha256:0ba4375a6dcf6117a8e7729fbed36d9220f8ad98dbc7aabc16186f22917caead 3.16.3: sha256:266f7698c56a724fddd3a2f2b862ad496c4338dce79f0282fdbc6e23e1738608 3.16.2: sha256:32a1b6073064a4a86d2a684180b6662ea202d1294b09ca52a6ba9d4cf071fec7 3.16.1: sha256:9f0178957c94516eff9a3897778edb93d78fab1f76751bd282883f584ea81c23 3.16.0: sha256:d13a4b87b31a5b50c8d93dd9988dfb312a61e56504102f466a4004e5a3ab8e9e cri_dockerd_archive_checksums: arm64: 0.3.24: sha256:c783a03735887c4a8fc894bd4cf7a1c0defef3ecf50a4d79ff31eed45c26b17e 0.3.23: sha256:a78037d2d2e9c52c48372a5cbba7b94b1c57be5759449beef29cfe03cbe6f14b 0.3.22: sha256:3260b214c9b12dbf0cbf4d60410c45aacfc31ba52aa7b74164135968e8950cb6 0.3.21: sha256:35de6b1e8eba11d8ba6d71fa7499cb3d610a1e7b866c9d43b7f87029e3a769cd 0.3.20: sha256:e6b4661c51c832ee1cbbb75d1c8b086fa803acc153d400454c3b8cf324547d89 0.3.18: sha256:d16204a4f01685ba67319adb3acc6a6f3e62d8bcfd87bc67f5e08f7332515a9d 0.3.17: sha256:761ee6db946ff2c8da04e57833dbbca8e5c38bc3f5b8e84bda689b1b7260c36f 0.3.16: sha256:0ab930b6f7ab87697cd94c954d01e19fdd5b5d65810662effb957c5db49a55d4 0.3.15: sha256:39e430378c9c08a1d5056426e3c772ae50416ffeadbe720fa63a853c475fd5a4 0.3.14: sha256:12c5538887ccea5ec363337dbb50ff52f3b020c93d65649ee1a906c4ec994bf7 0.3.13: sha256:a4a3d18f26902631b8323c6b6d87925f8ceff626b21dbc1d160aca84454f9b69 0.3.12: sha256:23520fc42dcdcb9f44b0edb555a5d760eedde52a230750e21227fbb258f514b5 0.3.11: sha256:877f635a7005b393f7aab24ca4b1cd7bdfb3b967d055e858408240c86e3cab9a 0.3.10: sha256:24d2d9cdbb4ed4bda4b0838edb52104ac7a4e2212a0ee05b177de0ae5b6a4a9a 0.3.9: sha256:f5051002b4f95b0e8fe7fbd5f8de4493350e010834d2a8b647f2b26c45c6c203 0.3.8: sha256:64286af171785f0facb72cf364867600b4db19f43a01db49b8b364f5d04aadae 0.3.7: sha256:8da54563ee7ddee36b1adf1f96b3b7b97ec2bc0ec23559b89d9af8eae5e62d9e 0.3.6: sha256:793b8f57cecf734c47bface10387a8e90994c570b516cb755900f21ebd0a663b 0.3.5: sha256:c20014dc5a71e6991a3bd7e1667c744e3807b5675b1724b26bb7c70093582cfe amd64: 0.3.24: sha256:dd4b7f514c248a3aaca398f467430a4c58aae9a77ea8b96a2f5b5d6fba0948d1 0.3.23: sha256:c7fe5db7f9396186193b58ded0e62a31eca7b3c58ad8691d57017986f96482ee 0.3.22: sha256:6621a96a885c82844d12318de00f510eae3459871cf1ad47317f38dd242f9a03 0.3.21: sha256:6c35838bc4b1aef74f9113670e114ca729a5f295f9457b226791e18e86e91698 0.3.20: sha256:2ce46d6bbd7f6a7e06e211836c201fdc2311111913eccc63a03f6ef4fe1958fc 0.3.18: sha256:937578ddcdb28c71afded3fda25d555e0c9e6d396668977ff98228d55886dc79 0.3.17: sha256:5568d571c2cfee7a31ce0b35b7fcc65b96c85b573ee6645151c4d022ed92a626 0.3.16: sha256:cc7f181ce850130dc375515c54cd8a27e1e862252abc5b7eade7b4a03ddabd8e 0.3.15: sha256:4779b7c3663f002871e79ecf6aa8eb48d0bb74df035baecf56b816deb21d12c4 0.3.14: sha256:89096dff11b60c8d4f061ee88751c895650a3ef6db5c19d63ab22a1aeb42a5b3 0.3.13: sha256:b902eef9bd02529e73a03078a1234c038cd9a12cd86af19970e4f8596f95df26 0.3.12: sha256:bf0890018cb339a313b6d52e994fe2628f5b6d5f005cec39adb171f055691c96 0.3.11: sha256:b2475988f3b86d85c7835269121171e35c92454ad5f4cd6252183b0fccd74d63 0.3.10: sha256:3e19ef525e02d2d1dfd42e8d661ee45b4bc8a49a6dcafd8baa578bdb3a23aeb6 0.3.9: sha256:a6d9b4b796e9eff830311a2349d259507302cb3955dd07b78296b91e40e8b433 0.3.8: sha256:e12ea6df8228b7d0794c930d32117c4e5a3dcf25a56c3facdf7006289ec6383c 0.3.7: sha256:518c5d5345085f36d311f274208705d7fdb79337a80c256871ce941d5a7d47a1 0.3.6: sha256:cf271d65abee88c0c0a6d9dacb151913bf37d25d45913a7e04b09efe408eae18 0.3.5: sha256:30d47bd89998526d51a8518f9e8ef10baed408ab273879ee0e30350702092938 runc_checksums: arm64: 1.3.4: sha256:d6dcab36d1b6af1b72c7f0662e5fcf446a291271ba6006532b95c4144e19d428 1.3.3: sha256:3c9a8e9e6dafd00db61f4611692447ebab4a56388bae4f82192aed67b66df712 1.3.2: sha256:06fbccb4528ecd490f3f333d6dcf22c876bd72a024813a0c0a46312121f4c5fd 1.3.1: sha256:e531c276312ec701be7edf2b998ac4220c858d02d274bef5546945964792d074 1.3.0: sha256:85c5e4e4f72e442c8c17bac07527cd4f961ee48e4f2b71797f7533c94f4a52b9 1.2.9: sha256:c155ae4716480be73a1c3bbedfa23df179800ba39ca83eb59f5b1b9b72a7ecbd 1.2.8: sha256:7c463d49d83f2da9f5ba6db395600c8da6c5445fd4597c2de8e8b0f82eb1bebb 1.2.7: sha256:b540e52e2e2d285b7ac9336d788195eb832292e4aa311dbd936ff7e8514e6cb7 1.2.6: sha256:12c612e2ebe6ca198de676ce75ed557e79fe6109032209bb8e25166c967fe170 1.2.5: sha256:bfc6575f4c601740539553b639ad6f635c23f76695ed484171bd864df6a23f76 1.2.4: sha256:285f6c4c3de1d78d9f536a0299ae931219527b2ebd9ad89df5a1072896b7e82a 1.2.3: sha256:4ef19ab21ce1ae5a01e1d3fa5b005e45cdf59f5d3ab32541c9e262cb2b2d3451 1.2.2: sha256:bfd3e6c58bd6060eaa725520c31cbc8f6386ac7606e65bfa7fe9084100aa1789 1.2.1: sha256:8c0d81c80ffdaab986629a9c787d8468ab41851e7aab8f9617a4c3674e192aaa 1.2.0: sha256:3d4f66dc1d91f1b2a46713d185a506a604f1fe9f2f2b89c281eb1c5c13677ff0 1.1.15: sha256:c680f8c470ffb228944ca80e1a4dbb6768b3ad97057350852e128847f9dd10bc 1.1.14: sha256:050ee97c266bf7d31e1474568ffcbb2a3ff2208087aaa238c8bbe7e398414126 1.1.13: sha256:4b93701752f5338ed51592b38e039aef8c1a59856d1225df21eba84c2830743c 1.1.12: sha256:879f910a05c95c10c64ad8eb7d5e3aa8e4b30e65587b3d68e009a3565aed5bb8 1.1.11: sha256:9f1ee53f06b78cc4a115ca6ae4eec10567999539ce828a22c5351edba043ed12 1.1.10: sha256:4830afd426bdeacbdf9cb8729524aa2ed51790b8c4b28786995925593708f1c8 1.1.9: sha256:b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0 1.1.8: sha256:7c22cb618116d1d5216d79e076349f93a672253d564b19928a099c20e4acd658 amd64: 1.3.4: sha256:5966ca40b6187b30e33bfc299c5f1fe72e8c1aa01cf3fefdadf391668f47f103 1.3.3: sha256:8781ab9f71c12f314d21c8e85f13ca1a82d90cf475aa5131a7b543fcc5487543 1.3.2: sha256:e7a8e30bd6d248f494aae9163521ff4eb112a30602ac56ada0871e3531269c2d 1.3.1: sha256:53bfce31ca047e537e0767b21c9d529d4b5b3e1cb9c590ca81654f9a5615d80d 1.3.0: sha256:028986516ab5646370edce981df2d8e8a8d12188deaf837142a02097000ae2f2 1.2.9: sha256:b21fda1d5a1510b19c3f6fb43ecb3a92887654a9d0c2b64e84b8b08f24ff0e30 1.2.8: sha256:09d2b25f31d3a819c97854c223ff9b36be7fdb0ce57fec709af499e1e19e8e89 1.2.7: sha256:7c6564ab7b562a275778e59dd6bd43404768e5462e31cbac6b15e36e14693ca8 1.2.6: sha256:0774f49d1b1eebb5849e644db5e4dc6f2b06cee05f13b3d17d5d6ba62d6f2ebc 1.2.5: sha256:fbd851fce6a8e0d67a9d184ea544c2abf67c9fd29b80fcc1adf67dfe9eb036a1 1.2.4: sha256:e83565aa78ec8f52a4d2b4eb6c4ca262b74c5f6770c1f43670c3029c20175502 1.2.3: sha256:e6e8c8049b1910fce58fa68c057aaa5f42cee2a73834df5e59e5da7612d2739d 1.2.2: sha256:a34f5ab4fc1df1f456293c3d797a76f2d41cf3cd970bb49fc53ba94bbc8a5cf6 1.2.1: sha256:b106d49c60e688022f5909432a77bd3260f29687199d47213ed87269588af781 1.2.0: sha256:3bbb68e49bc89dd2607f11d2ff0fa699963ebada39c32ad8a6aab0d40435c1ed 1.1.15: sha256:d218e1f8be4dcb1f288dea754faee342375a36f695eac5ab37fc8b7270a78763 1.1.14: sha256:a83c0804ebc16826829e7925626c4793da89a9b225bbcc468f2b338ea9f8e8a8 1.1.13: sha256:bcfc299c1ab255e9d045ffaf2e324c0abaf58f599831a7c2c4a80b33f795de94 1.1.12: sha256:aadeef400b8f05645768c1476d1023f7875b78f52c7ff1967a6dbce236b8cbd8 1.1.11: sha256:77ae134de014613c44d25e6310a57a219a7a91155cd47d069a0f22a2cad5caea 1.1.10: sha256:81f73a59be3d122ab484d7dfe9ddc81030f595cc59968f61c113a9a38a2c113a 1.1.9: sha256:b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43 1.1.8: sha256:1d05ed79854efc707841dfc7afbf3b86546fc1d0b3a204435ca921c14af8385b ppc64le: 1.3.4: sha256:268d9be1188f3efa82cad0d8e6b938d8da0d741427660d874ca9386c68d72937 1.3.3: sha256:c42394e7cf7cd508a91b090b72d57ff4df262effde742d5e29ea607e65f38b43 1.3.2: sha256:9373062bc547b5afe44fb0122a12aaa980763969d4b69dd17134a6a292838ce5 1.3.1: sha256:a1365d76be84e7ec06340a7e9fd31c0dc89f4e704b7198f2fb0c35788dbacc6f 1.3.0: sha256:156601012e6c473f2a5c7dbabbd08d8c56f151256433e6010bf4e5f6e569b5b6 1.2.9: sha256:cb1b63c25a102d7ccd7c590ebb57b530551134f2cd40fc9cfce39e1741b68c9f 1.2.8: sha256:36fa5415fc001bbf0b702c249819dd20d33d739199e0f5c65502fd213fbf5670 1.2.7: sha256:09fc9f9cfdca8820227a8fa76536b2f08488a51d34e987686b7f5d9d2d9c2d3c 1.2.6: sha256:0d7fffba4f89920edd3246afd4f07b18a975d0d97193ffae418e8418c236c168 1.2.5: sha256:3764385971ac719535425629e1ac4d451934392993779ee9e8e8ed7566715f5f 1.2.4: sha256:141fa41c1f382483ccf374827f99c7843414fceb95e8ceb710aba8bac984d016 1.2.3: sha256:6d1b771096000a14faae660465faf9626a76afe994cbe60581ec4eac1718f12d 1.2.2: sha256:9af46fe0bdc654c72593a937806ca034ffbbf4f62f25c1de7a40b5b0f4374de7 1.2.1: sha256:652920e145b461151b7e87b28b339594e62129cfc87370b03651a37c39bbc0df 1.2.0: sha256:0bd876309958ec00a0e86df3f549f025ad7ae32d981536c1a2932465b479be70 1.1.15: sha256:b4c8dbbd4973b1cc69fcade012db690e26e0b354d5fcdf04a12ffe972d5ab098 1.1.14: sha256:31630474455c4208594623f1337a06556c7fe42d3983000ca800ad3bef6d8164 1.1.13: sha256:4675d51dc0b08ad8e17d3065f2e4ce47760728945f33d3092385e792357e6519 1.1.12: sha256:4069d1d57724126e116ad6dbd84409082d1b0afee1ee960b17558f146a742bb6 1.1.11: sha256:e3d1da41f97db1bb7e9a8d96c9092747c14ee53bc9f160048828e63f3a2d0896 1.1.10: sha256:94a091c06c363e4af7be398dc31fa6e02576d5ecda6de1cbf3a08fe8662bf678 1.1.9: sha256:065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6 1.1.8: sha256:a816cd654e804249c4f757cc6bf2aa2c128e4b8e6a993067d44c63c891c081ab crun_checksums: arm64: '1.17': sha256:3049017b99208f5ecd15c1366f47a77dace87f42dccf317ad40a07f1a867518c 1.16.1: sha256:973817340e6da12c90c751b011c797396940cca965cefa74557bd1c0939f4042 '1.16': sha256:4595ff16487b16d2158fa8c3452bc0e1ecdc177ab2ace40fc02cd6e49838ff67 amd64: '1.17': sha256:e9512a3e034e781b2396d068fd24eafcd5788e410403da886df9dc8871d504a5 1.16.1: sha256:7b6f1791fb9b2c49ec959b9384b3c4e2ec8c69945fd5292a179d23eb62422eb3 '1.16': sha256:7f53bffd6b0e216f8f6d6472bb73dc4c6c4ea2c2e7342c52d4bee2972798ce68 ppc64le: '1.17': sha256:ca8ee0fabcac57b61b80f6c234ae20b3b9821433fdf1a6306be5defeac11930e 1.16.1: sha256:9590ce79697c5509731f8e58d1733b7051c36f92104925221ca8bda800afee41 '1.16': sha256:fc7199a2faac1ca0e3e58dee4dd369b9065aa0d95f3257d8803e521213f1bd9b youki_checksums: amd64: 0.5.7: sha256:10077b1a4f013990a416acf15e6b397cf64b5d62008516a9711abc22730d8203 0.5.6: sha256:c372ad70d4107174064225848e686e84ddb597cb44ac7a540e0c47a86e6e6081 0.5.5: sha256:7a6844c2e529daa54c6d7558572269508faa26e3b14b5440f9502c413fef558c 0.5.4: sha256:69709a8739c868071e8e7e4bdcbcab41b28524c2adb2f0af0c1a4d6ba4691ae0 0.5.3: sha256:173b8998cd0abf22e38e36611b34cc19a16431b353dd893e3d988cfc77b4e6ac 0.5.2: sha256:361c7187939eab02039fa4289d33158fe4bf4d21ab8cc7139ace8f52081524ba 0.5.1: sha256:554dea487dcb54b34fff003fc047458cecf073337d20b77e69c22918c2986aaf 0.5.0: sha256:d7ec0cd5f69ecd96b41df13cc38a970ab217aa9c4b2262ca4843a68595975d91 0.4.1: sha256:6f7ea3651b284a808f344ad40f9f8315b3b02b76a3ee6c6af7bfff65753284c9 0.4.0: sha256:7cf3ae3d1be19a731378b289000cb36ba9cedcbff8a0cabe38edb782e8c55f72 arm64: 0.5.7: sha256:d51990751e796391254c36d40f05b0624017cb544284f9721a645d05747b3d84 0.5.6: sha256:df078232fd5e4af822be2d35e0614715dfa781a3e5790b66b71d1d89a2247692 0.5.5: sha256:83069dd0ac11dca04f5a2376f27538f86ae5304db0ba148b20d342afe62d6069 0.5.4: sha256:2a8d55a7127c751d088ab9ef989496b700afde414296139d5461606527869c42 0.5.3: sha256:a15dfe9a1eec2d595b9a972a8a0fa1a919ee3d3523e77ca8c22099bfadf7e88d 0.5.2: sha256:8df7e4898088e6fcf942f2e145b0a50c1ad81f5109044e48897c0af9bb3f83e6 0.5.1: sha256:ed8e2f5ac03553d0a8ac641339a9f94f42baee652e2afdc1afae08529807a796 0.5.0: sha256:1e504e01caf396e8ab87de24bcdde0adaf6251ff2a3dcd0d8622cba0132caadc 0.4.1: sha256:c954397353340a013d77d63cda9b074e5f1de57e06fe6f8d44f508b5dc03c998 0.4.0: sha256:0d897debaee98a937188317304a9caafb0da5bf798717aefedf113c108f73263 kata_containers_binary_checksums: amd64: 3.7.0: sha256:bebf218cafdc082476c7dabbcc5439aee6a41d6dda24dd3cfffbe0a6ae94e23d 3.6.0: sha256:d418859e123d04edd71e79c8f532213e5281ec7170b97cdfe3cf85487187d60f 3.5.0: sha256:cd0e6a70828db3f9e27390f08c863edda36824c201c35eaa0a4c7498b7e4c9e3 arm64: 3.7.0: sha256:23e0d5c4e38c6944729a11c631f947abc5aab19427a44d8691bfb9ad1c38b831 3.6.0: sha256:38277c8d1b9f3c4ea1089338f9251238551b88a5259e8e8ed45fe392ec1ed4bd 3.5.0: sha256:fa4cf67d010244c4f8d0e6d450d04e28d1bbce5ad1a3cbc0154adff628d56c0c gvisor_runsc_binary_checksums: arm64: '20260209.1': sha512:e95170b4f70688d014c795ffa9b3d583753f865edfd8afb4e2969490869bdb46b60672f641741f788e2ffee8f29751a017e9a68b98c1e44f5194da9a64b0ff28 '20260202.0': sha512:5fbb9c68efdf3a404217fb57be55051b4b5f8b83ca631101204615b87ff5b6ea8680cd6599e434f1d87fecb9071367b65e90cd8ad5df3f0b9f0101796ecc8c43 '20260126.0': sha512:c1b42f5789c09a68eb006964048448c058776440477fac83c7fd9cef879cec40878fb2f5f2450315ca0e7f568889f0b52c842b84929784a57023961f6eb77d04 '20260112.0': sha512:3b7925d26d71fdcb8cb552950c88bcfed658c06ad6b1211906bfe86d13bc56d8005ac90a4d9ab4c8b6a48eb62ec51ebcdfd45a64067ac5190274e710961e51ea '20260105.0': sha512:cc98ad73e8d181f4738c97883180bc76cf8b2eb773c11f3a44f1636d0b0e00f2ee9228e4eecd414f94d6410f4877e6c93260b8070130fba767583026115d1038 '20251215.0': sha512:5e7d6206bce4164c9109d37dfb0b169d1c59cc256910de42799a868c3f9ba5560ef5c05c0de3fad4f0856f906463588ff25c9bce3b25e0d3f20874521dffe767 '20251208.0': sha512:db07dc2def9b1e0b13e17bec5f98e9cd794159955ac999432fad16d1ec747924a05cd5e854b4d45f11147c090208c0ce7d915a0734cf2960047bd4daaba0465b '20251201.0': sha512:fb527cea4d165478f297a918734f10acabf5230a4a0d29b19709cb6a69a389d32c2a0da328146f72ef0d8776aca35d97647db82ff46be60e85ad02305f631896 '20251118.0': sha512:80a2970cb966d69d59313ec64583174189293db24605f8309a9e8b230e3be6f0e7e387bc11dd5db1896a8c308dd81da8778c0f0418d7ebfdda6b26f03c8d499f '20251110.0': sha512:297d42a46463d5b68c4786bcd448fe0914d7af91cd62f3c51b494b94e1c91d7eda68c6f21cfb1a66ac4d45aadcf20f7410291c0f3f17b090799f9cfa676cb563 '20251103.0': sha512:a124f892b6f937ff88f9833cba78bc22d2cf869a205e20694374cbad575c9c9dd501cd4b8897aa4338622b6171d7cf4ce8d5a9b3b259c60450a6ade5b3fbc4f5 '20251020.0': sha512:683b465e68310c9e633213b8663095a8967529ffc1b1b44f7be31eabc53805b46f590b5da1782390f039fa71cd9566eeeebf0c58def1eb59482ca2ff0c7601df '20251013.0': sha512:5b27f834190173fcae4ac7e1f0658bed093b1aa18884338474989977f252974ab856242abe5e4fff3e40e9e7e10f15391745805f17dd9414f6ec8f52bc1f2d7c '20251006.0': sha512:c4d3eab7b556d5f8108083b398f8a967da26ea28a7f61dd56d690aca4a9123f629278a1a63bf8e93d586830fbd496760a412d7103c406226c9dea9443dbb5a0c '20250922.0': sha512:a31a71a24d20e5800c76bf44a9cf33af0dec063c8657fac65805994eef6b136bc66bf4780a4c8acfe0c9d87733c30432b1e4c31cf9994a23d448c8af0865dcd7 '20250915.0': sha512:a31a71a24d20e5800c76bf44a9cf33af0dec063c8657fac65805994eef6b136bc66bf4780a4c8acfe0c9d87733c30432b1e4c31cf9994a23d448c8af0865dcd7 '20250820.0': sha512:02767f2682b088558d55b2874732e2668803eee06608654a27429b2f17428b469ea3d166f0d281fd0e081e2b57df86e50dea5bd9fb2286d08edc120b26516ead '20250811.0': sha512:f37d5629be1877937ab906a1a5047430a9c16bf15688e3068b589e3659c4ad176faced985f16abdb120ecdceb1c149439f4c2ff557c40a7699502f11edeff9d0 '20250804.0': sha512:18c66aff5fa090f7929c57a30157b577ec5322718693caa4d9fb404e13ab531e0e77fc55ad3470217b3808ee68876670f38ec950147d9ac13857ded758f09528 '20250721.0': sha512:e419753ffe2fea7ba74a033156d445236f531a52cf4976b4b40d50a4071fc1b29ac5faac9604fd069e1a6d9553b66f75f649b3437c6cc5afe9e7d071c1415dbe '20250715.0': sha512:e49ef9b557c9800f16156c9b79b44b2c3ab2779c0bf4ed5ed2a9a666c4e63684bd954485e4db94dde8cb69f46fb59794d9716c453921b8c69ddd40d8b01c1e40 '20250707.0': sha512:c2f7539edb022c01912eee358f57f0fdee2c7333323e46f53a3892a3d367746cbb5cb472070a3ebabbfde4491009613a663f9ab5f2bdea7059e21ea2764614a0 '20250625.0': sha512:3016f92fdcf4e0badc801ffa5502751c0746efce50dfbc2e58613ee0aaeb44e4ea2dffc01a52703d24348f815c1c16af1eb77ec463b4ad0f790ac54fffd041bf '20250616.0': sha512:b1e9af177411faaab3349e1e04924ad4b7b19103c974962fe65235943a5856e87b8f9f9261bc6b1f014237dfe443c833c1ae12139228ddb49c4745e6c848f414 '20250611.0': sha512:6eba75bd816329bdb909c1d7f80dfd01aea88aa8f880740a1eefa348d2211b1460975da559d851609e3b68008ebf89643a23a8854413b032e5855e32fbb8246a '20250512.0': sha512:00e9edeb4a9ae702c9617a583f2978a042f20a05807acfa992bc76de4fae2e6e1e994d34ad6f21c826d2cfdea89f6a163c69c0750cf4d90135146438587a3a8c '20250505.0': sha512:1611599c6788d3c3f7495b5054aaf9ec81e7a714061582f913359886452fa14f8e65b2bd2d139bc24b5955749167f0db03aefaa6b3ae175296b56814f53d7898 '20250429.0': sha512:bd58d212088263ad998fa62dbc7f2a8f74ea3914e8a7a319813c3e461f297dcdbc3e85069aacbcaa8c2e573b0e7b17d730d21ab96f8c3ca9516bd43acc070330 '20250421.0': sha512:647127e139c77d5d360db915d64a21f461fc11ea47d3660feb48952a70639155cd8c19e2bbe16d190a1666c6f689c45bda2aa5d3440596ef174983fe41d8539d '20250414.0': sha512:d1ba68b20057622e58e886f472e021a473222590c936a86951005d7b97366b446ef0342b91457ffc0d7e543d54c9c06a363f2883bdd6c594799c4ca1091dabd5 '20250407.0': sha512:cb590f72b0fbda45e89a2300e9247f12ff295a8c52653c8cf815c662d3fbbc774f9b915cdd4fad59e30694d8cc8737fe2a1a8186ab5136f7701bd6e6877a1662 amd64: '20260209.1': sha512:1e0e42f7d3f4b3eded4e96be5af4dcdbecc9bca7ce40f5b9fa191210690397d71771c7c0e0835c32221261b004250fe513a9265447e62d9bf92fb6a5f7276a68 '20260202.0': sha512:f7bb9cc5e3f5e36a6788f959361415f6d7f7cd0225b8b4d99728da4b1ac7e5c7ce9c72b4c61e424ba93db77c983109d56b54907a3b2e2b982b34058410611023 '20260126.0': sha512:cce974fa832c50d26c6ccc08ce50b4972921cd0818ebe8007587211d360cbc828ceea4ec8296703200afa208b679437d24f27a6dca31887b3c0fc6ee8be5eb05 '20260112.0': sha512:b36de90cdad4cfe0b9b66318407da79c035dd6dcf4c1374250011f34e511c0a29e335fe04eabb0d3fe7140131925f619f724a4702b37c49557bdeb25924b4dc8 '20260105.0': sha512:15c8adabc9f1006d469177b0ec3962d4993e01c85be17d381a4979029eacc7db37ef354e3eafd279573135a1adf81baffc5c19f2bbfac932c79386f6ac74e52f '20251215.0': sha512:ea82bb66ce61a80adb6edaa61e2f2b1cd6339c504a55dd6663555010ed7f96c6234ac787bd9ecdb29ed4058e806e829fa45f14093466913dafc44d56055a5acb '20251208.0': sha512:4b9a29a6f887aedbc10de5f5f0900eb64026c3472b5522ee21a6d2b3d30ac3ebc084a78b97e371d3bf830dcba4f61a5809922ea768650d52ef120221b4a9b19f '20251201.0': sha512:8534bc833d9b1e286b8876abb17dd6fe202c40a75a36dd62b0ce892bf9dceb1773e71447848e7acab120ce99283c22d2f4e4a6171008c9c5f3d5fe6ad6f1cc75 '20251118.0': sha512:cc95eec3e22a574ac533278ee8c72672542edf0ab467a89c13f02abea6404ffe20ea4a538a3482b072a8e45222a13cefbf9e7f44bada35b436769e04b12ab970 '20251110.0': sha512:40d9ec839850cb1994321f0716026b6149adb712bab576b157be2c31b832e68e11475647b2776499fca4c52e96fc7489877fff2fb1985d5f1e128d14f776bd6b '20251103.0': sha512:01a465bb5bb37d3c6343a33420b6badfe6e5d8a5ff522f1fb2c183a6e24559cc660373137adc8f5a8c8c362c573a2d01ef6936e126c74c810b16bf9bd19bfd04 '20251020.0': sha512:8ca31116bdaf6ed4ed6f5475438ac55392b822e98227338a967e4d47bad84c4a96a194f9f33659b7d52c18417a4d55b727b2573a1c8eb8db569cc65e85c38984 '20251013.0': sha512:923362961cfca5594468669a3ba59fa3f0e210ff9d24b175ad4ff767334d1440fda08423a4a598c64ba2b3e29c09a82eb947a2134d8eb8156e96e699a3b692ae '20251006.0': sha512:2086280196df63be2a983e15b594bd9dfbb90a177160e3dfcbfc73bf38dcd31705e62a4ac6d2a3c17e9d2afbd6699dc9780e4cc2bb0ae877ea375d10925882cd '20250922.0': sha512:aa008e497d50cc97eb3103645a80ede5f40f13b0d55c675ef3774ecc60cac299e3941253daf9fcbbd097294d7826a2dcf2343eec39a6b52646ec9deaceb3c9cc '20250915.0': sha512:aa008e497d50cc97eb3103645a80ede5f40f13b0d55c675ef3774ecc60cac299e3941253daf9fcbbd097294d7826a2dcf2343eec39a6b52646ec9deaceb3c9cc '20250820.0': sha512:d6c12a4cb4f714bfcba6fd6611ad4ca73fd88dce790a083d2ceda807cd7e074c0131d5dd2a3490399e8be91feed9afe450793e9708dacddc4afc99ee6e5c3d2e '20250811.0': sha512:95cc8973a8ba6fdea608c36288afe83e17a890398d387de89dfd1457e902ab1d73fd3bd52a4fc2b923accd36ad5d1e76b5ea373e9c68d9821efb1785f830892d '20250804.0': sha512:c9572008a35e812277b158933e7f549b734ac0e52349398067f3de1bd42572dee6a0911e85d2737b33e8adbf94515b630d57936058cdbb199b02a290a906ae5b '20250721.0': sha512:cafd0256341a5e6b32b81cdb8664943d82a80f55ce10fb4d5061e1ac0c8a767946d8665055bb1529226813b215dc545067a3a428b9673aaec85a32455a82a11a '20250715.0': sha512:06b3c36e6230a4f106914b7a82c5218724ca574b6c4590bb9f50796e4aa257bfae8b621b346561ed9a6e31c1328d447cf7594c6d0fc26532c3ba15171f18e7f0 '20250707.0': sha512:348687c9a10c23a51da5dcffcd9e1866250b2a964aa1f599ccd706c41bf0b85823875d6edd5b4dbc1f2e9229eed6d9cb13193fc13b988d3ea614bed9b4ebe955 '20250625.0': sha512:2193da9a4a2a072bfb1fb314964528ac0da4bd56482552458da6ecb557aaa2f2e939fede3545933608ce2daf43897fff173f1c272375a331647a2bd27588f3be '20250616.0': sha512:516e39e16ed05c69d4173a408b0a4ab9a24ac3d1129a35f01173c76e174d3aa9652371c85445e5d45aed6de1a0f37507b4f735184f35ee8e6f3d9af968a6659e '20250611.0': sha512:669c9eac780242ef966ac09804a9448faace00f91e2b1f2f5b79d88f214333e84f8bab4304ee48db59c715dc89685787adbc5fc34f8ad3bb379852a88b1d9bdb '20250512.0': sha512:981a554ad63f7ed082a43be646b8e910481e4bfc837c5ee5dd5a1353a47b0ae337f9b02700649a542db864ae35af6981e6bdef86c6a48a5e47dacfb97be9b7b0 '20250505.0': sha512:25705616c3cfc82bb5772e815b2b6b030664dccba7a0db9babcfad5de46d16ce8bff8cd9cc11d366da4acd0f01fb04a0d95bbae070aa923f1492d2f142f271c3 '20250429.0': sha512:b91d0351907290fe159cf041dbd332f8d2d4151d6a7aaafe161cd842452551b98fa1122e195e2cf42801eb8ff38716270de4f33331dde784cbfc452ec1e368a0 '20250421.0': sha512:419f80c01cef46aaab0a0eaf9be4bc20fd3aba94e8d0dd8ceacd3b166139d5bc8e701964feb11bf6de7a4274924692a7d0b5bcf5de34f5dfaeec57f7f1ecd88f '20250414.0': sha512:cc629d16ed0483bfa42ec270f409349f02f3bbd27db1fa384c3146cb54e85cabbfeaf920d1fd189fcae2ab11a54f9ce481d3c4fb306ac18e035943aa02144fbe '20250407.0': sha512:097259d6d93548bf669e21cfec5ba6a47081e43f61d22c5d8a8a4c0c209c81ac9c4454162b826f98cec49e047bbdc29c270113ab6db5519ef3e6a90f302fa47b gvisor_containerd_shim_binary_checksums: arm64: '20260209.1': sha512:714ad3a53a28aa4acd891553d848278f5a873d0a1733836382eaf2bf701d62ece9cef324390602d2676af5e2e3a3d329486d2b18803c9cef5685220764757eb4 '20260202.0': sha512:714ad3a53a28aa4acd891553d848278f5a873d0a1733836382eaf2bf701d62ece9cef324390602d2676af5e2e3a3d329486d2b18803c9cef5685220764757eb4 '20260126.0': sha512:84abf41b68ba450ed2cbbdf544e7d347d30f6fd577572e2e58f2fa8e038689f557953148287e26c8f4ee5040c1e928670f113bebca6d81ed7ce014ec4e0ad256 '20260112.0': sha512:3215952718bd1636173649c4742e3d8e1978c410abd71bb8252c8ad6d28130cb6d66684aa089f61a0eda0b8786553620a08a9f1b5ab824bb27b1b0cf47bfb25b '20260105.0': sha512:cfe8a07c304dca21171e5a76614ac3605f5b1ec8f9ed2eeac014a44bc00821864f219db0e25fcc1c56cedbe335bbf34a7fa6bc57335888dcd04278bc0263f5cc '20251215.0': sha512:2b3a00ec2d646a1c26c1944781b5caf039ce7035dd72281ccff8e244af55606e01667de311febee1a0a03ebd2633af6ebb0ad72d27b8a966743ffe31563b3a5a '20251208.0': sha512:f3a6d9ff32dae45c62ae831580e5dfbd28fed38f1ca9daf09e6a9960a5373da7e29fbf61e0846676102f053ed38f23a0ad41349f5326fb3a2991b296d33c853a '20251201.0': sha512:9546236a7ddad9a2ccd51c41f2f309b7f4016fdf489581f77b1b803ed73ca72501af2de3e3d0b58daa633384baa0d46ecd515760165ed51bfb6c0900649c6306 '20251118.0': sha512:0179f0b049c882703758d5cba387e1e4fd0300aef20197e35e2886f480f0668fecb8deb3aa84341d0b874127d88b337fbcf609f563c3310b47520e8144e9d55a '20251110.0': sha512:8f2b16ad59e9ffdafd1218851cba9d007d4ffb15c5ec2003e0c691eb048935a82f9e8b578c051b05738e3b4e1f141ed893c73415313ec639f348fe989659b893 '20251103.0': sha512:8f2b16ad59e9ffdafd1218851cba9d007d4ffb15c5ec2003e0c691eb048935a82f9e8b578c051b05738e3b4e1f141ed893c73415313ec639f348fe989659b893 '20251020.0': sha512:8f2b16ad59e9ffdafd1218851cba9d007d4ffb15c5ec2003e0c691eb048935a82f9e8b578c051b05738e3b4e1f141ed893c73415313ec639f348fe989659b893 '20251013.0': sha512:038c3a7180520538d488d6dead7c04555d82060d301e3d785b4b6315a33e51b20b7895caa8dedfcf8cfeb0f355ea7aea579c2ea34ea46c5aad408cc962dd282f '20251006.0': sha512:eb7b3aa338ad44a72f1944eb06edeb73c1a1633ec4f7961e4e6262420a288319e2bb30101802b497e1aaf5b34a7438dd1733571249db74f00846e7a20cacd681 '20250922.0': sha512:038c3a7180520538d488d6dead7c04555d82060d301e3d785b4b6315a33e51b20b7895caa8dedfcf8cfeb0f355ea7aea579c2ea34ea46c5aad408cc962dd282f '20250915.0': sha512:038c3a7180520538d488d6dead7c04555d82060d301e3d785b4b6315a33e51b20b7895caa8dedfcf8cfeb0f355ea7aea579c2ea34ea46c5aad408cc962dd282f '20250820.0': sha512:0015d061af2369a8e5e21dad6f69f0e3cb03c6e396090d0e127c6638d646dac8f4e8bd60ab901ea87f6ebbd093e66db86efbccb1f225dc1dfccc76b8861976a7 '20250811.0': sha512:70a3c4212bc4a5ced31313640d5696af5d3bd11fc06221af2ac5441b2527b82d0c1671ad76ba5c5b120108912744477a2a922cc8444661fc50594b72f146c2f7 '20250804.0': sha512:a6f7c8e76969588de24f75965ddfb62eec592d03f1882bfb746d7cf7a8a86949f8f441a4a063d7a5c766e6cf576c8d2b9ca7a1918de4902d1053938b6cc382c0 '20250721.0': sha512:ff38aa2218ae24f7747a85c8d5f5e8a466a68a73c2b7f8a55c1cc8cf7eb054e70924685b491f860bab613169bd67c7bc9113e45884028625d6598c9fd8dcf802 '20250715.0': sha512:8a92f6334b93fabcbe251c3d0877d7e7dfc22cd26b317a758e20f4cf884002edbfbef3137fb899bf9515f72d2d8aeb4b115902e2decff0e136979f4045d0e7e9 '20250707.0': sha512:2f891792ae6610b471111ad917ddf69b2504ba741490e8e120b268b5e1340c445af89512c52a5c3376ca31dd857722b69f5dd0e79e3cf81d9ecde021054a0f73 '20250625.0': sha512:fad846df0dcd29ff84280fb2e01e3edf51a8648cb24baf53d9aad46e35891fef215863a4a3a42c4a96bc3c359495def4d3ee25985ae4e99df2db7bdbb318498b '20250616.0': sha512:5de1584ae09bf56d40af000f656f1cef9357d69d223ae40938a77b05fcc8326a3ef5058f7a9860471205df072923c2e1a776b85ddafdddad0d20f33810456307 '20250611.0': sha512:b84bd331fb9abb13c5f2260de04d536b134b3e71e9b6575700eed64441023c34cc1164125bc0485014782683e4d177b3c85c19e5c1ed218820a505c57e286922 '20250512.0': sha512:43daf4b8b0e094ebf2cede8bbbf89ee0695ff31924e140bdfcff529296e8f004b457485b9f991ae9ec93cf6150535e297db00a92be8a054589b3316841fbc056 '20250505.0': sha512:42cd72f9b2011a8ad166d9dc246fdb46ef602aae43127373750a7ff65be84f8b300c50e4977495ce59670af5fc5f92c3c5ba96c5d751cb4e6e2fcce373210e06 '20250429.0': sha512:9a9a2c351789e6a14896ec5e56ebe7ca1dc7424087d13c175e38a4522a7e6f1533ac8ef5aeea0bcbd554cd5e4d6b6d7ac3df2dfebcbea3e7164bf00fa823c310 '20250421.0': sha512:c86577ddb8b7b46b5b050000e242dc09bebeffa7cb9d21acb84c4ef896cfa340f024e2b9f463fd4f7945683854c524f4a45de3ff3917f4ba65552cede4229974 '20250414.0': sha512:33b9c67bc7b73ca49154aff48da52029414a707b6a3a25eb4f71e861a94dec8fce220e63a162841670ddd4876f45b0e39abdf9f8c3235019c89f209684d3007d '20250407.0': sha512:1c3838e10c905af0cb52697712bf6bd76b94c9e9d3d07a7643cd43dc2f8dab03b4ed4693c117e555e07a158e04ee583b6b1f1cf2fb9705244ffa5fdc4af67248 amd64: '20260209.1': sha512:bd21b80502be25484d8b43168c88d66b6f3e853c78c0ae5b5206c5625e2a365e98c8b3ba259453d18c01d1aa08fb7c8c1e7f122fdcd7ef806bfc2f44f5837b5e '20260202.0': sha512:bd21b80502be25484d8b43168c88d66b6f3e853c78c0ae5b5206c5625e2a365e98c8b3ba259453d18c01d1aa08fb7c8c1e7f122fdcd7ef806bfc2f44f5837b5e '20260126.0': sha512:51c3b4bc21cb5c3d4e3baf9f43e5fecd86c327abf0c84d492510f480cdfb38c90d43f3b0dbf1887ada8846d3806da79a73729acaedc570894ba6ed7cf9e083ed '20260112.0': sha512:89f55750488559796fe51d2c10c289a8b0617fb9f6498714c026825268eeed449941d23e8cd5b285b69c1b032005ddeec278345198301c50d89ff6d3f66871a5 '20260105.0': sha512:7f3f5a864fda5f4e2de9db20dd5edad60b6aa467cc7c22d13f40cdce811783d66018f2c28fb74b907c6d6ac0e39f6d0e1047f1f33447b8a8682f1fbaa25edeb4 '20251215.0': sha512:538a04d88a39de1679afd9868806bd5fdc63737a4871955fc8a8c8e183942c6cc3dbd6b34b2f5589f5f474b4826427f149d5c6abec4ca8d09db363ff5f149b4f '20251208.0': sha512:8f1e41374785bdfdf69c5798cfbdec53a833ff6724d36dd644a387b2f6e151c513389b2e5b3c0d5347c5c2ff214910db3c8c164b4d5bb2fe963c5a5eb70ca1a4 '20251201.0': sha512:216a937437cb1747d5e84edd9ae7274c5a2c4f712f4601e7e0ca06e0a688bebfac267707028b78845276302023d305ec9a93f5b200c9c3c3cdf86a2f41817703 '20251118.0': sha512:b0f0fa1ee431c63cfbb9007a62c49a374bcbfbbbf5997e63c827d1673f6933d65044ca4f06608bb494f870ced97295dc065810f5e905dfd4a632fe4d61faff7f '20251110.0': sha512:56a27dab74191db97f888c936b53861248851a2579d838073f528db7cb9353da5a919a27a38a48447b0a81bd42ab92873c480be769a9818a464ba9cf27872581 '20251103.0': sha512:56a27dab74191db97f888c936b53861248851a2579d838073f528db7cb9353da5a919a27a38a48447b0a81bd42ab92873c480be769a9818a464ba9cf27872581 '20251020.0': sha512:56a27dab74191db97f888c936b53861248851a2579d838073f528db7cb9353da5a919a27a38a48447b0a81bd42ab92873c480be769a9818a464ba9cf27872581 '20251013.0': sha512:e5478ab26ed5e20a503661ccf23b39a45a2ef593e5ad2d8db36c4d44917747efdc22f6af8e76e0c0eeec91e28cc8efedd9c0ed640cde7c9d1843293d47b19196 '20251006.0': sha512:5849b045a3aa9d26e5850b18b96db8566781398a26db43f02087089e388b6fad05072dc16f1e6024ff725a1d60dd14c3428a3611c781060f973b196db594c304 '20250922.0': sha512:e5478ab26ed5e20a503661ccf23b39a45a2ef593e5ad2d8db36c4d44917747efdc22f6af8e76e0c0eeec91e28cc8efedd9c0ed640cde7c9d1843293d47b19196 '20250915.0': sha512:e5478ab26ed5e20a503661ccf23b39a45a2ef593e5ad2d8db36c4d44917747efdc22f6af8e76e0c0eeec91e28cc8efedd9c0ed640cde7c9d1843293d47b19196 '20250820.0': sha512:06ae1d7647d3ae2155fe374339ad7ec2fcc0d84b3d8d4cef72e1c788d6fe17d6aa227e6dfe1214335617d8d42a06aeb495c65fa53cfb4444e01dc7ab2f22a86b '20250811.0': sha512:e8fbe414831e50d20c9f7e3046674a2b609b01a8a7814334ba33e1d6d4ab370525cb494069a782f180de82db5fb6a72c35d9e8cdfd9eba2b43937c474f8c59e0 '20250804.0': sha512:474113f2c6454738b276023e96548f4fa1bb84017160bf06bb0aa2c6c5b27c12fd1d5fa0d5f9c39e0e91ccbb69525b303018a13b24eec18c3139db94128c0a74 '20250721.0': sha512:c6305e9e5b0f20ef11e681f0901c6fdd245a79880fe75861f8489a816b4c26f6080d54e210cb827150a4347167c25f2a698d586bed8cc265095bbbcd678d28fe '20250715.0': sha512:c6ffccc15f47a9ff69c6f9a5843c93025e8bd2684846a973a8b483a2e8adabd153029864124404de66606a5707706ec79eaa5d0a9ed5df77a3bf2730f488c7c2 '20250707.0': sha512:c171aeba73e56ad81e55dbd31b518401ca88ae3bcfa9d8209d3b28122b06e5c79e0fdb8bc25a748bc0426241bf5225f466f5ad138f61b488046e258bcf70a30d '20250625.0': sha512:61f19bc35df8b15669d822df1fc2ef1e0e9fbac1451be4aa16979df42bd100889e960df689dba9d40d35c700408e1071da41c9dddb426332daef6f2939c6067f '20250616.0': sha512:1622ac317d7dff4ca7a5e56bff396a50fbc90f061b12289944d3946acdafb9814a2ea0bd2900e90f0d924f9c6f9450a29dd9bd83a7d976b30ce3cb3a85a2c168 '20250611.0': sha512:04c9ff5f2e344fe45d743c093ec80948817e7b9af8de5899bd1d3de893cb29bfbcad42f244afea030ac64ddf86a33ee45e22c956162d9991fc6b388cc96c81af '20250512.0': sha512:eb7acb5bbd24dd208643b0e91b2195fabd1ca3887612ad33bc34d62a86e4944f3ad80e7592ee5a49cbd6a12aeaa466127a7a220722c2ea64f37df96bebba4ac2 '20250505.0': sha512:11a1b003a73b2ae8924b03adc557966d815b79d756c9e40adc505c11ffe6f8e30153e5d133566bced39797fbd41651680fc17c0d7686d2ab3cf63b466e68dcc1 '20250429.0': sha512:42b16d541d589d96075c29e4bf7005bc429c28f411c928412fcd18f093b98f3a7969d799d567730e08f379ce9c2ba7c02bb1e8d10b7fa72179349ac2f40c8d7f '20250421.0': sha512:eda25a84342130d3fe7f23ec3abad56de0fb08ac36c430b423c2d51cc21a75e902a4671ffac9481bf04f8985ded12110e65aa8a2032bda2699083d1b9b07a672 '20250414.0': sha512:93fba15f7bd00dec79fbf132d586497f7981cb4bc3ef6cd77e150367cb45f5f2b8aedb00952ff652ec7438d742b773a19ee2b6eae1bf98facd192a5c233d38a3 '20250407.0': sha512:09acdc895cea6706ba528939da2e6ddab148dfee56addb0d52d7af74378454f4e05cfd47cbb29ad0569139c49cf298be9d4b94a3c2d28b75c05f713e425746e8 nerdctl_archive_checksums: arm: 2.2.1: sha256:8d49681ac806dd3acb2477675daba3574b7d019aea26513ac1960549473738df 2.2.0: sha256:91f286e4babbd6e000e743f55e2ec6fd6b93f5b227386175f7932d247ab5a431 2.1.6: sha256:9523cce6ed87d379fe06d9b043936398f1b047917037d0faae151de83acb3b4d 2.1.5: sha256:a946db17dca42c0835d4ef891057afda03998501f600db3825f8f421f9c0180e 2.1.4: sha256:dcb2786c45913903e84ee2cbc064f9812b3f7e82bb208b14d622cd4707882063 2.1.3: sha256:5176c4fc55d2d769f619da6cc577c5e4b1fa64d5ff9911df269980084fd9708b 2.1.2: sha256:8dab9e2cfe58b94e3e0de837c2ba4dec6864c014848f59bc764a2b8db44322b5 2.1.1: sha256:06ec5d079166a69ea0326af7f30dd7306c03a25dcf27b186bc125230f5d827c3 2.0.5: sha256:128bb220999c69cad054cf6394cd70d0665891257db4a7dcbedc5ced409eccb6 2.0.4: sha256:89e540cb1ac0ed37ec50afc578970a0c9b6a7f1c1b684368da9a726259a3d359 2.0.3: sha256:d95f238738623ae1f4fb01b6a7f287436ba85493700a9de263b3efbff57424d4 2.0.2: sha256:910619da11b90d71758e6843543ab2106c20b5149f353289bd6d553151b540ca 2.0.1: sha256:8f42611dc1554b29dfe990f058ed12920be9cdd78798dcfc6b3845e613eb1252 2.0.0: sha256:c68d19a66b6163e67290397ff24660b63c8e6ba6ae3ce4deff33b5a9f7df51d3 1.7.7: sha256:26582565426152dc426fd47d8090547128e415314f36710bc58ab475160ab0a4 1.7.6: sha256:4c48463659b09636aa23b50825f85cdc38901b6c42e321f69a589d89f6e1d0d5 1.7.5: sha256:2d258a7d67e9fa808424ad42f9299a0feb318cafd2758f0287748acedeee4c0d 1.7.4: sha256:91d3a8bcc2247dd80f8f5769419e6f344dea412937de4c318f65d8e9bf01355b 1.7.3: sha256:44369f34a98e5955eb02e41779b1a470332194e4c2bef136fe471943eaf8057a 1.7.2: sha256:d952c1cbe3d25478bbed5f4ee7af4bb52fa4ed47e43802dc5eb2888a4c8da704 1.7.1: sha256:799d35de7a182da35d850308c7f1787cd7321404348ff2d5ba64ad43b06b395a 1.7.0: sha256:8b9e7cccbcc0a472685d1bc285f591f41005f8699e7265ea5438a3e06aefdcfd arm64: 2.2.1: sha256:abc83c9ac3d843c3442eedfb61c6456b8b59b1e4cd69f69598ca1582acc7c094 2.2.0: sha256:37b353122e0785578d1680fb1d7be546f4c64d0a4aed7875d3a216b2c44be76d 2.1.6: sha256:5c30be3ec118eb222bf635f8049c2f96b4c46d9343ec445058e9bc2ee9531c28 2.1.5: sha256:d8d7caea291e0ad828dfb885fed2b9ee6703432be54ff3b62ed81aedd8431f63 2.1.4: sha256:aaf5acbbb044d82038518780cebebcc2a901afde2db465d3581b8987c2d6f6fc 2.1.3: sha256:62e34ce6156c942368968c0d88c731cb2f3b341785d584e56a2c175916d5f88d 2.1.2: sha256:af6b8c9028fb1adc84b64c9a63da0f369bfa5cfc9176c8b7c58a573d30bd8a74 2.1.1: sha256:648a053c9b5f4e0da1b474fdec40ea7f5b2eb2c1ce88c7881a7c6c2cd11c6478 2.0.5: sha256:ecf57d202fe6fb1fadb68f25f3d1702a424490f8ca0817a902689f1e1fae9e4d 2.0.4: sha256:1f394e3aabc2b202ad17a5ece0495ca554ae3ca346a5dd0aa8344a891734763d 2.0.3: sha256:f2c3f12c99e112cd82ba19ca9b875045c44b2f5a19cecc295ed8d61d415e8851 2.0.2: sha256:c50ba98be0ef05684948f7873078558504a7cc46ff92cffc764c1625b1cd0d40 2.0.1: sha256:cbca59744f6e9dea962e1d3a754294b5e64b53b82f4f7f7d603a591f38545fd5 2.0.0: sha256:568b0383996cb51e1a5f97abb3973ebac2af1a26268fc7d9ae0d34165ba033e1 1.7.7: sha256:230ad8f4f88100774d123213a427d3d43071e0dcf9f70efbea50ab9efff1cd4c 1.7.6: sha256:4fa0a6e936c7a9cb9bb81e784fddaa593cb00afb48b08842e3f0503748c21348 1.7.5: sha256:a53d87fc7d1f4ffeec55e5e08d2397b02ada0d334874c3cece306ad36f828a6c 1.7.4: sha256:d8df47708ca57b9cd7f498055126ba7dcfc811d9ba43aae1830c93a09e70e22d 1.7.3: sha256:e4f16b78d884768f6997558130146ba9bd7846828b19fa2ca8e8eda988953fd7 1.7.2: sha256:de68d5380d65604cd26c164988547cf46b698f7819a5d51d98e3a0f031f5594d 1.7.1: sha256:46affa0564bb74f595a817e7d5060140099d9cfd9e00e1272b4dbe8b0b85c655 1.7.0: sha256:1255eea5bc2dbac9339d0a9acfb0651dda117504d52cd52b38cf3c2251db4f39 amd64: 2.2.1: sha256:34144de7f12756aa4b9dc42a907fd95b0c5eb82a63566a650ca10c8abe7a26a0 2.2.0: sha256:1b3390a832eaeaa1459cf42357da983205da2dd72300a015ad018b3499fc455e 2.1.6: sha256:22857b373edea479a4534ba62cae1c77f2af38f0aac4c91c1c68cf09e29d6f9e 2.1.5: sha256:9ff862624084fa1b2c6272c4498754801bff3f4ce09421ac6eba58a760878544 2.1.4: sha256:d6e91d3e275bfb3404959b1f95ed25ff6cd83e3181d17b93afe2d39cd025501d 2.1.3: sha256:227390fb16c20d6ab3a7588e5ce72202df1bec7960a2a091599ce3c1bf3fd1d8 2.1.2: sha256:1a08c35d16a0db0b4ac298adb8e4dab4293803d492cbba7aaf862a48a04c463d 2.1.1: sha256:4fe308bbadf7dd079c058a34cd0bacef3b35c46da88c37f5f125044d90941595 2.0.5: sha256:a029af80fd4b3de096d1a18779e7ff8369fbd1285944ebc50bdd22ad41438b5d 2.0.4: sha256:2f9d22179868db4f0a1daf2fd65f58a24f8a78efd2d9b17659f56bcdce85efd0 2.0.3: sha256:95ff850688a73eace7453f19e74bf4cc8a1f3e458eeb97ef7a6b74de9825df16 2.0.2: sha256:1ba015dba039cf6ec2434e88d97707f0b715790e6b7f2e7b6ff7be9200f47bc1 2.0.1: sha256:96e5e3ed79f189a986cd33a40b0c817d7b6c7d9238f51a0737213f409e5d82af 2.0.0: sha256:b3920c4a04dc7972d9e2977147c1ef5c415551802b3c4b0b613ba9d0c46cfbbf 1.7.7: sha256:298bb95aee485b24d566115ef7e4e90951dd232447b05de5646a652a23db70a9 1.7.6: sha256:0326d6a42dbec5c104ed9d7aa8cbc62727433dbe000cf21cc29d06b22507e0f0 1.7.5: sha256:775f8bddd5e93acc4162c568dd6b84d5925549180991b83cfd4b7e33d4844921 1.7.4: sha256:71aee9d987b7fad0ff2ade50b038ad7e2356324edc02c54045960a3521b3e6a7 1.7.3: sha256:ee93ffe6f90e50bde153a9a0dd779594e0bc13a26949053965958b91b6dffdd0 1.7.2: sha256:aed7d33d645bfb97c8df978d952a1e1f7e02b0b3ed2c0089ee4285af7f8f971b 1.7.1: sha256:5fc0a6e8c3a71cbba95fbdb6833fb8a7cd8e78f53de10988362d4029c14b905a 1.7.0: sha256:844c47b175a3d6bc8eaad0c51f23624a5ef10c09e55607803ec2bc846fb04df9 ppc64le: 2.2.1: sha256:05c3573e0468fbe6ccecce497b8129beec0fa1d8afadeba244e3d5ac63047fce 2.2.0: sha256:cc9f55ffec892498bb27db1f6b0eef16b591ee4ce873b61f2fd9a9a30930c620 2.1.6: sha256:807678bc5042cccf81dbe13b00bbe8e18dc24412481c3cc68eafa316ae43842d 2.1.5: sha256:18c72aa80d974394452058472e0dfcfe6200307969a17a33ffe1a85606a2663c 2.1.4: sha256:82b3c3ccbf314e27591977268c81f92660bf6b78fc568e4e7dc1b583f61b622d 2.1.3: sha256:a3ca5c287bc3dad0d23ee8c10a9acfd86910bc120737763bd4cefb872b33b958 2.1.2: sha256:2008c94b18900f3a58439ae9dcb7aa6659da82e2d0c9d67ffbecdaa152b9b0bd 2.1.1: sha256:f277823a3814c6309bf4d31e94e2b8fec24c99326e517ec81c325e081dd7c20a 2.0.5: sha256:01bec466e0184945d8138ef0bc7e6689e61c4c8f56bf0968825c7f5a956c076d 2.0.4: sha256:93c5bd5f32a3b821fd462a96c6943b06663ff03e8fb16327957fe99576855116 2.0.3: sha256:8d6283b1fe871e319a2f5cf96fe97aba649eeaac0a2a22c81b9b4d3c613c210a 2.0.2: sha256:1baed7f4312404da966155856aa1e4b4f48bae73d64fd2cf6c41ef9326a07b10 2.0.1: sha256:78a3846cacc570e8ee4a1d60928a55954fb4fd1b3b731c0c975a808134166fab 2.0.0: sha256:879efec2963c9854aa1491d80d341570c44dc12b9728084ce64a986ac4fa9bfd 1.7.7: sha256:8236524c4cc6c91007c794160065716516a4f7b63270dc00b796df11776b4f19 1.7.6: sha256:89906f9bcdf8d5bd866646c43e14c0ae15a83ba3ebed44b06c3629a11517e242 1.7.5: sha256:8e0891f608144d8d751070edea5cd98d2a76a053ad7fa6b9d4aae94a700aaea2 1.7.4: sha256:97c99ab6030ffac1fb780fe012de06a36512b17b13de5c99445468b5a5fe5a62 1.7.3: sha256:e63ae0a8f5ccd12877ff944b609d0a4c55c97ba79808ab16c7dc7e99fd8f3dd6 1.7.2: sha256:e5c01702d3cec0763d28bd3cf6ea9c3efc58662a93cb4e15669a839782af10d7 1.7.1: sha256:09fd0cbef25c98e08c5cc2d1e39da279cbf66c430fdf6c8738e56ce8f949dad9 1.7.0: sha256:e421ae655ff68461bad04b4a1a0ffe40c6f0fcfb0847d5730d66cd95a7fd10cd containerd_archive_checksums: arm64: 2.2.1: sha256:dac15a0d412a24be8bfe6a40cec8f51829062725169f1e72ac7d120a891ef5b6 2.2.0: sha256:8805c2123d3b7c7ee2030e9f8fc07a1167d8a3f871d6a7d7ec5d1deb0b51a4a7 2.1.6: sha256:88d6e32348c36628c8500a630c6dd4b3cb8c680b1d18dc8d1d19041f67757c6e 2.1.5: sha256:fe81122c0cc8222470fa3be51f42fa918ac29ffd956ccd2fc408c1997babd2ca 2.1.4: sha256:846d13bc2bf1c01ae2f20d13beb9b3a1e50b52c86e955b4ac7d658f5847f2b0e 2.1.3: sha256:7e423abc7bf52ff6cb724f44995cca335b40331efa727415a5efc99ca34ac8d5 2.1.2: sha256:57fa4005ed3bb648f4a2ff3ef2f9ce12b27ee1397225626e3165b9ef4af45530 2.1.1: sha256:4e3c8c0c2e61438bb393a9ea6bb94f8f56b559ec3243d7b1a2943117bca4dcb4 2.1.0: sha256:f6c3972347848177805eed8a9c282fca6aaec0e6fd28701579e63cb20bdbce07 2.0.7: sha256:e590e39956a451ff692454ea991961b08627c514dcc253e8845d2a57f9a78e2f 2.0.6: sha256:0f308f386b1ee24712875f02bded92ce7099a707ed43f57b3fb9c934dcb6bed1 2.0.5: sha256:36eaf77dc65df4b60d6e06204631a4105b4e942dd2704d618758a2aa0eecc264 2.0.4: sha256:0fde98b24bb55363a54150732e0ac99a43bccf2a9711371bd5470f32790316f2 2.0.3: sha256:3701008e72e983259afaa594cca5d8126e78e38cf0a586a1f6971cb3f61c4b6b 2.0.2: sha256:14a2a9f7f75f73e5bcfb8b183d0b84830c54b98ef8c5f6ed70e51f1a230c673e 2.0.1: sha256:b07120ae227b52edfdb54131d44b13b987b39e8c1f740b0c969b7701e0fad4fa 2.0.0: sha256:2a00b1553f38aa9e716d61316b661961c2fbfbb7aad7bd73b377be5725ecc0f1 1.7.30: sha256:a09c3b01b3b6935e839c8a9588b5528c57ebfca4747d816654a7d1e7575c0a63 1.7.29: sha256:176d523a6d6dc5520e0c35b8eb0de6e54bd4d00486d5fbbeb4163f30e2962f17 1.7.28: sha256:97457594ff8549cb82d664306593cafd3d2c781c706f9fffed885a46d8919bec 1.7.27: sha256:3f03ea60c7dacddf890be3ab18f7ef859d9d104b19627f52038d7984361912bc 1.7.26: sha256:adea067914e678ac37d5091ead66f1e36e5cced4d395bbd2be60772495e09eff 1.7.25: sha256:e9201d478e4c931496344b779eb6cb40ce5084ec08c8fff159a02cabb0c6b9bf 1.7.24: sha256:420406d2b34ebb422ab3755fbeede59bf3bfcfccf5cfa584b558c93769d99064 1.7.23: sha256:6a66b5e63a5e88ff7eeb478ccaca9083d44e51e1d7261ae183fe5951a6226ccd 1.7.22: sha256:48d0a8461ae829b12b07c3663b14b70287d0607a0792719c51b4e4dd700b02ce 1.7.21: sha256:7b6b67d998eb86856d23df5d57269c054539072bbb27677975cf78269b2c5c10 1.7.20: sha256:cf80cd305f7d1c23aaf0c57bc1c1e37089cad9130d533db6fe968cdebd16c759 1.7.19: sha256:1839e6f7cd7c62d9df3ef3deac3f404cdd5cd47bbdf8acfeb0b0f3776eb20002 1.7.18: sha256:e80ce87b469af03b3bdcf68b95f0f4a303787ae247581bcd42f04acf1ad4c24d 1.7.17: sha256:8d9749985796a208e860afe331ec77cb485566104e5cc7c0b5e9e82ec7681969 1.7.16: sha256:2d4373de40a6f58cd0f29377c0257b35697a987248e6268520586996771d7a75 1.7.15: sha256:5cc8bd8f3d9803ef0ef701596e89d62ad6850a2544e722842f4533642df36d87 1.7.14: sha256:44df66d0a0332465e7d15e90b974cd4f08d059dfa26652218ed9485390f47f9e 1.7.13: sha256:118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d 1.7.12: sha256:8a1b35a521d071a8828f63fe007a51e5b7ac863a1195f5dee32543b1a9d5f2b6 1.7.11: sha256:5eae27cce38a14be5390d4035127aa11416bc5ae592a9ff25b11870872ce1159 1.7.10: sha256:0667b12a04a896a61cf508a4a77190c280f4a1fa35f38c8a4ba63f605b5ec375 1.7.9: sha256:09ca326dee14e00c439137071747c15cc280480e2c26c1e82698c992dd1889c6 1.7.8: sha256:3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1 1.7.7: sha256:0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3 1.7.6: sha256:d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce 1.7.5: sha256:98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9 1.7.4: sha256:ea5a04379bd4252fc1e0b7b37f69cd516350c5269054483535d6eab7a0c79d2e 1.7.3: sha256:85d2eaedabff57ac1d7cd3884bf232155c4c46491f6b071982e4f7b684b74445 1.7.2: sha256:d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6 1.7.1: sha256:1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b 1.7.0: sha256:e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2 amd64: 2.2.1: sha256:f5d8e90ecb6c1c7e33ecddf8cc268a93b9e5b54e0e850320d765511d76624f41 2.2.0: sha256:b9626a94ab93b00bcbcbf13d98deef972c6fb064690e57940632df54ad39ee71 2.1.6: sha256:4793dc5c1f34ebf8402990d0050f3c294aa3c794cd5a4baa403c1cf10602326d 2.1.5: sha256:403af72d9f956ed8a5ad5b0ac0f1e8e371a1488f2b9edf9b4ba13db0653936ea 2.1.4: sha256:316d510a0428276d931023f72c09fdff1a6ba81d6cc36f31805fea6a3c88f515 2.1.3: sha256:436cc160c33b37ec25b89fb5c72fc879ab2b3416df5d7af240c3e9c2f4065d3c 2.1.2: sha256:87c18b2686f38ee6f738492d04fc849f80567b7849d0710ee9d19fac3454adc4 2.1.1: sha256:918e88fd393c28c89424e6535df0546ca36c1dfa7d8a5d685dee70b449380a9b 2.1.0: sha256:0e5359e957b66b679be807563a543c7416e305e3aafcf56bad90ef87a917014d 2.0.7: sha256:6dc663cd245d19c3e68260ca86247469563a31decac925087b62a2a819dfcbee 2.0.6: sha256:a545471a67b8508a3c58ad01a4e6bb2921ace1e59e00a0a2d2e784c1f4fa8caa 2.0.5: sha256:88ab31f3e78e4d2fa12dcb933032122d11d441c83b79a89c6c8076f871e50df8 2.0.4: sha256:e1c64c5fd60ecd555e750744eaef150b6f78d7f750da5c08c52825aa6b791737 2.0.3: sha256:ac70856f1d8bd3aa9ca5d62db5516b86dfa0f934c1fd1d1c5fa4422dd12ba45e 2.0.2: sha256:9bd5b6a1bdf505d520d9a329c520258ed0a17faa9fe3db12712ee858ad59aae3 2.0.1: sha256:85061a5ce1b306292d5a64f85d5cd3aff93d0982737a1069d370dd6cb7bbfd09 2.0.0: sha256:6f8da716941f7e89315cefaa6e5a8f1ff10b323ff46611313c455df7ab1ebee1 1.7.30: sha256:ca0f27e34411504acd1cd24fcbaa71b9c47d31ce9408c47c54a2dc1810ceb1df 1.7.29: sha256:71d9f6e4ea4a9e108e2172b0e7f6fa137e086808db3e6874dbdb91c01102d3d4 1.7.28: sha256:7a8c262deb63becc877e82d23749e4f99f4a17e8e660f9b8c257ca87a5c056b6 1.7.27: sha256:5b038fb22ab5dbb1ce57dd3d8f102460cd8619ff2afc78870837b06e8c4e840a 1.7.26: sha256:fdf1fb17086b62fc861103da4e3fda3d79bc543b42d2acef5d07e76b13d35d19 1.7.25: sha256:02990fa281c0a2c4b073c6d2415d264b682bd693aa7d86c5d8eb4b86d684a18c 1.7.24: sha256:1a94f15139f37633f39e24f08a4071f4533b285df3cbee6478972d26147bcaef 1.7.23: sha256:8a0de43d9313aef2ebdccc0ffa49461a4a28139a2c0ef104c3c847f6f37c8119 1.7.22: sha256:f8b2d935d1f86003f4e0c1af3b9f0d2820bacabe6dc9f562785b74af24c5e468 1.7.21: sha256:3d1fcdfd0b141f4dc4916b7aee7f9a7773dc344baffc8954e1ca66b1adc5c120 1.7.20: sha256:e09410787b6f392748959177a84e024424f75d7aff33ea1c5b783f2260edce67 1.7.19: sha256:97f75e60f0ad19d335b1d23385835df721cad4492740d50576997f2717dc3f94 1.7.18: sha256:a24b05b341c155a0ec367d3d0fd1d437c09a0261dffdecc0e44e9abbf2d02aca 1.7.17: sha256:04cf937349f82d29fe98553ff45a7e9ea2ed6b81fe6514e3679cf263b50409ff 1.7.16: sha256:4f4f2c3c7d14fd59a404961a3a3341303c2fdeeba0e78808c209f606e828f99c 1.7.15: sha256:ea27e6454954bd9cb62a70b0a40eb085ae9c96cb8c075a74910102b33586e07d 1.7.14: sha256:48e0d9747cd51cb90e0b278d100397653d9f2e765effca194427e4796395b240 1.7.13: sha256:c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23 1.7.12: sha256:6a24d8b996533fa1b0d7348fe9813a78cd01fa16cff865a961ad0d556f5cd665 1.7.11: sha256:d66161d54546fad502fd50a13fcb79efff033fcd895adc9c44762680dcde4e69 1.7.10: sha256:eacb0296bff2ae5225a18492dcb32fb28ad4a1fe0a39ea9073367c7e43dc5838 1.7.9: sha256:ccd5b434393666f6ebbc90eea959ffd3e61958a1e3e1cc830a678f040142d4b0 1.7.8: sha256:5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9 1.7.7: sha256:371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba 1.7.6: sha256:58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe 1.7.5: sha256:33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25 1.7.4: sha256:fc070fabfe3539d46ae5db160d18381270928b3f912e2e800947e9fbd43f510c 1.7.3: sha256:de7f61aacba88ee647a7dcde1ca77672ec44ab9fb3e58ae90c0efc9b2d8f3068 1.7.2: sha256:2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7 1.7.1: sha256:9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85 1.7.0: sha256:b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0 ppc64le: 2.2.1: sha256:3de0f215ea649952a9e99040cb3888d8059bd3d35b04edbe6afb916c763f9ea7 2.2.0: sha256:e4ecd0b03200864e117371b25cce5335e39ce0b0a168a01d2ba6562a05020f0b 2.1.6: sha256:aef2b639a14ae79f2bbe43356b25e84ecfb2c7f269c87f41e41585e724073e54 2.1.5: sha256:dc95edc01958d18f8475ab4d415e8c92cb3bad580167db8b0054374fd9004f78 2.1.4: sha256:d519e40e266f39cdd68f2c31e2e4e9b70eda09b96f3c3de343a7a3e11d49ad4c 2.1.3: sha256:e517a6d936ffb6d2292e9c6560aa363382b1457eba34cad8289f6f3f76201588 2.1.2: sha256:9d8a02413050ae234eeef4152fb703b3d093b5809411f0e905f098554fc066d6 2.1.1: sha256:36c90c9993e9f2142aa014cbd352bd7d3db6b8daa2990b4a9f59e706db78deaf 2.1.0: sha256:3e380629da9d21366c916ec3873022a2a5700584d409297a1bc183b11fcb0809 2.0.7: sha256:15186b11d555f61e495e94d5a99b2eb2c7f21a5e569eca02170c4bc12fa8c7f6 2.0.6: sha256:20df16cac3a912d6df34040dc81c7e5e2c95a06b2b21fc2ffb4372b6d41274e4 2.0.5: sha256:09773a42829c0ac9b8dd449753c755b3ba65cb7e8d06485950f99d32fd6c1e0d 2.0.4: sha256:ca970d9a53ae504bc36197d6daa931338c387c83b6948b9f9bfdd1a75e25dcf6 2.0.3: sha256:2f0faa0086ae81d00680367ee9d75aafd3c4ca4535362db83fea62dd19c47079 2.0.2: sha256:1b19d31bb8a7f9d26d9b50675e78f397d0b01fa635c33cca456f91c412fa6df1 2.0.1: sha256:09a25357343c7336fe519e5fd1a9dd0f22da869e9deda50c2bc61b6e8c9384be 2.0.0: sha256:2e7f4b15ac85c22c1ced102bbb424124078248f0af3183425ff335a998079809 1.7.30: sha256:2edba94f31ed0c32cde0cca2e79e4426224eece4ef4fa1f194b2df4c811417fe 1.7.29: sha256:8c08edb22b53a44cf7b5a5e31ca8fb86d1d6df378724048ae0d9890a3b66f081 1.7.28: sha256:e8f64abf81503aeee0db0d5682197e9ce377ffeb858313c5bc9fc3d7faa4b85f 1.7.27: sha256:ccdfa16e4bba3a993d74fac794d22ddadc1013d351cd099ea933827050ef05a0 1.7.26: sha256:34a86b1bd598b34e8c05956c5976fb0c0b347937d3cd0837edbcebc7f9e7e53f 1.7.25: sha256:0934176e32eace1c23dcb9edff0e78f872bf8f7152b5e6f622e9ccf1ddce8722 1.7.24: sha256:2ca4d527dac68132a2a6b3971d82ddfd18edc7fa838b7cfcfe6eb11efd017871 1.7.23: sha256:00dd8a1145d7392ffe1e2b74da147b896e4387afb5e73ed6e5cd3744add32826 1.7.22: sha256:6747b7291ffbfde2c0bf0031978985df92ac74414f09bf190afda0fc9e797146 1.7.21: sha256:5ce0c1125e8d9ca04e2b524a2bac8b1eb97876c073023d5e083f7da64fcd8207 1.7.20: sha256:dc611df0baa90509dda35e0be993da52f42b067514329fcf538d000b110364e8 1.7.19: sha256:f41c2f28ee933a9ca24ff02cca159099fbcf798850e56cf0b7a6047ebe21fa86 1.7.18: sha256:d6cfb3bc8fbdead7d435d5f3f6b1913b5896f7f97102c1bbad206f9123c2a5d3 1.7.17: sha256:873b76a507d362eec73887f61fa1400f3a892c7dbed1759f5dad2b654095b534 1.7.16: sha256:d0add7a55a5d4411cafb276469d2b78bc3ada11cb4b444b9e35f9ef60c00960d 1.7.15: sha256:b38641d9bd18139495cf9839999039b19941f53d36a6d72efe4577c489dfda0c 1.7.14: sha256:b84b523909b9dd0c0b2bc40bd2b9af543ec9f1186df69c220ae3749e34623dbb 1.7.13: sha256:89605ed2365d5eb779477d11947101236eb44e5244f1e58bb162a9e68d242798 1.7.12: sha256:80f16891b387d86712026234de7d4d0365a38106dbe5e51b65b1200b24822721 1.7.11: sha256:6f91c5dabdccd1fc75aae8687381bb185b9eb4200beb29d0993dea8175f5fa61 1.7.10: sha256:15a5191bf7c555956a8565d8786399d51b13f2718d59b1a5b2bd380fc420bf8a 1.7.9: sha256:174b8af2d878ad8410205b9ba44fa8d2a9683a521abf13f168f67b7f7375d5b3 1.7.8: sha256:2b563df9e1bddc96a99a023963c99b5faf3066d3fcbc23ff44ba24229e939444 1.7.7: sha256:0335e7447ed84757489337686a709e95ffa379a8780f238725abb10facaeaa7f 1.7.6: sha256:956fadb01b35c3214f2b6f82abc0dda3e1b754cb223cd24e818334b08cb09fb2 1.7.5: sha256:2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc 1.7.4: sha256:c3397f67fb5756e6336ff76eeb34dfad7ad66235877a4186d82044c7a4caf482 1.7.3: sha256:d1977922e74147782dd5bb488f260ee14d758d29a7651cd97bc2e6c7cc1a3cce 1.7.2: sha256:cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e 1.7.1: sha256:17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342 1.7.0: sha256:051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889 containerd_static_archive_checksums: arm64: 2.2.1: sha256:6b3b011ee388638eace05ac3be0eb32dfb4a43086695c29d06e997febd336f2e 2.2.0: sha256:5f2a7f451231ff35d8306f874c51606fc9da1e2db56048834a23260f68a78eef 2.1.6: sha256:9da292010d36d80afa3bb48cbd15f65d3bf38177217060272a1c3fd65351cfa4 2.1.5: sha256:d1a1e64c4334e17d6f9f40093d5ff9f810b95ac34c7dcba55e7d2226d2a8ab79 2.1.4: sha256:c5f0957064e6ed5a67905ea3f8e451dadd16530334b86baaad678dd357205c30 2.1.3: sha256:74703e628223c6f19ab2df8497a061d08dc1b81c03c720cbc3d66fedaddf9ca5 2.1.2: sha256:3f1dcd2bc02cbf9e8fde308c144551153f19e15105a26cebf213c8e232a27f5e 2.1.1: sha256:f4525d8adc4445f8d623c6fd91e7dd750189b96539c29ede2636583ac5e4cf7b 2.1.0: sha256:1239f60717f4ec2e06e51d2ed86c43d8a0ee10880c73131e58e1e0689e9ad049 2.0.7: sha256:cb46d915f8dea6e94fccd204f29bdb3cbe502d7deb9220a6e662cb33f29784fb 2.0.6: sha256:4b6e773839b153a189935d76aace6e965811fbacce5cc938d74910582f2c0de3 2.0.5: sha256:302a57f36c80c547c57e439e5f4d5d4e6ce73a9c7266a6787c739a3b9d4dd813 2.0.4: sha256:3529558b49e87a220205b632fea1c4b2cdc60f1d93eab3d4c307f66b91df189d 2.0.3: sha256:f4946c0a73f966d47ea56dd806a4e466447f194d9d472fe3f59477c6b1510dee 2.0.2: sha256:da5631d38702257674da542969c285adfdbdccffc12c5ea39de4db6113de51bc 2.0.1: sha256:6022bd160b6d83f13fccd87b1c3854b0294a940335fdf014c10c80d71b279115 2.0.0: sha256:428cd0b08eab57003db8a98742d8404b4b69dcd335c5f0f66ceec5fe3b9b31b6 1.7.30: sha256:1458b3c4a20eb1305bb1d4680c660d0726fa7709acdd95ec7c0fd6bdd632c36a 1.7.29: sha256:d428ad3fb7036c07c141f6666de575d8c7fd862e658a28da5ae20b737fdf7d75 1.7.28: sha256:360307a347f1114d6f67fe261674033661c56e6deb3c2529705fc834aba98ff1 1.7.27: sha256:ce2b308e81dc1e633362a45b64fbefc58a7c521e7e060cf95a7709bc1704b402 1.7.26: sha256:f35d8eb8467b7875ab768b4b869c9905616d998549e1e0ed993a52eec319dc51 1.7.25: sha256:34bd6c26f07011af71a4b31d6637b28d693b0c3a0e847e9df10cac1ccfd17db1 1.7.24: sha256:16d13febaa1da15c3ca9fa662f6ae08ca7dc4274f1d6f15f478a61238470c2c0 1.7.23: sha256:e7b21b7a487f6102693e115d9150976a3d4812084ec5ec950ff0840fb97631c5 1.7.22: sha256:a4cc5c572b587799550d4fdb27b3f3f86bc0fc9d93a7b9b468e704df9d052c5c 1.7.21: sha256:5f36f5381fa5db7e35eec8e2bf342dd26ff9825f24caf7b41f72778a97e9c757 1.7.20: sha256:9082a99262f8b007bc19856ebef2c48b7b78d50c17b80f8835fffcbdd9f490ea 1.7.19: sha256:efd641f5443614b4c17b33c185b62b3e5f67dccc9d140c48d67783c685f62532 1.7.18: sha256:1eba98be0632d30911ae4bf6ceb27cda32007f5c0d7701a94623f59848dc7cdf 1.7.17: sha256:53fb903fa47b4816308297ae9a0a2ead8cbe3028fbba573d0d5054f722c70663 1.7.16: sha256:6d64cdda15b0c0a5de0a644e7f0b0704582eed2e388a75a88f8b6dce8f3f7ebc 1.7.15: sha256:5b8affc00938d2e74331ad54e15b89a209b4e0adf0c81256b07b4363ce0a657b 1.7.14: sha256:5f2add641528921a42451409db305f64c397421a4571466c800bef22f055a0f8 1.7.13: sha256:551bcbace1865c645285e2e8458cbb22ca5674de776e409ce75c81a2c2124f70 1.7.12: sha256:d1617a272be304d0b4ed423c0bfa905ea68d979fdac777ec4136eacafabdde5e 1.7.11: sha256:bcc0ce5b485ce4b82ed3e585ed7093f4dfda9c9828d1b4caf53efb414bc0ed18 1.7.10: sha256:4601af2e1cbb08125c1d72ae7c5dbd5dc1f6e47a2ff7b9d42409b494e8d2f75b 1.7.9: sha256:5b76cf84a4f7707ad9ea411c16d5d74b07b06faab38e089b145697ef615bdf3f 1.7.8: sha256:661fc140d9054c2a2717491ce268596619b16958a7344a144ba5401a8e7d536e 1.7.7: sha256:b9bb0aeaf86a5f70015033651e471d067266747a8a39d6fbfd96b8f9c654b891 1.7.6: sha256:6f0fa35e7b3ca528986ea6962fac883a250b06d2f86ac8a4e9f41384929d7648 1.7.5: sha256:851da2b4a43acfeaa6eb17296b20b5b7ddc4bf48e255209351ee8217cc7e99c7 1.7.4: sha256:4557030cf4d3fe7ce52c8b299fa365340302f027311e0968103b3df3d88dacb8 1.7.3: sha256:188ee47a27966608db91b147187a76c367357a3138add5c6ef35fa9bb1a4bb96 1.7.2: sha256:0dee8e1862f92af95ac4b29eb23738cc7eaec2e61cb05a93be27137014e4af97 1.7.1: sha256:f0435e7cda3c3abc40d3f27d403a8e24bd0b927a8a893a7e4dfaec5996fa9731 1.7.0: sha256:6e648cd832f026e23eb6998191e618da7c1ec0c0373263d503ff464e0ae3977a amd64: 2.2.1: sha256:af3e82bac6abed58d45956c653244aa2be583359a9753614278ef652012f2883 2.2.0: sha256:2d20037947cbb0def12b8ac0c572b212284c1832bf3c921df1e58975515d1d08 2.1.6: sha256:577900a5a8684c27e344aeeb1fc64e355745f58cba7f83c53649235ba25abbbf 2.1.5: sha256:9389a4bff0112258bd953c66382709c2a4a11e25e376c8c4c7075b39de156882 2.1.4: sha256:50e53500800f4f74d0d8b2e57e939297eab68b0fe11a0957b771d5faa61fae5b 2.1.3: sha256:1bb0c910e8fdf623fac2305ec66e72c4afbf612de282577dfdbebf08360937d5 2.1.2: sha256:5b2b8e82c5d3e6019428db55296b5169e748b94601c0c944c1d780904aa35543 2.1.1: sha256:9ab7df748ae3e9c2a513586668b46175c27290eda6028609223d9065eb777964 2.1.0: sha256:e8e5da8ca6586bf2023940ec5bc5d4e949924ac8dca2a9f0dc469bb9a297146f 2.0.7: sha256:23371338ea3d930f020f6ad476a608aa8196ecdd016f9f5cd511c87cf37f86b7 2.0.6: sha256:e6d69c5931af2a36c98cd066c96d60ee8b05a961d1f54d3d13d80c9afcd74a16 2.0.5: sha256:3506b47f8f0806c86ab315ee50e09580fb6f21e1ed3a30051105c98c5592c57b 2.0.4: sha256:b8785bc2ca5417fe62fcd5efc687e630cce11ca57e50edd6b9baca10d41b4c1c 2.0.3: sha256:866c05ef1c7f22512e1abed271a1b4b6a8ab9c02e8e802da536f7043767a0e8a 2.0.2: sha256:7cd4de9c8ad37f3248a45b9d6347b7628e4d77d1c8e35c1f80343450fa47dc00 2.0.1: sha256:22b2a7df86fe3e53e219af22a2b5e81d1b67e67d55ec3a18f89990b161fb2157 2.0.0: sha256:e72cc69db9984a8d46a34495c302d2b50188ee2dd5c7000a7b471d0350e14ad1 1.7.30: sha256:290eea517a7aa919aa319563f8270d284ab5197736362285da09b198b76a34b5 1.7.29: sha256:ba4a2e919d2882dba707d8c9c436ed3496e3bb99f5cfb13a31b29b497cf7ec2e 1.7.28: sha256:659eceb8a5831a704089825d980f63abfb686650646751e13f9f9fac780ea08e 1.7.27: sha256:e3ea27eb0e7b8dd92ba7a5ecdd363372ad3c30f9cdecbec75e60ae4a43ca93b9 1.7.26: sha256:e19d790ee965e39d0eaa03d471ac4d3a8dfffab0a7ebba9f5deab77b6eb66f0a 1.7.25: sha256:53f30807bceaefc3d294cc223893e3897062c99c4fc0ff6b0fd60a81e7ec0bee 1.7.24: sha256:c42230baff057c606cd4eb9b71007fb8e2e55e951ad901f0ae0d96bc55ccf85e 1.7.23: sha256:625c317ca556b39852bd1260196a2b9d18ab44c8980cea31f4fc58cf9bdc9b47 1.7.22: sha256:d42192685f9fc8b6761c648ed5f0a4ea10b21c4288444d575c5608b42b20eb17 1.7.21: sha256:24527d4b0696ef16717d19d983d70a031c77c979e235acb90653480c71b9ec6f 1.7.20: sha256:61929db32711d62d1c95400a44606daca2992c18a090135c2a42e41f75777b48 1.7.19: sha256:7a79ea6953d02dba76e08213fb5f9ecd6397440d832aadc621d255b25a62414e 1.7.18: sha256:ef5b06fbc09b2e83ee739e49e2ec71bd18bb9e334eb79c5c073cd23d800260e7 1.7.17: sha256:b4ccad6a3c14a30c9a14571527c2eab0ea2231e27d187750d9ac6d5f477938e5 1.7.16: sha256:ef763349371986fd9723eea2b7ce98610328283efc1fdd65754302cce4539db9 1.7.15: sha256:089901b4356d59f5e9ab8f06ee28e3f4a5896cd97b4aa76fdc35a891f4aff48e 1.7.14: sha256:091b374cdac4b0539b7362ce7167b79de0708a42d06baf558f67bd81a385af9a 1.7.13: sha256:b0a9bb64ee5c0347978f02b97dbb31665cee9254e674da54ffbc3bb4b5fda34a 1.7.12: sha256:1c1df44243bb6923ade7c22c8d9776470e538a683be6f68347bb4f0d82149152 1.7.11: sha256:f1d6a1b1bc1a6b7d4daadb3c474110cd2172071c9b34f90856ea305d1a2d9e18 1.7.10: sha256:2d4318cd0229b1dc757a3eaa37cd1f6df58bb876371ce59ecc3571e0f2fca446 1.7.9: sha256:1d550bd05e4f997e61ab6f3e9f4e19cdc660521e37f3373bd9c7d1804d37043a 1.7.8: sha256:3e13f8ee1657b874bf4741975d094d7a9df9051c400b3f822e310c91997c1441 1.7.7: sha256:e3c68cf9e01fb150d81764355718a5c31367d3129d76d108bb3b63b368ad2d1e 1.7.6: sha256:ee3d1e2a10e7b8b4deface422fc3b4c3700bcac8ae9497c0ade22d5b76649c71 1.7.5: sha256:c32df30af783dd67f7640458881b0dbb06f076b9ffe384d710d2d1a24cdad2b2 1.7.4: sha256:04f4b3034b2b39e40d621ef24abae8daadfe2d06f1bd480f633869944c379c9c 1.7.3: sha256:66eeb863c375dc832589e852798f9ec47bf5da8b53c7ab513f63a30adb13618f 1.7.2: sha256:56d2b16560592c42eee2eb758c51815a37f1de3aa3bf5c57073b4d6ded761358 1.7.1: sha256:8b4e8ed8a650ea435aa71e115fa1a70701ab98bc1836b3ed33341af35bf85a3a 1.7.0: sha256:64ad6428cc4aca486db3a6148682052955d1e3134b69f079edf686c21d123fcd ppc64le: 2.2.1: sha256:fc9235be9a3dd3005e7fe6a9d7bb80e42dbfbff4b119cdce6ea3ee66bc7ae9ca 2.2.0: sha256:d15a4edfe689ce71df8cc5a0c1837856f54aba8d7336170600e6592c2fbf3d8d 2.1.6: sha256:c64312b87181d900452b5c3360a90578acd39ec7664d0c2e060183b24a708766 2.1.5: sha256:4404b918b9e101274baa072188054766a1af16be8d22f02a51a5f6ee4e5d159f 2.1.4: sha256:9bc1ac45ba197873a4d47045313e0cc55910802937739bf57aded125abe55c8c 2.1.3: sha256:0084e26bcf5a2653278766662d5adc27cb00a17a21413cc3fbe1e99d9dacf174 2.1.2: sha256:4e61d5f9b5789e27c28b94fe0901552fb0070aa0c36aeaabc6e2786f3d5828ff 2.1.1: sha256:32c307e4b5cf014b594a9e7c22c610238143da8797f2bf4058dfb94225606ad4 2.1.0: sha256:df73c5ce713196be46f4b20d3ff081d67dabb5c730f61da7a3a65a9029b4f762 2.0.7: sha256:3f5993c7717216ae82083ce8c96af4d471da6e55affb03bfe4b86c59c6f54781 2.0.6: sha256:15a9af201730810fa2b2943c94a2651063a7d09b684f706520965c02000bbd0a 2.0.5: sha256:7e16bdcc5ceab50a880f6a9c7885b15f98ab60f0b963b3092928f89a7b26c8be 2.0.4: sha256:740018f44d2dccbd48d24c2362aae5098c459eda4602f5bc5f85738dccc2ead7 2.0.3: sha256:328c4647cee948a9062b8c17b5d1e75125cab1c025a2087dcc7bb1eef2d666de 2.0.2: sha256:4085c490ad5afb40b34782d427130396513a7eb35a49da8a6b7ade946dd309ce 2.0.1: sha256:6e8608e1993099f4ad44d77765be76fd67147108a6247f6886af33be08287a84 2.0.0: sha256:c389b68c9ca7774efdbc9e479d9a3be14b71d60a2a8c23f1d8764ec9598f3d55 1.7.30: sha256:95f12a656977f516b257f6447c197a30bd75b8b9d494a741a7f1f486a069de8c 1.7.29: sha256:5f4a351d4ecf0aedda743b2ea4b7304a72cf534f4f83c976de3a0d5adf006445 1.7.28: sha256:94dcce54f82bcd34a171af082cd035163cbb281923afc09454e0539eeef73974 1.7.27: sha256:6a140413dd38954078dddb1b1405e403a92173fe15d094fe0bcbac93a1ff7039 1.7.26: sha256:79fb1499e436265a71a31d1ecb69d8b5630df168555786fde8797c91320ad21f 1.7.25: sha256:6df5a58d88dba8e3b932fb6b5be1ff780848cdcdf32bddfaf5358ef3d45b62ab 1.7.24: sha256:c91562864a6f7e037b10689838fc1dd4c3051e82dc5c1ba0a409019d72816315 1.7.23: sha256:4757c76855adc70860a2bcfc583379ac97220c563f356c5eef06d663762b6931 1.7.22: sha256:b8fa787859cfb8e9c245036d21c9096ef835247349b0126eb15d6ae94a693924 1.7.21: sha256:544a2e9c265d643ad2c96aea2a8d33cb0709681f19186abc4a3bc0779dbe9360 1.7.20: sha256:b8dc431a83d46a29cdbda9ad1842e3409b9d8ec5fdf629d02a7e77f1f4e47dfb 1.7.19: sha256:e6d5ffcc15f80d506293eea7ca0e123fd63151f49f52b6b56c08d2b07eea97b7 1.7.18: sha256:978c938b0b02de4041be758207462a0337d798caa5bb04dd10ff7d2c4ce0c9e4 1.7.17: sha256:8154423f54a786bfd1cb6b5e41a47181e80ce7a52a9b28c205b6e81e4d6ec0c7 1.7.16: sha256:d02dce8cc44715bc437e546f669615f8c632788542845be224bc0604ad4de857 1.7.15: sha256:7b7a1c7d1d1483ff4b9849a6064aa709ee5bc4b8835dfe75714665550fdac89d 1.7.14: sha256:2dd693054275fe476a7d7efeae9b103557c51574aa702c38890b737f1a2be7f7 1.7.13: sha256:b97a754c5207562f70d30fa6a700cc90e65bb1bdfe876b5e42ba098e6f261b3b 1.7.12: sha256:e9ca2e6abb8993d04e41555fc5bdce6f44a4cebe3dc54ca9f113445716c1a71d 1.7.11: sha256:451578368acba17569ceb186511929afa7ad1c4a3af3d02983c0c1b5f5f7f21e 1.7.10: sha256:6c874d49da4a004158000611fc06d36432a8a888a7fefd19e524d4165bd86141 1.7.9: sha256:9b3afba45aa5facc3b0695db48d4da0acf94673a373b72e34ea45f8eeb946e5e 1.7.8: sha256:fe38a1d4c868b0756220ce6ea32862fcf9d980a98407740d8a4a1702807792ac 1.7.7: sha256:3a2e43b7324aedbf2ca49a533a956c83a448f99a2bb07b3320d407cf8e28ad55 1.7.6: sha256:5377b4fa4a8759c18d211afd7767d87b17b8f042bdc9901f9b2544261d28c5f3 1.7.5: sha256:258aa8a6dec6a80ec9fc152786e15cf4e2a165147c2eac9a78aeec869292f558 1.7.4: sha256:493cdd9e5ae57722ddb718b59a790ec07c5aa328254602f6df25b83f40e6a1cd 1.7.3: sha256:56ed334b3b8c1c191694c49efc123c4b8ee8244674ff7dbb2bd309da9eac4b57 1.7.2: sha256:d5c32307d7982a5314946e3724534739c433222ead3c36c7afeff2165653d2df 1.7.1: sha256:53c4fa2d544a74ee813e1881c06fa23cb7cf58d9fbd2dd7d5a20f38fc308ac44 1.7.0: sha256:316aca35c3e046d99adc2fb9771e6c3c42ebfc35332e42cfc467708d8bc0d60e skopeo_binary_checksums: arm64: 1.16.1: sha256:3272f15f469af843d325134ff8a77a069d647c5f247766715c098b8f0622b627 1.16.0: sha256:331b09b3b6e6550c178ea1c2fb2bdc5bdbd90c6f6e8d86a974f1117d6ab2fabe 1.15.2: sha256:f81487af3104e37537ff21f1b2527b294f5cc4e7988941a1655ded97c027ac1d 1.15.1: sha256:e20e34f96b5545bacd469b0d85ccce811ffbe2809db36248a3becb4638276959 1.15.0: sha256:bde8cc7e764d246281430d5da07ca906ee0838803199e3a6136a58802b2e0207 1.14.5: sha256:23e157de988c6020f1300b5d73d84d2fed2823ed61dbc6828de3552e9c77a6db 1.14.4: sha256:d825f93b28cf7502569fe75c46aa78187bb63b6bc06036621de7b63290b51058 1.14.3: sha256:e93a82b88e9bff46cbe4e68f96e265d934026a845b76ce51672c7cce26fba164 1.14.2: sha256:364c46085de31edf4b312f13587442f4eade1f181bc5a9ea2ab2ffab5b575916 1.14.1: sha256:fd4fc0adae14f27788fd52cf0d23be2cfd1963e184c4af689de30185455e29a6 amd64: 1.16.1: sha256:8813fb7fcd7a723196ac287683dd929d280f6fe7f0782eace452fe1e3ff2b7eb 1.16.0: sha256:7bc31ed810d1366304d2e975c2910cea5e22cbd68f8316f14cacf44f6c0bd1d2 1.15.2: sha256:6b84d1158f29610f692f24c82459a865c2a21911647cc0cdf44027e7a59f73ba 1.15.1: sha256:d45a93dab851f072fe5d3f0419f5c8bb3ee48069b588c211cccebd023fd5ae3a 1.15.0: sha256:3cdbcde0163abb4c942f62d0302479d5aa4d31c5970d712841cf5d5f76edc594 1.14.5: sha256:180c2d7e8bc00685ba291572db6ddd90acadf03af7595521da17ae1f2c28f4b1 1.14.4: sha256:4c6f8f7c6e5f01675adff8c5bbb542d8d02b9bbdecf0d2abac1e99b8a34a9768 1.14.3: sha256:2db7e036e99ad3b808aaffbafc5267391bd3ba2f45ff03dd0090686eb3eb0f1e 1.14.2: sha256:51218f93a2b079e36a36f7fbe2d2d86778be0a6947653031b4f9e254e2469224 1.14.1: sha256:6b7776bcdf0c92af5d3f3c91a959d091011b42d839025b90f12b7201a083f308 ppc64le: 1.16.1: sha256:248f8f601e4c40dd6d603b66ac26246f96d18451cc3642718c59afb6c2403cf7 1.16.0: sha256:24f1266d6146c27143b5002387c5b68086f1355de7db5c9bfe820928e3b8e298 1.15.2: sha256:5b123d38c34024e8b62b3bc94abfeea3007291743260bf7f62b2a1d935f1c3f9 1.15.1: sha256:39a4a6d77daca09a93a0b490285f48cd9040da1ba9c05b1f9709483e4f65c318 1.15.0: sha256:fb7f390f52f4b81f85d9bdce8715af5e27ee3969eff236b5f3c0f3a0b5a182e1 1.14.5: sha256:4ed476c46fabb3b320aac9b88ddc1b7a2665cb151a93482db7cb98e5768a768f 1.14.4: sha256:f1b37ad1b83bd43bada6e49518165cf41d727d0662351dc5fcc9a46f0c3b4482 1.14.3: sha256:9028b7c4aafe235f1ba4efd57435b97ace341e544d3a6807440ac3b0f32d7d73 yq_checksums: arm: 4.42.1: sha256:4e3fe0c37793d28e96d465d9958fbf679d8c616e1857d0faf7980ad087f32aee 4.41.1: sha256:ccd50344652c02574ca7dd123c7d66a06b391838e8ca6088b688e6edf2e25d0c 4.40.7: sha256:fb922bb1e3974fbd15957feafb5e9bbabe43f4192999cf9b3e0e470815f2e0da 4.40.5: sha256:c587b2411e43d3fbcdd24c233fb558a362b5111a8446b23f9ce9a4a5665a7041 4.40.4: sha256:2ff3f17483f2172a20130b16328114bfe6abd7d3068d66d8194a5093079e8529 4.40.3: sha256:6a97856e8b4ef992ce08dcfdf97fec517cf612b1a89078406f401673f126c21c arm64: 4.42.1: sha256:16a57531a594b66c3e0981cd93f9e9cd4b684a347b86eaf5e3f409074ad67eb8 4.41.1: sha256:066aa930d74e39a25447b1900d8cbb3e1c7df72cd75bc203bc6ae5ee577a5b4a 4.40.7: sha256:a84f2c8f105b70cd348c3bf14048aeb1665c2e7314cbe9aaff15479f268b8412 4.40.5: sha256:9431f0fa39a0af03a152d7fe19a86e42e9ff28d503ed4a70598f9261ec944a97 4.40.4: sha256:79c61a1ebfedb165ec8c4678777775b52e2c581801f5d4cd80f97300852fe0f0 4.40.3: sha256:44a5cca10d33019b8a46212882197be4f961dfe7deddde0af497065aa980a6a4 amd64: 4.42.1: sha256:1a95960dddd426321354d58d2beac457717f7c49a9ec0806749a5a9e400eb45e 4.41.1: sha256:ce0d5a61c256a463fd32f67f133e0c2948bc2cf77d44c42ff335a40e6bef34bf 4.40.7: sha256:4f13ee9303a49f7e8f61e7d9c87402e07cc920ae8dfaaa8c10d7ea1b8f9f48ed 4.40.5: sha256:0d6aaf1cf44a8d18fbc7ed0ef14f735a8df8d2e314c4cc0f0242d35c0a440c95 4.40.4: sha256:f9163412d9aa2aa55e888fdcaf2b4053ada20074be35f701424caa7163100704 4.40.3: sha256:6e9a5ed9591dbf1d13aaec4efaaf0ecdaf4945ea393b9ce01f4c3dea22311470 ppc64le: 4.42.1: sha256:d0d1cdbd2c4a7e6995433baf879cadaa47f6f12290e1661ea11933ed90baccb6 4.41.1: sha256:eed2af79d0ad787878b2d5c7c592e43ac152208d9ed432b42a43663167e276e8 4.40.7: sha256:ac0e8d06a7ed9afc108b4e2e9d6900312b01757f61b75fcecb809f15c39b10e7 4.40.5: sha256:a1df9d2b872fbb30583526bf4f37f737dc1913b28606dfc1dafeaf56a8862b3d 4.40.4: sha256:c67379085a44558825a60a8af3b59b400852b168356070829bc0f45c70553f45 4.40.3: sha256:2fe818a0b141913a41548e0e727267479d0f755221c73f9e304788c8e9139a45 gateway_api_standard_crds_checksums: no_arch: 1.4.1: sha256:73b91b77f6be023a8c92c969fc664e5bd3b1a28aea59eac9ebc904607354dad2 1.4.0: sha256:6a4029e661446d64add866a00ecdc40c14219b68777ab614c5cdaac0adb481f1 1.3.0: sha256:78796d5c51450fc55d8dc8092ba8137f8c807982d7508d7875d5c537a24082b9 1.2.1: sha256:97598bf6ab3b33b9b5c5432bdd24de091e4e9c3aa0575ebb0710a2a19cd64d64 1.2.0: sha256:38ed055bb25dc580c0366899c0bed9b9e92dfcd1c180a569133f3946026cf102 1.1.1: sha256:ffbfc11c5d1a11e8fd03de12a1b48f55ee782646d84b630068f48fdde86a60cf 1.1.0: sha256:c411805475d430a34242623a8e17153a7c40e946497bfd494e558b0d1a8858b3 1.0.0: sha256:23e4e1095c72a0587474f7fb3f85c319cdec77a083ab91237ffbdec1f1834d2a gateway_api_experimental_crds_checksums: no_arch: 1.4.1: sha256:553327e0ff32a1a2be446bf93823c8413cf9253ac6a6d5407eebd1e8d269f69e 1.4.0: sha256:0414b160767377e85fd362855501200c6b83b84758bcd532652e3fe1cc677e49 1.3.0: sha256:3e7a27e4456ff3d68606a6a8516306aaff354d6f0950b32bb31930669b7bf8b8 1.2.1: sha256:d3aa6723a3306770cffb601ee22af3d35da43acfa1ca547fc0d3bce08dad66e7 1.2.0: sha256:4369188e63b9ab5a35b5a83032c94d871159dece086b908b6ea18ea321ca06a9 1.1.1: sha256:529011bdf6c71ad6200bcd483ce4f248bc45309207d294bedf24e45a7563a9b0 1.1.0: sha256:10f322744a005d4e73e2b067e95fecd4cfec619dc7564930b488c296bfa3bec1 1.0.0: sha256:6c601dced7872a940d76fa667ae126ba718cb4c6db970d0bab49128ecc1192a3 prometheus_operator_crds_checksums: no_arch: 0.88.1: sha256:b827b8ec478e6b31cc1b85c1736570a3575953fe9f470fc29d0ffdb2803d94c4 0.88.0: sha256:11ee66653657f3abc1bc8c41e17aa950eadb66035edb7f84cd3a1cbe4c67b2a4 0.87.1: sha256:62490f7c1863539d61295f53784e27d70deec96a3b465832ba3cf96120e298b5 0.87.0: sha256:a5282133ffa634405b0414d2fdc07e6fe393124d1d5072073af363689dac6a62 0.86.2: sha256:7c9d455333ac5ea7837d5f0e4edd966698e44edd79108bafdd8508f2da503b5b 0.86.1: sha256:9a30912ba9970a2968d7a8bf030a9f6579a5e8b312961018b5fe4c1153fc5fce 0.86.0: sha256:0d2a590b288c79a98515e9fc4315451cfbde964c7977eb527696f7c2ebf47f58 0.85.0: sha256:30e1b1b034ebc750d50a77dc19841176d698d524edf677276a760f9e228e1208 0.84.1: sha256:f4a186ac58f354793e27a0b4b6f8baf5a31a9d10045e5085c23b0570dbfd30dd 0.84.0: sha256:8990f6837ccff4461df9abe19d31d532fef11386d85d861b392249fff2502255 argocd_install_checksums: no_arch: 2.14.21: sha256:ee39d40847bbb36154ebcdf2f5c93e0a9001ab60131afb60dbe42981a069699e 2.14.20: sha256:747db8bb6d1591b49bb6266412e6dcaccf3c7bd3bd81ef7d63b0ab5bfe6af951 2.14.19: sha256:c0f4c5331837a01a9764853ba451993a61f403e9e520986479e7777af5ef5a1f 2.14.18: sha256:ea09a2a920d234ee99468bcf8da2851f909906e9152178b5d06efb282af0f33e 2.14.17: sha256:d62bbea850dc8b515aa0d990fa52d9296d99542f8446b5d309c59e8a1fa29b21 2.14.16: sha256:f8788883040870835c008ce11942a8d5d8b250df4a314e3009dfb30cfb963725 2.14.15: sha256:0368b8a0adbb673408f2cc2367302ad1068d12cd9ab17cf6680bcb5fdba7c381 2.14.14: sha256:36315fcd4bc86e41b89f322daabf0bab20ffccc98976f882dba56f79c00e65ed 2.14.13: sha256:2b96310c5f52f23aad8643fc4536fc4ba45d2a41384bd71f191dc4ed2939b166 2.14.12: sha256:c1156a3bd9f76397757d3c42f024aaff67cac718518138c65e78b5997c37d6fc 2.14.11: sha256:9fd3317476ddad309954b7f99dbf53712b3611f0fc713390b5f7fdc68b536e2f 2.14.10: sha256:ddb73ff9b1ddfda4f882175a2069be163ba3ff7eb9c64622415778bb34e43955 2.14.9: sha256:b2c8093169039e3e516cb82cd98a64ca9ca0f116d7302a6ad074ffc7b28d85c2 2.14.8: sha256:f436b495681b2bdc1388c3ce0f708c6744b771ae637c9f93b2e50e166fac2dbd 2.14.7: sha256:4cffc79a9967cdd48dec48f0dd5f5b257057d0c193b3f4a9209b4d2ed614952d 2.14.6: sha256:549de9af3ff1a244e50e39ef523034788edc25e268c7be12afcf401e94cbc916 2.14.5: sha256:247ccda29c9faac4e0c8598680f5ebefff9911e957e3aeaf838eb4bbf455f2f4 ================================================ FILE: roles/kubespray_defaults/vars/main/main.yml ================================================ --- # Internal version manipulation tooling # Get kubernetes major version (i.e. 1.17.4 => 1.17) kube_major_version: "{{ (kube_version | split('.'))[:-1] | join('.') }}" kube_next: "{{ ((kube_version | split('.'))[1] | int) + 1 }}" kube_major_next_version: "1.{{ kube_next }}" pod_infra_supported_versions: '1.35': '3.10.1' '1.34': '3.10.1' '1.33': '3.10' etcd_supported_versions: '1.35': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.7', '<'))[0] }}" '1.34': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}" '1.33': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}" # Kubespray constants kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}" # The lowest version allowed to upgrade from (same as calico_version in the previous branch) calico_min_version_required: "3.27.0" containerd_min_version_required: "1.3.7" # mixed kube_service_addresses/kube_service_addresses_ipv6 for a variety of network stacks(dualstack, ipv6only, ipv4only) kube_service_subnets: >- {%- if ipv4_stack and ipv6_stack -%} {{ kube_service_addresses }},{{ kube_service_addresses_ipv6 }} {%- elif ipv4_stack -%} {{ kube_service_addresses }} {%- else -%} {{ kube_service_addresses_ipv6 }} {%- endif -%} # mixed kube_pods_subnet/kube_pods_subnet_ipv6 for a variety of network stacks(dualstack, ipv6only, ipv4only) kube_pods_subnets: >- {%- if ipv4_stack and ipv6_stack -%} {{ kube_pods_subnet }},{{ kube_pods_subnet_ipv6 }} {%- elif ipv4_stack -%} {{ kube_pods_subnet }} {%- else -%} {{ kube_pods_subnet_ipv6 }} {%- endif -%} ================================================ FILE: roles/network_facts/defaults/main.yml ================================================ --- # Additional string host to inject into NO_PROXY additional_no_proxy: "" additional_no_proxy_list: "{{ additional_no_proxy | split(',') }}" no_proxy_exclude_workers: false ================================================ FILE: roles/network_facts/meta/main.yml ================================================ --- dependencies: - role: kubespray_defaults ================================================ FILE: roles/network_facts/tasks/main.yaml ================================================ --- - name: Gather node IPs setup: gather_subset: '!all,!min,network' filter: "ansible_default_ip*" when: ansible_default_ipv4 is not defined or ansible_default_ipv6 is not defined ignore_unreachable: true - name: Set computed IPs variables vars: fallback_ip: "{{ ansible_default_ipv4.address | d('127.0.0.1') }}" fallback_ip6: "{{ ansible_default_ipv6.address | d('::1') }}" # Set 127.0.0.1 as fallback IP if we do not have host facts for host # ansible_default_ipv4 isn't what you think. _ipv4: "{{ ip | default(fallback_ip) }}" _access_ipv4: "{{ access_ip | default(_ipv4) }}" _ipv6: "{{ ip6 | default(fallback_ip6) }}" _access_ipv6: "{{ access_ip6 | default(_ipv6) }}" _access_ips: - "{{ _access_ipv4 if ipv4_stack }}" - "{{ _access_ipv6 if ipv6_stack }}" _ips: - "{{ _ipv4 if ipv4_stack }}" - "{{ _ipv6 if ipv6_stack }}" set_fact: cacheable: true main_access_ip: "{{ _access_ipv4 if ipv4_stack else _access_ipv6 }}" main_ip: "{{ _ipv4 if ipv4_stack else _ipv6 }}" # Mixed IPs - for dualstack main_access_ips: "{{ _access_ips | select }}" main_ips: "{{ _ips | select }}" - name: Set no_proxy to all assigned cluster IPs and hostnames when: - http_proxy is defined or https_proxy is defined - no_proxy is not defined vars: groups_with_no_proxy: - kube_control_plane - "{{ '' if no_proxy_exclude_workers else 'kube_node' }}" # TODO: exclude by a boolean in inventory rather than global variable - etcd - calico_rr hosts_with_no_proxy: "{{ groups_with_no_proxy | select | map('extract', groups) | select('defined') | flatten }}" _hostnames: "{{ (hosts_with_no_proxy + (hosts_with_no_proxy | map('extract', hostvars, morekeys=['ansible_hostname']) | select('defined'))) | unique }}" no_proxy_prepare: - "{{ apiserver_loadbalancer_domain_name | d('') }}" - "{{ loadbalancer_apiserver.address if loadbalancer_apiserver is defined else '' }}" - "{{ hosts_with_no_proxy | map('extract', hostvars, morekeys=['main_access_ip']) }}" - "{{ _hostnames }}" - "{{ _hostnames | map('regex_replace', '$', '.' + dns_domain ) }}" - "{{ additional_no_proxy_list }}" - 127.0.0.1 - localhost - "{{ kube_service_subnets }}" - "{{ kube_pods_subnets }}" - svc - "svc.{{ dns_domain }}" set_fact: no_proxy: "{{ no_proxy_prepare | select | flatten | unique | join(',') }}" run_once: true ================================================ FILE: roles/network_plugin/calico/files/openssl.conf ================================================ req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [ v3_req ] basicConstraints = CA:FALSE keyUsage = digitalSignature, keyEncipherment [ ssl_client ] extendedKeyUsage = clientAuth, serverAuth basicConstraints = CA:FALSE subjectKeyIdentifier=hash authorityKeyIdentifier=keyid,issuer [ v3_ca ] basicConstraints = CA:TRUE keyUsage = cRLSign, digitalSignature, keyCertSign subjectKeyIdentifier=hash authorityKeyIdentifier=keyid:always,issuer [ ssl_client_apiserver ] extendedKeyUsage = clientAuth, serverAuth basicConstraints = CA:FALSE subjectKeyIdentifier=hash authorityKeyIdentifier=keyid,issuer subjectAltName = DNS:calico-api.calico-apiserver.svc ================================================ FILE: roles/network_plugin/calico/handlers/main.yml ================================================ --- - name: Delete 10-calico.conflist file: path: /etc/cni/net.d/10-calico.conflist state: absent listen: Reset_calico_cni when: calico_cni_config is defined - name: Calico | delete calico-node docker containers shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" args: executable: /bin/bash register: docker_calico_node_remove until: docker_calico_node_remove is succeeded retries: 5 when: - container_manager in ["docker"] - calico_cni_config is defined listen: Reset_calico_cni - name: Calico | delete calico-node crio/containerd containers shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name calico-node-* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"' args: executable: /bin/bash register: crictl_calico_node_remove until: crictl_calico_node_remove is succeeded retries: 5 when: - container_manager in ["crio", "containerd"] - calico_cni_config is defined listen: Reset_calico_cni ================================================ FILE: roles/network_plugin/calico/meta/main.yml ================================================ --- dependencies: - role: network_plugin/calico_defaults ================================================ FILE: roles/network_plugin/calico/rr/defaults/main.yml ================================================ --- # Global as_num (/calico/bgp/v1/global/as_num) # should be the same as in calico role global_as_num: "64512" calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}" ================================================ FILE: roles/network_plugin/calico/rr/tasks/main.yml ================================================ --- - name: Calico-rr | Pre-upgrade tasks include_tasks: pre.yml - name: Calico-rr | Configuring node tasks include_tasks: update-node.yml - name: Calico-rr | Set label for route reflector command: >- {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} 'i-am-a-route-reflector=true' --overwrite changed_when: false register: calico_rr_label until: calico_rr_label is succeeded delay: "{{ retry_stagger | random + 3 }}" retries: 10 ================================================ FILE: roles/network_plugin/calico/rr/tasks/pre.yml ================================================ --- - name: Calico-rr | Disable calico-rr service if it exists service: name: calico-rr state: stopped enabled: false failed_when: false - name: Calico-rr | Delete obsolete files file: path: "{{ item }}" state: absent with_items: - /etc/calico/calico-rr.env - /etc/systemd/system/calico-rr.service ================================================ FILE: roles/network_plugin/calico/rr/tasks/update-node.yml ================================================ --- # Workaround to retry a block of tasks, ansible doesn't have a direct way to do it, # you can follow the block loop request in: https://github.com/ansible/ansible/issues/46203 - name: Calico-rr | Configure route reflector block: - name: Set the retry count set_fact: retry_count: "{{ 0 if retry_count is undefined else retry_count | int + 1 }}" - name: Calico | Set label for route reflector # noqa command-instead-of-shell shell: "{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-rr-id={{ calico_rr_id }} --overwrite" changed_when: false register: calico_rr_id_label until: calico_rr_id_label is succeeded delay: "{{ retry_stagger | random + 3 }}" retries: 10 when: calico_rr_id is defined - name: Calico-rr | Fetch current node object command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson" changed_when: false register: calico_rr_node until: calico_rr_node is succeeded delay: "{{ retry_stagger | random + 3 }}" retries: 10 - name: Calico-rr | Set route reflector cluster ID # noqa: jinja[spacing] set_fact: calico_rr_node_patched: >- {{ calico_rr_node.stdout | from_json | combine({ 'spec': { 'bgp': { 'routeReflectorClusterID': cluster_id }}}, recursive=True) }} - name: Calico-rr | Configure route reflector # noqa command-instead-of-shell shell: "{{ bin_dir }}/calicoctl.sh replace -f-" args: stdin: "{{ calico_rr_node_patched | to_json }}" rescue: - name: Fail if retry limit is reached fail: msg: Ended after 10 retries when: retry_count | int == 10 - name: Retrying node configuration debug: msg: "Failed to configure route reflector - Retrying..." - name: Retry node configuration include_tasks: update-node.yml ================================================ FILE: roles/network_plugin/calico/tasks/calico_apiserver_certs.yml ================================================ --- - name: Calico | Check if calico apiserver exists command: "{{ kubectl }} -n calico-apiserver get secret calico-apiserver-certs" register: calico_apiserver_secret changed_when: false failed_when: false - name: Calico | Create ns manifests template: src: "calico-apiserver-ns.yml.j2" dest: "{{ kube_config_dir }}/calico-apiserver-ns.yml" mode: "0644" - name: Calico | Apply ns manifests kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/calico-apiserver-ns.yml" state: "latest" - name: Calico | Ensure calico certs dir file: path: /etc/calico/certs state: directory mode: "0755" when: calico_apiserver_secret.rc != 0 - name: Calico | Copy ssl script for apiserver certs template: src: make-ssl-calico.sh.j2 dest: "{{ bin_dir }}/make-ssl-apiserver.sh" mode: "0755" when: calico_apiserver_secret.rc != 0 - name: Calico | Copy ssl config for apiserver certs copy: src: openssl.conf dest: /etc/calico/certs/openssl.conf mode: "0644" when: calico_apiserver_secret.rc != 0 - name: Calico | Generate apiserver certs command: >- {{ bin_dir }}/make-ssl-apiserver.sh -f /etc/calico/certs/openssl.conf -c {{ kube_cert_dir }} -d /etc/calico/certs -s apiserver when: calico_apiserver_secret.rc != 0 - name: Calico | Create calico apiserver generic secrets command: >- {{ kubectl }} -n calico-apiserver create secret generic {{ item.name }} --from-file={{ item.cert }} --from-file={{ item.key }} with_items: - name: calico-apiserver-certs cert: /etc/calico/certs/apiserver.crt key: /etc/calico/certs/apiserver.key when: calico_apiserver_secret.rc != 0 ================================================ FILE: roles/network_plugin/calico/tasks/check.yml ================================================ --- - name: Stop if legacy encapsulation variables are detected (ipip) assert: that: - ipip is not defined msg: "'ipip' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Stop if legacy encapsulation variables are detected (ipip_mode) assert: that: - ipip_mode is not defined msg: "'ipip_mode' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Stop if legacy encapsulation variables are detected (calcio_ipam_autoallocateblocks) assert: that: - calcio_ipam_autoallocateblocks is not defined msg: "'calcio_ipam_autoallocateblocks' configuration variable is deprecated, it's a typo, please configure your inventory with 'calico_ipam_autoallocateblocks' set to 'true' or 'false' according to your specific needs" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Stop if supported Calico versions assert: that: - "calico_version in calico_crds_checksums.no_arch.keys()" msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_checksums.no_arch.keys() }}" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Check if calicoctl.sh exists stat: path: "{{ bin_dir }}/calicoctl.sh" register: calicoctl_sh_exists run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Check if calico ready command: "{{ bin_dir }}/calicoctl.sh get ClusterInformation default" register: calico_ready run_once: true ignore_errors: true retries: 5 delay: 10 until: calico_ready.rc == 0 delegate_to: "{{ groups['kube_control_plane'][0] }}" when: calicoctl_sh_exists.stat.exists - name: Check that current calico version is enough for upgrade run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" when: calicoctl_sh_exists.stat.exists and calico_ready.rc == 0 block: - name: Get current calico version shell: "set -o pipefail && {{ bin_dir }}/calicoctl.sh version | grep 'Client Version:' | awk '{ print $3}'" args: executable: /bin/bash register: calico_version_on_server changed_when: false check_mode: false - name: Assert that current calico version is enough for upgrade assert: that: - calico_version_on_server.stdout.removeprefix('v') is version(calico_min_version_required, '>=') msg: > Your version of calico is not fresh enough for upgrade. Minimum version is {{ calico_min_version_required }} supported by the previous kubespray release. But current version is {{ calico_version_on_server.stdout }}. - name: "Check that cluster_id is set and a valid IPv4 address if calico_rr enabled" assert: that: - cluster_id is defined - cluster_id is ansible.utils.ipv4 msg: "A unique cluster_id is required if using calico_rr, and it must be a valid IPv4 address" when: - peer_with_calico_rr - inventory_hostname == groups['kube_control_plane'][0] run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check that calico_rr nodes are in k8s_cluster group" assert: that: - '"k8s_cluster" in group_names' msg: "calico_rr must be a child group of k8s_cluster group" when: - '"calico_rr" in group_names' run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check vars defined correctly" assert: that: - "calico_pool_name is defined" - "calico_pool_name is match('^[a-zA-Z0-9-_\\\\.]{2,63}$')" msg: "calico_pool_name contains invalid characters" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check calico network backend defined correctly" assert: that: - "calico_network_backend in ['bird', 'vxlan', 'none']" msg: "calico network backend is not 'bird', 'vxlan' or 'none'" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check ipip and vxlan mode defined correctly" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" assert: that: - "calico_ipip_mode in ['Always', 'CrossSubnet', 'Never']" - "calico_vxlan_mode in ['Always', 'CrossSubnet', 'Never']" msg: "calico inter host encapsulation mode is not 'Always', 'CrossSubnet' or 'Never'" - name: "Check ipip and vxlan mode if simultaneously enabled" assert: that: - "calico_vxlan_mode in ['Never']" msg: "IP in IP and VXLAN mode is mutualy exclusive modes" when: - "calico_ipip_mode in ['Always', 'CrossSubnet']" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check ipip and vxlan mode if simultaneously enabled" assert: that: - "calico_ipip_mode in ['Never']" msg: "IP in IP and VXLAN mode is mutualy exclusive modes" when: - "calico_vxlan_mode in ['Always', 'CrossSubnet']" run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Get Calico {{ calico_pool_name }} configuration" command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }} -o json" failed_when: false changed_when: false check_mode: false register: calico run_once: true when: ipv4_stack | bool delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Set calico_pool_conf" set_fact: calico_pool_conf: '{{ calico.stdout | from_json }}' when: - ipv4_stack | bool - calico is defined - calico.rc == 0 and calico.stdout run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check if inventory match current cluster configuration" assert: that: - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize | int - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet)) - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode msg: "Your inventory doesn't match the current cluster configuration" when: - ipv4_stack | bool - calico_pool_conf is defined run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Get Calico {{ calico_pool_name }}-ipv6 configuration" command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }}-ipv6 -o json" failed_when: false changed_when: false check_mode: false register: calico_ipv6 run_once: true when: ipv6_stack | bool delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Set calico_pool_ipv6_conf" set_fact: calico_pool_conf: '{{ calico_ipv6.stdout | from_json }}' when: - ipv6_stack | bool - alico_ipv6 is defined - calico_ipv6.rc == 0 and calico_ipv6.stdout run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check if ipv6 inventory match current cluster configuration" assert: that: - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize_ipv6 | int - calico_pool_conf.spec.cidr == (calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6)) - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode_ipv6 - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode_ipv6 msg: "Your ipv6 inventory doesn't match the current cluster configuration" when: - ipv6_stack | bool - calico_pool_ipv6_conf is defined run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check kdd calico_datastore if calico_apiserver_enabled" assert: that: calico_datastore == "kdd" msg: "When using calico apiserver you need to use the kubernetes datastore" when: - calico_apiserver_enabled run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check kdd calico_datastore if typha_enabled" assert: that: calico_datastore == "kdd" msg: "When using typha you need to use the kubernetes datastore" when: - typha_enabled run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: "Check ipip mode is Never for calico ipv6" assert: that: - "calico_ipip_mode_ipv6 in ['Never']" msg: "Calico doesn't support ipip tunneling for the IPv6" when: ipv6_stack | bool run_once: true delegate_to: "{{ groups['kube_control_plane'][0] }}" ================================================ FILE: roles/network_plugin/calico/tasks/install.yml ================================================ --- - name: Calico | Install Wireguard packages package: name: "{{ item }}" state: present with_items: "{{ calico_wireguard_packages }}" register: calico_package_install until: calico_package_install is succeeded retries: 4 when: calico_wireguard_enabled - name: Calico | Copy calicoctl binary from download dir copy: src: "{{ downloads.calicoctl.dest }}" dest: "{{ bin_dir }}/calicoctl" mode: "0755" remote_src: true - name: Calico | Create calico certs directory file: dest: "{{ calico_cert_dir }}" state: directory mode: "0750" owner: root group: root when: calico_datastore == "etcd" - name: Calico | Link etcd certificates for calico-node file: src: "{{ etcd_cert_dir }}/{{ item.s }}" dest: "{{ calico_cert_dir }}/{{ item.d }}" state: hard mode: "0640" force: true with_items: - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} when: calico_datastore == "etcd" - name: Calico | Generate typha certs include_tasks: typha_certs.yml when: - typha_secure - inventory_hostname == groups['kube_control_plane'][0] - name: Calico | Generate apiserver certs include_tasks: calico_apiserver_certs.yml when: - calico_apiserver_enabled - inventory_hostname == groups['kube_control_plane'][0] - name: Calico | Install calicoctl wrapper script template: src: "calicoctl.{{ calico_datastore }}.sh.j2" dest: "{{ bin_dir }}/calicoctl.sh" mode: "0755" owner: root group: root - name: Calico | wait for etcd uri: url: "{{ etcd_access_addresses.split(',') | first }}/health" validate_certs: false client_cert: "{{ calico_cert_dir }}/cert.crt" client_key: "{{ calico_cert_dir }}/key.pem" register: result until: result.status == 200 or result.status == 401 retries: 10 delay: 5 run_once: true when: calico_datastore == "etcd" - name: Calico | Check if calico network pool has already been configured # noqa risky-shell-pipe - grep will exit 1 if no match found shell: > {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l args: executable: /bin/bash register: calico_conf retries: 4 until: calico_conf.rc == 0 delay: "{{ retry_stagger | random + 3 }}" changed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - ipv4_stack | bool - name: Calico | Ensure that calico_pool_cidr is within kube_pods_subnet when defined assert: that: "[calico_pool_cidr] | ansible.utils.ipaddr(kube_pods_subnet) | length == 1" msg: "{{ calico_pool_cidr }} is not within or equal to {{ kube_pods_subnet }}" when: - inventory_hostname == groups['kube_control_plane'][0] - ipv4_stack | bool - calico_pool_cidr is defined - 'calico_conf.stdout == "0"' - name: Calico | Check if calico IPv6 network pool has already been configured # noqa risky-shell-pipe - grep will exit 1 if no match found shell: > {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}" | wc -l args: executable: /bin/bash register: calico_conf_ipv6 retries: 4 until: calico_conf_ipv6.rc == 0 delay: "{{ retry_stagger | random + 3 }}" changed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - ipv6_stack - name: Calico | Ensure that calico_pool_cidr_ipv6 is within kube_pods_subnet_ipv6 when defined assert: that: "[calico_pool_cidr_ipv6] | ansible.utils.ipaddr(kube_pods_subnet_ipv6) | length == 1" msg: "{{ calico_pool_cidr_ipv6 }} is not within or equal to {{ kube_pods_subnet_ipv6 }}" when: - inventory_hostname == groups['kube_control_plane'][0] - ipv6_stack | bool - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == "0" - calico_pool_cidr_ipv6 is defined - name: Calico | kdd specific configuration when: - ('kube_control_plane' in group_names) - calico_datastore == "kdd" block: - name: Calico | Create calico manifests for kdd copy: src: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/crds.yaml" dest: "{{ kube_config_dir }}/kdd-crds.yml" mode: "0644" remote_src: true - name: Calico | Create Calico Kubernetes datastore resources kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/kdd-crds.yml" state: "latest" register: kubectl_result until: kubectl_result is succeeded retries: 5 when: - inventory_hostname == groups['kube_control_plane'][0] - name: Calico | Configure Felix when: - inventory_hostname == groups['kube_control_plane'][0] block: - name: Calico | Get existing FelixConfiguration command: "{{ bin_dir }}/calicoctl.sh get felixconfig default -o json" register: _felix_cmd ignore_errors: true changed_when: false - name: Calico | Set kubespray FelixConfiguration set_fact: _felix_config: > { "kind": "FelixConfiguration", "apiVersion": "projectcalico.org/v3", "metadata": { "name": "default", }, "spec": { "ipipEnabled": {{ calico_ipip_mode != 'Never' }}, "reportingInterval": "{{ calico_felix_reporting_interval }}", "bpfLogLevel": "{{ calico_bpf_log_level }}", "bpfEnabled": {{ calico_bpf_enabled | bool }}, "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}", "wireguardEnabled": {{ calico_wireguard_enabled | bool }}, "logSeverityScreen": "{{ calico_felix_log_severity_screen }}", "vxlanEnabled": {{ calico_vxlan_mode != 'Never' }}, "featureDetectOverride": "{{ calico_feature_detect_override }}", "floatingIPs": "{{ calico_felix_floating_ips }}" } } - name: Calico | Process FelixConfiguration set_fact: _felix_config: "{{ _felix_cmd.stdout | from_json | combine(_felix_config, recursive=True) }}" when: - _felix_cmd is success - name: Calico | Configure calico FelixConfiguration command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ _felix_config is string | ternary(_felix_config, _felix_config | to_json) }}" changed_when: false - name: Calico | Configure Calico IP Pool when: - inventory_hostname == groups['kube_control_plane'][0] - ipv4_stack | bool block: - name: Calico | Get existing calico network pool command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }} -o json" register: _calico_pool_cmd ignore_errors: true changed_when: false - name: Calico | Set kubespray calico network pool set_fact: _calico_pool: > { "kind": "IPPool", "apiVersion": "projectcalico.org/v3", "metadata": { "name": "{{ calico_pool_name }}", }, "spec": { "blockSize": {{ calico_pool_blocksize }}, "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ calico_ipip_mode }}", "vxlanMode": "{{ calico_vxlan_mode }}", "natOutgoing": {{ nat_outgoing | default(false) }} } } - name: Calico | Process calico network pool when: - _calico_pool_cmd is success block: - name: Calico | Get current calico network pool blocksize set_fact: _calico_blocksize: > { "spec": { "blockSize": {{ (_calico_pool_cmd.stdout | from_json).spec.blockSize }} } } - name: Calico | Merge calico network pool set_fact: _calico_pool: "{{ _calico_pool_cmd.stdout | from_json | combine(_calico_pool, _calico_blocksize, recursive=True) }}" - name: Calico | Configure calico network pool command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ _calico_pool is string | ternary(_calico_pool, _calico_pool | to_json) }}" changed_when: false - name: Calico | Configure Calico IPv6 Pool when: - inventory_hostname == groups['kube_control_plane'][0] - ipv6_stack | bool block: - name: Calico | Get existing calico ipv6 network pool command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }}-ipv6 -o json" register: _calico_pool_ipv6_cmd ignore_errors: true changed_when: false - name: Calico | Set kubespray calico network pool set_fact: _calico_pool_ipv6: > { "kind": "IPPool", "apiVersion": "projectcalico.org/v3", "metadata": { "name": "{{ calico_pool_name }}-ipv6", }, "spec": { "blockSize": {{ calico_pool_blocksize_ipv6 }}, "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}", "ipipMode": "{{ calico_ipip_mode_ipv6 }}", "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}", "natOutgoing": {{ nat_outgoing_ipv6 | default(false) }} } } - name: Calico | Process calico ipv6 network pool when: - _calico_pool_ipv6_cmd is success block: - name: Calico | Get current calico ipv6 network pool blocksize set_fact: _calico_blocksize_ipv6: > { "spec": { "blockSize": {{ (_calico_pool_ipv6_cmd.stdout | from_json).spec.blockSize }} } } - name: Calico | Merge calico ipv6 network pool set_fact: _calico_pool_ipv6: "{{ _calico_pool_ipv6_cmd.stdout | from_json | combine(_calico_pool_ipv6, _calico_blocksize_ipv6, recursive=True) }}" - name: Calico | Configure calico ipv6 network pool command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ _calico_pool_ipv6 is string | ternary(_calico_pool_ipv6, _calico_pool_ipv6 | to_json) }}" changed_when: false - name: Populate Service External IPs set_fact: _service_external_ips: "{{ _service_external_ips | default([]) + [{'cidr': item}] }}" with_items: "{{ calico_advertise_service_external_ips }}" run_once: true - name: Populate Service LoadBalancer IPs set_fact: _service_loadbalancer_ips: "{{ _service_loadbalancer_ips | default([]) + [{'cidr': item}] }}" with_items: "{{ calico_advertise_service_loadbalancer_ips }}" run_once: true - name: "Determine nodeToNodeMesh needed state" set_fact: nodeToNodeMeshEnabled: "false" when: - peer_with_router | default(false) or peer_with_calico_rr | default(false) - ('k8s_cluster' in group_names) run_once: true - name: Calico | Configure Calico BGP when: - inventory_hostname == groups['kube_control_plane'][0] block: - name: Calico | Get existing BGP Configuration command: "{{ bin_dir }}/calicoctl.sh get bgpconfig default -o json" register: _bgp_config_cmd ignore_errors: true changed_when: false - name: Calico | Set kubespray BGP Configuration set_fact: # noqa: jinja[spacing] _bgp_config: > { "kind": "BGPConfiguration", "apiVersion": "projectcalico.org/v3", "metadata": { "name": "default", }, "spec": { "listenPort": {{ calico_bgp_listen_port }}, "logSeverityScreen": "Info", {% if not calico_no_global_as_num | default(false) %}"asNumber": {{ global_as_num }},{% endif %} "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled | default('true') }} , {% if calico_advertise_cluster_ips | default(false) %} "serviceClusterIPs": {%- if ipv4_stack and ipv6_stack-%} [{"cidr": "{{ kube_service_addresses }}", "cidr": "{{ kube_service_addresses_ipv6 }}"}], {%- elif ipv6_stack-%} [{"cidr": "{{ kube_service_addresses_ipv6 }}"}], {%- else -%} [{"cidr": "{{ kube_service_addresses }}"}], {%- endif -%} {% endif %} {% if calico_advertise_service_loadbalancer_ips | length > 0 %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %} "serviceExternalIPs": {{ _service_external_ips | default([]) }} } } - name: Calico | Process BGP Configuration set_fact: _bgp_config: "{{ _bgp_config_cmd.stdout | from_json | combine(_bgp_config, recursive=True) }}" when: - _bgp_config_cmd is success - name: Calico | Set up BGP Configuration command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ _bgp_config is string | ternary(_bgp_config, _bgp_config | to_json) }}" changed_when: false - name: Calico | Create calico manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: calico-config, file: calico-config.yml, type: cm} - {name: calico-node, file: calico-node.yml, type: ds} - {name: calico, file: calico-node-sa.yml, type: sa} - {name: calico, file: calico-cr.yml, type: clusterrole} - {name: calico, file: calico-crb.yml, type: clusterrolebinding} - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm } register: calico_node_manifests when: - ('kube_control_plane' in group_names) - rbac_enabled or item.type not in rbac_resources - name: Calico | Create calico manifests for typha template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: calico, file: calico-typha.yml, type: typha} register: calico_node_typha_manifest when: - ('kube_control_plane' in group_names) - typha_enabled - name: Calico | get calico apiserver caBundle command: "{{ bin_dir }}/kubectl get secret -n calico-apiserver calico-apiserver-certs -o jsonpath='{.data.apiserver\\.crt}'" changed_when: false register: calico_apiserver_cabundle when: - inventory_hostname == groups['kube_control_plane'][0] - calico_apiserver_enabled - name: Calico | set calico apiserver caBundle fact set_fact: calico_apiserver_cabundle: "{{ calico_apiserver_cabundle.stdout }}" when: - inventory_hostname == groups['kube_control_plane'][0] - calico_apiserver_enabled - name: Calico | Create calico manifests for apiserver template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: calico, file: calico-apiserver.yml, type: calico-apiserver} register: calico_apiserver_manifest when: - ('kube_control_plane' in group_names) - calico_apiserver_enabled - name: Start Calico resources kube: name: "{{ item.item.name }}" namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ calico_node_manifests.results }}" - "{{ calico_node_typha_manifest.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" - name: Start Calico apiserver resources kube: name: "{{ item.item.name }}" namespace: "calico-apiserver" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: - "{{ calico_apiserver_manifest.results }}" when: - inventory_hostname == groups['kube_control_plane'][0] - not item is skipped loop_control: label: "{{ item.item.file }}" - name: Wait for calico kubeconfig to be created wait_for: path: /etc/cni/net.d/calico-kubeconfig timeout: "{{ calico_kubeconfig_wait_timeout }}" when: - inventory_hostname not in groups['kube_control_plane'] - calico_datastore == "kdd" - name: Calico | Create Calico ipam manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: calico, file: calico-ipamconfig.yml, type: ipam} when: - ('kube_control_plane' in group_names) - calico_datastore == "kdd" - name: Calico | Create ipamconfig resources kube: kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/calico-ipamconfig.yml" state: "latest" register: resource_result until: resource_result is succeeded retries: 4 when: - inventory_hostname == groups['kube_control_plane'][0] - calico_datastore == "kdd" - name: Calico | Peer with Calico Route Reflector include_tasks: peer_with_calico_rr.yml when: - peer_with_calico_rr | default(false) - name: Calico | Peer with the router include_tasks: peer_with_router.yml when: - peer_with_router | default(false) ================================================ FILE: roles/network_plugin/calico/tasks/main.yml ================================================ --- - name: Calico Pre tasks import_tasks: pre.yml - name: Calico repos import_tasks: repos.yml - name: Calico install include_tasks: install.yml ================================================ FILE: roles/network_plugin/calico/tasks/peer_with_calico_rr.yml ================================================ --- - name: Calico | Set label for groups nodes command: "{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite" changed_when: false register: calico_group_id_label until: calico_group_id_label is succeeded delay: "{{ retry_stagger | random + 3 }}" retries: 10 when: - calico_group_id is defined - name: Calico | Configure peering with route reflectors at global scope command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" # revert when it's already a string stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" vars: stdin: > {"apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { "name": "{{ calico_rr_id }}-to-node" }, "spec": { "peerSelector": "calico-rr-id == '{{ calico_rr_id }}'", "nodeSelector": "calico-group-id == '{{ calico_group_id }}'" }} register: output retries: 4 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" when: - calico_rr_id is defined - calico_group_id is defined - ('calico_rr' in group_names) - name: Calico | Configure peering with route reflectors at global scope command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" # revert when it's already a string stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" vars: stdin: > {"apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { "name": "peer-to-rrs" }, "spec": { "nodeSelector": "!has(i-am-a-route-reflector)", "peerSelector": "has(i-am-a-route-reflector)" }} register: output retries: 4 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" with_items: - "{{ groups['calico_rr'] | default([]) }}" when: - inventory_hostname == groups['kube_control_plane'][0] - calico_rr_id is not defined or calico_group_id is not defined - name: Calico | Configure route reflectors to peer with each other command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" # revert when it's already a string stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" vars: stdin: > {"apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { "name": "rr-mesh" }, "spec": { "nodeSelector": "has(i-am-a-route-reflector)", "peerSelector": "has(i-am-a-route-reflector)" }} register: output retries: 4 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" with_items: - "{{ groups['calico_rr'] | default([]) }}" when: - inventory_hostname == groups['kube_control_plane'][0] ================================================ FILE: roles/network_plugin/calico/tasks/peer_with_router.yml ================================================ --- - name: Calico | Configure peering with router(s) at global scope command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" vars: stdin: > {"apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { "name": "global-{{ item.name | default(item.router_id | replace(':', '-')) }}" }, "spec": { "asNumber": "{{ item.as }}", "peerIP": "{{ item.router_id }}" }} register: output retries: 4 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" with_items: - "{{ peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'global') | list }}" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Calico | Get node for per node peering command: cmd: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }}" register: output_get_node when: - ('k8s_cluster' in group_names) - local_as is defined - groups['calico_rr'] | default([]) | length == 0 delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Calico | Patch node asNumber for per node peering command: cmd: |- {{ bin_dir }}/calicoctl.sh patch node "{{ inventory_hostname }}" --patch '{{ patch is string | ternary(patch, patch | to_json) }}' vars: patch: > {"spec": { "bgp": { "asNumber": "{{ local_as }}" }, "orchRefs": [{"nodeName": "{{ inventory_hostname }}", "orchestrator": "k8s"}] }} register: output retries: 0 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" when: - ('k8s_cluster' in group_names) - local_as is defined - groups['calico_rr'] | default([]) | length == 0 - output_get_node.rc == 0 - name: Calico | Configure node asNumber for per node peering command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" vars: stdin: > {"apiVersion": "projectcalico.org/v3", "kind": "Node", "metadata": { "name": "{{ inventory_hostname }}" }, "spec": { "bgp": { "asNumber": "{{ local_as }}" }, "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] }} register: output retries: 4 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" when: - ('k8s_cluster' in group_names) - local_as is defined - groups['calico_rr'] | default([]) | length == 0 - output_get_node.rc != 0 - name: Calico | Configure peering with router(s) at node scope command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" vars: stdin: > {"apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id | replace(':', '-')) }}" }, "spec": { "asNumber": "{{ item.as }}", "node": "{{ inventory_hostname }}", "peerIP": "{{ item.router_id }}", {% if calico_version is version('3.26.0', '>=') and (item.filters | default([]) | length > 0) %} "filters": {{ item.filters }}, {% endif %} {% if calico_version is version('3.23.0', '>=') and (item.numallowedlocalasnumbers | default(0) > 0) %} "numAllowedLocalASNumbers": {{ item.numallowedlocalasnumbers }}, {% endif %} "sourceAddress": "{{ item.sourceaddress | default('UseNodeIP') }}" }} register: output retries: 4 until: output.rc == 0 delay: "{{ retry_stagger | random + 3 }}" with_items: - "{{ peers | default([]) | selectattr('scope', 'undefined') | list | union(peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'node') | list ) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" when: - ('k8s_cluster' in group_names) ================================================ FILE: roles/network_plugin/calico/tasks/pre.yml ================================================ --- - name: Slurp CNI config slurp: src: /etc/cni/net.d/10-calico.conflist register: calico_cni_config_slurp failed_when: false - name: Gather calico facts tags: - facts when: calico_cni_config_slurp.content is defined block: - name: Set fact calico_cni_config from slurped CNI config set_fact: calico_cni_config: "{{ calico_cni_config_slurp['content'] | b64decode | from_json }}" - name: Set fact calico_datastore to etcd if needed set_fact: calico_datastore: etcd when: - "'plugins' in calico_cni_config" - "'etcd_endpoints' in calico_cni_config.plugins.0" - name: Calico | Gather os specific variables include_vars: "{{ item }}" with_first_found: - files: - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" - "{{ ansible_distribution | lower }}.yml" - "{{ ansible_os_family | lower }}-{{ ansible_architecture }}.yml" - "{{ ansible_os_family | lower }}.yml" - defaults.yml paths: - ../vars skip: true ================================================ FILE: roles/network_plugin/calico/tasks/repos.yml ================================================ --- - name: Calico | Add wireguard yum repo when: - calico_wireguard_enabled block: - name: Calico | Add wireguard yum repo yum_repository: name: copr:copr.fedorainfracloud.org:jdoss:wireguard file: _copr:copr.fedorainfracloud.org:jdoss:wireguard description: Copr repo for wireguard owned by jdoss baseurl: "{{ calico_wireguard_repo }}" gpgcheck: true gpgkey: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/pubkey.gpg skip_if_unavailable: true enabled: true repo_gpgcheck: false when: - ansible_os_family in ['RedHat'] - ansible_distribution not in ['Fedora'] - ansible_facts['distribution_major_version'] | int < 9 ================================================ FILE: roles/network_plugin/calico/tasks/reset.yml ================================================ --- - name: Reset | check vxlan.calico network device stat: path: /sys/class/net/vxlan.calico get_attributes: false get_checksum: false get_mime: false register: vxlan - name: Reset | remove the network vxlan.calico device created by calico command: ip link del vxlan.calico when: vxlan.stat.exists - name: Reset | check dummy0 network device stat: path: /sys/class/net/dummy0 get_attributes: false get_checksum: false get_mime: false register: dummy0 - name: Reset | remove the network device created by calico command: ip link del dummy0 when: dummy0.stat.exists - name: Reset | get and remove remaining routes set by bird shell: set -o pipefail && ip route show proto bird | xargs -i bash -c "ip route del {} proto bird " args: executable: /bin/bash changed_when: false ================================================ FILE: roles/network_plugin/calico/tasks/typha_certs.yml ================================================ --- - name: Calico | Check if typha-server exists command: "{{ kubectl }} -n kube-system get secret typha-server" register: typha_server_secret changed_when: false failed_when: false - name: Calico | Ensure calico certs dir file: path: /etc/calico/certs state: directory mode: "0755" when: typha_server_secret.rc != 0 - name: Calico | Copy ssl script for typha certs template: src: make-ssl-calico.sh.j2 dest: "{{ bin_dir }}/make-ssl-typha.sh" mode: "0755" when: typha_server_secret.rc != 0 - name: Calico | Copy ssl config for typha certs copy: src: openssl.conf dest: /etc/calico/certs/openssl.conf mode: "0644" when: typha_server_secret.rc != 0 - name: Calico | Generate typha certs command: >- {{ bin_dir }}/make-ssl-typha.sh -f /etc/calico/certs/openssl.conf -c {{ kube_cert_dir }} -d /etc/calico/certs -s typha when: typha_server_secret.rc != 0 - name: Calico | Create typha tls secrets command: >- {{ kubectl }} -n kube-system create secret tls {{ item.name }} --cert {{ item.cert }} --key {{ item.key }} with_items: - name: typha-server cert: /etc/calico/certs/typha-server.crt key: /etc/calico/certs/typha-server.key - name: typha-client cert: /etc/calico/certs/typha-client.crt key: /etc/calico/certs/typha-client.key when: typha_server_secret.rc != 0 ================================================ FILE: roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2 ================================================ # This is a tech-preview manifest which installs the Calico API server. Note that this manifest is liable to change # or be removed in future releases without further warning. # # Namespace and namespace-scoped resources. apiVersion: v1 kind: Namespace metadata: labels: name: calico-apiserver name: calico-apiserver ================================================ FILE: roles/network_plugin/calico/templates/calico-apiserver.yml.j2 ================================================ # Policy to ensure the API server isn't cut off. Can be modified, but ensure # that the main API server is always able to reach the Calico API server. kind: NetworkPolicy apiVersion: networking.k8s.io/v1 metadata: name: allow-apiserver namespace: calico-apiserver spec: podSelector: matchLabels: apiserver: "true" ingress: - ports: - protocol: TCP port: 5443 --- apiVersion: v1 kind: Service metadata: name: calico-api namespace: calico-apiserver spec: ports: - name: apiserver port: 443 protocol: TCP targetPort: 5443 selector: apiserver: "true" type: ClusterIP --- apiVersion: apps/v1 kind: Deployment metadata: labels: apiserver: "true" k8s-app: calico-apiserver name: calico-apiserver namespace: calico-apiserver spec: replicas: 1 selector: matchLabels: apiserver: "true" strategy: type: Recreate template: metadata: labels: apiserver: "true" k8s-app: calico-apiserver name: calico-apiserver namespace: calico-apiserver spec: containers: - args: - --secure-port=5443 env: - name: DATASTORE_TYPE value: kubernetes image: {{ calico_apiserver_image_repo }}:{{ calico_apiserver_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} livenessProbe: httpGet: path: /version port: 5443 scheme: HTTPS initialDelaySeconds: 90 periodSeconds: 10 name: calico-apiserver {% if calico_version is version('3.28.0', '>=') %} readinessProbe: httpGet: path: /readyz port: 5443 scheme: HTTPS timeoutSeconds: 5 periodSeconds: 60 {% else %} readinessProbe: exec: command: - /code/filecheck failureThreshold: 5 initialDelaySeconds: 5 periodSeconds: 10 {% endif %} securityContext: privileged: false runAsUser: 0 volumeMounts: - mountPath: /code/apiserver.local.config/certificates name: calico-apiserver-certs dnsPolicy: ClusterFirst nodeSelector: kubernetes.io/os: linux restartPolicy: Always serviceAccount: calico-apiserver serviceAccountName: calico-apiserver tolerations: - effect: NoSchedule key: node-role.kubernetes.io/control-plane volumes: - name: calico-apiserver-certs secret: secretName: calico-apiserver-certs --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-apiserver namespace: calico-apiserver --- # Cluster-scoped resources below here. apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: v3.projectcalico.org spec: group: projectcalico.org groupPriorityMinimum: 1500 caBundle: {{ calico_apiserver_cabundle }} service: name: calico-api namespace: calico-apiserver port: 443 version: v3 versionPriority: 200 --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: calico-crds rules: - apiGroups: - extensions - networking.k8s.io - "" resources: - networkpolicies - nodes - namespaces - pods - serviceaccounts verbs: - get - list - watch - apiGroups: - crd.projectcalico.org resources: - globalnetworkpolicies - networkpolicies - clusterinformations - hostendpoints - globalnetworksets - networksets - bgpconfigurations - bgppeers - bgpfilters - felixconfigurations - kubecontrollersconfigurations - ippools - ipamconfigs - ipreservations - ipamblocks - blockaffinities - caliconodestatuses - tiers - stagednetworkpolicies - stagedglobalnetworkpolicies - stagedkubernetesnetworkpolicies verbs: - get - list - watch - create - update - delete {% if calico_version is version('3.28.0', '>=') %} - apiGroups: - policy resourceNames: - calico-apiserver resources: - podsecuritypolicies verbs: - use {% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: calico-extension-apiserver-auth-access rules: - apiGroups: - "" resourceNames: - extension-apiserver-authentication resources: - configmaps verbs: - list - watch - get - apiGroups: - rbac.authorization.k8s.io resources: - clusterroles - clusterrolebindings - roles - rolebindings verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: calico-webhook-reader rules: - apiGroups: - admissionregistration.k8s.io resources: - mutatingwebhookconfigurations - validatingwebhookconfigurations - validatingadmissionpolicies # Required for Kubernetes 1.33+ - validatingadmissionpolicybindings # Required for Kubernetes 1.33+ verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-apiserver-access-crds roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-crds subjects: - kind: ServiceAccount name: calico-apiserver namespace: calico-apiserver --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-apiserver-delegate-auth roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: calico-apiserver namespace: calico-apiserver --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-apiserver-webhook-reader roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-webhook-reader subjects: - kind: ServiceAccount name: calico-apiserver namespace: calico-apiserver --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-extension-apiserver-auth-access roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-extension-apiserver-auth-access subjects: - kind: ServiceAccount name: calico-apiserver namespace: calico-apiserver ================================================ FILE: roles/network_plugin/calico/templates/calico-config.yml.j2 ================================================ kind: ConfigMap apiVersion: v1 metadata: name: calico-config namespace: kube-system data: {% if calico_datastore == "etcd" %} etcd_endpoints: "{{ etcd_access_addresses }}" etcd_ca: "/calico-secrets/ca_cert.crt" etcd_cert: "/calico-secrets/cert.crt" etcd_key: "/calico-secrets/key.pem" {% elif calico_datastore == "kdd" and typha_enabled %} # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is # essential. typha_service_name: "calico-typha" {% endif %} {% if calico_network_backend == 'bird' %} cluster_type: "kubespray,bgp" calico_backend: "bird" {% else %} cluster_type: "kubespray" calico_backend: "{{ calico_network_backend }}" {% endif %} {% if inventory_hostname in groups['k8s_cluster'] and peer_with_router | default(false) %} as: "{{ local_as | default(global_as_num) }}" {% endif -%} # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. cni_network_config: |- { "name": "{{ calico_cni_name }}", "cniVersion":"0.3.1", "plugins":[ { {% if calico_datastore == "kdd" %} "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", {% endif %} "type": "calico", "log_level": "info", {% if calico_cni_log_file_path %} "log_file_path": "{{ calico_cni_log_file_path }}", {% endif %} {% if calico_datastore == "etcd" %} "etcd_endpoints": "{{ etcd_access_addresses }}", "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt", "etcd_key_file": "{{ calico_cert_dir }}/key.pem", "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", {% endif %} {% if calico_ipam_host_local %} "ipam": { "type": "host-local", "subnet": "usePodCidr" }, {% else %} "ipam": { "type": "calico-ipam", {% if ipv4_stack %} "assign_ipv4": "true"{{ ',' if (ipv6_stack and ipv4_stack) }} {% endif %} {% if ipv6_stack %} "assign_ipv6": "true" {% endif %} }, {% endif %} {% if calico_allow_ip_forwarding %} "container_settings": { "allow_ip_forwarding": true }, {% endif %} {% if (calico_feature_control is defined) and (calico_feature_control | length > 0) %} "feature_control": { {% for fc in calico_feature_control -%} {% set fcval = calico_feature_control[fc] -%} "{{ fc }}": {{ (fcval | string | lower) if (fcval == true or fcval == false) else "\"" + fcval + "\"" }}{{ "," if not loop.last else "" }} {% endfor -%} {{- "" }} }, {% endif %} {% if enable_network_policy %} "policy": { "type": "k8s" }, {% endif %} {% if calico_mtu is defined and calico_mtu is number %} "mtu": {{ calico_mtu }}, {% endif %} "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { "type":"portmap", "capabilities": { "portMappings": true } }, { "type":"bandwidth", "capabilities": { "bandwidth": true } } ] } ================================================ FILE: roles/network_plugin/calico/templates/calico-cr.yml.j2 ================================================ --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-cni-plugin rules: - apiGroups: [""] resources: - pods - nodes - namespaces verbs: - get - apiGroups: [""] resources: - pods/status verbs: - patch - apiGroups: [""] resources: - nodes/status verbs: - update - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles - clusterinformations - ippools - ipreservations - ipamconfigs verbs: - get - list - create - update - delete --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node namespace: kube-system rules: - apiGroups: [""] resources: - pods - nodes - namespaces - configmaps verbs: - get # EndpointSlices are used for Service-based network policy rule # enforcement. - apiGroups: ["discovery.k8s.io"] resources: - endpointslices verbs: - watch - list - apiGroups: [""] resources: - endpoints - services verbs: - watch - list {% if calico_datastore == "kdd" %} # Used to discover Typhas. - get {% endif %} - apiGroups: [""] resources: - nodes/status verbs: # Needed for clearing NodeNetworkUnavailable flag. - patch {% if calico_datastore == "kdd" %} # Calico stores some configuration information in node annotations. - update # Watch for changes to Kubernetes NetworkPolicies. - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: - watch - list # Watch for changes to Kubernetes AdminNetworkPolicies. - apiGroups: ["policy.networking.k8s.io"] resources: - adminnetworkpolicies - baselineadminnetworkpolicies verbs: - watch - list # Used by Calico for policy information. - apiGroups: [""] resources: - pods - namespaces - serviceaccounts verbs: - list - watch # The CNI plugin patches pods/status. - apiGroups: [""] resources: - pods/status verbs: - patch # Calico monitors various CRDs for config. - apiGroups: ["crd.projectcalico.org"] resources: - globalfelixconfigs - felixconfigurations - bgppeers - bgpfilters - globalbgpconfigs - bgpconfigurations - ippools - ipreservations - ipamblocks - globalnetworkpolicies - stagedglobalnetworkpolicies - networkpolicies - stagednetworkpolicies - stagedkubernetesnetworkpolicies - globalnetworksets - networksets - clusterinformations - hostendpoints - blockaffinities - caliconodestatuses - tiers verbs: - get - list - watch # Calico creates some tiers on startup. - apiGroups: ["crd.projectcalico.org"] resources: - tiers verbs: - create # Calico must create and update some CRDs on startup. - apiGroups: ["crd.projectcalico.org"] resources: - ippools - felixconfigurations - clusterinformations verbs: - create - update # Calico must update some CRDs. - apiGroups: [ "crd.projectcalico.org" ] resources: - caliconodestatuses verbs: - update # Calico stores some configuration information on the node. - apiGroups: [""] resources: - nodes verbs: - get - list - watch # These permissions are only required for upgrade from v2.6, and can # be removed after upgrade or on fresh installations. - apiGroups: ["crd.projectcalico.org"] resources: - bgpconfigurations - bgppeers verbs: - create - update # These permissions are required for Calico CNI to perform IPAM allocations. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities - ipamblocks - ipamhandles verbs: - get - list - create - update - delete - apiGroups: ["crd.projectcalico.org"] resources: - ipamconfigs verbs: - get - create # Block affinities must also be watchable by confd for route aggregation. - apiGroups: ["crd.projectcalico.org"] resources: - blockaffinities verbs: - watch # The Calico IPAM migration needs to get daemonsets. These permissions can be # removed if not upgrading from an installation using host-local IPAM. - apiGroups: ["apps"] resources: - daemonsets verbs: - get {% endif %} # Used for creating service account tokens to be used by the CNI plugin - apiGroups: [""] resources: - serviceaccounts/token resourceNames: - calico-cni-plugin verbs: - create {% if calico_version is version('3.29.0', '>=') %} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-tier-getter rules: - apiGroups: - "projectcalico.org" resources: - "tiers" verbs: - "get" {% endif %} ================================================ FILE: roles/network_plugin/calico/templates/calico-crb.yml.j2 ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-node roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-node subjects: - kind: ServiceAccount name: calico-node namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-cni-plugin roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-cni-plugin subjects: - kind: ServiceAccount name: calico-cni-plugin namespace: kube-system {% if calico_version is version('3.29.0', '>=') %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: calico-tier-getter roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: calico-tier-getter subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: system:kube-controller-manager {% endif %} ================================================ FILE: roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2 ================================================ apiVersion: crd.projectcalico.org/v1 kind: IPAMConfig metadata: name: default spec: autoAllocateBlocks: {{ calico_ipam_autoallocateblocks }} strictAffinity: {{ calico_ipam_strictaffinity }} maxBlocksPerHost: {{ calico_ipam_maxblocksperhost }} ================================================ FILE: roles/network_plugin/calico/templates/calico-node-sa.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-node namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: calico-cni-plugin namespace: kube-system ================================================ FILE: roles/network_plugin/calico/templates/calico-node.yml.j2 ================================================ --- # This manifest installs the calico/node container, as well # as the Calico CNI plugins and network config on # each control plane and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: apps/v1 metadata: name: calico-node namespace: kube-system labels: k8s-app: calico-node spec: selector: matchLabels: k8s-app: calico-node updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 template: metadata: labels: k8s-app: calico-node annotations: {% if calico_datastore == "etcd" %} kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}" {% endif %} {% if calico_felix_prometheusmetricsenabled %} prometheus.io/scrape: 'true' prometheus.io/port: "{{ calico_felix_prometheusmetricsport }}" {% endif %} spec: nodeSelector: {{ calico_ds_nodeselector }} priorityClassName: system-node-critical hostNetwork: true serviceAccountName: calico-node tolerations: # Make sure calico-node gets scheduled on all nodes. - effect: NoSchedule operator: Exists # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists - effect: NoExecute operator: Exists # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 initContainers: {% if calico_datastore == "kdd" and not calico_ipam_host_local %} # This container performs upgrade from host-local IPAM to calico-ipam. # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend volumeMounts: - mountPath: /var/lib/cni/networks name: host-local-net-dir - mountPath: /host/opt/cni/bin name: cni-bin-dir securityContext: privileged: true {% endif %} # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/opt/cni/bin/install"] envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: configMapKeyRef: name: calico-config key: cni_network_config # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-calico.conflist" {% if calico_mtu is defined %} # CNI MTU Config variable - name: CNI_MTU value: "{{ calico_veth_mtu | default(calico_mtu) }}" {% endif %} # Prevents the container from sleeping forever. - name: SLEEP value: "false" {% if calico_datastore == "etcd" %} - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: calico-config key: etcd_endpoints {% endif %} {% if calico_datastore == "kdd" %} # Set the hostname based on the k8s node name. - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName {% endif %} volumeMounts: - mountPath: /host/etc/cni/net.d name: cni-net-dir - mountPath: /host/opt/cni/bin name: cni-bin-dir securityContext: privileged: true # This init container mounts the necessary filesystems needed by the BPF data plane # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - name: "mount-bpffs" image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["calico-node", "-init", "-best-effort"] volumeMounts: - mountPath: /sys/fs name: sys-fs # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host # so that it outlives the init container. mountPropagation: Bidirectional - mountPath: /var/run/calico name: var-run-calico # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host # so that it outlives the init container. mountPropagation: Bidirectional # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. - mountPath: /nodeproc name: nodeproc readOnly: true securityContext: privileged: true containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # The location of the Calico etcd cluster. {% if calico_datastore == "etcd" %} - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: calico-config key: etcd_endpoints # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: configMapKeyRef: name: calico-config key: etcd_ca # Location of the client key for etcd. - name: ETCD_KEY_FILE valueFrom: configMapKeyRef: name: calico-config key: etcd_key # Location of the client certificate for etcd. - name: ETCD_CERT_FILE valueFrom: configMapKeyRef: name: calico-config key: etcd_cert {% elif calico_datastore == "kdd" %} # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" {% if typha_enabled %} # Typha support: controlled by the ConfigMap. - name: FELIX_TYPHAK8SSERVICENAME valueFrom: configMapKeyRef: name: calico-config key: typha_service_name {% if typha_secure %} - name: FELIX_TYPHACN value: typha-server - name: FELIX_TYPHACAFILE value: /etc/typha-ca/ca.crt - name: FELIX_TYPHACERTFILE value: /etc/typha-client/typha-client.crt - name: FELIX_TYPHAKEYFILE value: /etc/typha-client/typha-client.key {% endif %} {% endif %} # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" {% endif %} {% if calico_network_backend == 'vxlan' %} - name: FELIX_VXLANVNI value: "{{ calico_vxlan_vni }}" - name: FELIX_VXLANPORT value: "{{ calico_vxlan_port }}" {% endif %} # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: configMapKeyRef: name: calico-config key: calico_backend # Cluster type to identify the deployment type - name: CLUSTER_TYPE value: "k8s,bgp" # Set noderef for node controller. - name: CALICO_K8S_NODE_REF valueFrom: fieldRef: fieldPath: spec.nodeName # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" # Set Felix endpoint to host default action to ACCEPT. - name: FELIX_DEFAULTENDPOINTTOHOSTACTION value: "{{ calico_endpoint_to_host_action | default('RETURN') }}" - name: FELIX_HEALTHHOST value: "{{ calico_healthhost }}" {% if kube_proxy_mode == 'ipvs' and kube_apiserver_node_port_range is defined %} - name: FELIX_KUBENODEPORTRANGES value: "{{ kube_apiserver_node_port_range.split('-')[0] }}:{{ kube_apiserver_node_port_range.split('-')[1] }}" {% endif %} - name: FELIX_IPTABLESBACKEND value: "{{ calico_iptables_backend }}" - name: FELIX_IPTABLESLOCKTIMEOUTSECS value: "{{ calico_iptables_lock_timeout_secs }}" # The default IPv4 pool to create on startup if none exists. Pod IPs will be # chosen from this range. Changing this value after installation will have # no effect. This should fall within `--cluster-cidr`. # - name: CALICO_IPV4POOL_CIDR # value: "192.168.0.0/16" - name: CALICO_IPV4POOL_IPIP value: "{{ calico_ipv4pool_ipip }}" # Enable or Disable VXLAN on the default IP pool. - name: CALICO_IPV4POOL_VXLAN value: "Never" - name: FELIX_IPV6SUPPORT value: "{{ ipv6_stack | default(false) }}" # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN value: "{{ calico_loglevel }}" # Set Calico startup logging to "error" - name: CALICO_STARTUP_LOGLEVEL value: "{{ calico_node_startup_loglevel }}" # Enable or disable usage report - name: FELIX_USAGEREPORTINGENABLED value: "{{ calico_usage_reporting }}" {% if calico_version is version('3.29.0', '>=') %} - name: FELIX_NFTABLESMODE value: "{{ calico_nftable_mode }}" {% endif %} # Set MTU for tunnel device used if ipip is enabled {% if calico_mtu is defined %} # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU value: "{{ calico_veth_mtu | default(calico_mtu) }}" # Set MTU for the VXLAN tunnel device. - name: FELIX_VXLANMTU value: "{{ calico_veth_mtu | default(calico_mtu) }}" # Set MTU for the Wireguard tunnel device. - name: FELIX_WIREGUARDMTU value: "{{ calico_veth_mtu | default(calico_mtu) }}" {% endif %} - name: FELIX_CHAININSERTMODE value: "{{ calico_felix_chaininsertmode }}" - name: FELIX_PROMETHEUSMETRICSENABLED value: "{{ calico_felix_prometheusmetricsenabled }}" - name: FELIX_PROMETHEUSMETRICSPORT value: "{{ calico_felix_prometheusmetricsport }}" - name: FELIX_PROMETHEUSGOMETRICSENABLED value: "{{ calico_felix_prometheusgometricsenabled }}" - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED value: "{{ calico_felix_prometheusprocessmetricsenabled }}" {% if calico_ip_auto_method is defined %} - name: IP_AUTODETECTION_METHOD value: "{{ calico_ip_auto_method }}" {% else %} - name: NODEIP valueFrom: fieldRef: fieldPath: status.hostIP - name: IP_AUTODETECTION_METHOD value: "can-reach=$(NODEIP)" {% endif %} {% if ipv4_stack %} - name: IP value: "autodetect" {% else %} - name: IP value: none {% endif %} {% if ipv6_stack %} - name: IP6 value: autodetect {% endif %} {% if calico_ip6_auto_method is defined and ipv6_stack %} - name: IP6_AUTODETECTION_METHOD value: "{{ calico_ip6_auto_method }}" {% endif %} {% if calico_felix_mtu_iface_pattern is defined %} - name: FELIX_MTUIFACEPATTERN value: "{{ calico_felix_mtu_iface_pattern }}" {% endif %} {% if calico_use_default_route_src_ipaddr | default(false) %} - name: FELIX_DEVICEROUTESOURCEADDRESS valueFrom: fieldRef: fieldPath: status.hostIP {% endif %} - name: NODENAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: FELIX_HEALTHENABLED value: "true" - name: FELIX_IGNORELOOSERPF value: "{{ calico_node_ignorelooserpf }}" - name: CALICO_MANAGE_CNI value: "true" {% if calico_ipam_host_local %} - name: USE_POD_CIDR value: "true" {% endif %} {% if calico_node_extra_envs is defined %} {% for key in calico_node_extra_envs %} - name: {{ key }} value: "{{ calico_node_extra_envs[key] }}" {% endfor %} {% endif %} securityContext: privileged: true resources: limits: {% if calico_node_cpu_limit != "0" %} cpu: {{ calico_node_cpu_limit }} {% endif %} memory: {{ calico_node_memory_limit }} requests: cpu: {{ calico_node_cpu_requests }} memory: {{ calico_node_memory_requests }} lifecycle: preStop: exec: command: - /bin/calico-node - -shutdown livenessProbe: exec: command: - /bin/calico-node - -felix-live {% if calico_network_backend == "bird" %} - -bird-live {% endif %} periodSeconds: 10 initialDelaySeconds: 10 timeoutSeconds: {{ calico_node_livenessprobe_timeout | default(10) }} failureThreshold: 6 readinessProbe: exec: command: - /bin/calico-node {% if calico_network_backend == "bird" %} - -bird-ready {% endif %} - -felix-ready periodSeconds: 10 timeoutSeconds: {{ calico_node_readinessprobe_timeout | default(10) }} failureThreshold: 6 volumeMounts: - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /var/run/calico name: var-run-calico readOnly: false - mountPath: /var/lib/calico name: var-lib-calico readOnly: false {% if calico_datastore == "etcd" %} - mountPath: /calico-secrets name: etcd-certs readOnly: true {% endif %} - name: xtables-lock mountPath: /run/xtables.lock readOnly: false # For maintaining CNI plugin API credentials. - mountPath: /host/etc/cni/net.d name: cni-net-dir readOnly: false {% if typha_secure %} - name: typha-client mountPath: /etc/typha-client readOnly: true - name: typha-cacert subPath: ca.crt mountPath: /etc/typha-ca/ca.crt readOnly: true {% endif %} - name: policysync mountPath: /var/run/nodeagent # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the # parent directory. - name: bpffs mountPath: /sys/fs/bpf - name: cni-log-dir mountPath: /var/log/calico/cni readOnly: true volumes: # Used by calico/node. - name: lib-modules hostPath: path: /lib/modules - name: var-run-calico hostPath: path: /var/run/calico type: DirectoryOrCreate - name: var-lib-calico hostPath: path: /var/lib/calico type: DirectoryOrCreate # Used to install CNI. - name: cni-net-dir hostPath: path: /etc/cni/net.d - name: cni-bin-dir hostPath: path: /opt/cni/bin type: DirectoryOrCreate {% if calico_datastore == "etcd" %} # Mount in the etcd TLS secrets. - name: etcd-certs hostPath: path: "{{ calico_cert_dir }}" {% endif %} # Mount the global iptables lock file, used by calico/node - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate {% if calico_datastore == "kdd" and not calico_ipam_host_local %} # Mount in the directory for host-local IPAM allocations. This is # used when upgrading from host-local to calico-ipam, and can be removed # if not using the upgrade-ipam init container. - name: host-local-net-dir hostPath: path: /var/lib/cni/networks {% endif %} {% if typha_enabled and typha_secure %} - name: typha-client secret: secretName: typha-client items: - key: tls.crt path: typha-client.crt - key: tls.key path: typha-client.key - name: typha-cacert hostPath: path: "/etc/kubernetes/ssl/" {% endif %} - name: sys-fs hostPath: path: /sys/fs/ type: DirectoryOrCreate - name: bpffs hostPath: path: /sys/fs/bpf type: Directory # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. - name: nodeproc hostPath: path: /proc # Used to access CNI logs. - name: cni-log-dir hostPath: path: /var/log/calico/cni # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: type: DirectoryOrCreate path: /var/run/nodeagent ================================================ FILE: roles/network_plugin/calico/templates/calico-typha.yml.j2 ================================================ # This manifest creates a Service, which will be backed by Calico's Typha daemon. # Typha sits in between Felix and the API server, reducing Calico's load on the API server. apiVersion: v1 kind: Service metadata: name: calico-typha namespace: kube-system labels: k8s-app: calico-typha spec: ports: - port: 5473 protocol: TCP targetPort: calico-typha name: calico-typha {% if typha_prometheusmetricsenabled %} - port: {{ typha_prometheusmetricsport }} protocol: TCP targetPort: http-metrics name: metrics {% endif %} selector: k8s-app: calico-typha --- # This manifest creates a Deployment of Typha to back the above service. apiVersion: apps/v1 kind: Deployment metadata: name: calico-typha namespace: kube-system labels: k8s-app: calico-typha spec: # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the # typha_service_name variable in the calico-config ConfigMap above. # # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. replicas: {{ typha_replicas }} revisionHistoryLimit: 2 selector: matchLabels: k8s-app: calico-typha template: metadata: labels: k8s-app: calico-typha annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' {% if typha_prometheusmetricsenabled %} prometheus.io/scrape: 'true' prometheus.io/port: "{{ typha_prometheusmetricsport }}" {% endif %} spec: nodeSelector: kubernetes.io/os: linux hostNetwork: true tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node priorityClassName: system-cluster-critical # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 securityContext: fsGroup: 65534 containers: - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} name: calico-typha ports: - containerPort: 5473 name: calico-typha protocol: TCP {% if typha_prometheusmetricsenabled %} - containerPort: {{ typha_prometheusmetricsport }} name: http-metrics protocol: TCP {% endif %} envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. name: kubernetes-services-endpoint optional: true env: # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - name: TYPHA_LOGSEVERITYSCREEN value: "info" # Disable logging to file and syslog since those don't make sense in Kubernetes. - name: TYPHA_LOGFILEPATH value: "none" - name: TYPHA_LOGSEVERITYSYS value: "none" # Monitor the Kubernetes API to find the number of running instances and rebalance # connections. - name: TYPHA_CONNECTIONREBALANCINGMODE value: "kubernetes" - name: TYPHA_DATASTORETYPE value: "kubernetes" - name: TYPHA_HEALTHENABLED value: "true" - name: TYPHA_MAXCONNECTIONSLOWERLIMIT value: "{{ typha_max_connections_lower_limit }}" {% if typha_secure %} - name: TYPHA_CAFILE value: /etc/ca/ca.crt - name: TYPHA_CLIENTCN value: typha-client - name: TYPHA_SERVERCERTFILE value: /etc/typha/server_certificate.pem - name: TYPHA_SERVERKEYFILE value: /etc/typha/server_key.pem {% endif %} {% if typha_prometheusmetricsenabled %} # Since Typha is host-networked, # this opens a port on the host, which may need to be secured. - name: TYPHA_PROMETHEUSMETRICSENABLED value: "true" - name: TYPHA_PROMETHEUSMETRICSPORT value: "{{ typha_prometheusmetricsport }}" {% endif %} {% if calico_ipam_host_local %} - name: USE_POD_CIDR value: "true" {% endif %} {% if typha_secure %} volumeMounts: - mountPath: /etc/typha name: typha-server readOnly: true - mountPath: /etc/ca/ca.crt subPath: ca.crt name: cacert readOnly: true {% endif %} livenessProbe: httpGet: path: /liveness port: 9098 host: localhost periodSeconds: 30 initialDelaySeconds: 30 readinessProbe: httpGet: path: /readiness port: 9098 host: localhost periodSeconds: 10 {% if typha_secure %} volumes: - name: typha-server secret: secretName: typha-server items: - key: tls.crt path: server_certificate.pem - key: tls.key path: server_key.pem - name: cacert hostPath: path: "{{ kube_cert_dir }}" {% endif %} --- # This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-typha namespace: kube-system labels: k8s-app: calico-typha spec: maxUnavailable: 1 selector: matchLabels: k8s-app: calico-typha ================================================ FILE: roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2 ================================================ #!/bin/bash ETCD_ENDPOINTS={{ etcd_access_addresses }} \ ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ {{ bin_dir }}/calicoctl --allow-version-mismatch "$@" ================================================ FILE: roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2 ================================================ #!/bin/bash DATASTORE_TYPE=kubernetes \ {% if inventory_hostname in groups['kube_control_plane'] %} KUBECONFIG=/etc/kubernetes/admin.conf \ {% else %} KUBECONFIG=/etc/cni/net.d/calico-kubeconfig \ {% endif %} {{ bin_dir }}/calicoctl --allow-version-mismatch "$@" ================================================ FILE: roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 ================================================ --- apiVersion: v1 kind: ConfigMap metadata: namespace: kube-system name: kubernetes-services-endpoint data: {% if calico_bpf_enabled or loadbalancer_apiserver_localhost %} KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" {% endif %} ================================================ FILE: roles/network_plugin/calico/templates/make-ssl-calico.sh.j2 ================================================ #!/bin/bash # Author: Smana smainklh@gmail.com # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o pipefail usage() { cat << EOF Create self signed certificates Usage : $(basename $0) -f [-d ] -h | --help : Show this message -f | --config : Openssl configuration file -d | --ssldir : Directory where the certificates will be installed -c | --cadir : Directory where the existing CA is located -s | --service : Service for the ca ex : $(basename $0) -f openssl.conf -d /srv/ssl EOF } # Options parsing while (($#)); do case "$1" in -h | --help) usage; exit 0;; -f | --config) CONFIG=${2}; shift 2;; -d | --ssldir) SSLDIR="${2}"; shift 2;; -c | --cadir) CADIR="${2}"; shift 2;; -s | --service) SERVICE="${2}"; shift 2;; *) usage echo "ERROR : Unknown option" exit 3 ;; esac done if [ -z ${CONFIG} ]; then echo "ERROR: the openssl configuration file is missing. option -f" exit 1 fi if [ -z ${SSLDIR} ]; then SSLDIR="/etc/calico/certs" fi tmpdir=$(mktemp -d /tmp/calico_${SERVICE}_certs.XXXXXX) trap 'rm -rf "${tmpdir}"' EXIT cd "${tmpdir}" mkdir -p ${SSLDIR} ${CADIR} # Root CA if [ -e "$CADIR/ca.key" ]; then # Reuse existing CA cp $CADIR/{ca.crt,ca.key} . else openssl genrsa -out ca.key {{certificates_key_size}} > /dev/null 2>&1 openssl req -x509 -new -nodes -key ca.key -days {{certificates_duration}} -out ca.crt -subj "/CN=calico-${SERVICE}-ca" > /dev/null 2>&1 fi if [ $SERVICE == "typha" ]; then # Typha server openssl genrsa -out typha-server.key {{certificates_key_size}} > /dev/null 2>&1 openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1 openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 # Typha client openssl genrsa -out typha-client.key {{certificates_key_size}} > /dev/null 2>&1 openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1 openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 elif [ $SERVICE == "apiserver" ]; then # calico-apiserver openssl genrsa -out apiserver.key {{certificates_key_size}} > /dev/null 2>&1 openssl req -new -key apiserver.key -out apiserver.csr -subj "/CN=calico-apiserver" -config ${CONFIG} > /dev/null 2>&1 openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out apiserver.crt -days {{certificates_duration}} -extensions ssl_client_apiserver -extfile ${CONFIG} > /dev/null 2>&1 else echo "ERROR: the openssl configuration file is missing. option -s" exit 1 fi # Install certs if [ -e "$CADIR/ca.key" ]; then # No pass existing CA rm -f ca.crt ca.key fi mv {*.crt,*.key} ${SSLDIR}/ ================================================ FILE: roles/network_plugin/calico/vars/amazon.yml ================================================ --- calico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-7-$basearch/ calico_wireguard_packages: - wireguard-dkms - wireguard-tools ================================================ FILE: roles/network_plugin/calico/vars/centos-9.yml ================================================ --- calico_wireguard_packages: - wireguard-tools ================================================ FILE: roles/network_plugin/calico/vars/debian.yml ================================================ --- calico_wireguard_packages: - wireguard ================================================ FILE: roles/network_plugin/calico/vars/fedora.yml ================================================ --- calico_wireguard_packages: - wireguard-tools ================================================ FILE: roles/network_plugin/calico/vars/opensuse.yml ================================================ --- calico_wireguard_packages: - wireguard-tools ================================================ FILE: roles/network_plugin/calico/vars/redhat-9.yml ================================================ --- calico_wireguard_packages: - wireguard-tools ================================================ FILE: roles/network_plugin/calico/vars/redhat.yml ================================================ --- calico_wireguard_packages: - wireguard-dkms - wireguard-tools ================================================ FILE: roles/network_plugin/calico/vars/rocky-9.yml ================================================ --- calico_wireguard_packages: - wireguard-tools ================================================ FILE: roles/network_plugin/calico_defaults/defaults/main.yml ================================================ --- # the default value of name calico_cni_name: k8s-pod-network # Enables Internet connectivity from containers nat_outgoing: true nat_outgoing_ipv6: false # add default ippool name calico_pool_name: "default-pool" calico_ipv4pool_ipip: "Off" # Change encapsulation mode, by default we enable vxlan which is the most mature and well tested mode calico_ipip_mode: Never # valid values are 'Always', 'Never' and 'CrossSubnet' calico_vxlan_mode: Always # valid values are 'Always', 'Never' and 'CrossSubnet' # add default ippool blockSize calico_pool_blocksize: 26 # Calico doesn't support ipip tunneling for the IPv6. calico_ipip_mode_ipv6: Never calico_vxlan_mode_ipv6: Always # add default ipv6 ippool blockSize calico_pool_blocksize_ipv6: 122 # Calico network backend can be 'bird', 'vxlan' and 'none' calico_network_backend: vxlan calico_cert_dir: /etc/calico/certs # Global as_num (/calico/bgp/v1/global/as_num) global_as_num: "64512" # You can set MTU value here. If left undefined or empty, it will # not be specified in calico CNI config, so Calico will use built-in # defaults. The value should be a number, not a string. # calico_mtu: 1500 # Advertise Service External IPs calico_advertise_service_external_ips: [] # Advertise Service LoadBalancer IPs calico_advertise_service_loadbalancer_ips: [] # Calico eBPF support calico_bpf_enabled: false calico_bpf_log_level: "" # Valid option for service mode: Tunnel (default), DSR=Direct Server Return calico_bpf_service_mode: Tunnel # Calico floatingIPs support # Valid option for floatingIPs: Disabled (default), Enabled calico_felix_floating_ips: Disabled # Limits for apps calico_node_memory_limit: 500M calico_node_cpu_limit: "0" calico_node_memory_requests: 64M calico_node_cpu_requests: 150m calico_felix_chaininsertmode: Insert # Calico daemonset nodeselector calico_ds_nodeselector: "kubernetes.io/os: linux" # Virtual network ID to use for VXLAN traffic. A value of 0 means “use the kernel default”. calico_vxlan_vni: 4096 # Port to use for VXLAN traffic. A value of 0 means “use the kernel default”. calico_vxlan_port: 4789 # Enable Prometheus Metrics endpoint for felix calico_felix_prometheusmetricsenabled: false calico_felix_prometheusmetricsport: 9091 calico_felix_prometheusgometricsenabled: true calico_felix_prometheusprocessmetricsenabled: true # Set the agent log level. Can be debug, warning, info or fatal calico_loglevel: info calico_node_startup_loglevel: error # Set log path for calico CNI plugin. Set to false to disable logging to disk. calico_cni_log_file_path: /var/log/calico/cni/cni.log # Enable or disable usage report to 'usage.projectcalico.org' calico_usage_reporting: false # Should calico ignore kernel's RPF check setting, # see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198 calico_node_ignorelooserpf: false # Define address on which Felix will respond to health requests calico_healthhost: "localhost" # Configure time in seconds that calico will wait for the iptables lock calico_iptables_lock_timeout_secs: 10 # Choose Calico iptables backend: "Legacy", "Auto" or "NFT" (FELIX_IPTABLESBACKEND) calico_iptables_backend: "Auto" # Calico NFTable Mode Support (tech preview 3.29) # Valid option: Disabled (default), Enabled calico_nftable_mode: "Disabled" # Calico Wireguard support calico_wireguard_enabled: false calico_wireguard_packages: [] calico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-{{ ansible_distribution_major_version }}-$basearch/ # If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of: # * can-reach=DESTINATION # * interface=INTERFACE-REGEX # see https://projectcalico.docs.tigera.io/reference/node/configuration#ip-autodetection-methods # calico_ip_auto_method: "interface=eth.*" # calico_ip6_auto_method: "interface=eth.*" # Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection. # see https://projectcalico.docs.tigera.io/reference/felix/configuration # calico_felix_mtu_iface_pattern: "^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)" calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}" kube_etcd_cacert_file: ca.pem kube_etcd_cert_file: node-{{ inventory_hostname }}.pem kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem # Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) # The default value for calico_datastore is set in role kubespray-default # Use typha (only with kdd) typha_enabled: false typha_prometheusmetricsenabled: false typha_prometheusmetricsport: 9093 # Scaling typha: 1 replica per 100 nodes is adequate # Number of typha replicas typha_replicas: 1 # Set max typha connections typha_max_connections_lower_limit: 300 # Generate certifcates for typha<->calico-node communication typha_secure: false calico_feature_control: {} # Calico default BGP port calico_bgp_listen_port: 179 # Calico FelixConfiguration options calico_felix_reporting_interval: 0s calico_felix_log_severity_screen: Info # Calico container settings calico_allow_ip_forwarding: false # Calico IPAM strictAffinity calico_ipam_strictaffinity: false # Calico IPAM autoAllocateBlocks calico_ipam_autoallocateblocks: true # Calico IPAM maxBlocksPerHost, default 0 calico_ipam_maxblocksperhost: 0 # Calico host local IPAM (use node .spec.podCIDR) calico_ipam_host_local: false # Calico apiserver (only with kdd) calico_apiserver_enabled: false # Calico feature detect override calico_feature_detect_override: "" # Calico kubeconfig wait timeout in seconds calico_kubeconfig_wait_timeout: 300 ================================================ FILE: roles/network_plugin/cilium/defaults/main.yml ================================================ --- cilium_min_version_required: "1.15" # Log-level cilium_debug: false cilium_mtu: "0" cilium_enable_ipv4: "{{ ipv4_stack }}" cilium_enable_ipv6: "{{ ipv6_stack }}" # Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/ cilium_l2announcements: false # Cilium agent health port cilium_agent_health_port: "9879" # Etcd SSL dirs cilium_cert_dir: /etc/cilium/certs kube_etcd_cacert_file: ca.pem kube_etcd_cert_file: node-{{ inventory_hostname }}.pem kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem # Limits for apps cilium_memory_limit: 500M cilium_cpu_limit: 500m cilium_memory_requests: 64M cilium_cpu_requests: 100m # Overlay Network Mode cilium_tunnel_mode: vxlan # LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode cilium_loadbalancer_mode: snat # -- Configure Loadbalancer IP Pools cilium_loadbalancer_ip_pools: [] # Optional features cilium_enable_prometheus: false # Enable if you want to make use of hostPort mappings cilium_enable_portmap: false # Monitor aggregation level (none/low/medium/maximum) cilium_monitor_aggregation: medium # Kube Proxy Replacement mode (true/false) cilium_kube_proxy_replacement: false # If not defined `cilium_dns_proxy_enable_transparent_mode`, it will following the Cilium behavior. # When Cilium is configured to replace kube-proxy, it automatically enables dnsProxy, which will conflict with nodelocaldns. # You can set `false` avoid conflict with nodelocaldns. # https://github.com/cilium/cilium/issues/33144 # cilium_dns_proxy_enable_transparent_mode: # If upgrading from Cilium < 1.5, you may want to override some of these options # to prevent service disruptions. See also: # http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action cilium_preallocate_bpf_maps: false # Auto direct nodes routes can be used to advertise pods routes in your cluster # without any tunelling (with `cilium_tunnel_mode` sets to `disabled`). # This works only if you have a L2 connectivity between all your nodes. # You wil also have to specify the variable `cilium_native_routing_cidr` to # make this work. Please refer to the cilium documentation for more # information about this kind of setups. cilium_auto_direct_node_routes: false # Allows to explicitly specify the IPv4 CIDR for native routing. # When specified, Cilium assumes networking for this CIDR is preconfigured and # hands traffic destined for that range to the Linux network stack without # applying any SNAT. # Generally speaking, specifying a native routing CIDR implies that Cilium can # depend on the underlying networking stack to route packets to their # destination. To offer a concrete example, if Cilium is configured to use # direct routing and the Kubernetes CIDR is included in the native routing CIDR, # the user must configure the routes to reach pods, either manually or by # setting the auto-direct-node-routes flag. cilium_native_routing_cidr: "" # Allows to explicitly specify the IPv6 CIDR for native routing. cilium_native_routing_cidr_ipv6: "" # Enable transparent network encryption. cilium_encryption_enabled: false # Encryption method. Can be either ipsec or wireguard. # Only effective when `cilium_encryption_enabled` is set to true. cilium_encryption_type: "ipsec" # Enable encryption for pure node to node traffic. # This option is only effective when `cilium_encryption_type` is set to `wireguard`. cilium_encryption_node_encryption: false # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. # When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, # it will fallback on the wireguard-go user-space implementation of WireGuard. # This option is only effective when `cilium_encryption_type` is set to `wireguard`. cilium_wireguard_userspace_fallback: false # Enable Bandwidth Manager # Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. # Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies. # In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods. # Bandwidth Manager requires a v5.1.x or more recent Linux kernel. cilium_enable_bandwidth_manager: false cilium_enable_bandwidth_manager_bbr: false # IP Masquerade Agent # https://docs.cilium.io/en/stable/concepts/networking/masquerading/ # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded cilium_ip_masq_agent_enable: false ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded cilium_non_masquerade_cidrs: - 10.0.0.0/8 - 172.16.0.0/12 - 192.168.0.0/16 - 100.64.0.0/10 - 192.0.0.0/24 - 192.0.2.0/24 - 192.88.99.0/24 - 198.18.0.0/15 - 198.51.100.0/24 - 203.0.113.0/24 - 240.0.0.0/4 ### Indicates whether to masquerade traffic to the link local prefix. ### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list. cilium_masq_link_local: false cilium_masq_link_local_ipv6: false ### A time interval at which the agent attempts to reload config from disk cilium_ip_masq_resync_interval: 60s # Hubble ### Enable Hubble without install cilium_enable_hubble: false ### Enable Hubble-ui cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}" ### Enable Hubble Metrics (deprecated) cilium_enable_hubble_metrics: false ### if cilium_enable_hubble_metrics: true cilium_hubble_metrics: [] # - dns # - drop # - tcp # - flow # - icmp # - http ### Enable Hubble install cilium_hubble_install: false ### Enable auto generate certs if cilium_hubble_install: true cilium_hubble_tls_generate: false cilium_hubble_export_file_max_backups: "5" cilium_hubble_export_file_max_size_mb: "10" cilium_hubble_export_dynamic_enabled: false cilium_hubble_export_dynamic_config_content: - name: all fieldMask: [] includeFilters: [] excludeFilters: [] filePath: "/var/run/cilium/hubble/events.log" # Override the DNS suffix that Hubble-Relay uses to resolve its peer service. # It defaults to the inventory's `dns_domain`. cilium_hubble_peer_service_cluster_domain: "{{ dns_domain }}" ### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535 ### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095) # cilium_hubble_event_buffer_capacity: 4095 ### Buffer size of the channel to receive monitor events. # cilium_hubble_event_queue_size: 50 cilium_gateway_api_enabled: false # The default IP address management mode is "Cluster Scope". # https://docs.cilium.io/en/stable/concepts/networking/ipam/ cilium_ipam_mode: cluster-pool # Cluster Pod CIDRs use the kube_pods_subnet value by default. # If your node network is in the same range you will lose connectivity to other nodes. # Defaults to kube_pods_subnet if not set. # cilium_pool_cidr: 10.233.64.0/18 # When cilium_enable_ipv6 is used, you need to set the IPV6 value. Defaults to kube_pods_subnet_ipv6 if not set. # cilium_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # When cilium IPAM uses the "Cluster Scope" mode, it will pre-allocate a segment of IP to each node, # schedule the Pod to this node, and then allocate IP from here. cilium_pool_mask_size Specifies # the size allocated from cluster Pod CIDR to node.ipam.podCIDRs # Defaults to kube_network_node_prefix if not set. # cilium_pool_mask_size: "24" # cilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cluster Pod IPV6 CIDR # Defaults to kube_network_node_prefix_ipv6 if not set. # cilium_pool_mask_size_ipv6: "120" # Extra arguments for the Cilium agent cilium_agent_custom_args: [] # deprecated cilium_agent_extra_args: [] # For adding and mounting extra volumes to the cilium agent cilium_agent_extra_volumes: [] cilium_agent_extra_volume_mounts: [] cilium_agent_extra_env_vars: [] cilium_operator_replicas: 2 # The address at which the cillium operator bind health check api cilium_operator_api_serve_addr: "127.0.0.1:9234" ## A dictionary of extra config variables to add to cilium-config, formatted like: ## cilium_config_extra_vars: ## var1: "value1" ## var2: "value2" cilium_config_extra_vars: {} # For adding and mounting extra volumes to the cilium operator cilium_operator_extra_volumes: [] cilium_operator_extra_volume_mounts: [] # Extra arguments for the Cilium Operator cilium_operator_custom_args: [] # deprecated cilium_operator_extra_args: [] # Tolerations of the cilium operator cilium_operator_tolerations: - operator: "Exists" # Unique ID of the cluster. Must be unique across all connected # clusters and in the range of 1 to 255. Only required for Cluster Mesh, # may be 0 if Cluster Mesh is not used. cilium_cluster_id: 0 # Name of the cluster. Only relevant when building a mesh of clusters. # The "default" name cannot be used if the Cluster ID is different from 0. cilium_cluster_name: default # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. # This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. # Available for Cilium v1.10 and up. cilium_cni_exclusive: true # Configure the log file for CNI logging with retention policy of 7 days. # Disable CNI file logging by setting this field to empty explicitly. # Available for Cilium v1.12 and up. cilium_cni_log_file: "/var/run/cilium/cilium-cni.log" # -- Configure cgroup related configuration # -- Enable auto mount of cgroup2 filesystem. # When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at # `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod. # If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted # cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the # volume will be mounted inside the cilium agent pod at the same path. # Available for Cilium v1.11 and up cilium_cgroup_auto_mount: true # -- Configure cgroup root where cgroup2 filesystem is mounted on the host cilium_cgroup_host_root: "/run/cilium/cgroupv2" # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. cilium_bpf_map_dynamic_size_ratio: "0.0025" # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. # Available for Cilium v1.10 and up cilium_enable_ipv4_masquerade: true # -- Enables masquerading of IPv6 traffic leaving the node from endpoints. # Available for Cilium v1.10 and up cilium_enable_ipv6_masquerade: true # -- Enable native IP masquerade support in eBPF cilium_enable_bpf_masquerade: false # -- Configure whether direct routing mode should route traffic via # host stack (true) or directly and more efficiently out of BPF (false) if # the kernel supports it. The latter has the implication that it will also # bypass netfilter in the host namespace. cilium_enable_host_legacy_routing: false # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity cilium_enable_remote_node_identity: true # -- Enable the use of well-known identities. cilium_enable_well_known_identities: false # The monitor aggregation flags determine which TCP flags which, upon the # first observation, cause monitor notifications to be generated. # # Only effective when monitor aggregation is set to "medium" or higher. cilium_monitor_aggregation_flags: "all" # -- Enable BGP Control Plane cilium_enable_bgp_control_plane: false # -- Configure BGP Instances (New bgpv2 API v1.16+) cilium_bgp_cluster_configs: [] # -- Configure BGP Peers (New bgpv2 API v1.16+) cilium_bgp_peer_configs: [] # -- Configure BGP Advertisements (New bgpv2 API v1.16+) cilium_bgp_advertisements: [] # -- Configure BGP Node Config Overrides (New bgpv2 API v1.16+) cilium_bgp_node_config_overrides: [] # -- Configure BGP Peers (Legacy < v1.16) cilium_bgp_peering_policies: [] # -- Whether to enable CNP status updates. cilium_disable_cnp_status_updates: true # Configure how long to wait for the Cilium DaemonSet to be ready again cilium_rolling_restart_wait_retries_count: 30 cilium_rolling_restart_wait_retries_delay_seconds: 10 # Cilium changed the default metrics exporter ports in 1.12 cilium_agent_scrape_port: "9962" cilium_operator_scrape_port: "9963" cilium_hubble_scrape_port: "9965" # Cilium certgen args for generate certificate for hubble mTLS cilium_certgen_args: cilium-namespace: kube-system ca-reuse-secret: true ca-secret-name: hubble-ca-secret ca-generate: true ca-validity-duration: 94608000s hubble-server-cert-generate: true hubble-server-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io' hubble-server-cert-validity-duration: 94608000s hubble-server-cert-secret-name: hubble-server-certs hubble-relay-client-cert-generate: true hubble-relay-client-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io' hubble-relay-client-cert-validity-duration: 94608000s hubble-relay-client-cert-secret-name: hubble-relay-client-certs hubble-relay-server-cert-generate: false cilium_enable_host_firewall: false cilium_policy_audit_mode: false # Cilium extra install flags cilium_install_extra_flags: "" # Cilium extra values, use any values from cilium Helm Chart # ref: https://docs.cilium.io/en/stable/helm-reference/ cilium_extra_values: {} ================================================ FILE: roles/network_plugin/cilium/tasks/apply.yml ================================================ --- - name: Check if Cilium Helm release exists (via cilium version) command: "{{ bin_dir }}/cilium version" register: cilium_release_info when: inventory_hostname == groups['kube_control_plane'][0] failed_when: false changed_when: false - name: Set action to install or upgrade set_fact: cilium_action: "{{ 'install' if ('release: not found' in cilium_release_info.stderr | default('') or 'release: not found' in cilium_release_info.stdout | default('')) else 'upgrade' }}" - name: Cilium | Install environment: "{{ proxy_env }}" command: "{{ bin_dir }}/cilium {{ cilium_action }} --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml -f {{ kube_config_dir }}/cilium-extra-values.yaml {{ cilium_install_extra_flags }}" when: inventory_hostname == groups['kube_control_plane'][0] - name: Cilium | Wait for pods to run command: "{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa literal-compare register: pods_not_ready until: pods_not_ready.stdout.find("cilium")==-1 retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: inventory_hostname == groups['kube_control_plane'][0] - name: Cilium | Wait for CiliumLoadBalancerIPPool CRD to be present command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumloadbalancerippools.cilium.io" register: cillium_lbippool_crd_ready retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0) - name: Cilium | Create CiliumLoadBalancerIPPool manifests template: src: "{{ item.name }}/{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" mode: "0644" with_items: - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0 - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0) - name: Cilium | Apply CiliumLoadBalancerIPPool from cilium_loadbalancer_ip_pools kube: name: "{{ item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.type }}" filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" state: "latest" loop: - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0 - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0) - name: Cilium | Wait for CiliumBGPPeeringPolicy CRD to be present command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeeringpolicies.cilium.io" register: cillium_bgpppolicy_crd_ready retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0) - name: Cilium | Create CiliumBGPPeeringPolicy manifests template: src: "{{ item.name }}/{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" mode: "0644" with_items: - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0 - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0) - name: Cilium | Apply CiliumBGPPeeringPolicy from cilium_bgp_peering_policies kube: name: "{{ item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.type }}" filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" state: "latest" loop: - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0 - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0) - name: Cilium | Wait for CiliumBGPClusterConfig CRD to be present command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpclusterconfigs.cilium.io" register: cillium_bgpcconfig_crd_ready retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0) - name: Cilium | Create CiliumBGPClusterConfig manifests template: src: "{{ item.name }}/{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" mode: "0644" with_items: - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0 - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0) - name: Cilium | Apply CiliumBGPClusterConfig from cilium_bgp_cluster_configs kube: name: "{{ item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.type }}" filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" state: "latest" loop: - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0 - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0) - name: Cilium | Wait for CiliumBGPPeerConfig CRD to be present command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeerconfigs.cilium.io" register: cillium_bgppconfig_crd_ready retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0) - name: Cilium | Create CiliumBGPPeerConfig manifests template: src: "{{ item.name }}/{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" mode: "0644" with_items: - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0 - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0) - name: Cilium | Apply CiliumBGPPeerConfig from cilium_bgp_peer_configs kube: name: "{{ item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.type }}" filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" state: "latest" loop: - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0 - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0) - name: Cilium | Wait for CiliumBGPAdvertisement CRD to be present command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpadvertisements.cilium.io" register: cillium_bgpadvert_crd_ready retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0) - name: Cilium | Create CiliumBGPAdvertisement manifests template: src: "{{ item.name }}/{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" mode: "0644" with_items: - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0 - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0) - name: Cilium | Apply CiliumBGPAdvertisement from cilium_bgp_advertisements kube: name: "{{ item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.type }}" filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" state: "latest" loop: - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement} when: - inventory_hostname == groups['kube_control_plane'][0] - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0 - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0) - name: Cilium | Wait for CiliumBGPNodeConfigOverride CRD to be present command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpnodeconfigoverrides.cilium.io" register: cilium_bgp_node_config_crd_ready retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" failed_when: false when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0) - name: Cilium | Create CiliumBGPNodeConfigOverride manifests template: src: "{{ item.name }}/{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" mode: "0644" with_items: - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride} when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0 - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0) - name: Cilium | Apply CiliumBGPNodeConfigOverride from cilium_bgp_node_config_overrides kube: name: "{{ item.name }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.type }}" filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" state: "latest" loop: - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride} when: - inventory_hostname == groups['kube_control_plane'][0] - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0 - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0) ================================================ FILE: roles/network_plugin/cilium/tasks/check.yml ================================================ --- - name: Cilium | Check Cilium encryption `cilium_ipsec_key` for ipsec assert: that: - "cilium_ipsec_key is defined" msg: "cilium_ipsec_key should be defined to enable encryption using ipsec" when: - cilium_encryption_enabled - cilium_encryption_type == "ipsec" - cilium_tunnel_mode in ['vxlan'] # TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled` - name: Stop if `cilium_ipsec_enabled` is defined and `cilium_encryption_type` is not `ipsec` assert: that: cilium_encryption_type == 'ipsec' msg: > It is not possible to use `cilium_ipsec_enabled` when `cilium_encryption_type` is set to {{ cilium_encryption_type }}. when: - cilium_ipsec_enabled is defined - cilium_ipsec_enabled - kube_network_plugin == 'cilium' or cilium_deploy_additionally - name: Stop if kernel version is too low for Cilium Wireguard encryption assert: that: ansible_kernel.split('-')[0] is version('5.6.0', '>=') when: - kube_network_plugin == 'cilium' or cilium_deploy_additionally - cilium_encryption_enabled - cilium_encryption_type == "wireguard" - not ignore_assert_errors - name: Stop if bad Cilium identity allocation mode assert: that: cilium_identity_allocation_mode in ['crd', 'kvstore'] msg: "cilium_identity_allocation_mode must be either 'crd' or 'kvstore'" - name: Stop if bad Cilium Cluster ID assert: that: - cilium_cluster_id <= 255 - cilium_cluster_id >= 0 msg: "'cilium_cluster_id' must be between 1 and 255" when: cilium_cluster_id is defined - name: Stop if bad encryption type assert: that: cilium_encryption_type in ['ipsec', 'wireguard'] msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'" when: cilium_encryption_enabled - name: Stop if cilium_version is < {{ cilium_min_version_required }} assert: that: cilium_version is version(cilium_min_version_required, '>=') msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}" # TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled` - name: Set `cilium_encryption_type` to "ipsec" and if `cilium_ipsec_enabled` is true set_fact: cilium_encryption_type: ipsec cilium_encryption_enabled: true when: - cilium_ipsec_enabled is defined - cilium_ipsec_enabled - name: Stop if cilium_hubble_event_buffer_capacity is not a power of 2 minus 1 and is not between 1 and 65535 assert: that: "cilium_hubble_event_buffer_capacity in [1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, 65535]" msg: "Error: cilium_hubble_event_buffer_capacity:{{ cilium_hubble_event_buffer_capacity }} is not a power of 2 minus 1 and it should be between 1 and 65535." when: cilium_hubble_event_buffer_capacity is defined ================================================ FILE: roles/network_plugin/cilium/tasks/install.yml ================================================ --- - name: Cilium | Ensure BPFFS mounted ansible.posix.mount: fstype: bpf path: /sys/fs/bpf src: bpffs state: mounted - name: Cilium | Create Cilium certs directory file: dest: "{{ cilium_cert_dir }}" state: directory mode: "0750" owner: root group: root when: - cilium_identity_allocation_mode == "kvstore" - name: Cilium | Link etcd certificates for cilium file: src: "{{ etcd_cert_dir }}/{{ item.s }}" dest: "{{ cilium_cert_dir }}/{{ item.d }}" mode: "0644" state: hard force: true loop: - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} when: - cilium_identity_allocation_mode == "kvstore" - name: Cilium | Render values template: src: values.yaml.j2 dest: "{{ kube_config_dir }}/cilium-values.yaml" mode: "0644" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Cilium | Copy extra values copy: content: "{{ cilium_extra_values | to_nice_yaml(indent=2) }}" dest: "{{ kube_config_dir }}/cilium-extra-values.yaml" mode: "0644" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Cilium | Copy Ciliumcli binary from download dir copy: src: "{{ local_release_dir }}/cilium" dest: "{{ bin_dir }}/cilium" mode: "0755" remote_src: true ================================================ FILE: roles/network_plugin/cilium/tasks/main.yml ================================================ --- - name: Cilium check import_tasks: check.yml - name: Cilium install include_tasks: install.yml - name: Cilium apply include_tasks: apply.yml ================================================ FILE: roles/network_plugin/cilium/tasks/reset.yml ================================================ --- - name: Reset | check and remove devices if still present include_tasks: reset_iface.yml vars: iface: "{{ item }}" loop: - cilium_host - cilium_net - cilium_vxlan ================================================ FILE: roles/network_plugin/cilium/tasks/reset_iface.yml ================================================ --- - name: "Reset | check if network device {{ iface }} is present" stat: path: "/sys/class/net/{{ iface }}" get_attributes: false get_checksum: false get_mime: false register: device_remains - name: "Reset | remove network device {{ iface }}" command: "ip link del {{ iface }}" when: device_remains.stat.exists ================================================ FILE: roles/network_plugin/cilium/templates/cilium/cilium-bgp-advertisement.yml.j2 ================================================ {% for cilium_bgp_advertisement in cilium_bgp_advertisements %} --- apiVersion: "cilium.io/v2" kind: CiliumBGPAdvertisement metadata: name: "{{ cilium_bgp_advertisement.name }}" {% if cilium_bgp_advertisement.labels %} labels: {{ cilium_bgp_advertisement.labels | to_yaml }} {% endif %} spec: {{ cilium_bgp_advertisement.spec | to_yaml | indent(4) }} {% endfor %} ================================================ FILE: roles/network_plugin/cilium/templates/cilium/cilium-bgp-cluster-config.yml.j2 ================================================ {% for cilium_bgp_cluster_config in cilium_bgp_cluster_configs %} --- apiVersion: "cilium.io/v2" kind: CiliumBGPClusterConfig metadata: name: "{{ cilium_bgp_cluster_config.name }}" spec: {{ cilium_bgp_cluster_config.spec | to_yaml | indent(2) }} {% endfor %} ================================================ FILE: roles/network_plugin/cilium/templates/cilium/cilium-bgp-node-config-override.yml.j2 ================================================ {% for cilium_bgp_node_config_override in cilium_bgp_node_config_overrides %} --- apiVersion: "cilium.io/v2" kind: CiliumBGPNodeConfigOverride metadata: name: "{{ cilium_bgp_node_config_override.name }}" spec: {{ cilium_bgp_node_config_override.spec | to_yaml | indent(2) }} {% endfor %} ================================================ FILE: roles/network_plugin/cilium/templates/cilium/cilium-bgp-peer-config.yml.j2 ================================================ {% for cilium_bgp_peer_config in cilium_bgp_peer_configs %} --- apiVersion: "cilium.io/v2" kind: CiliumBGPPeerConfig metadata: name: "{{ cilium_bgp_peer_config.name }}" spec: {{ cilium_bgp_peer_config.spec | to_yaml | indent(2) }} {% endfor %} ================================================ FILE: roles/network_plugin/cilium/templates/cilium/cilium-bgp-peering-policy.yml.j2 ================================================ {% for cilium_bgp_peering_policy in cilium_bgp_peering_policies %} --- apiVersion: "cilium.io/v2alpha1" kind: CiliumBGPPeeringPolicy metadata: name: "{{ cilium_bgp_peering_policy.name }}" spec: {{ cilium_bgp_peering_policy.spec | to_yaml | indent(2) }} {% endfor %} ================================================ FILE: roles/network_plugin/cilium/templates/cilium/cilium-loadbalancer-ip-pool.yml.j2 ================================================ {% for cilium_loadbalancer_ip_pool in cilium_loadbalancer_ip_pools %} --- apiVersion: "cilium.io/v2" kind: CiliumLoadBalancerIPPool metadata: name: "{{ cilium_loadbalancer_ip_pool.name }}" spec: blocks: {% for cblock in cilium_loadbalancer_ip_pool.cidrs | default([]) %} - cidr: "{{ cblock }}" {% endfor %} {% for rblock in cilium_loadbalancer_ip_pool.ranges | default([]) %} - start: "{{ rblock.start }}" stop: "{{ rblock.stop | default(rblock.start) }}" {% endfor %} {% endfor %} ================================================ FILE: roles/network_plugin/cilium/templates/values.yaml.j2 ================================================ #jinja2: trim_blocks: True, lstrip_blocks: True MTU: {{ cilium_mtu }} debug: enabled: {{ cilium_debug | to_json }} image: repository: {{ cilium_image_repo }} tag: {{ cilium_image_tag }} k8sServiceHost: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" k8sServicePort: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" ipv4: enabled: {{ cilium_enable_ipv4 | to_json }} ipv6: enabled: {{ cilium_enable_ipv6 | to_json }} l2announcements: enabled: {{ cilium_l2announcements | to_json }} bgpControlPlane: enabled: {{ cilium_enable_bgp_control_plane | to_json }} healthPort: {{ cilium_agent_health_port }} identityAllocationMode: {{ cilium_identity_allocation_mode }} tunnelProtocol: {{ cilium_tunnel_mode }} loadBalancer: mode: {{ cilium_loadbalancer_mode }} kubeProxyReplacement: {{ cilium_kube_proxy_replacement | to_json }} {% if cilium_dns_proxy_enable_transparent_mode is defined %} dnsProxy: enableTransparentMode: {{ cilium_dns_proxy_enable_transparent_mode | to_json }} {% endif %} extraVolumes: {{ cilium_agent_extra_volumes | to_nice_yaml(indent=2) | indent(2) }} extraVolumeMounts: {{ cilium_agent_extra_volume_mounts | to_nice_yaml(indent=2) | indent(2) }} extraArgs: {{ cilium_agent_extra_args | to_nice_yaml(indent=2) | indent(2) }} bpf: masquerade: {{ cilium_enable_bpf_masquerade | to_json }} hostLegacyRouting: {{ cilium_enable_host_legacy_routing | to_json }} monitorAggregation: {{ cilium_monitor_aggregation }} preallocateMaps: {{ cilium_preallocate_bpf_maps | to_json }} mapDynamicSizeRatio: {{ cilium_bpf_map_dynamic_size_ratio }} cni: exclusive: {{ cilium_cni_exclusive | to_json }} logFile: {{ cilium_cni_log_file }} {% if cilium_enable_portmap %} chainingMode: portmap {% endif %} autoDirectNodeRoutes: {{ cilium_auto_direct_node_routes | to_json }} ipv4NativeRoutingCIDR: "{{ cilium_native_routing_cidr }}" ipv6NativeRoutingCIDR: "{{ cilium_native_routing_cidr_ipv6 }}" encryption: enabled: {{ cilium_encryption_enabled | to_json }} {% if cilium_encryption_enabled %} type: {{ cilium_encryption_type }} {% if cilium_encryption_type == 'wireguard' %} nodeEncryption: {{ cilium_encryption_node_encryption | to_json }} {% endif %} {% endif %} bandwidthManager: enabled: {{ cilium_enable_bandwidth_manager | to_json }} bbr: {{ cilium_enable_bandwidth_manager_bbr | to_json }} ipMasqAgent: enabled: {{ cilium_ip_masq_agent_enable | to_json }} {% if cilium_ip_masq_agent_enable %} config: nonMasqueradeCIDRs: {{ cilium_non_masquerade_cidrs }} masqLinkLocal: {{ cilium_masq_link_local | to_json }} masqLinkLocalIPv6: {{ cilium_masq_link_local_ipv6 | to_json }} # cilium_ip_masq_resync_interval {% endif %} hubble: peerService: clusterDomain: {{ cilium_hubble_peer_service_cluster_domain }} enabled: {{ cilium_enable_hubble | to_json }} relay: enabled: {{ cilium_enable_hubble | to_json }} image: repository: {{ cilium_hubble_relay_image_repo }} tag: {{ cilium_hubble_relay_image_tag }} ui: enabled: {{ cilium_enable_hubble_ui | to_json }} backend: image: repository: {{ cilium_hubble_ui_backend_image_repo }} tag: {{ cilium_hubble_ui_backend_image_tag }} frontend: image: repository: {{ cilium_hubble_ui_image_repo }} tag: {{ cilium_hubble_ui_image_tag }} metrics: enabled: {{ cilium_hubble_metrics | to_json }} export: {% if cilium_version is version('1.18.0', '>=') %} static: fileMaxBackups: {{ cilium_hubble_export_file_max_backups }} fileMaxSizeMb: {{ cilium_hubble_export_file_max_size_mb }} {% else %} fileMaxBackups: {{ cilium_hubble_export_file_max_backups }} fileMaxSizeMb: {{ cilium_hubble_export_file_max_size_mb }} {% endif %} dynamic: enabled: {{ cilium_hubble_export_dynamic_enabled | to_json }} config: content: {{ cilium_hubble_export_dynamic_config_content | to_nice_yaml(indent=10) | indent(10) }} gatewayAPI: enabled: {{ cilium_gateway_api_enabled | to_json }} ipam: mode: {{ cilium_ipam_mode }} operator: clusterPoolIPv4PodCIDRList: - {{ cilium_pool_cidr | default(kube_pods_subnet) }} clusterPoolIPv4MaskSize: {{ cilium_pool_mask_size | default(kube_network_node_prefix) }} clusterPoolIPv6PodCIDRList: - {{ cilium_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }} clusterPoolIPv6MaskSize: {{ cilium_pool_mask_size_ipv6 | default(kube_network_node_prefix_ipv6) }} cgroup: autoMount: enabled: {{ cilium_cgroup_auto_mount | to_json }} hostRoot: {{ cilium_cgroup_host_root }} resources: limits: memory: "{{ cilium_memory_limit }}" cpu: "{{ cilium_cpu_limit }}" requests: memory: "{{ cilium_memory_requests }}" cpu: "{{ cilium_cpu_requests }}" operator: image: repository: {{ cilium_operator_image_repo }} tag: {{ cilium_operator_image_tag }} replicas: {{ cilium_operator_replicas }} extraArgs: {{ cilium_operator_extra_args | to_nice_yaml(indent=2) | indent(4) }} extraVolumes: {{ cilium_operator_extra_volumes | to_nice_yaml(indent=2) | indent(4) }} extraVolumeMounts: {{ cilium_operator_extra_volume_mounts | to_nice_yaml(indent=2) | indent(4) }} tolerations: {{ cilium_operator_tolerations | to_nice_yaml(indent=2) | indent(4) }} cluster: id: {{ cilium_cluster_id }} name: {{ cilium_cluster_name }} enableIPv4Masquerade: {{ cilium_enable_ipv4_masquerade | to_json }} enableIPv6Masquerade: {{ cilium_enable_ipv6_masquerade | to_json }} hostFirewall: enabled: {{ cilium_enable_host_firewall | to_json }} policyAuditMode: {{ cilium_policy_audit_mode | to_json }} certgen: image: repository: {{ cilium_hubble_certgen_image_repo }} tag: {{ cilium_hubble_certgen_image_tag }} envoy: image: repository: {{ cilium_hubble_envoy_image_repo }} tag: {{ cilium_hubble_envoy_image_tag }} extraConfig: {{ cilium_config_extra_vars | to_yaml | indent(2) }} ================================================ FILE: roles/network_plugin/cni/defaults/main.yml ================================================ --- cni_bin_owner: "{{ kube_owner }}" ================================================ FILE: roles/network_plugin/cni/tasks/main.yml ================================================ --- - name: CNI | make sure /opt/cni/bin exists file: path: /opt/cni/bin state: directory mode: "0755" owner: "{{ cni_bin_owner }}" recurse: true - name: CNI | Copy cni plugins unarchive: src: "{{ downloads.cni.dest }}" dest: "/opt/cni/bin" mode: "0755" owner: "{{ cni_bin_owner }}" remote_src: true ================================================ FILE: roles/network_plugin/custom_cni/defaults/main.yml ================================================ --- custom_cni_manifests: [] custom_cni_chart_namespace: kube-system custom_cni_chart_release_name: "" custom_cni_chart_repository_name: "" custom_cni_chart_repository_url: "" custom_cni_chart_ref: "" custom_cni_chart_version: "" custom_cni_chart_values: {} ================================================ FILE: roles/network_plugin/custom_cni/meta/main.yml ================================================ --- dependencies: - role: helm-apps when: - inventory_hostname == groups['kube_control_plane'][0] - custom_cni_chart_release_name | length > 0 environment: http_proxy: "{{ http_proxy | default('') }}" https_proxy: "{{ https_proxy | default('') }}" release_common_opts: {} releases: - name: "{{ custom_cni_chart_release_name }}" namespace: "{{ custom_cni_chart_namespace }}" chart_ref: "{{ custom_cni_chart_ref }}" chart_version: "{{ custom_cni_chart_version }}" wait: true create_namespace: true values: "{{ custom_cni_chart_values }}" repositories: - name: "{{ custom_cni_chart_repository_name }}" url: "{{ custom_cni_chart_repository_url }}" ================================================ FILE: roles/network_plugin/custom_cni/tasks/main.yml ================================================ --- - name: Custom CNI | Manifest deployment when: not custom_cni_chart_release_name | length > 0 block: - name: Custom CNI | Check Custom CNI Manifests assert: that: - "custom_cni_manifests | length > 0" msg: "custom_cni_manifests should not be empty" - name: Custom CNI | Copy Custom manifests template: src: "{{ item }}" dest: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}" mode: "0644" loop: "{{ custom_cni_manifests }}" delegate_to: "{{ groups['kube_control_plane'] | first }}" run_once: true - name: Custom CNI | Start Resources kube: namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}" state: "latest" wait: true loop: "{{ custom_cni_manifests }}" delegate_to: "{{ groups['kube_control_plane'] | first }}" run_once: true ================================================ FILE: roles/network_plugin/flannel/defaults/main.yml ================================================ --- # Flannel public IP # The address that flannel should advertise as how to access the system # Disabled until https://github.com/coreos/flannel/issues/712 is fixed # flannel_public_ip: "{{ main_access_ip }}" ## interface that should be used for flannel operations ## This is actually an inventory cluster-level item # flannel_interface: ## Select interface that should be used for flannel operations by regexp on Name or IP ## This is actually an inventory cluster-level item ## example: select interface with ip from net 10.0.0.0/23 ## single quote and escape backslashes # flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}' # You can choose what type of flannel backend to use # please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md flannel_backend_type: "vxlan" flannel_vxlan_vni: 1 flannel_vxlan_port: 8472 flannel_vxlan_direct_routing: false # Limits for apps flannel_memory_limit: 500M flannel_cpu_limit: 300m flannel_memory_requests: 64M flannel_cpu_requests: 150m ================================================ FILE: roles/network_plugin/flannel/meta/main.yml ================================================ --- dependencies: - role: network_plugin/cni ================================================ FILE: roles/network_plugin/flannel/tasks/main.yml ================================================ --- - name: Flannel | Stop if kernel version is too low for Flannel Wireguard encryption assert: that: ansible_kernel.split('-')[0] is version('5.6.0', '>=') when: - kube_network_plugin == 'flannel' - flannel_backend_type == 'wireguard' - not ignore_assert_errors - name: Flannel | Create Flannel manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: flannel, file: cni-flannel-rbac.yml, type: sa} - {name: kube-flannel, file: cni-flannel.yml, type: ds} register: flannel_node_manifests when: - inventory_hostname == groups['kube_control_plane'][0] - name: Flannel | Start Resources kube: name: "{{ item.item.name }}" namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: "{{ flannel_node_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped - name: Flannel | Wait for flannel subnet.env file presence wait_for: path: /run/flannel/subnet.env delay: 5 timeout: 600 ================================================ FILE: roles/network_plugin/flannel/tasks/reset.yml ================================================ --- - name: Reset | check cni network device stat: path: /sys/class/net/cni0 get_attributes: false get_checksum: false get_mime: false register: cni - name: Reset | remove the network device created by the flannel command: ip link del cni0 when: cni.stat.exists - name: Reset | check flannel network device stat: path: /sys/class/net/flannel.1 get_attributes: false get_checksum: false get_mime: false register: flannel - name: Reset | remove the network device created by the flannel command: ip link del flannel.1 when: flannel.stat.exists ================================================ FILE: roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: flannel namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel rules: - apiGroups: - "" resources: - pods verbs: - get - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - "" resources: - nodes/status verbs: - patch - apiGroups: - "networking.k8s.io" resources: - clustercidrs verbs: - list - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: flannel subjects: - kind: ServiceAccount name: flannel namespace: kube-system ================================================ FILE: roles/network_plugin/flannel/templates/cni-flannel.yml.j2 ================================================ --- kind: ConfigMap apiVersion: v1 metadata: name: kube-flannel-cfg namespace: kube-system labels: tier: node app: flannel data: cni-conf.json: | { "name": "cbr0", "cniVersion": "0.3.1", "plugins": [ { "type": "flannel", "delegate": { "hairpinMode": true, "isDefaultGateway": true } }, { "type": "portmap", "capabilities": { "portMappings": true } } ] } net-conf.json: | { {% if ipv4_stack %} "Network": "{{ kube_pods_subnet }}", "EnableIPv4": true, {% endif %} {% if ipv6_stack %} "EnableIPv6": true, "IPv6Network": "{{ kube_pods_subnet_ipv6 }}", {% endif %} "Backend": { "Type": "{{ flannel_backend_type }}"{% if flannel_backend_type == "vxlan" %}, "VNI": {{ flannel_vxlan_vni }}, "Port": {{ flannel_vxlan_port }}, "DirectRouting": {{ flannel_vxlan_direct_routing | to_json }} {% endif %} } } {% for arch in ['amd64', 'arm64', 'arm', 'ppc64le', 's390x'] %} --- apiVersion: apps/v1 kind: DaemonSet metadata: {% if arch == 'amd64' %} name: kube-flannel {% else %} name: kube-flannel-ds-{{ arch }} {% endif %} namespace: kube-system labels: tier: node app: flannel spec: selector: matchLabels: app: flannel template: metadata: labels: tier: node app: flannel spec: priorityClassName: system-node-critical serviceAccountName: flannel containers: - name: kube-flannel image: {{ flannel_image_repo }}:{{ flannel_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} resources: limits: cpu: {{ flannel_cpu_limit }} memory: {{ flannel_memory_limit }} requests: cpu: {{ flannel_cpu_requests }} memory: {{ flannel_memory_requests }} command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %}{% if flannel_interface_regexp is defined %}, "--iface-regex={{ flannel_interface_regexp }}"{% endif %} ] securityContext: privileged: false capabilities: add: ["NET_ADMIN", "NET_RAW"] env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: EVENT_QUEUE_DEPTH value: "5000" volumeMounts: - name: run mountPath: /run/flannel - name: flannel-cfg mountPath: /etc/kube-flannel/ - name: xtables-lock mountPath: /run/xtables.lock affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/os operator: In values: - linux - key: kubernetes.io/arch operator: In values: - {{ arch }} initContainers: - name: install-cni-plugin image: {{ flannel_init_image_repo }}:{{ flannel_init_image_tag }} command: - cp args: - -f - /flannel - /opt/cni/bin/flannel volumeMounts: - name: cni-plugin mountPath: /opt/cni/bin - name: install-cni image: {{ flannel_image_repo }}:{{ flannel_image_tag }} command: - cp args: - -f - /etc/kube-flannel/cni-conf.json - /etc/cni/net.d/10-flannel.conflist volumeMounts: - name: cni mountPath: /etc/cni/net.d - name: flannel-cfg mountPath: /etc/kube-flannel/ hostNetwork: true dnsPolicy: ClusterFirstWithHostNet tolerations: - operator: Exists volumes: - name: run hostPath: path: /run/flannel - name: cni hostPath: path: /etc/cni/net.d - name: flannel-cfg configMap: name: kube-flannel-cfg - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate - name: cni-plugin hostPath: path: /opt/cni/bin updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate {% endfor %} ================================================ FILE: roles/network_plugin/kube-ovn/defaults/main.yml ================================================ --- kube_ovn_db_cpu_request: 500m kube_ovn_db_memory_request: 200Mi kube_ovn_db_cpu_limit: 3000m kube_ovn_db_memory_limit: 3000Mi kube_ovn_node_cpu_request: 200m kube_ovn_node_memory_request: 200Mi kube_ovn_node_cpu_limit: 1000m kube_ovn_node_memory_limit: 800Mi kube_ovn_cni_server_cpu_request: 200m kube_ovn_cni_server_memory_request: 200Mi kube_ovn_cni_server_cpu_limit: 1000m kube_ovn_cni_server_memory_limit: 1Gi kube_ovn_controller_cpu_request: 200m kube_ovn_controller_memory_request: 200Mi kube_ovn_controller_cpu_limit: 1000m kube_ovn_controller_memory_limit: 1Gi kube_ovn_pinger_cpu_request: 100m kube_ovn_pinger_memory_request: 200Mi kube_ovn_pinger_cpu_limit: 200m kube_ovn_pinger_memory_limit: 400Mi kube_ovn_monitor_memory_request: 200Mi kube_ovn_monitor_cpu_request: 200m kube_ovn_monitor_memory_limit: 200Mi kube_ovn_monitor_cpu_limit: 200m kube_ovn_dpdk_node_cpu_request: 1000m kube_ovn_dpdk_node_memory_request: 2Gi kube_ovn_dpdk_node_cpu_limit: 1000m kube_ovn_dpdk_node_memory_limit: 2Gi kube_ovn_central_hosts: "{{ groups['kube_control_plane'] }}" kube_ovn_central_replics: "{{ kube_ovn_central_hosts | length }}" kube_ovn_controller_replics: "{{ kube_ovn_central_hosts | length }}" kube_ovn_central_ips: |- {% for item in kube_ovn_central_hosts -%} {{ hostvars[item]['main_ip'] }}{% if not loop.last %},{% endif %} {%- endfor %} kube_ovn_ic_enable: false kube_ovn_ic_autoroute: true kube_ovn_ic_dbhost: "127.0.0.1" kube_ovn_ic_zone: "kubernetes" # geneve or vlan kube_ovn_network_type: geneve # geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module kube_ovn_tunnel_type: geneve ## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use. # kube_ovn_iface: eth1 ## The MTU used by pod iface in overlay networks (default iface MTU - 100) # kube_ovn_mtu: 1333 ## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port. kube_ovn_hw_offload: false # traffic mirror kube_ovn_traffic_mirror: false # kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 # kube_ovn_default_interface_name: eth0 kube_ovn_external_address: 8.8.8.8 kube_ovn_external_address_ipv6: 2400:3200::1 kube_ovn_external_address_merged: >- {%- if ipv4_stack and ipv6_stack -%} {{ kube_ovn_external_address }},{{ kube_ovn_external_address_ipv6 }} {%- elif ipv4_stack -%} {{ kube_ovn_external_address }} {%- else -%} {{ kube_ovn_external_address_ipv6 }} {%- endif -%} kube_ovn_external_dns: alauda.cn # kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0 kube_ovn_default_gateway_check: true kube_ovn_default_logical_gateway: false # u2o_interconnection kube_ovn_u2o_interconnection: false # kube_ovn_default_exclude_ips: 10.16.0.1 kube_ovn_node_switch_cidr: 100.64.0.0/16 kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64 kube_ovn_node_switch_cidr_merged: >- {%- if ipv4_stack and ipv6_stack -%} {{ kube_ovn_node_switch_cidr }},{{ kube_ovn_node_switch_cidr_ipv6 }} {%- elif ipv4_stack -%} {{ kube_ovn_node_switch_cidr }} {%- else -%} {{ kube_ovn_node_switch_cidr_ipv6 }} {%- endif -%} ## vlan config, set default interface name and vlan id # kube_ovn_default_interface_name: eth0 kube_ovn_default_vlan_id: 100 kube_ovn_vlan_name: product ## pod nic type, support: veth-pair or internal-port kube_ovn_pod_nic_type: veth_pair ## Enable load balancer kube_ovn_enable_lb: true ## Enable network policy support kube_ovn_enable_np: true ## Enable external vpc support kube_ovn_enable_external_vpc: true ## Enable checksum kube_ovn_encap_checksum: true ## enable ssl kube_ovn_enable_ssl: false ## dpdk kube_ovn_dpdk_enabled: false kube_ovn_dpdk_tunnel_iface: br-phy ## bind local ip kube_ovn_bind_local_ip_enabled: true ## eip snat kube_ovn_eip_snat_enabled: true # ls dnat mod dl dst kube_ovn_ls_dnat_mod_dl_dst: true ## keep vm ip kube_ovn_keep_vm_ip: true ## cni config priority, default: 01 kube_ovn_cni_config_priority: '01' ================================================ FILE: roles/network_plugin/kube-ovn/tasks/main.yml ================================================ --- - name: Kube-OVN | Label ovn-db node command: "{{ kubectl }} label --overwrite node {{ item }} kube-ovn/role=master" loop: "{{ kube_ovn_central_hosts }}" when: - inventory_hostname == groups['kube_control_plane'][0] - name: Kube-OVN | Create Kube-OVN manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: kube-ovn-crd, file: cni-kube-ovn-crd.yml} - {name: ovn, file: cni-ovn.yml} - {name: kube-ovn, file: cni-kube-ovn.yml} register: kube_ovn_node_manifests - name: Kube-OVN | Start Resources kube: name: "{{ item.item.name }}" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" with_items: "{{ kube_ovn_node_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped ================================================ FILE: roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 ================================================ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vpc-dnses.kubeovn.io spec: group: kubeovn.io names: plural: vpc-dnses singular: vpc-dns shortNames: - vpc-dns kind: VpcDns listKind: VpcDnsList scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .status.active name: Active type: boolean - jsonPath: .spec.vpc name: Vpc type: string - jsonPath: .spec.subnet name: Subnet type: string name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: spec: type: object properties: vpc: type: string subnet: type: string replicas: type: integer minimum: 1 maximum: 3 status: type: object properties: active: type: boolean conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: switch-lb-rules.kubeovn.io spec: group: kubeovn.io names: plural: switch-lb-rules singular: switch-lb-rule shortNames: - slr kind: SwitchLBRule listKind: SwitchLBRuleList scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .spec.vip name: vip type: string - jsonPath: .status.ports name: port(s) type: string - jsonPath: .status.service name: service type: string - jsonPath: .metadata.creationTimestamp name: age type: date name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: spec: type: object properties: namespace: type: string vip: type: string sessionAffinity: type: string ports: items: properties: name: type: string port: type: integer minimum: 1 maximum: 65535 protocol: type: string targetPort: type: integer minimum: 1 maximum: 65535 type: object type: array selector: items: type: string type: array endpoints: items: type: string type: array status: type: object properties: ports: type: string service: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vpc-nat-gateways.kubeovn.io spec: group: kubeovn.io names: plural: vpc-nat-gateways singular: vpc-nat-gateway shortNames: - vpc-nat-gw kind: VpcNatGateway listKind: VpcNatGatewayList scope: Cluster versions: - additionalPrinterColumns: - jsonPath: .spec.vpc name: Vpc type: string - jsonPath: .spec.subnet name: Subnet type: string - jsonPath: .spec.lanIp name: LanIP type: string name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: status: type: object properties: externalSubnets: items: type: string type: array selector: type: array items: type: string qosPolicy: type: string tolerations: type: array items: type: object properties: key: type: string operator: type: string enum: - Equal - Exists value: type: string effect: type: string enum: - NoExecute - NoSchedule - PreferNoSchedule tolerationSeconds: type: integer affinity: properties: nodeAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: preference: properties: matchExpressions: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchFields: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array type: object weight: format: int32 type: integer required: - preference - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: properties: nodeSelectorTerms: items: properties: matchExpressions: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchFields: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array type: object type: array required: - nodeSelectorTerms type: object type: object podAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: podAffinityTerm: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object weight: format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: items: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object type: array type: object podAntiAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: podAffinityTerm: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object weight: format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: items: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object type: array type: object type: object spec: type: object properties: lanIp: type: string subnet: type: string externalSubnets: items: type: string type: array vpc: type: string selector: type: array items: type: string qosPolicy: type: string tolerations: type: array items: type: object properties: key: type: string operator: type: string enum: - Equal - Exists value: type: string effect: type: string enum: - NoExecute - NoSchedule - PreferNoSchedule tolerationSeconds: type: integer affinity: properties: nodeAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: preference: properties: matchExpressions: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchFields: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array type: object weight: format: int32 type: integer required: - preference - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: properties: nodeSelectorTerms: items: properties: matchExpressions: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchFields: items: properties: key: type: string operator: type: string values: items: type: string type: array required: - key - operator type: object type: array type: object type: array required: - nodeSelectorTerms type: object type: object podAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: podAffinityTerm: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object weight: format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: items: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object type: array type: object podAntiAffinity: properties: preferredDuringSchedulingIgnoredDuringExecution: items: properties: podAffinityTerm: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object weight: format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: items: properties: labelSelector: properties: matchExpressions: items: properties: key: type: string x-kubernetes-patch-strategy: merge x-kubernetes-patch-merge-key: key operator: type: string values: items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string type: object type: object namespaces: items: type: string type: array topologyKey: type: string required: - topologyKey type: object type: array type: object type: object --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-eips.kubeovn.io spec: group: kubeovn.io names: plural: iptables-eips singular: iptables-eip shortNames: - eip kind: IptablesEIP listKind: IptablesEIPList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .status.ip name: IP type: string - jsonPath: .spec.macAddress name: Mac type: string - jsonPath: .status.nat name: Nat type: string - jsonPath: .spec.natGwDp name: NatGwDp type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean ip: type: string nat: type: string redo: type: string qosPolicy: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: v4ip: type: string v6ip: type: string macAddress: type: string natGwDp: type: string qosPolicy: type: string externalSubnet: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-fip-rules.kubeovn.io spec: group: kubeovn.io names: plural: iptables-fip-rules singular: iptables-fip-rule shortNames: - fip kind: IptablesFIPRule listKind: IptablesFIPRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.eip name: Eip type: string - jsonPath: .status.v4ip name: V4ip type: string - jsonPath: .spec.internalIp name: InternalIp type: string - jsonPath: .status.v6ip name: V6ip type: string - jsonPath: .status.ready name: Ready type: boolean - jsonPath: .status.natGwDp name: NatGwDp type: string schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string natGwDp: type: string redo: type: string internalIp: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: eip: type: string internalIp: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-dnat-rules.kubeovn.io spec: group: kubeovn.io names: plural: iptables-dnat-rules singular: iptables-dnat-rule shortNames: - dnat kind: IptablesDnatRule listKind: IptablesDnatRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.eip name: Eip type: string - jsonPath: .spec.protocol name: Protocol type: string - jsonPath: .status.v4ip name: V4ip type: string - jsonPath: .status.v6ip name: V6ip type: string - jsonPath: .spec.internalIp name: InternalIp type: string - jsonPath: .spec.externalPort name: ExternalPort type: string - jsonPath: .spec.internalPort name: InternalPort type: string - jsonPath: .status.natGwDp name: NatGwDp type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string natGwDp: type: string redo: type: string protocol: type: string internalIp: type: string internalPort: type: string externalPort: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: eip: type: string externalPort: type: string protocol: type: string internalIp: type: string internalPort: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: iptables-snat-rules.kubeovn.io spec: group: kubeovn.io names: plural: iptables-snat-rules singular: iptables-snat-rule shortNames: - snat kind: IptablesSnatRule listKind: IptablesSnatRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.eip name: EIP type: string - jsonPath: .status.v4ip name: V4ip type: string - jsonPath: .status.v6ip name: V6ip type: string - jsonPath: .spec.internalCIDR name: InternalCIDR type: string - jsonPath: .status.natGwDp name: NatGwDp type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4ip: type: string v6ip: type: string natGwDp: type: string redo: type: string internalCIDR: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: eip: type: string internalCIDR: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ovn-eips.kubeovn.io spec: group: kubeovn.io names: plural: ovn-eips singular: ovn-eip shortNames: - oeip kind: OvnEip listKind: OvnEipList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .status.v4Ip name: V4IP type: string - jsonPath: .status.v6Ip name: V6IP type: string - jsonPath: .status.macAddress name: Mac type: string - jsonPath: .status.type name: Type type: string - jsonPath: .status.nat name: Nat type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: type: type: string nat: type: string ready: type: boolean v4Ip: type: string v6Ip: type: string macAddress: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: externalSubnet: type: string type: type: string v4Ip: type: string v6Ip: type: string macAddress: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ovn-fips.kubeovn.io spec: group: kubeovn.io names: plural: ovn-fips singular: ovn-fip shortNames: - ofip kind: OvnFip listKind: OvnFipList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .status.vpc name: Vpc type: string - jsonPath: .status.v4Eip name: V4Eip type: string - jsonPath: .status.v4Ip name: V4Ip type: string - jsonPath: .status.ready name: Ready type: boolean - jsonPath: .spec.ipType name: IpType type: string - jsonPath: .spec.ipName name: IpName type: string schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4Eip: type: string v4Ip: type: string vpc: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: ovnEip: type: string ipType: type: string ipName: type: string vpc: type: string v4Ip: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ovn-snat-rules.kubeovn.io spec: group: kubeovn.io names: plural: ovn-snat-rules singular: ovn-snat-rule shortNames: - osnat kind: OvnSnatRule listKind: OvnSnatRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .status.vpc name: Vpc type: string - jsonPath: .status.v4Eip name: V4Eip type: string - jsonPath: .status.v4IpCidr name: V4IpCidr type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4Eip: type: string v4IpCidr: type: string vpc: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: ovnEip: type: string vpcSubnet: type: string ipName: type: string vpc: type: string v4IpCidr: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ovn-dnat-rules.kubeovn.io spec: group: kubeovn.io names: plural: ovn-dnat-rules singular: ovn-dnat-rule shortNames: - odnat kind: OvnDnatRule listKind: OvnDnatRuleList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .status.vpc name: Vpc type: string - jsonPath: .spec.ovnEip name: Eip type: string - jsonPath: .status.protocol name: Protocol type: string - jsonPath: .status.v4Eip name: V4Eip type: string - jsonPath: .status.v4Ip name: V4Ip type: string - jsonPath: .status.internalPort name: InternalPort type: string - jsonPath: .status.externalPort name: ExternalPort type: string - jsonPath: .spec.ipName name: IpName type: string - jsonPath: .status.ready name: Ready type: boolean schema: openAPIV3Schema: type: object properties: status: type: object properties: ready: type: boolean v4Eip: type: string v4Ip: type: string vpc: type: string externalPort: type: string internalPort: type: string protocol: type: string ipName: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: ovnEip: type: string ipType: type: string ipName: type: string externalPort: type: string internalPort: type: string protocol: type: string vpc: type: string v4Ip: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vpcs.kubeovn.io spec: group: kubeovn.io versions: - additionalPrinterColumns: - jsonPath: .status.enableExternal name: EnableExternal type: boolean - jsonPath: .status.enableBfd name: EnableBfd type: boolean - jsonPath: .status.standby name: Standby type: boolean - jsonPath: .status.subnets name: Subnets type: string - jsonPath: .status.extraExternalSubnets name: ExtraExternalSubnets type: string - jsonPath: .spec.namespaces name: Namespaces type: string name: v1 schema: openAPIV3Schema: properties: spec: properties: enableExternal: type: boolean enableBfd: type: boolean namespaces: items: type: string type: array extraExternalSubnets: items: type: string type: array staticRoutes: items: properties: policy: type: string cidr: type: string nextHopIP: type: string ecmpMode: type: string bfdId: type: string routeTable: type: string type: object type: array policyRoutes: items: properties: priority: type: integer action: type: string match: type: string nextHopIP: type: string type: object type: array vpcPeerings: items: properties: remoteVpc: type: string localConnectIP: type: string type: object type: array type: object status: properties: conditions: items: properties: lastTransitionTime: type: string lastUpdateTime: type: string message: type: string reason: type: string status: type: string type: type: string type: object type: array default: type: boolean defaultLogicalSwitch: type: string router: type: string standby: type: boolean enableExternal: type: boolean enableBfd: type: boolean subnets: items: type: string type: array extraExternalSubnets: items: type: string type: array vpcPeerings: items: type: string type: array tcpLoadBalancer: type: string tcpSessionLoadBalancer: type: string udpLoadBalancer: type: string udpSessionLoadBalancer: type: string sctpLoadBalancer: type: string sctpSessionLoadBalancer: type: string type: object type: object served: true storage: true subresources: status: {} names: kind: Vpc listKind: VpcList plural: vpcs shortNames: - vpc singular: vpc scope: Cluster --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ips.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: V4IP type: string jsonPath: .spec.v4IpAddress - name: V6IP type: string jsonPath: .spec.v6IpAddress - name: Mac type: string jsonPath: .spec.macAddress - name: Node type: string jsonPath: .spec.nodeName - name: Subnet type: string jsonPath: .spec.subnet schema: openAPIV3Schema: type: object properties: spec: type: object properties: podName: type: string namespace: type: string subnet: type: string attachSubnets: type: array items: type: string nodeName: type: string ipAddress: type: string v4IpAddress: type: string v6IpAddress: type: string attachIps: type: array items: type: string macAddress: type: string attachMacs: type: array items: type: string containerID: type: string podType: type: string scope: Cluster names: plural: ips singular: ip kind: IP shortNames: - ip --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vips.kubeovn.io spec: group: kubeovn.io names: plural: vips singular: vip shortNames: - vip kind: Vip listKind: VipList scope: Cluster versions: - name: v1 served: true storage: true additionalPrinterColumns: - name: V4IP type: string jsonPath: .status.v4ip - name: V6IP type: string jsonPath: .status.v6ip - name: Mac type: string jsonPath: .status.mac - name: PMac type: string jsonPath: .spec.parentMac - name: Subnet type: string jsonPath: .spec.subnet - jsonPath: .status.ready name: Ready type: boolean - jsonPath: .status.type name: Type type: string schema: openAPIV3Schema: type: object properties: status: type: object properties: type: type: string ready: type: boolean v4ip: type: string v6ip: type: string mac: type: string pv4ip: type: string pv6ip: type: string pmac: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: namespace: type: string subnet: type: string type: type: string attachSubnets: type: array items: type: string v4ip: type: string macAddress: type: string v6ip: type: string parentV4ip: type: string parentMac: type: string parentV6ip: type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: subnets.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - name: Provider type: string jsonPath: .spec.provider - name: Vpc type: string jsonPath: .spec.vpc - name: Protocol type: string jsonPath: .spec.protocol - name: CIDR type: string jsonPath: .spec.cidrBlock - name: Private type: boolean jsonPath: .spec.private - name: NAT type: boolean jsonPath: .spec.natOutgoing - name: Default type: boolean jsonPath: .spec.default - name: GatewayType type: string jsonPath: .spec.gatewayType - name: V4Used type: number jsonPath: .status.v4usingIPs - name: V4Available type: number jsonPath: .status.v4availableIPs - name: V6Used type: number jsonPath: .status.v6usingIPs - name: V6Available type: number jsonPath: .status.v6availableIPs - name: ExcludeIPs type: string jsonPath: .spec.excludeIps - name: U2OInterconnectionIP type: string jsonPath: .status.u2oInterconnectionIP schema: openAPIV3Schema: type: object properties: metadata: type: object properties: name: type: string pattern: ^[^0-9] status: type: object properties: v4availableIPs: type: number v4usingIPs: type: number v6availableIPs: type: number v6usingIPs: type: number activateGateway: type: string dhcpV4OptionsUUID: type: string dhcpV6OptionsUUID: type: string u2oInterconnectionIP: type: string u2oInterconnectionVPC: type: string v4usingIPrange: type: string v4availableIPrange: type: string v6usingIPrange: type: string v6availableIPrange: type: string natOutgoingPolicyRules: type: array items: type: object properties: ruleID: type: string action: type: string enum: - nat - forward match: type: object properties: srcIPs: type: string dstIPs: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: vpc: type: string default: type: boolean protocol: type: string enum: - IPv4 - IPv6 - Dual cidrBlock: type: string namespaces: type: array items: type: string gateway: type: string provider: type: string excludeIps: type: array items: type: string vips: type: array items: type: string gatewayType: type: string allowSubnets: type: array items: type: string gatewayNode: type: string natOutgoing: type: boolean externalEgressGateway: type: string policyRoutingPriority: type: integer minimum: 1 maximum: 32765 policyRoutingTableID: type: integer minimum: 1 maximum: 2147483647 not: enum: - 252 # compat - 253 # default - 254 # main - 255 # local mtu: type: integer minimum: 68 maximum: 65535 private: type: boolean vlan: type: string logicalGateway: type: boolean disableGatewayCheck: type: boolean disableInterConnection: type: boolean enableDHCP: type: boolean dhcpV4Options: type: string dhcpV6Options: type: string enableIPv6RA: type: boolean ipv6RAConfigs: type: string acls: type: array items: type: object properties: direction: type: string enum: - from-lport - to-lport priority: type: integer minimum: 0 maximum: 32767 match: type: string action: type: string enum: - allow-related - allow-stateless - allow - drop - reject natOutgoingPolicyRules: type: array items: type: object properties: action: type: string enum: - nat - forward match: type: object properties: srcIPs: type: string dstIPs: type: string u2oInterconnection: type: boolean u2oInterconnectionIP: type: string enableLb: type: boolean enableEcmp: type: boolean enableMulticastSnoop: type: boolean routeTable: type: string scope: Cluster names: plural: subnets singular: subnet kind: Subnet shortNames: - subnet --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: ippools.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - name: Subnet type: string jsonPath: .spec.subnet - name: IPs type: string jsonPath: .spec.ips - name: V4Used type: number jsonPath: .status.v4UsingIPs - name: V4Available type: number jsonPath: .status.v4AvailableIPs - name: V6Used type: number jsonPath: .status.v6UsingIPs - name: V6Available type: number jsonPath: .status.v6AvailableIPs schema: openAPIV3Schema: type: object properties: spec: type: object properties: subnet: type: string x-kubernetes-validations: - rule: "self == oldSelf" message: "This field is immutable." namespaces: type: array x-kubernetes-list-type: set items: type: string ips: type: array minItems: 1 x-kubernetes-list-type: set items: type: string anyOf: - format: ipv4 - format: ipv6 - format: cidr - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.\.(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))\.\.((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ required: - subnet - ips status: type: object properties: v4AvailableIPs: type: number v4UsingIPs: type: number v6AvailableIPs: type: number v6UsingIPs: type: number v4AvailableIPRange: type: string v4UsingIPRange: type: string v6AvailableIPRange: type: string v6UsingIPRange: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string scope: Cluster names: plural: ippools singular: ippool kind: IPPool shortNames: - ippool --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: vlans.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: spec: type: object properties: id: type: integer minimum: 0 maximum: 4095 provider: type: string vlanId: type: integer description: Deprecated in favor of id providerInterfaceName: type: string description: Deprecated in favor of provider required: - provider status: type: object properties: subnets: type: array items: type: string additionalPrinterColumns: - name: ID type: string jsonPath: .spec.id - name: Provider type: string jsonPath: .spec.provider scope: Cluster names: plural: vlans singular: vlan kind: Vlan shortNames: - vlan --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: provider-networks.kubeovn.io spec: group: kubeovn.io versions: - name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: metadata: type: object properties: name: type: string maxLength: 12 not: enum: - int spec: type: object properties: defaultInterface: type: string maxLength: 15 pattern: '^[^/\s]+$' customInterfaces: type: array items: type: object properties: interface: type: string maxLength: 15 pattern: '^[^/\s]+$' nodes: type: array items: type: string exchangeLinkName: type: boolean excludeNodes: type: array items: type: string required: - defaultInterface status: type: object properties: ready: type: boolean readyNodes: type: array items: type: string notReadyNodes: type: array items: type: string vlans: type: array items: type: string conditions: type: array items: type: object properties: node: type: string type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string additionalPrinterColumns: - name: DefaultInterface type: string jsonPath: .spec.defaultInterface - name: Ready type: boolean jsonPath: .status.ready scope: Cluster names: plural: provider-networks singular: provider-network kind: ProviderNetwork listKind: ProviderNetworkList --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: security-groups.kubeovn.io spec: group: kubeovn.io names: plural: security-groups singular: security-group shortNames: - sg kind: SecurityGroup listKind: SecurityGroupList scope: Cluster versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object properties: ingressRules: type: array items: type: object properties: ipVersion: type: string protocol: type: string priority: type: integer remoteType: type: string remoteAddress: type: string remoteSecurityGroup: type: string portRangeMin: type: integer portRangeMax: type: integer policy: type: string egressRules: type: array items: type: object properties: ipVersion: type: string protocol: type: string priority: type: integer remoteType: type: string remoteAddress: type: string remoteSecurityGroup: type: string portRangeMin: type: integer portRangeMax: type: integer policy: type: string allowSameGroupTraffic: type: boolean status: type: object properties: portGroup: type: string allowSameGroupTraffic: type: boolean ingressMd5: type: string egressMd5: type: string ingressLastSyncSuccess: type: boolean egressLastSyncSuccess: type: boolean subresources: status: {} conversion: strategy: None --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: qos-policies.kubeovn.io spec: group: kubeovn.io names: plural: qos-policies singular: qos-policy shortNames: - qos kind: QoSPolicy listKind: QoSPolicyList scope: Cluster versions: - name: v1 served: true storage: true subresources: status: {} additionalPrinterColumns: - jsonPath: .spec.shared name: Shared type: string - jsonPath: .spec.bindingType name: BindingType type: string schema: openAPIV3Schema: type: object properties: status: type: object properties: shared: type: boolean bindingType: type: string bandwidthLimitRules: type: array items: type: object properties: name: type: string interface: type: string rateMax: type: string burstMax: type: string priority: type: integer direction: type: string matchType: type: string matchValue: type: string conditions: type: array items: type: object properties: type: type: string status: type: string reason: type: string message: type: string lastUpdateTime: type: string lastTransitionTime: type: string spec: type: object properties: shared: type: boolean bindingType: type: string bandwidthLimitRules: type: array items: type: object properties: name: type: string interface: type: string rateMax: type: string burstMax: type: string priority: type: integer direction: type: string matchType: type: string matchValue: type: string required: - name x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map ================================================ FILE: roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 ================================================ --- kind: ConfigMap apiVersion: v1 metadata: name: ovn-vpc-nat-config namespace: kube-system annotations: kubernetes.io/description: | kube-ovn vpc-nat common config data: image: {{ kube_ovn_vpc_container_image_repo }}:{{ kube_ovn_vpc_container_image_tag }} --- kind: ConfigMap apiVersion: v1 metadata: name: ovn-vpc-nat-gw-config namespace: kube-system data: enable-vpc-nat-gw: "true" --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-ovn-cni namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.k8s.io/system-only: "true" name: system:kube-ovn-cni rules: - apiGroups: - "kubeovn.io" resources: - subnets - vlans - provider-networks verbs: - get - list - watch - apiGroups: - "" - "kubeovn.io" resources: - ovn-eips - ovn-eips/status - nodes - pods - vlans verbs: - get - list - patch - watch - apiGroups: - "kubeovn.io" resources: - ips verbs: - get - update - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - "" resources: - configmaps verbs: - get - list - watch - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kube-ovn-cni roleRef: name: system:kube-ovn-cni kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: kube-ovn-cni namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: kube-ovn-cni namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: kube-ovn-cni namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-ovn-app namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.k8s.io/system-only: "true" name: system:kube-ovn-app rules: - apiGroups: - "" resources: - pods - nodes verbs: - get - list - apiGroups: - apps resources: - daemonsets verbs: - get - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kube-ovn-app roleRef: name: system:kube-ovn-app kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: kube-ovn-app namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: kube-ovn-app namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: kube-ovn-app namespace: kube-system --- kind: Deployment apiVersion: apps/v1 metadata: name: kube-ovn-controller namespace: kube-system annotations: kubernetes.io/description: | kube-ovn controller spec: replicas: {{ kube_ovn_controller_replics }} selector: matchLabels: app: kube-ovn-controller strategy: rollingUpdate: maxSurge: 0% maxUnavailable: 100% type: RollingUpdate template: metadata: labels: app: kube-ovn-controller component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - preference: matchExpressions: - key: "ovn.kubernetes.io/ic-gw" operator: NotIn values: - "true" weight: 100 podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: kube-ovn-controller topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: kube-ovn-controller image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - /kube-ovn/start-controller.sh - --default-cidr={{ kube_pods_subnets }} - --default-gateway={% if kube_ovn_default_gateway is defined %}{{ kube_ovn_default_gateway }}{% endif %}{{ '' }} - --default-gateway-check={{ kube_ovn_default_gateway_check | string }} - --default-logical-gateway={{ kube_ovn_default_logical_gateway | string }} - --default-u2o-interconnection={{ kube_ovn_u2o_interconnection }} - --default-exclude-ips={% if kube_ovn_default_exclude_ips is defined %}{{ kube_ovn_default_exclude_ips }}{% endif %}{{ '' }} - --node-switch-cidr={{ kube_ovn_node_switch_cidr_merged }} - --service-cluster-ip-range={{ kube_service_subnets }} - --network-type={{ kube_ovn_network_type }} - --default-interface-name={{ kube_ovn_default_interface_name | default('') }} - --default-vlan-id={{ kube_ovn_default_vlan_id }} - --ls-dnat-mod-dl-dst={{ kube_ovn_ls_dnat_mod_dl_dst }} - --pod-nic-type={{ kube_ovn_pod_nic_type }} - --enable-lb={{ kube_ovn_enable_lb | string }} - --enable-np={{ kube_ovn_enable_np | string }} - --enable-eip-snat={{ kube_ovn_eip_snat_enabled }} - --enable-external-vpc={{ kube_ovn_enable_external_vpc | string }} - --logtostderr=false - --alsologtostderr=true - --gc-interval=360 - --inspect-interval=20 - --log_file=/var/log/kube-ovn/kube-ovn-controller.log - --log_file_max_size=0 - --enable-lb-svc=false - --keep-vm-ip={{ kube_ovn_keep_vm_ip }} securityContext: runAsUser: 0 privileged: false capabilities: add: - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: KUBE_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: OVN_DB_IPS value: "{{ kube_ovn_central_ips }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: fieldPath: status.podIPs - name: ENABLE_BIND_LOCAL_IP value: "{{ kube_ovn_bind_local_ip_enabled }}" volumeMounts: - mountPath: /etc/localtime name: localtime - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /var/log/ovn name: ovn-log - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - /kube-ovn/kube-ovn-healthcheck - --port=10660 - --tls=false periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - /kube-ovn/kube-ovn-healthcheck - --port=10660 - --tls=false initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 timeoutSeconds: 45 resources: requests: cpu: {{ kube_ovn_controller_cpu_request }} memory: {{ kube_ovn_controller_memory_request }} limits: cpu: {{ kube_ovn_controller_cpu_limit }} memory: {{ kube_ovn_controller_memory_limit }} nodeSelector: kubernetes.io/os: "linux" volumes: - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-log hostPath: path: /var/log/kube-ovn - name: ovn-log hostPath: path: /var/log/ovn - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: DaemonSet apiVersion: apps/v1 metadata: name: kube-ovn-cni namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the kube-ovn cni daemon. spec: selector: matchLabels: app: kube-ovn-cni template: metadata: labels: app: kube-ovn-cni component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists priorityClassName: system-node-critical serviceAccountName: kube-ovn-cni hostNetwork: true hostPID: true initContainers: - name: install-cni image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/kube-ovn/install-cni.sh"] securityContext: runAsUser: 0 privileged: true volumeMounts: - mountPath: /opt/cni/bin name: cni-bin - mountPath: /usr/local/bin name: local-bin containers: - name: cni-server image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: - bash - /kube-ovn/start-cniserver.sh args: - --enable-mirror={{ kube_ovn_traffic_mirror | lower }} - --encap-checksum={{ kube_ovn_encap_checksum | lower }} - --service-cluster-ip-range={{ kube_service_subnets }} - --iface={{ kube_ovn_iface | default('') }} - --dpdk-tunnel-iface={{ kube_ovn_dpdk_tunnel_iface }} - --network-type={{ kube_ovn_network_type }} - --default-interface-name={{ kube_ovn_default_interface_name | default('') }} {% if kube_ovn_mtu is defined %} - --mtu={{ kube_ovn_mtu }} {% endif %} - --cni-conf-name={{ kube_ovn_cni_config_priority }}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log - --log_file_max_size=0 securityContext: runAsUser: 0 privileged: false capabilities: add: - NET_ADMIN - NET_BIND_SERVICE - NET_RAW - SYS_ADMIN env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: MODULES value: kube_ovn_fastpath.ko - name: RPMS value: openvswitch-kmod - name: POD_IPS valueFrom: fieldRef: fieldPath: status.podIPs - name: ENABLE_BIND_LOCAL_IP value: "{{ kube_ovn_bind_local_ip_enabled }}" - name: DBUS_SYSTEM_BUS_ADDRESS value: "unix:path=/host/var/run/dbus/system_bus_socket" volumeMounts: - name: host-modules mountPath: /lib/modules readOnly: true - name: shared-dir mountPath: $KUBELET_DIR/pods - mountPath: /etc/openvswitch name: systemid readOnly: true - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs mountPropagation: HostToContainer - mountPath: /run/ovn name: host-run-ovn - mountPath: /host/var/run/dbus name: host-dbus mountPropagation: HostToContainer - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime readOnly: true - mountPath: /tmp name: tmp livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 exec: command: - /kube-ovn/kube-ovn-healthcheck - --port=10665 - --tls=false timeoutSeconds: 5 readinessProbe: failureThreshold: 3 periodSeconds: 7 successThreshold: 1 exec: command: - /kube-ovn/kube-ovn-healthcheck - --port=10665 - --tls=false timeoutSeconds: 5 resources: requests: cpu: {{ kube_ovn_cni_server_cpu_request }} memory: {{ kube_ovn_cni_server_memory_request }} limits: cpu: {{ kube_ovn_cni_server_cpu_limit }} memory: {{ kube_ovn_cni_server_memory_limit }} nodeSelector: kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: path: /lib/modules - name: shared-dir hostPath: path: /var/lib/kubelet/pods - name: systemid hostPath: path: /etc/origin/openvswitch - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: cni-conf hostPath: path: /etc/cni/net.d - name: cni-bin hostPath: path: /opt/cni/bin - name: host-ns hostPath: path: /var/run/netns - name: host-dbus hostPath: path: /var/run/dbus - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: kube-ovn-log hostPath: path: /var/log/kube-ovn - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: tmp hostPath: path: /tmp - name: local-bin hostPath: path: /usr/local/bin --- kind: DaemonSet apiVersion: apps/v1 metadata: name: kube-ovn-pinger namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the openvswitch daemon. spec: selector: matchLabels: app: kube-ovn-pinger updateStrategy: type: RollingUpdate template: metadata: labels: app: kube-ovn-pinger component: network type: infra spec: priorityClassName: system-node-critical serviceAccountName: ovn hostPID: true containers: - name: pinger image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} command: - /kube-ovn/kube-ovn-pinger args: - --external-address={{ kube_ovn_external_address_merged }} - --external-dns={{ kube_ovn_external_dns }} - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log - --log_file_max_size=0 imagePullPolicy: {{ k8s_image_pull_policy }} securityContext: runAsUser: 0 privileged: false env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: HOST_IP valueFrom: fieldRef: fieldPath: status.hostIP - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /var/log/openvswitch name: host-log-ovs readOnly: true - mountPath: /var/log/ovn name: host-log-ovn readOnly: true - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /etc/localtime name: localtime readOnly: true - mountPath: /var/run/tls name: kube-ovn-tls resources: requests: cpu: {{ kube_ovn_pinger_cpu_request }} memory: {{ kube_ovn_pinger_memory_request }} limits: cpu: {{ kube_ovn_pinger_cpu_limit }} memory: {{ kube_ovn_pinger_memory_limit }} nodeSelector: kubernetes.io/os: "linux" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: kube-ovn-log hostPath: path: /var/log/kube-ovn - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: Deployment apiVersion: apps/v1 metadata: name: kube-ovn-monitor namespace: kube-system annotations: kubernetes.io/description: | Metrics for OVN components: northd, nb and sb. spec: replicas: 1 strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate selector: matchLabels: app: kube-ovn-monitor template: metadata: labels: app: kube-ovn-monitor component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: kube-ovn-monitor topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: kube-ovn-monitor image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/kube-ovn/start-ovn-monitor.sh"] args: - --secure-serving=false - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log - --logtostderr=false - --alsologtostderr=true - --log_file_max_size=200 securityContext: runAsUser: 0 privileged: false env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: fieldPath: status.podIPs - name: ENABLE_BIND_LOCAL_IP value: "{{ kube_ovn_bind_local_ip_enabled }}" resources: requests: cpu: {{ kube_ovn_monitor_cpu_request }} memory: {{ kube_ovn_monitor_memory_request }} limits: cpu: {{ kube_ovn_monitor_cpu_limit }} memory: {{ kube_ovn_monitor_memory_limit }} volumeMounts: - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/ovn name: host-log-ovn readOnly: true - mountPath: /etc/localtime name: localtime readOnly: true - mountPath: /var/run/tls name: kube-ovn-tls - mountPath: /var/log/kube-ovn name: kube-ovn-log livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 exec: command: - /kube-ovn/kube-ovn-healthcheck - --port=10661 - --tls=false timeoutSeconds: 5 readinessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 exec: command: - /kube-ovn/kube-ovn-healthcheck - --port=10661 - --tls=false timeoutSeconds: 5 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls - name: kube-ovn-log hostPath: path: /var/log/kube-ovn --- kind: Service apiVersion: v1 metadata: name: kube-ovn-monitor namespace: kube-system labels: app: kube-ovn-monitor spec: ports: - name: metrics port: 10661 type: ClusterIP {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: kube-ovn-monitor sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: kube-ovn-pinger namespace: kube-system labels: app: kube-ovn-pinger spec: {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: kube-ovn-pinger ports: - port: 8080 name: metrics --- kind: Service apiVersion: v1 metadata: name: kube-ovn-controller namespace: kube-system labels: app: kube-ovn-controller spec: {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: kube-ovn-controller ports: - port: 10660 name: metrics --- kind: Service apiVersion: v1 metadata: name: kube-ovn-cni namespace: kube-system labels: app: kube-ovn-cni spec: {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: kube-ovn-cni ports: - port: 10665 name: metrics {% if kube_ovn_ic_enable %} --- kind: ConfigMap apiVersion: v1 metadata: name: ovn-ic-config namespace: kube-system data: enable-ic: "{{ kube_ovn_ic_enable | lower }}" az-name: "{{ kube_ovn_ic_zone }}" ic-db-host: "{{ kube_ovn_ic_dbhost }}" ic-nb-port: "6645" ic-sb-port: "6646" gw-nodes: "{{ kube_ovn_central_hosts | join(',') }}" auto-route: "{{ kube_ovn_ic_autoroute | lower }}" {% endif %} ================================================ FILE: roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: ovn-ovs namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.k8s.io/system-only: "true" name: system:ovn-ovs rules: - apiGroups: - "" resources: - pods verbs: - get - patch - apiGroups: - "" resources: - services - endpoints verbs: - get - apiGroups: - apps resources: - controllerrevisions verbs: - get - list --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: ovn-ovs roleRef: name: system:ovn-ovs kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: ovn-ovs namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: ovn namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.k8s.io/system-only: "true" name: system:ovn rules: - apiGroups: - "kubeovn.io" resources: - vpcs - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status - subnets - subnets/status - ippools - ippools/status - ips - vips - vips/status - vlans - vlans/status - provider-networks - provider-networks/status - security-groups - security-groups/status - iptables-eips - iptables-fip-rules - iptables-dnat-rules - iptables-snat-rules - iptables-eips/status - iptables-fip-rules/status - iptables-dnat-rules/status - iptables-snat-rules/status - ovn-eips - ovn-fips - ovn-snat-rules - ovn-eips/status - ovn-fips/status - ovn-snat-rules/status - ovn-dnat-rules - ovn-dnat-rules/status - switch-lb-rules - switch-lb-rules/status - vpc-dnses - vpc-dnses/status - qos-policies - qos-policies/status verbs: - "*" - apiGroups: - "" resources: - pods - namespaces verbs: - get - list - patch - watch - apiGroups: - "" resources: - nodes verbs: - get - list - patch - update - watch - apiGroups: - "" resources: - pods/exec verbs: - create - apiGroups: - "k8s.cni.cncf.io" resources: - network-attachment-definitions verbs: - get - apiGroups: - "" - networking.k8s.io resources: - networkpolicies - configmaps verbs: - get - list - watch - apiGroups: - apps resources: - daemonsets verbs: - get - apiGroups: - "" resources: - services - services/status verbs: - get - list - update - create - delete - watch - apiGroups: - "" resources: - endpoints verbs: - create - update - get - list - watch - apiGroups: - apps resources: - statefulsets - deployments - deployments/scale verbs: - get - list - create - delete - update - apiGroups: - "" resources: - events verbs: - create - patch - update - apiGroups: - coordination.k8s.io resources: - leases verbs: - "*" - apiGroups: - "kubevirt.io" resources: - virtualmachines - virtualmachineinstances verbs: - get - list - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: ovn roleRef: name: system:ovn kind: ClusterRole apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: ovn namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ovn namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: ovn namespace: kube-system --- kind: Service apiVersion: v1 metadata: name: ovn-nb namespace: kube-system spec: ports: - name: ovn-nb protocol: TCP port: 6641 targetPort: 6641 type: ClusterIP {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: ovn-central ovn-nb-leader: "true" sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: ovn-sb namespace: kube-system spec: ports: - name: ovn-sb protocol: TCP port: 6642 targetPort: 6642 type: ClusterIP {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: ovn-central ovn-sb-leader: "true" sessionAffinity: None --- kind: Service apiVersion: v1 metadata: name: ovn-northd namespace: kube-system spec: ports: - name: ovn-northd protocol: TCP port: 6643 targetPort: 6643 type: ClusterIP {% if ipv6_stack %} ipFamilyPolicy: PreferDualStack {% endif %} selector: app: ovn-central ovn-northd-leader: "true" sessionAffinity: None --- kind: Deployment apiVersion: apps/v1 metadata: name: ovn-central namespace: kube-system annotations: kubernetes.io/description: | OVN components: northd, nb and sb. spec: replicas: {{ kube_ovn_central_replics }} strategy: rollingUpdate: maxSurge: 0 maxUnavailable: 1 type: RollingUpdate selector: matchLabels: app: ovn-central template: metadata: labels: app: ovn-central component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: ovn-central topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn-ovs hostNetwork: true containers: - name: ovn-central image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/kube-ovn/start-db.sh"] securityContext: capabilities: add: - NET_BIND_SERVICE - SYS_NICE env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" - name: NODE_IPS value: "{{ kube_ovn_central_ips }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: POD_IPS valueFrom: fieldRef: fieldPath: status.podIPs - name: ENABLE_BIND_LOCAL_IP value: "{{ kube_ovn_bind_local_ip_enabled }}" - name: PROBE_INTERVAL value: "180000" - name: OVN_NORTHD_PROBE_INTERVAL value: "5000" - name: OVN_LEADER_PROBE_INTERVAL value: "5" resources: requests: cpu: {{ kube_ovn_db_cpu_request }} memory: {{ kube_ovn_db_memory_request }} limits: cpu: {{ kube_ovn_db_cpu_limit }} memory: {{ kube_ovn_db_memory_limit }} volumeMounts: - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - bash - /kube-ovn/ovn-healthcheck.sh periodSeconds: 15 timeoutSeconds: 45 livenessProbe: exec: command: - bash - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 15 failureThreshold: 5 timeoutSeconds: 45 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- kind: DaemonSet apiVersion: apps/v1 metadata: name: ovs-ovn namespace: kube-system annotations: kubernetes.io/description: | This daemon set launches the openvswitch daemon. spec: selector: matchLabels: app: ovs updateStrategy: type: RollingUpdate rollingUpdate: maxSurge: 1 maxUnavailable: 0 template: metadata: labels: app: ovs component: network type: infra spec: tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - key: CriticalAddonsOnly operator: Exists priorityClassName: system-node-critical serviceAccountName: ovn-ovs hostNetwork: true hostPID: true containers: - name: openvswitch image: {% if kube_ovn_dpdk_enabled %}{{ kube_ovn_dpdk_container_image_repo }}:{{ kube_ovn_dpdk_container_image_tag }}{% else %}{{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}{% endif %} imagePullPolicy: {{ k8s_image_pull_policy }} command: [{% if kube_ovn_dpdk_enabled %}"/kube-ovn/start-ovs-dpdk.sh"{% else %}"/kube-ovn/start-ovs.sh"{% endif %}] securityContext: runAsUser: 0 privileged: false capabilities: add: - NET_ADMIN - NET_BIND_SERVICE - SYS_MODULE - SYS_NICE env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace {% if not kube_ovn_dpdk_enabled %} - name: HW_OFFLOAD value: "{{ kube_ovn_hw_offload | string | lower }}" - name: TUNNEL_TYPE value: "{{ kube_ovn_tunnel_type }}" {% endif %} - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: OVN_DB_IPS value: "{{ kube_ovn_central_ips }}" volumeMounts: - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer - mountPath: /lib/modules name: host-modules readOnly: true - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - mountPath: /var/log/openvswitch name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn {% if kube_ovn_dpdk_enabled %} - mountPath: /opt/ovs-config name: host-config-ovs - mountPath: /dev/hugepages name: hugepage {% endif %} - mountPath: /etc/localtime name: localtime - mountPath: /var/run/tls name: kube-ovn-tls - mountPath: /var/run/containerd name: cruntime readOnly: true readinessProbe: exec: command: - bash {% if kube_ovn_dpdk_enabled %} - /kube-ovn/ovs-dpdk-healthcheck.sh {% else %} - /kube-ovn/ovs-healthcheck.sh {% endif %} periodSeconds: 5 timeoutSeconds: 45 livenessProbe: exec: command: - bash {% if kube_ovn_dpdk_enabled %} - /kube-ovn/ovs-dpdk-healthcheck.sh {% else %} - /kube-ovn/ovs-healthcheck.sh {% endif %} initialDelaySeconds: 60 periodSeconds: 5 failureThreshold: 5 timeoutSeconds: 45 resources: {% if kube_ovn_dpdk_enabled %} requests: cpu: {{ kube_ovn_dpdk_node_cpu_request }} memory: {{ kube_ovn_dpdk_node_memory_request }} limits: cpu: {{ kube_ovn_dpdk_node_cpu_limit }} memory: {{ kube_ovn_dpdk_node_memory_limit }} hugepages-1Gi: 1Gi {% else %} requests: cpu: {{ kube_ovn_node_cpu_request }} memory: {{ kube_ovn_node_memory_request }} limits: cpu: {{ kube_ovn_node_cpu_limit }} memory: {{ kube_ovn_node_memory_limit }} {% endif %} nodeSelector: kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: path: /lib/modules - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-ns hostPath: path: /var/run/netns - name: cni-conf hostPath: path: /etc/cni/net.d - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - name: host-config-ovn hostPath: path: /etc/origin/ovn - name: host-log-ovs hostPath: path: /var/log/openvswitch - name: host-log-ovn hostPath: path: /var/log/ovn {% if kube_ovn_dpdk_enabled %} - name: host-config-ovs hostPath: path: /opt/ovs-config type: DirectoryOrCreate - name: hugepage emptyDir: medium: HugePages {% endif %} - name: localtime hostPath: path: /etc/localtime - name: cruntime hostPath: path: /var/run/containerd - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls ================================================ FILE: roles/network_plugin/kube-router/defaults/main.yml ================================================ --- # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP kube_router_run_router: true # Enables Network Policy -- sets up iptables to provide ingress firewall for pods kube_router_run_firewall: true # Enables Service Proxy -- sets up IPVS for Kubernetes Services # see docs/kube-router.md "Caveats" section kube_router_run_service_proxy: false # Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers. kube_router_advertise_cluster_ip: false # Add External IP of service to the RIB so that it gets advertised to the BGP peers. kube_router_advertise_external_ip: false # Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers. kube_router_advertise_loadbalancer_ip: false # Enables BGP graceful restarts kube_router_bgp_graceful_restart: true # Adjust manifest of kube-router daemonset template with DSR needed changes kube_router_enable_dsr: false # Array of arbitrary extra arguments to kube-router, see # https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md kube_router_extra_args: [] # ASN number of the cluster, used when communicating with external BGP routers kube_router_cluster_asn: ~ # ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. kube_router_peer_router_asns: ~ # The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. kube_router_peer_router_ips: ~ # The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used. kube_router_peer_router_ports: ~ # Setups node CNI to allow hairpin mode, requires node reboots, see # https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode kube_router_support_hairpin_mode: false # Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc. kube_router_dns_policy: ClusterFirstWithHostNet # Adds annotations to kubernetes nodes for advanced configuration of BGP Peers. # https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md # Array of annotations for master kube_router_annotations_master: [] # Array of annotations for every node kube_router_annotations_node: [] # Array of common annotations for every node kube_router_annotations_all: [] # Enables scraping kube-router metrics with Prometheus kube_router_enable_metrics: false # Path to serve Prometheus metrics on kube_router_metrics_path: /metrics # Prometheus metrics port to use kube_router_metrics_port: 9255 ================================================ FILE: roles/network_plugin/kube-router/handlers/main.yml ================================================ --- - name: Kube-router | delete kube-router docker containers shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f" args: executable: /bin/bash register: docker_kube_router_remove until: docker_kube_router_remove is succeeded retries: 5 when: container_manager in ["docker"] listen: Reset_kube_router - name: Kube-router | delete kube-router crio/containerd containers shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"' args: executable: /bin/bash register: crictl_kube_router_remove until: crictl_kube_router_remove is succeeded retries: 5 when: container_manager in ["crio", "containerd"] listen: Reset_kube_router ================================================ FILE: roles/network_plugin/kube-router/meta/main.yml ================================================ --- dependencies: - role: network_plugin/cni ================================================ FILE: roles/network_plugin/kube-router/tasks/annotate.yml ================================================ --- - name: Kube-router | Add annotations on kube_control_plane command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}" with_items: - "{{ kube_router_annotations_master }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" when: kube_router_annotations_master is defined and 'kube_control_plane' in group_names - name: Kube-router | Add annotations on kube_node command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}" with_items: - "{{ kube_router_annotations_node }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" when: kube_router_annotations_node is defined and 'kube_node' in group_names - name: Kube-router | Add common annotations on all servers command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}" with_items: - "{{ kube_router_annotations_all }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" when: kube_router_annotations_all is defined and 'k8s_cluster' in group_names ================================================ FILE: roles/network_plugin/kube-router/tasks/main.yml ================================================ --- - name: Kube-router | Create annotations import_tasks: annotate.yml tags: annotate - name: Kube-router | Create config directory file: path: /var/lib/kube-router state: directory owner: "{{ kube_owner }}" recurse: true mode: "0755" - name: Kube-router | Create kubeconfig template: src: kubeconfig.yml.j2 dest: /var/lib/kube-router/kubeconfig mode: "0644" owner: "{{ kube_owner }}" notify: - Reset_kube_router - name: Kube-router | Slurp cni config slurp: src: /etc/cni/net.d/10-kuberouter.conflist register: cni_config_slurp ignore_errors: true # noqa ignore-errors - name: Kube-router | Set cni_config variable set_fact: cni_config: "{{ cni_config_slurp.content | b64decode | from_json }}" when: - not cni_config_slurp.failed - name: Kube-router | Set host_subnet variable when: - cni_config is defined - cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | length > 0 set_fact: host_subnet: "{{ cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | first }}" - name: Kube-router | Create cni config template: src: cni-conf.json.j2 dest: /etc/cni/net.d/10-kuberouter.conflist mode: "0644" owner: "{{ kube_owner }}" notify: - Reset_kube_router - name: Kube-router | Delete old configuration file: path: /etc/cni/net.d/10-kuberouter.conf state: absent - name: Kube-router | Create manifest template: src: kube-router.yml.j2 dest: "{{ kube_config_dir }}/kube-router.yml" mode: "0644" delegate_to: "{{ groups['kube_control_plane'] | first }}" run_once: true - name: Kube-router | Start Resources kube: name: "kube-router" kubectl: "{{ bin_dir }}/kubectl" filename: "{{ kube_config_dir }}/kube-router.yml" resource: "ds" namespace: "kube-system" state: "latest" delegate_to: "{{ groups['kube_control_plane'] | first }}" run_once: true - name: Kube-router | Wait for kube-router pods to be ready command: "{{ kubectl }} -n kube-system get pods -l k8s-app=kube-router -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors register: pods_not_ready until: pods_not_ready.stdout.find("kube-router")==-1 retries: 30 delay: 10 ignore_errors: true delegate_to: "{{ groups['kube_control_plane'] | first }}" run_once: true changed_when: false ================================================ FILE: roles/network_plugin/kube-router/tasks/reset.yml ================================================ --- - name: Reset | check kube-dummy-if network device stat: path: /sys/class/net/kube-dummy-if get_attributes: false get_checksum: false get_mime: false register: kube_dummy_if - name: Reset | remove the network device created by kube-router command: ip link del kube-dummy-if when: kube_dummy_if.stat.exists - name: Check kube-bridge exists stat: path: /sys/class/net/kube-bridge get_attributes: false get_checksum: false get_mime: false register: kube_bridge_if - name: Reset | donw the network bridge create by kube-router command: ip link set kube-bridge down when: kube_bridge_if.stat.exists - name: Reset | remove the network bridge create by kube-router command: ip link del kube-bridge when: kube_bridge_if.stat.exists ================================================ FILE: roles/network_plugin/kube-router/templates/cni-conf.json.j2 ================================================ { "cniVersion":"0.3.0", "name":"kubernetes", "plugins":[ { "name":"kubernetes", "type":"bridge", "bridge":"kube-bridge", "isDefaultGateway":true, {% if kube_router_support_hairpin_mode %} "hairpinMode":true, {% endif %} "ipam":{ {% if host_subnet is defined %} "subnet": "{{ host_subnet }}", {% endif %} "type":"host-local" } }, { "type":"portmap", "capabilities":{ "portMappings":true } } ] } ================================================ FILE: roles/network_plugin/kube-router/templates/kube-router.yml.j2 ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: labels: k8s-app: kube-router tier: node name: kube-router namespace: kube-system spec: minReadySeconds: 3 updateStrategy: rollingUpdate: maxUnavailable: 1 type: RollingUpdate selector: matchLabels: k8s-app: kube-router tier: node template: metadata: labels: k8s-app: kube-router tier: node annotations: {% if kube_router_enable_metrics %} prometheus.io/path: {{ kube_router_metrics_path }} prometheus.io/port: "{{ kube_router_metrics_port }}" prometheus.io/scrape: "true" {% endif %} spec: priorityClassName: system-node-critical serviceAccountName: kube-router containers: - name: kube-router image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - --run-router={{ kube_router_run_router | bool }} - --run-firewall={{ kube_router_run_firewall | bool }} - --run-service-proxy={{ kube_router_run_service_proxy | bool }} - --kubeconfig=/var/lib/kube-router/kubeconfig - --bgp-graceful-restart={{ kube_router_bgp_graceful_restart }} {% if kube_router_advertise_cluster_ip %} - --advertise-cluster-ip {% endif %} {% if kube_router_advertise_external_ip %} - --advertise-external-ip {% endif %} {% if kube_router_advertise_loadbalancer_ip %} - --advertise-loadbalancer-ip {% endif %} {% if kube_router_cluster_asn %} - --cluster-asn={{ kube_router_cluster_asn }} {% endif %} {% if kube_router_peer_router_asns %} - --peer-router-asns={{ kube_router_peer_router_asns }} {% endif %} {% if kube_router_peer_router_ips %} - --peer-router-ips={{ kube_router_peer_router_ips }} {% endif %} {% if kube_router_peer_router_ports %} - --peer-router-ports={{ kube_router_peer_router_ports }} {% endif %} {% if kube_router_enable_metrics %} - --metrics-path={{ kube_router_metrics_path }} - --metrics-port={{ kube_router_metrics_port }} {% endif %} {% if kube_router_enable_dsr %} {% if container_manager == "docker" %} - --runtime-endpoint=unix:///var/run/docker.sock {% endif %} {% if container_manager == "containerd" %} {% endif %} - --runtime-endpoint=unix:///run/containerd/containerd.sock {% endif %} {% for arg in kube_router_extra_args %} - "{{ arg }}" {% endfor %} env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: KUBE_ROUTER_CNI_CONF_FILE value: /etc/cni/net.d/10-kuberouter.conflist livenessProbe: httpGet: path: /healthz port: 20244 initialDelaySeconds: 10 periodSeconds: 3 resources: requests: cpu: 250m memory: 250Mi securityContext: privileged: true volumeMounts: {% if kube_router_enable_dsr %} {% if container_manager == "docker" %} - name: docker-socket mountPath: /var/run/docker.sock readOnly: true {% endif %} {% if container_manager == "containerd" %} - name: containerd-socket mountPath: /run/containerd/containerd.sock readOnly: true {% endif %} {% endif %} - name: lib-modules mountPath: /lib/modules readOnly: true - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kubeconfig mountPath: /var/lib/kube-router readOnly: true - name: xtables-lock mountPath: /run/xtables.lock readOnly: false {% if kube_router_enable_metrics %} ports: - containerPort: {{ kube_router_metrics_port }} hostPort: {{ kube_router_metrics_port }} name: metrics protocol: TCP {% endif %} hostNetwork: true dnsPolicy: {{ kube_router_dns_policy }} {% if kube_router_enable_dsr %} hostIPC: true hostPID: true {% endif %} tolerations: - operator: Exists volumes: {% if kube_router_enable_dsr %} {% if container_manager == "docker" %} - name: docker-socket hostPath: path: /var/run/docker.sock type: Socket {% endif %} {% if container_manager == "containerd" %} - name: containerd-socket hostPath: path: /run/containerd/containerd.sock type: Socket {% endif %} {% endif %} - name: lib-modules hostPath: path: /lib/modules - name: cni-conf-dir hostPath: path: /etc/cni/net.d - name: kubeconfig hostPath: path: /var/lib/kube-router - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-router namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-router namespace: kube-system rules: - apiGroups: - "" resources: - namespaces - pods - services - nodes - endpoints verbs: - list - get - watch - apiGroups: - "networking.k8s.io" resources: - networkpolicies verbs: - list - get - watch - apiGroups: - extensions resources: - networkpolicies verbs: - get - list - watch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kube-router roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: kube-router subjects: - kind: ServiceAccount name: kube-router namespace: kube-system ================================================ FILE: roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 ================================================ apiVersion: v1 kind: Config clusterCIDR: {{ kube_pods_subnets }} clusters: - name: cluster cluster: certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt server: {{ kube_apiserver_endpoint }} users: - name: kube-router user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token contexts: - context: cluster: cluster user: kube-router name: kube-router-context current-context: kube-router-context ================================================ FILE: roles/network_plugin/macvlan/defaults/main.yml ================================================ --- macvlan_interface: eth0 enable_nat_default_gateway: true # sysctl_file_path to add sysctl conf to sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" ================================================ FILE: roles/network_plugin/macvlan/files/ifdown-local ================================================ #!/bin/bash POSTDOWNNAME="/etc/sysconfig/network-scripts/post-down-$1" if [ -x $POSTDOWNNAME ]; then exec $POSTDOWNNAME fi ================================================ FILE: roles/network_plugin/macvlan/files/ifdown-macvlan ================================================ #!/bin/bash # # initscripts-macvlan # Copyright (C) 2014 Lars Kellogg-Stedman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . . /etc/init.d/functions cd /etc/sysconfig/network-scripts . ./network-functions [ -f ../network ] && . ../network CONFIG=${1} need_config ${CONFIG} source_config OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}" if [ ! -x ${OTHERSCRIPT} ]; then OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-eth" fi ${OTHERSCRIPT} ${CONFIG} ip link del ${DEVICE} type ${TYPE:-macvlan} ================================================ FILE: roles/network_plugin/macvlan/files/ifup-local ================================================ #!/bin/bash POSTUPNAME="/etc/sysconfig/network-scripts/post-up-$1" if [ -x $POSTUPNAME ]; then exec $POSTUPNAME fi ================================================ FILE: roles/network_plugin/macvlan/files/ifup-macvlan ================================================ #!/bin/bash # # initscripts-macvlan # Copyright (C) 2014 Lars Kellogg-Stedman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . . /etc/init.d/functions cd /etc/sysconfig/network-scripts . ./network-functions [ -f ../network ] && . ../network CONFIG=${1} need_config ${CONFIG} source_config OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}" if [ ! -x ${OTHERSCRIPT} ]; then OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-eth" fi ip link add \ link ${MACVLAN_PARENT} \ name ${DEVICE} \ type ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private} ${OTHERSCRIPT} ${CONFIG} ================================================ FILE: roles/network_plugin/macvlan/handlers/main.yml ================================================ --- - name: Macvlan | reload network service: # noqa: jinja[spacing] name: >- {% if ansible_os_family == "RedHat" -%} network {%- elif ansible_distribution == "Ubuntu" and ansible_distribution_release == "bionic" -%} systemd-networkd {%- elif ansible_os_family == "Debian" -%} networking {%- endif %} state: restarted when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and kube_network_plugin not in ['calico'] listen: Macvlan | restart network ================================================ FILE: roles/network_plugin/macvlan/meta/main.yml ================================================ --- dependencies: - role: network_plugin/cni ================================================ FILE: roles/network_plugin/macvlan/tasks/main.yml ================================================ --- - name: Macvlan | Retrieve Pod Cidr command: "{{ kubectl }} get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'" changed_when: false register: node_pod_cidr_cmd delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Macvlan | set node_pod_cidr set_fact: node_pod_cidr: "{{ node_pod_cidr_cmd.stdout }}" - name: Macvlan | Retrieve default gateway network interface become: false raw: ip -4 route list 0/0 | sed 's/.*dev \([[:alnum:]]*\).*/\1/' changed_when: false register: node_default_gateway_interface_cmd - name: Macvlan | set node_default_gateway_interface set_fact: node_default_gateway_interface: "{{ node_default_gateway_interface_cmd.stdout | trim }}" - name: Macvlan | Install network gateway interface on debian template: src: debian-network-macvlan.cfg.j2 dest: /etc/network/interfaces.d/60-mac0.cfg mode: "0644" notify: Macvlan | restart network when: ansible_os_family in ["Debian"] - name: Install macvlan config on RH distros when: ansible_os_family == "RedHat" block: - name: Macvlan | Install macvlan script on centos copy: src: "{{ item }}" dest: /etc/sysconfig/network-scripts/ owner: root group: root mode: "0755" with_fileglob: - files/* - name: Macvlan | Install post-up script on centos copy: src: "files/ifup-local" dest: /sbin/ owner: root group: root mode: "0755" when: enable_nat_default_gateway - name: Macvlan | Install network gateway interface on centos template: src: "{{ item.src }}.j2" dest: "/etc/sysconfig/network-scripts/{{ item.dst }}" mode: "0644" with_items: - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 } - {src: centos-routes-macvlan.cfg, dst: route-mac0 } - {src: centos-postup-macvlan.cfg, dst: post-up-mac0 } notify: Macvlan | restart network - name: Install macvlan config on Flatcar when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] block: - name: Macvlan | Install service nat via gateway on Flatcar Container Linux template: src: coreos-service-nat_ouside.j2 dest: /etc/systemd/system/enable_nat_ouside.service mode: "0644" when: enable_nat_default_gateway - name: Macvlan | Enable service nat via gateway on Flatcar Container Linux command: "{{ item }}" with_items: - systemctl daemon-reload - systemctl enable enable_nat_ouside.service when: enable_nat_default_gateway - name: Macvlan | Install network gateway interface on Flatcar Container Linux template: src: "{{ item.src }}.j2" dest: "/etc/systemd/network/{{ item.dst }}" mode: "0644" with_items: - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev } - {src: coreos-interface-macvlan.cfg, dst: output.network } - {src: coreos-network-macvlan.cfg, dst: macvlan.network } notify: Macvlan | restart network - name: Macvlan | Install cni definition for Macvlan template: src: 10-macvlan.conf.j2 dest: /etc/cni/net.d/10-macvlan.conf mode: "0644" - name: Macvlan | Install loopback definition for Macvlan template: src: 99-loopback.conf.j2 dest: /etc/cni/net.d/99-loopback.conf mode: "0644" - name: Enable net.ipv4.conf.all.arp_notify in sysctl ansible.posix.sysctl: name: net.ipv4.conf.all.arp_notify value: 1 sysctl_set: true sysctl_file: "{{ sysctl_file_path }}" state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" ================================================ FILE: roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 ================================================ { "cniVersion": "0.4.0", "name": "mynet", "type": "macvlan", "master": "{{ macvlan_interface }}", "hairpinMode": true, "ipam": { "type": "host-local", "subnet": "{{ node_pod_cidr }}", "routes": [ { "dst": "0.0.0.0/0" } ], "gateway": "{{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}" } } ================================================ FILE: roles/network_plugin/macvlan/templates/99-loopback.conf.j2 ================================================ { "cniVersion": "0.2.0", "name": "lo", "type": "loopback" } ================================================ FILE: roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 ================================================ DEVICE=mac0 DEVICETYPE=macvlan TYPE=macvlan BOOTPROTO=none ONBOOT=yes NM_CONTROLLED=no MACVLAN_PARENT={{ macvlan_interface }} MACVLAN_MODE=bridge IPADDR={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }} NETMASK={{ node_pod_cidr|ansible.utils.ipaddr('netmask') }} NETWORK={{ node_pod_cidr|ansible.utils.ipaddr('network') }} ================================================ FILE: roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 ================================================ {% if enable_nat_default_gateway %} iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE {% endif %} ================================================ FILE: roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 ================================================ {% if enable_nat_default_gateway %} iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE {% endif %} ================================================ FILE: roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 ================================================ {% for host in groups['kube_node'] %} {% if hostvars[host]['access_ip'] is defined %} {% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} {% endif %} {% endif %} {% endfor %} ================================================ FILE: roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 ================================================ [NetDev] Name=mac0 Kind=macvlan [MACVLAN] Mode=bridge ================================================ FILE: roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 ================================================ [Match] Name={{ macvlan_interface }} [Network] MACVLAN=mac0 DHCP=yes ================================================ FILE: roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 ================================================ [Match] Name=mac0 [Network] Address={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}/{{ node_pod_cidr|ansible.utils.ipaddr('prefix') }} {% for host in groups['kube_node'] %} {% if hostvars[host]['access_ip'] is defined %} {% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} [Route] Gateway={{ hostvars[host]['access_ip'] }} Destination={{ hostvars[host]['node_pod_cidr'] }} GatewayOnlink=yes {% endif %} {% endif %} {% endfor %} ================================================ FILE: roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 ================================================ [Service] Type=oneshot ExecStart=/bin/bash -c "iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE" [Install] WantedBy=sys-subsystem-net-devices-mac0.device ================================================ FILE: roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 ================================================ auto mac0 iface mac0 inet static address {{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }} network {{ node_pod_cidr|ansible.utils.ipaddr('network') }} netmask {{ node_pod_cidr|ansible.utils.ipaddr('netmask') }} broadcast {{ node_pod_cidr|ansible.utils.ipaddr('broadcast') }} pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge {% for host in groups['kube_node'] %} {% if hostvars[host]['access_ip'] is defined %} {% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} {% endif %} {% endif %} {% endfor %} {% if enable_nat_default_gateway %} post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE {% endif %} {% for host in groups['kube_node'] %} {% if hostvars[host]['access_ip'] is defined %} {% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} {% endif %} {% endif %} {% endfor %} post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE post-down ip link delete mac0 ================================================ FILE: roles/network_plugin/multus/defaults/main.yml ================================================ --- multus_conf_file: "auto" multus_cni_conf_dir_host: "/etc/cni/net.d" multus_cni_bin_dir_host: "/opt/cni/bin" multus_cni_run_dir_host: "/run" multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}" multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}" multus_cni_run_dir: "{{ ('/host', multus_cni_run_dir_host) | join }}" multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}" multus_namespace_isolation: false ================================================ FILE: roles/network_plugin/multus/files/multus-clusterrole.yml ================================================ --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: multus rules: - apiGroups: ["k8s.cni.cncf.io"] resources: - '*' verbs: - '*' - apiGroups: - "" resources: - pods - pods/status verbs: - get - update - apiGroups: - "" - events.k8s.io resources: - events verbs: - create - patch - update ================================================ FILE: roles/network_plugin/multus/files/multus-clusterrolebinding.yml ================================================ --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: multus roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: multus subjects: - kind: ServiceAccount name: multus namespace: kube-system ================================================ FILE: roles/network_plugin/multus/files/multus-crd.yml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: network-attachment-definitions.k8s.cni.cncf.io spec: group: k8s.cni.cncf.io scope: Namespaced names: plural: network-attachment-definitions singular: network-attachment-definition kind: NetworkAttachmentDefinition shortNames: - net-attach-def versions: - name: v1 served: true storage: true schema: openAPIV3Schema: description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing Working Group to express the intent for attaching pods to one or more logical or physical networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' type: object properties: apiVersion: description: 'APIVersion defines the versioned schema of this represen tation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' type: object properties: config: description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' type: string ================================================ FILE: roles/network_plugin/multus/files/multus-serviceaccount.yml ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: multus namespace: kube-system ================================================ FILE: roles/network_plugin/multus/meta/main.yml ================================================ --- dependencies: - role: network_plugin/cni ================================================ FILE: roles/network_plugin/multus/tasks/main.yml ================================================ --- - name: Multus | Copy manifest files copy: src: "{{ item.file }}" dest: "{{ kube_config_dir }}" mode: "0644" with_items: - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition} - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount} - {name: multus-clusterrole, file: multus-clusterrole.yml, type: clusterrole} - {name: multus-clusterrolebinding, file: multus-clusterrolebinding.yml, type: clusterrolebinding} register: multus_manifest_1 when: inventory_hostname == groups['kube_control_plane'][0] - name: Multus | Check container engine type set_fact: container_manager_types: "{{ ansible_play_hosts_all | map('extract', hostvars, ['container_manager']) | list | unique }}" - name: Multus | Copy manifest templates template: src: multus-daemonset.yml.j2 dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: multus-daemonset-containerd, file: multus-daemonset-containerd.yml, type: daemonset, engine: containerd } - {name: multus-daemonset-docker, file: multus-daemonset-docker.yml, type: daemonset, engine: docker } - {name: multus-daemonset-crio, file: multus-daemonset-crio.yml, type: daemonset, engine: crio } register: multus_manifest_2 vars: host_query: "*|[?container_manager=='{{ container_manager }}']|[0].inventory_hostname" vars_from_node: "{{ hostvars | json_query(host_query) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" when: - item.engine in container_manager_types - hostvars[inventory_hostname].container_manager == item.engine - inventory_hostname == vars_from_node - name: Multus | Start resources kube: name: "{{ item.item.name }}" namespace: "kube-system" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true with_items: "{{ (multus_manifest_1.results | default([])) + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2') | map('default', []) | list | json_query('[].results')) }}" loop_control: label: "{{ item.item.name if item != None else 'skipped' }}" vars: multus_nodes_list: "{{ groups['k8s_cluster'] if ansible_play_batch | length == ansible_play_hosts_all | length else ansible_play_batch }}" when: - not item is skipped ================================================ FILE: roles/network_plugin/multus/templates/multus-daemonset.yml.j2 ================================================ --- kind: DaemonSet apiVersion: apps/v1 metadata: {% if container_manager_types | length >= 2 %} name: kube-multus-{{ container_manager }}-{{ image_arch }} {% else %} name: kube-multus-ds-{{ image_arch }} {% endif %} namespace: kube-system labels: tier: node app: multus spec: selector: matchLabels: tier: node app: multus template: metadata: labels: tier: node app: multus spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet priorityClassName: system-node-critical nodeSelector: kubernetes.io/arch: {{ image_arch }} {% if container_manager_types | length >= 2 %} kubespray.io/container_manager: {{ container_manager }} {% endif %} tolerations: - operator: Exists serviceAccountName: multus initContainers: - name: install-multus-binary image: {{ multus_image_repo }}:{{ multus_image_tag }} command: ["/install_multus"] args: - "--type" - "thin" resources: requests: cpu: "10m" memory: "15Mi" securityContext: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cnibin mountPath: {{ multus_cni_bin_dir }} mountPropagation: Bidirectional containers: - name: kube-multus image: {{ multus_image_repo }}:{{ multus_image_tag }} command: ["/thin_entrypoint"] args: - "--cni-conf-dir={{ multus_cni_conf_dir }}" - "--multus-autoconfig-dir={{ multus_cni_conf_dir }}" - "--cni-bin-dir={{ multus_cni_bin_dir }}" - "--multus-conf-file={{ multus_conf_file }}" - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}" - "--namespace-isolation={{ multus_namespace_isolation | string | lower }}" resources: requests: cpu: "100m" memory: "90Mi" limits: cpu: "100m" memory: "90Mi" securityContext: privileged: true {% if container_manager == 'crio' %} capabilities: add: ["SYS_ADMIN"] {% endif %} terminationMessagePolicy: FallbackToLogsOnError volumeMounts: {% if container_manager == 'crio' %} - name: run mountPath: {{ multus_cni_run_dir }} mountPropagation: HostToContainer {% endif %} - name: cni mountPath: {{ multus_cni_conf_dir }} - name: cnibin mountPath: {{ multus_cni_bin_dir }} volumes: {% if container_manager == 'crio' %} - name: run hostPath: path: {{ multus_cni_run_dir_host }} {% endif %} - name: cni hostPath: path: {{ multus_cni_conf_dir_host }} - name: cnibin hostPath: path: {{ multus_cni_bin_dir_host }} ================================================ FILE: roles/network_plugin/ovn4nfv/tasks/main.yml ================================================ --- - name: Ovn4nfv | Label control-plane node command: >- {{ kubectl }} label --overwrite node {{ groups['kube_control_plane'] | first }} ovn4nfv-k8s-plugin=ovn-control-plane when: - inventory_hostname == groups['kube_control_plane'][0] - name: Ovn4nfv | Create ovn4nfv-k8s manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" mode: "0644" with_items: - {name: ovn-daemonset, file: ovn-daemonset.yml} - {name: ovn4nfv-k8s-plugin, file: ovn4nfv-k8s-plugin.yml} register: ovn4nfv_node_manifests ================================================ FILE: roles/network_plugin/tasks/main.yml ================================================ --- - name: Container Network Interface plugin include_role: name: network_plugin/cni when: kube_network_plugin != 'none' - name: Network plugin include_role: name: "network_plugin/{{ kube_network_plugin }}" apply: tags: - "{{ kube_network_plugin }}" - network when: - kube_network_plugin != 'none' tags: - cilium - calico - flannel - macvlan - kube-ovn - kube-router - custom_cni - name: Cilium additional include_role: name: network_plugin/cilium apply: tags: - cilium - network when: - kube_network_plugin != 'cilium' - cilium_deploy_additionally tags: - cilium - name: Multus include_role: name: network_plugin/multus apply: tags: - multus - network when: kube_network_plugin_multus tags: - multus ================================================ FILE: roles/recover_control_plane/control-plane/defaults/main.yml ================================================ --- bin_dir: /usr/local/bin ================================================ FILE: roles/recover_control_plane/control-plane/tasks/main.yml ================================================ --- - name: Wait for apiserver command: "{{ kubectl }} get nodes" environment: KUBECONFIG: "{{ ansible_env.HOME | default('/root') }}/.kube/config" register: apiserver_is_ready until: apiserver_is_ready.rc == 0 retries: 6 delay: 10 changed_when: false when: groups['broken_kube_control_plane'] - name: Delete broken kube_control_plane nodes from cluster command: "{{ kubectl }} delete node {{ item }}" environment: KUBECONFIG: "{{ ansible_env.HOME | default('/root') }}/.kube/config" with_items: "{{ groups['broken_kube_control_plane'] }}" register: delete_broken_kube_control_plane_nodes failed_when: false when: groups['broken_kube_control_plane'] - name: Fail if unable to delete broken kube_control_plane nodes from cluster fail: msg: "Unable to delete broken kube_control_plane node: {{ item.item }}" loop: "{{ delete_broken_kube_control_plane_nodes.results }}" changed_when: false when: - groups['broken_kube_control_plane'] - "item.rc != 0 and not 'NotFound' in item.stderr" ================================================ FILE: roles/recover_control_plane/etcd/tasks/main.yml ================================================ --- - name: Get etcd endpoint health command: "{{ bin_dir }}/etcdctl endpoint health" register: etcd_endpoint_health ignore_errors: true # noqa ignore-errors changed_when: false check_mode: false environment: ETCDCTL_API: "3" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" when: - groups['broken_etcd'] - name: Set healthy fact set_fact: healthy: "{{ etcd_endpoint_health.stderr is match('Error: unhealthy cluster') }}" when: - groups['broken_etcd'] - name: Set has_quorum fact set_fact: has_quorum: "{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}" when: - groups['broken_etcd'] - name: Recover lost etcd quorum include_tasks: recover_lost_quorum.yml when: - groups['broken_etcd'] - not has_quorum - name: Remove etcd data dir file: path: "{{ etcd_data_dir }}" state: absent delegate_to: "{{ item }}" with_items: "{{ groups['broken_etcd'] }}" ignore_errors: true # noqa ignore-errors ignore_unreachable: true when: - groups['broken_etcd'] - has_quorum - name: Delete old certificates shell: "rm {{ etcd_cert_dir }}/*{{ item }}*" with_items: "{{ groups['broken_etcd'] }}" register: delete_old_cerificates ignore_errors: true when: groups['broken_etcd'] - name: Fail if unable to delete old certificates fail: msg: "Unable to delete old certificates for: {{ item.item }}" loop: "{{ delete_old_cerificates.results }}" changed_when: false when: - groups['broken_etcd'] - "item.rc != 0 and not 'No such file or directory' in item.stderr" - name: Get etcd cluster members command: "{{ bin_dir }}/etcdctl member list" register: member_list changed_when: false check_mode: false environment: ETCDCTL_API: "3" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" when: - groups['broken_etcd'] - not healthy - has_quorum - name: Remove broken cluster members command: "{{ bin_dir }}/etcdctl member remove {{ item[1].replace(' ', '').split(',')[0] }}" environment: ETCDCTL_API: "3" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" with_nested: - "{{ groups['broken_etcd'] }}" - "{{ member_list.stdout_lines }}" when: - groups['broken_etcd'] - not healthy - has_quorum - hostvars[item[0]]['etcd_member_name'] == item[1].replace(' ', '').split(',')[2] ================================================ FILE: roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml ================================================ --- - name: Save etcd snapshot command: "{{ bin_dir }}/etcdctl snapshot save /tmp/snapshot.db" environment: ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}" ETCDCTL_API: "3" when: etcd_snapshot is not defined - name: Transfer etcd snapshot to host copy: src: "{{ etcd_snapshot }}" dest: /tmp/snapshot.db mode: "0640" when: etcd_snapshot is defined - name: Stop etcd systemd_service: name: etcd state: stopped - name: Remove etcd data-dir file: path: "{{ etcd_data_dir }}" state: absent - name: Restore etcd snapshot # noqa command-instead-of-shell shell: "{{ bin_dir }}/etcdctl snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}" environment: ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" ETCDCTL_API: "3" - name: Remove etcd snapshot file: path: /tmp/snapshot.db state: absent - name: Change etcd data-dir owner file: path: "{{ etcd_data_dir }}" owner: etcd group: etcd recurse: true - name: Reconfigure etcd replace: path: /etc/etcd.env regexp: "^(ETCD_INITIAL_CLUSTER=).*" replace: '\1{{ etcd_member_name }}={{ etcd_peer_url }}' - name: Start etcd systemd_service: name: etcd state: started ================================================ FILE: roles/recover_control_plane/post-recover/tasks/main.yml ================================================ --- # TODO: Figure out why kubeadm does not fix this - name: Set etcd-servers fact set_fact: # noqa: jinja[spacing] etcd_servers: >- {% for host in groups['etcd'] -%} {% if not loop.last -%} https://{{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:2379, {%- endif -%} {%- if loop.last -%} https://{{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:2379 {%- endif -%} {%- endfor -%} - name: Update apiserver etcd-servers list replace: path: /etc/kubernetes/manifests/kube-apiserver.yaml regexp: "(etcd-servers=).*" replace: "\\1{{ etcd_servers }}" ================================================ FILE: roles/remove-node/post-remove/defaults/main.yml ================================================ --- delete_node_retries: 10 delete_node_delay_seconds: 3 ================================================ FILE: roles/remove-node/post-remove/tasks/main.yml ================================================ --- - name: Remove-node | Delete node command: "{{ kubectl }} delete node {{ kube_override_hostname }}" delegate_to: "{{ groups['kube_control_plane'] | first }}" when: - groups['kube_control_plane'] | length > 0 # ignore servers that are not nodes - ('k8s_cluster' in group_names) and kube_override_hostname in nodes.stdout_lines retries: "{{ delete_node_retries }}" # Sometimes the api-server can have a short window of indisponibility when we delete a control plane node delay: "{{ delete_node_delay_seconds }}" register: result until: result is not failed ================================================ FILE: roles/remove-node/pre-remove/tasks/main.yml ================================================ --- - name: Warn for usage of deprecated role fail: msg: remove-node/pre-remove is deprecated, switch to remove_node/pre_remove ignore_errors: true # noqa ignore-errors run_once: true - name: Compat for direct role import import_role: name: remove_node/pre_remove ================================================ FILE: roles/remove-node/remove-etcd-node/tasks/main.yml ================================================ --- - name: Remove etcd member from cluster environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ kube_cert_dir + '/etcd/server.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '.pem' }}" ETCDCTL_KEY: "{{ kube_cert_dir + '/etcd/server.key' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '-key.pem' }}" ETCDCTL_CACERT: "{{ kube_cert_dir + '/etcd/ca.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/ca.pem' }}" ETCDCTL_ENDPOINTS: "https://127.0.0.1:2379" delegate_to: "{{ groups['etcd'] | first }}" block: - name: Lookup members infos command: "{{ bin_dir }}/etcdctl member list -w json" register: etcd_members changed_when: false check_mode: false tags: - facts - name: Remove member from cluster command: argv: - "{{ bin_dir }}/etcdctl" - member - remove - "{{ '%x' | format(etcd_removed_nodes[0].ID) }}" vars: etcd_removed_nodes: "{{ (etcd_members.stdout | from_json).members | selectattr('peerURLs.0', '==', etcd_peer_url) }}" # This should always have at most one member, since the etcd_peer_url should be unique in the etcd cluster when: etcd_removed_nodes != [] register: etcd_removal_output changed_when: "'Removed member' in etcd_removal_output.stdout" ================================================ FILE: roles/remove_node/pre_remove/defaults/main.yml ================================================ --- allow_ungraceful_removal: false drain_grace_period: 300 drain_timeout: 360s drain_retries: 3 drain_retry_delay_seconds: 10 ================================================ FILE: roles/remove_node/pre_remove/tasks/main.yml ================================================ --- - name: Remove-node | List nodes command: >- {{ kubectl }} get nodes -o go-template={% raw %}'{{ range .items }}{{ .metadata.name }}{{ "\n" }}{{ end }}'{% endraw %} register: nodes when: - groups['kube_control_plane'] | length > 0 delegate_to: "{{ groups['kube_control_plane'] | first }}" changed_when: false run_once: true - name: Remove-node | Drain node except daemonsets resource command: >- {{ kubectl }} drain --force --ignore-daemonsets --grace-period {{ drain_grace_period }} --timeout {{ drain_timeout }} --delete-emptydir-data {{ kube_override_hostname }} async: "{{ (drain_timeout | regex_replace('s$', '') | int) + 120 }}" poll: 15 when: - groups['kube_control_plane'] | length > 0 # ignore servers that are not nodes - kube_override_hostname in nodes.stdout_lines register: result failed_when: result.rc != 0 and not allow_ungraceful_removal delegate_to: "{{ groups['kube_control_plane'] | first }}" until: result.rc == 0 or allow_ungraceful_removal retries: "{{ drain_retries }}" delay: "{{ drain_retry_delay_seconds }}" - name: Remove-node | Wait until Volumes will be detached from the node command: >- {{ kubectl }} get volumeattachments -o go-template={% raw %}'{{ range .items }}{{ .spec.nodeName }}{{ "\n" }}{{ end }}'{% endraw %} register: nodes_with_volumes delegate_to: "{{ groups['kube_control_plane'] | first }}" changed_when: false until: not (kube_override_hostname in nodes_with_volumes.stdout_lines) retries: 3 delay: "{{ drain_grace_period }}" when: - groups['kube_control_plane'] | length > 0 - not allow_ungraceful_removal - kube_override_hostname in nodes.stdout_lines ================================================ FILE: roles/reset/defaults/main.yml ================================================ --- flush_iptables: true reset_restart_network: true # crictl stop container grace period cri_stop_containers_grace_period: 0 ================================================ FILE: roles/reset/tasks/main.yml ================================================ --- - name: Reset | stop services service: name: "{{ item }}" state: stopped enabled: false with_items: - kubelet.service - cri-dockerd.service - cri-dockerd.socket failed_when: false tags: - services - name: Reset | remove services file: path: "/etc/systemd/system/{{ item }}" state: absent with_items: - kubelet.service - cri-dockerd.service - cri-dockerd.socket - calico-node.service - containerd.service.d/http-proxy.conf - crio.service.d/http-proxy.conf - k8s-certs-renew.service - k8s-certs-renew.timer register: services_removed tags: - services - containerd - crio - name: Reset | Remove Docker include_role: name: container-engine/docker tasks_from: reset when: container_manager == 'docker' tags: - docker - name: Reset | systemctl daemon-reload # noqa no-handler systemd_service: daemon_reload: true when: services_removed.changed - name: Reset | check if crictl is present stat: path: "{{ bin_dir }}/crictl" get_attributes: false get_checksum: false get_mime: false register: crictl - name: Reset | stop all cri containers shell: "set -o pipefail && {{ bin_dir }}/crictl ps -q | xargs -r {{ bin_dir }}/crictl -t 60s stop -t {{ cri_stop_containers_grace_period }}" args: executable: /bin/bash register: remove_all_cri_containers retries: 5 until: remove_all_cri_containers.rc == 0 delay: 5 tags: - crio - containerd when: - crictl.stat.exists - container_manager in ["crio", "containerd"] - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined ignore_errors: true # noqa ignore-errors - name: Reset | force remove all cri containers command: "{{ bin_dir }}/crictl rm -a -f" register: remove_all_cri_containers retries: 5 until: remove_all_cri_containers.rc == 0 delay: 5 tags: - crio - containerd when: - crictl.stat.exists - container_manager in ["crio", "containerd"] - deploy_container_engine - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined ignore_errors: true # noqa ignore-errors - name: Reset | stop and disable crio service service: name: crio state: stopped enabled: false failed_when: false tags: [ crio ] when: container_manager == "crio" - name: Reset | forcefully wipe CRI-O's container and image storage command: "crio wipe -f" failed_when: false tags: [ crio ] when: container_manager == "crio" - name: Reset | stop all cri pods shell: "set -o pipefail && {{ bin_dir }}/crictl pods -q | xargs -r {{ bin_dir }}/crictl -t 60s stopp" args: executable: /bin/bash register: remove_all_cri_containers retries: 5 until: remove_all_cri_containers.rc == 0 delay: 5 tags: [ containerd ] when: - crictl.stat.exists - container_manager == "containerd" - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined ignore_errors: true # noqa ignore-errors - name: Reset | force remove all cri pods block: - name: Reset | force remove all cri pods command: "{{ bin_dir }}/crictl rmp -a -f" register: remove_all_cri_containers retries: 5 until: remove_all_cri_containers.rc == 0 delay: 5 tags: [ containerd ] when: - crictl.stat.exists - container_manager == "containerd" - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined rescue: - name: Reset | force remove all cri pods (rescue) shell: "ip netns list | cut -d' ' -f 1 | xargs -n1 ip netns delete && {{ bin_dir }}/crictl rmp -a -f" ignore_errors: true # noqa ignore-errors changed_when: true - name: Reset | stop containerd and etcd services service: name: "{{ item }}" state: stopped enabled: false with_items: - containerd.service - etcd.service - etcd-events.service failed_when: false tags: - services - name: Reset | remove containerd and etcd services file: path: "/etc/systemd/system/{{ item }}" state: absent with_items: - containerd.service - etcd.service - etcd-events.service register: services_removed_secondary tags: - services - containerd - name: Reset | systemctl daemon-reload # noqa no-handler systemd_service: daemon_reload: true when: services_removed_secondary.changed - name: Reset | gather mounted kubelet dirs shell: set -o pipefail && mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac args: executable: /bin/bash check_mode: false register: mounted_dirs failed_when: false changed_when: false tags: - mounts - name: Reset | unmount kubelet dirs command: umount -f {{ item }} with_items: "{{ mounted_dirs.stdout_lines }}" register: umount_dir when: mounted_dirs retries: 4 until: umount_dir.rc == 0 delay: 5 tags: - mounts - name: Set IPv4 iptables default policies to ACCEPT iptables: chain: "{{ item }}" policy: ACCEPT with_items: - INPUT - FORWARD - OUTPUT when: flush_iptables | bool and ipv4_stack tags: - iptables - name: Flush iptables iptables: table: "{{ item }}" flush: true with_items: - filter - nat - mangle - raw when: flush_iptables | bool and ipv4_stack tags: - iptables - name: Delete IPv4 user-defined chains # noqa command-instead-of-module command: iptables -X when: flush_iptables | bool and ipv4_stack tags: - iptables - name: Set IPv6 ip6tables default policies to ACCEPT iptables: chain: "{{ item }}" policy: ACCEPT ip_version: ipv6 with_items: - INPUT - FORWARD - OUTPUT when: flush_iptables | bool and ipv6_stack tags: - ip6tables - name: Flush ip6tables iptables: table: "{{ item }}" flush: true ip_version: ipv6 with_items: - filter - nat - mangle - raw when: flush_iptables | bool and ipv6_stack tags: - ip6tables - name: Delete IPv6 user-defined chains # noqa command-instead-of-module command: ip6tables -X when: flush_iptables | bool and ipv6_stack tags: - ip6tables - name: Clear IPVS virtual server table command: "ipvsadm -C" ignore_errors: true # noqa ignore-errors when: - kube_proxy_mode == 'ipvs' and 'k8s_cluster' in group_names - name: Reset | check kube-ipvs0 network device stat: path: /sys/class/net/kube-ipvs0 get_attributes: false get_checksum: false get_mime: false register: kube_ipvs0 - name: Reset | Remove kube-ipvs0 command: "ip link del kube-ipvs0" when: - kube_proxy_mode == 'ipvs' - kube_ipvs0.stat.exists - name: Reset | check nodelocaldns network device stat: path: /sys/class/net/nodelocaldns get_attributes: false get_checksum: false get_mime: false register: nodelocaldns_device - name: Reset | Remove nodelocaldns command: "ip link del nodelocaldns" when: - enable_nodelocaldns | default(false) | bool - nodelocaldns_device.stat.exists - name: Reset | Check whether /var/lib/kubelet directory exists stat: path: /var/lib/kubelet get_attributes: false get_checksum: false get_mime: false register: var_lib_kubelet_directory - name: Reset | Find files/dirs with immutable flag in /var/lib/kubelet command: lsattr -laR /var/lib/kubelet/ become: true register: var_lib_kubelet_files_dirs_w_attrs changed_when: false no_log: true when: var_lib_kubelet_directory.stat.exists - name: Reset | Remove immutable flag from files/dirs in /var/lib/kubelet file: path: "{{ filedir_path }}" state: touch attributes: "-i" mode: "0644" loop: "{{ var_lib_kubelet_files_dirs_w_attrs.stdout_lines | select('search', 'Immutable') | list }}" loop_control: loop_var: file_dir_line label: "{{ filedir_path }}" vars: filedir_path: "{{ file_dir_line.split(' ')[0] }}" when: var_lib_kubelet_directory.stat.exists - name: Reset | delete some files and directories file: path: "{{ item }}" state: absent with_items: - "{{ kube_config_dir }}" - /var/lib/kubelet - "{{ containerd_storage_dir }}" - "{{ ansible_env.HOME | default('/root') }}/.kube" - "{{ ansible_env.HOME | default('/root') }}/.helm" - "{{ ansible_env.HOME | default('/root') }}/.config/helm" - "{{ ansible_env.HOME | default('/root') }}/.cache/helm" - "{{ ansible_env.HOME | default('/root') }}/.local/share/helm" - "{{ etcd_data_dir }}" - "{{ etcd_events_data_dir }}" - "{{ etcd_config_dir }}" - /var/log/calico - /var/log/openvswitch - /var/log/ovn - /var/log/kube-ovn - /var/log/containers - /etc/cni - /etc/nerdctl - "{{ nginx_config_dir }}" - /etc/systemd/resolved.conf.d/kubespray.conf - /etc/etcd.env - /etc/calico - /etc/NetworkManager/conf.d/calico.conf - /etc/NetworkManager/conf.d/dns.conf - /etc/NetworkManager/conf.d/k8s.conf - /opt/cni - /etc/dhcp/dhclient.d/zdnsupdate.sh - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate - /run/flannel - /etc/flannel - /run/kubernetes - /usr/local/share/ca-certificates/etcd-ca.crt - /usr/local/share/ca-certificates/kube-ca.crt - /etc/ssl/certs/etcd-ca.pem - /etc/ssl/certs/kube-ca.pem - /etc/pki/ca-trust/source/anchors/etcd-ca.crt - /etc/pki/ca-trust/source/anchors/kube-ca.crt - /var/log/pods/ - "{{ bin_dir }}/kubelet" - "{{ bin_dir }}/cri-dockerd" - "{{ bin_dir }}/etcd-scripts" - "{{ bin_dir }}/etcd" - "{{ bin_dir }}/etcd-events" - "{{ bin_dir }}/etcdctl" - "{{ bin_dir }}/etcdctl.sh" - "{{ bin_dir }}/kubernetes-scripts" - "{{ bin_dir }}/kubectl" - "{{ bin_dir }}/kubeadm" - "{{ bin_dir }}/helm" - "{{ bin_dir }}/calicoctl" - "{{ bin_dir }}/calicoctl.sh" - "{{ bin_dir }}/calico-upgrade" - "{{ bin_dir }}/crictl" - "{{ bin_dir }}/nerdctl" - "{{ bin_dir }}/netctl" - "{{ bin_dir }}/k8s-certs-renew.sh" - /var/lib/cni - /etc/openvswitch - /run/openvswitch - /var/lib/kube-router - /var/lib/calico - /etc/cilium - /run/calico - /etc/bash_completion.d/kubectl.sh - /etc/bash_completion.d/crictl - /etc/bash_completion.d/nerdctl - /etc/modules-load.d/kube_proxy-ipvs.conf - /etc/modules-load.d/kubespray-br_netfilter.conf - /etc/modules-load.d/kubespray-kata-containers.conf - /usr/libexec/kubernetes - /etc/origin/openvswitch - /etc/origin/ovn - "{{ sysctl_file_path }}" - /etc/crictl.yaml ignore_errors: true # noqa ignore-errors tags: - files - name: Reset | remove containerd binary files file: path: "{{ containerd_bin_dir }}/{{ item }}" state: absent with_items: - containerd - containerd-shim - containerd-shim-runc-v1 - containerd-shim-runc-v2 - containerd-stress - crictl - critest - ctd-decoder - ctr - runc ignore_errors: true # noqa ignore-errors when: container_manager == 'containerd' tags: - files - name: Reset | remove dns settings from dhclient.conf blockinfile: path: "{{ item }}" state: absent marker: "# Ansible entries {mark}" failed_when: false with_items: - /etc/dhclient.conf - /etc/dhcp/dhclient.conf tags: - files - dns - name: Reset | include file with reset tasks specific to the network_plugin if exists include_role: name: "network_plugin/{{ kube_network_plugin }}" tasks_from: reset when: - kube_network_plugin in ['flannel', 'cilium', 'kube-router', 'calico'] tags: - network - name: Reset | Restart network when: - ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - reset_restart_network | bool tags: - services - network block: - name: Gather active network services systemd: name: "{{ item }}" loop: - NetworkManager - systemd-networkd - networking - network register: service_status changed_when: false ignore_errors: true - name: Restart active network services systemd: name: "{{ item }}" state: restarted loop: "{{ service_status.results | selectattr('status.ActiveState', '==', 'active') | map(attribute='item') }}" ================================================ FILE: roles/system_packages/defaults/main.yml ================================================ --- # number of times package install task should be retried pkg_install_retries: 4 pkg_install_timeout: "{{ 5 * 60 }}" yum_repo_dir: /etc/yum.repos.d ================================================ FILE: roles/system_packages/tasks/main.yml ================================================ --- - name: Gather OS information setup: gather_subset: - distribution - pkg_mgr - name: Update package management cache (zypper) - SUSE command: zypper -n --gpg-auto-import-keys ref register: make_cache_output until: make_cache_output is succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: - ansible_pkg_mgr == 'zypper' tags: bootstrap_os - name: Remove legacy docker repo file file: path: "{{ yum_repo_dir }}/docker.repo" state: absent when: - ansible_os_family == "RedHat" - not is_fedora_coreos - name: Install epel-release on RHEL derivatives package: name: epel-release state: present when: - ansible_os_family == "RedHat" - not is_fedora_coreos - epel_enabled | bool tags: - bootstrap_os # Remove this after ansible-core >= 2.19.0 # See https://github.com/kubernetes-sigs/kubespray/pull/12138#issuecomment-3019304574 - name: Install python3-libdnf5 on Fedora >= 41 raw: > dnf install --assumeyes python3-libdnf5 become: true retries: "{{ pkg_install_retries }}" when: - ansible_distribution == "Fedora" - ansible_distribution_major_version | int >= 41 - name: Manage packages package: name: "{{ item.packages }}" state: "{{ item.state }}" update_cache: "{{ true if ansible_pkg_mgr in ['zypper', 'apt', 'dnf'] else omit }}" cache_valid_time: "{{ 86400 if ansible_pkg_mgr == 'apt' else omit }}" # 24h register: pkgs_task_result until: pkgs_task_result is succeeded retries: "{{ pkg_install_retries }}" delay: "{{ retry_stagger | random + 3 }}" when: - ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - not is_fedora_coreos - item.packages != [] loop: - packages: "{{ pkgs_to_remove | dict2items | selectattr('value', 'ansible.builtin.all') | map(attribute='key') }}" state: "absent" action_label: "remove" - packages: "{{ pkgs | dict2items | selectattr('value', 'ansible.builtin.all') | map(attribute='key') }}" state: "present" action_label: "install" loop_control: label: "{{ item.action_label }}" tags: - bootstrap_os timeout: "{{ pkg_install_timeout }}" ================================================ FILE: roles/system_packages/vars/main.yml ================================================ --- pkgs_to_remove: systemd-timesyncd: - "{{ ntp_enabled }}" - "{{ ntp_package == 'ntp' }}" - "{{ ansible_os_family == 'Debian' }}" pkgs: apparmor: - "{{ ansible_os_family == 'Debian' }}" apparmor-parser: - "{{ ansible_os_family == 'Suse' }}" apt-transport-https: - "{{ ansible_os_family == 'Debian' }}" aufs-tools: - "{{ ansible_os_family == 'Debian' }}" - "{{ ansible_distribution_major_version == '10' }}" - "{{ 'k8s_cluster' in group_names }}" bash-completion: [] chrony: - "{{ ntp_enabled }}" - "{{ ntp_package == 'chrony' }}" conntrack: - "{{ ansible_os_family in ['Debian', 'RedHat'] }}" - "{{ ansible_distribution != 'openEuler' }}" - "{{ 'k8s_cluster' in group_names }}" conntrack-tools: - "{{ ansible_os_family == 'Suse' or ansible_distribution in ['Amazon', 'openEuler'] }}" - "{{ 'k8s_cluster' in group_names }}" container-selinux: - "{{ ansible_os_family == 'RedHat' }}" - "{{ 'k8s_cluster' in group_names }}" containers-basic: - "{{ ansible_os_family == 'ClearLinux' }}" - "{{ 'k8s_cluster' in group_names }}" curl: [] device-mapper: - "{{ ansible_os_family == 'Suse' or ansible_distribution == 'openEuler' }}" - "{{ 'k8s_cluster' in group_names }}" device-mapper-libs: - "{{ ansible_os_family == 'RedHat' }}" - "{{ ansible_distribution != 'openEuler' }}" e2fsprogs: [] ebtables: [] gnupg: - "{{ ansible_distribution == 'Debian' }}" - "{{ ansible_distribution_major_version in ['11', '12'] }}" - "{{ 'k8s_cluster' in group_names }}" iproute: - "{{ ansible_os_family == 'RedHat' }}" iproute2: - "{{ ansible_os_family != 'RedHat' }}" ipset: - "{{ kube_proxy_mode != 'ipvs' }}" - "{{ 'k8s_cluster' in group_names }}" iptables: - "{{ ansible_os_family in ['Debian', 'RedHat', 'Suse'] }}" iputils: - "{{ not ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk', 'Debian'] }}" - "{{ main_access_ip is defined }}" - "{{ ping_access_ip }}" - "{{ not is_fedora_coreos }}" iputils-ping: - "{{ ansible_os_family == 'Debian' }}" - "{{ main_access_ip is defined }}" - "{{ ping_access_ip }}" ipvsadm: - "{{ kube_proxy_mode == 'ipvs' }}" - "{{ 'k8s_cluster' in group_names }}" libseccomp: - "{{ ansible_os_family == 'RedHat' }}" libseccomp2: - "{{ ansible_os_family in ['Debian', 'Suse'] }}" - "{{ 'k8s_cluster' in group_names }}" libselinux-python: - "{{ ansible_distribution == 'Amazon' }}" libselinux-python3: - "{{ ansible_distribution == 'Fedora' }}" mergerfs: - "{{ ansible_distribution == 'Debian' }}" - "{{ ansible_distribution_major_version == '12' }}" nftables: - "{{ kube_proxy_mode == 'nftables' }}" - "{{ 'k8s_cluster' in group_names }}" nss: - "{{ ansible_os_family == 'RedHat' }}" ntp: - "{{ ntp_enabled }}" - "{{ ntp_package == 'ntp' }}" ntpsec: - "{{ ntp_enabled }}" - "{{ ntp_package == 'ntpsec' }}" openssl: [] python-apt: - "{{ ansible_os_family == 'Debian' }}" - "{{ ansible_distribution_major_version == '10' }}" python-cryptography: - "{{ ansible_os_family == 'Suse' and ansible_distribution_version is version('15.4', '<') }}" python3-apt: - "{{ ansible_os_family == 'Debian' }}" - "{{ ansible_distribution_major_version != '10' }}" python3-cryptography: - "{{ ansible_os_family == 'Suse' and ansible_distribution_version is version('15.4', '>=') }}" python3-libselinux: - "{{ ansible_distribution in ['RedHat', 'CentOS'] }}" rsync: [] socat: [] software-properties-common: - "{{ ansible_os_family == 'Debian' }}" - "{{ ansible_distribution_major_version != '13' }}" tar: [] unzip: [] xfsprogs: [] ================================================ FILE: roles/upgrade/post-upgrade/defaults/main.yml ================================================ --- # how long to wait for cilium after upgrade before uncordoning upgrade_post_cilium_wait_timeout: 120s upgrade_node_post_upgrade_confirm: false upgrade_node_post_upgrade_pause_seconds: 0 ================================================ FILE: roles/upgrade/post-upgrade/tasks/main.yml ================================================ --- - name: Wait for cilium when: - needs_cordoning | default(false) - kube_network_plugin == 'cilium' command: > {{ kubectl }} wait pod -n kube-system -l k8s-app=cilium --field-selector 'spec.nodeName=={{ kube_override_hostname | default(inventory_hostname) }}' --for=condition=Ready --timeout={{ upgrade_post_cilium_wait_timeout }} delegate_to: "{{ groups['kube_control_plane'][0] }}" - name: Confirm node uncordon pause: echo: true prompt: "Ready to uncordon node {{ kube_override_hostname | default(inventory_hostname) }}?" when: - upgrade_node_post_upgrade_confirm - name: Wait before uncordoning node pause: seconds: "{{ upgrade_node_post_upgrade_pause_seconds }}" when: - not upgrade_node_post_upgrade_confirm - upgrade_node_post_upgrade_pause_seconds != 0 - name: Run post upgrade hooks before uncordon loop: "{{ post_upgrade_hooks | default([]) }}" ansible.builtin.include_tasks: "{{ item }}" - name: Uncordon node command: "{{ kubectl }} uncordon {{ kube_override_hostname | default(inventory_hostname) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" when: - needs_cordoning | default(false) ================================================ FILE: roles/upgrade/pre-upgrade/defaults/main.yml ================================================ --- drain_grace_period: 300 drain_timeout: 360s drain_pod_selector: "" drain_nodes: true drain_retries: 3 drain_retry_delay_seconds: 10 drain_fallback_enabled: false drain_fallback_grace_period: 300 drain_fallback_timeout: 360s drain_fallback_retries: 0 drain_fallback_retry_delay_seconds: 10 upgrade_node_always_cordon: false upgrade_node_uncordon_after_drain_failure: true upgrade_node_fail_if_drain_fails: true upgrade_node_confirm: false upgrade_node_pause_seconds: 0 ================================================ FILE: roles/upgrade/pre-upgrade/tasks/main.yml ================================================ --- # Wait for upgrade - name: Confirm node upgrade pause: echo: true prompt: "Ready to upgrade node {{ kube_override_hostname | default(inventory_hostname) }}? (Press Enter to continue or Ctrl+C for other options)" when: - upgrade_node_confirm - name: Wait before upgrade node pause: seconds: "{{ upgrade_node_pause_seconds }}" when: - not upgrade_node_confirm - upgrade_node_pause_seconds != 0 # Node Ready: type = ready, status = True # Node NotReady: type = ready, status = Unknown - name: See if node is in ready state command: > {{ kubectl }} get node {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{ range .status.conditions[?(@.type == "Ready")].status }{ @ }{ end }' register: kubectl_node_ready delegate_to: "{{ groups['kube_control_plane'][0] }}" failed_when: false changed_when: false # SchedulingDisabled: unschedulable = true # else unschedulable key doesn't exist - name: See if node is schedulable command: > {{ kubectl }} get node {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{ .spec.unschedulable }' register: kubectl_node_unschedulable delegate_to: "{{ groups['kube_control_plane'][0] }}" failed_when: false changed_when: false - name: Set if node needs cordoning set_fact: needs_cordoning: "{{ (kubectl_node_ready.stdout == 'True' and not kubectl_node_unschedulable.stdout) or upgrade_node_always_cordon }}" - name: Node draining delegate_to: "{{ groups['kube_control_plane'][0] }}" when: - needs_cordoning block: - name: Cordon node command: "{{ kubectl }} cordon {{ kube_override_hostname | default(inventory_hostname) }}" delegate_to: "{{ groups['kube_control_plane'][0] }}" changed_when: true - name: Drain node command: >- {{ kubectl }} drain --force --ignore-daemonsets --grace-period {{ drain_grace_period }} --timeout {{ drain_timeout }} --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }} {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %} async: "{{ (drain_timeout | regex_replace('s$', '') | int) + 120 }}" poll: 15 when: drain_nodes register: result failed_when: - result.rc != 0 - not drain_fallback_enabled until: result.rc == 0 retries: "{{ drain_retries }}" delay: "{{ drain_retry_delay_seconds }}" - name: Drain node - fallback with disabled eviction when: - drain_nodes - drain_fallback_enabled - result.rc != 0 command: >- {{ kubectl }} drain --force --ignore-daemonsets --grace-period {{ drain_fallback_grace_period }} --timeout {{ drain_fallback_timeout }} --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }} {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %} --disable-eviction async: "{{ (drain_fallback_timeout | regex_replace('s$', '') | int) + 120 }}" poll: 15 register: drain_fallback_result until: drain_fallback_result.rc == 0 retries: "{{ drain_fallback_retries }}" delay: "{{ drain_fallback_retry_delay_seconds }}" changed_when: drain_fallback_result.rc == 0 rescue: - name: Set node back to schedulable command: "{{ kubectl }} uncordon {{ kube_override_hostname | default(inventory_hostname) }}" when: upgrade_node_uncordon_after_drain_failure - name: Fail after rescue fail: msg: "Failed to drain node {{ kube_override_hostname | default(inventory_hostname) }}" when: upgrade_node_fail_if_drain_fails ================================================ FILE: roles/upgrade/system-upgrade/tasks/apt.yml ================================================ --- - name: APT Dist-Upgrade apt: update_cache: true upgrade: dist autoremove: true dpkg_options: force-confold,force-confdef register: apt_upgrade - name: Reboot after APT Dist-Upgrade # noqa no-handler when: - apt_upgrade.changed or system_upgrade_reboot == 'always' - system_upgrade_reboot != 'never' reboot: ================================================ FILE: roles/upgrade/system-upgrade/tasks/main.yml ================================================ --- - name: APT upgrade when: - system_upgrade - ansible_os_family == "Debian" include_tasks: apt.yml tags: - system-upgrade-apt - name: YUM upgrade when: - system_upgrade - ansible_os_family == "RedHat" - not is_fedora_coreos include_tasks: yum.yml tags: - system-upgrade-yum ================================================ FILE: roles/upgrade/system-upgrade/tasks/yum.yml ================================================ --- - name: YUM upgrade all packages # noqa package-latest yum: name: '*' state: latest register: yum_upgrade - name: Reboot after YUM upgrade # noqa no-handler when: - yum_upgrade.changed or system_upgrade_reboot == 'always' - system_upgrade_reboot != 'never' reboot: ================================================ FILE: roles/validate_inventory/meta/main.yml ================================================ --- dependencies: - role: kubespray_defaults ================================================ FILE: roles/validate_inventory/tasks/main.yml ================================================ --- # This should only contains check of: # - the inventory # - extra variables # - ansible commandline # -> nothing depending on facts or similar cluster state # Checks depending on current state (of the nodes or the cluster) # should be in roles/kubernetes/preinstall/tasks/0040-verify-settings.yml - name: Fail if removed variables are used vars: # Always remove items from this list after the release in comments removed_vars: - kubelet_static_pod_path # 2.31.0 removed_vars_found: "{{ query('varnames', '^' + (removed_vars | join('|')) + '$') }}" assert: that: removed_vars_found | length == 0 fail_msg: "Removed variables present: {{ removed_vars_found | join(', ') }}" run_once: true - name: Stop if kube_control_plane group is empty assert: that: groups.get( 'kube_control_plane' ) run_once: true when: not ignore_assert_errors - name: Stop if etcd group is empty in external etcd mode assert: that: groups.get('etcd') or etcd_deployment_type == 'kubeadm' fail_msg: "Group 'etcd' cannot be empty in external etcd mode" run_once: true when: - not ignore_assert_errors - name: Warn if `kube_network_plugin` is `none` debug: msg: | "WARNING! => `kube_network_plugin` is set to `none`. The network configuration will be skipped. The cluster won't be ready to use, we recommend to select one of the available plugins" when: - kube_network_plugin == 'none' - name: Stop if unsupported version of Kubernetes assert: that: kube_version is version(kube_version_min_required, '>=') msg: "The current release of Kubespray only support newer version of Kubernetes than {{ kube_version_min_required }} - You are trying to apply {{ kube_version }}" when: not ignore_assert_errors - name: "Stop if known booleans are set as strings (Use JSON format on CLI: -e \"{'key': true }\")" assert: that: - download_run_once | type_debug == 'bool' - download_always_pull | type_debug == 'bool' - helm_enabled | type_debug == 'bool' - openstack_lbaas_enabled | type_debug == 'bool' run_once: true when: not ignore_assert_errors - name: Stop if even number of etcd hosts assert: that: groups.get('etcd', groups.kube_control_plane) | length is not divisibleby 2 run_once: true when: - not ignore_assert_errors # This assertion will fail on the safe side: One can indeed schedule more pods # on a node than the CIDR-range has space for when additional pods use the host # network namespace. It is impossible to ascertain the number of such pods at # provisioning time, so to establish a guarantee, we factor these out. # NOTICE: the check blatantly ignores the inet6-case - name: Guarantee that enough network address space is available for all pods assert: that: "{{ (kubelet_max_pods | default(110)) | int <= (2 ** (32 - kube_network_node_prefix | int)) - 2 }}" msg: "Do not schedule more pods on a node than inet addresses are available." when: - not ignore_assert_errors - ('k8s_cluster' in group_names) - kube_network_plugin not in ['calico', 'none'] - ipv4_stack | bool - name: Check cloud_provider value assert: that: cloud_provider == 'external' when: - cloud_provider - not ignore_assert_errors - name: Check external_cloud_provider value assert: that: external_cloud_provider in ['hcloud', 'huaweicloud', 'oci', 'openstack', 'vsphere', 'manual'] when: - cloud_provider == 'external' - not ignore_assert_errors - name: "Check that kube_service_addresses is a network range" assert: that: - kube_service_addresses | ansible.utils.ipaddr('net') msg: "kube_service_addresses = '{{ kube_service_addresses }}' is not a valid network range" run_once: true when: ipv4_stack | bool - name: "Check that kube_pods_subnet is a network range" assert: that: - kube_pods_subnet | ansible.utils.ipaddr('net') msg: "kube_pods_subnet = '{{ kube_pods_subnet }}' is not a valid network range" run_once: true when: ipv4_stack | bool - name: "Check that kube_pods_subnet does not collide with kube_service_addresses" assert: that: - kube_pods_subnet | ansible.utils.ipaddr(kube_service_addresses) | string == 'None' msg: "kube_pods_subnet cannot be the same network segment as kube_service_addresses" run_once: true when: ipv4_stack | bool - name: "Check that ipv4 IP range is enough for the nodes" assert: that: - 2 ** (kube_network_node_prefix - kube_pods_subnet | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length msg: "Not enough ipv4 IPs are available for the desired node count." when: - ipv4_stack | bool - kube_network_plugin != 'calico' run_once: true - name: "Check that kube_service_addresses_ipv6 is a network range" assert: that: - kube_service_addresses_ipv6 | ansible.utils.ipaddr('net') msg: "kube_service_addresses_ipv6 = '{{ kube_service_addresses_ipv6 }}' is not a valid network range" run_once: true when: ipv6_stack | bool - name: "Check that kube_pods_subnet_ipv6 is a network range" assert: that: - kube_pods_subnet_ipv6 | ansible.utils.ipaddr('net') msg: "kube_pods_subnet_ipv6 = '{{ kube_pods_subnet_ipv6 }}' is not a valid network range" run_once: true when: ipv6_stack | bool - name: "Check that kube_pods_subnet_ipv6 does not collide with kube_service_addresses_ipv6" assert: that: - kube_pods_subnet_ipv6 | ansible.utils.ipaddr(kube_service_addresses_ipv6) | string == 'None' msg: "kube_pods_subnet_ipv6 cannot be the same network segment as kube_service_addresses_ipv6" run_once: true when: ipv6_stack | bool - name: "Check that ipv6 IP range is enough for the nodes" assert: that: - 2 ** (kube_network_node_prefix_ipv6 - kube_pods_subnet_ipv6 | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length msg: "Not enough ipv6 IPs are available for the desired node count." when: - ipv6_stack | bool - kube_network_plugin != 'calico' run_once: true - name: Stop if unsupported options selected assert: that: - kube_network_plugin in ['calico', 'flannel', 'cloud', 'cilium', 'cni', 'kube-ovn', 'kube-router', 'macvlan', 'custom_cni', 'none'] - dns_mode in ['coredns', 'coredns_dual', 'manual', 'none'] - kube_proxy_mode in ['iptables', 'ipvs', 'nftables'] - cert_management in ['script', 'none'] - resolvconf_mode in ['docker_dns', 'host_resolvconf', 'none'] - etcd_deployment_type in ['host', 'docker', 'kubeadm'] - etcd_deployment_type in ['host', 'kubeadm'] or container_manager == 'docker' - container_manager in ['docker', 'crio', 'containerd'] msg: The selected choice is not supported run_once: true - name: Warn if `enable_dual_stack_networks` is set debug: msg: "WARNING! => `enable_dual_stack_networks` deprecation. Please switch to using ipv4_stack and ipv6_stack." when: - enable_dual_stack_networks is defined - name: Stop if download_localhost is enabled but download_run_once is not assert: that: download_run_once msg: "download_localhost requires enable download_run_once" when: download_localhost - name: Stop if kata_containers_enabled is enabled when container_manager is docker assert: that: container_manager != 'docker' msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" when: kata_containers_enabled - name: Stop if gvisor_enabled is enabled when container_manager is not containerd assert: that: container_manager == 'containerd' msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details" when: gvisor_enabled - name: Ensure minimum containerd version assert: that: containerd_version is version(containerd_min_version_required, '>=') msg: "containerd_version is too low. Minimum version {{ containerd_min_version_required }}" run_once: true when: - containerd_version not in ['latest', 'edge', 'stable'] - container_manager == 'containerd' - name: Stop if auto_renew_certificates is enabled when certificates are managed externally (kube_external_ca_mode is true) assert: that: not auto_renew_certificates msg: "Variable auto_renew_certificates must be disabled when CA are managed externally: kube_external_ca_mode = true" when: - kube_external_ca_mode - not ignore_assert_errors - name: Download_file | Check if requested Kubernetes are supported assert: that: - kube_version in kubeadm_checksums[image_arch] - kube_version in kubelet_checksums[image_arch] - kube_version in kubectl_checksums[image_arch] msg: >- Kubernetes v{{ kube_version }} is not supported for {{ image_arch }}. Please check roles/kubespray_defaults/vars/main/checksums.yml for supported versions. ================================================ FILE: roles/win_nodes/kubernetes_patch/defaults/main.yml ================================================ --- kubernetes_user_manifests_path: "{{ ansible_env.HOME }}/kube-manifests" kube_proxy_nodeselector: "kubernetes.io/os" ================================================ FILE: roles/win_nodes/kubernetes_patch/tasks/main.yml ================================================ --- - name: Ensure that user manifests directory exists file: path: "{{ kubernetes_user_manifests_path }}/kubernetes" state: directory recurse: true tags: [init, cni] - name: Apply kube-proxy nodeselector tags: init when: - kube_proxy_deployed block: # Due to https://github.com/kubernetes/kubernetes/issues/58212 we cannot rely on exit code for "kubectl patch" - name: Check current nodeselector for kube-proxy daemonset command: >- {{ kubectl }} get ds kube-proxy --namespace=kube-system -o jsonpath={.spec.template.spec.nodeSelector.{{ kube_proxy_nodeselector | regex_replace('\.', '\\.') }}} register: current_kube_proxy_state retries: 60 delay: 5 until: current_kube_proxy_state is succeeded changed_when: false - name: Apply nodeselector patch for kube-proxy daemonset command: > {{ kubectl }} patch ds kube-proxy --namespace=kube-system --type=strategic -p '{"spec":{"template":{"spec":{"nodeSelector":{"{{ kube_proxy_nodeselector }}":"linux"} }}}}' register: patch_kube_proxy_state when: current_kube_proxy_state.stdout | trim | lower != "linux" - debug: # noqa name[missing] msg: "{{ patch_kube_proxy_state.stdout_lines }}" when: patch_kube_proxy_state is not skipped - debug: # noqa name[missing] msg: "{{ patch_kube_proxy_state.stderr_lines }}" when: patch_kube_proxy_state is not skipped ================================================ FILE: scale.yml ================================================ --- - name: Scale the cluster ansible.builtin.import_playbook: playbooks/scale.yml ================================================ FILE: scripts/Dockerfile.j2 ================================================ # syntax=docker/dockerfile:1 # Use immutable image tags rather than mutable tags (like ubuntu:24.04) FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b # Some tools like yamllint need this # Pip needs this as well at the moment to install ansible # (and potentially other packages) # See: https://github.com/pypa/pip/issues/10219 ENV LANG=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 WORKDIR /kubespray # hadolint ignore=DL3008 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update -q \ && apt-get install -yq --no-install-recommends \ curl \ python3 \ python3-pip \ sshpass \ vim \ rsync \ openssh-client \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/log/* RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \ --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \ pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt \ && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \; SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \ && curl -L "https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \ && echo "$(curl -L "https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl COPY *.yml ./ COPY *.cfg ./ COPY roles ./roles COPY contrib ./contrib COPY inventory ./inventory COPY library ./library COPY extra_playbooks ./extra_playbooks COPY playbooks ./playbooks COPY plugins ./plugins ================================================ FILE: scripts/assert-sorted-checksums.yml ================================================ #!/usr/bin/env ansible-playbook --- - name: Verify correct structure of Kubespray variables hosts: localhost connection: local gather_facts: false vars: fallback_ip: 'bypass tasks in kubespray_defaults' _keys: "{{ query('ansible.builtin.varnames', '^.+_checksums$') }}" _values: "{{ query('ansible.builtin.vars', *_keys) | map('dict2items') }}" _components_archs_values: "{{ _keys | zip(_values) | community.general.dict | dict2items | subelements('value') }}" _minimal_data_needed: "{{ _components_archs_values | map(attribute='0.key') | zip(_components_archs_values | map(attribute='1')) }}" roles: - kubespray_defaults tasks: - name: Check all versions are strings assert: that: "{{ item.1.value | reject('string') == [] }}" quiet: true loop: "{{ _minimal_data_needed }}" loop_control: label: "{{ item.0 }}:{{ item.1.key }}" - name: Check all checksums are sorted by version vars: actual: "{{ item.1.value.keys() | map('string') | reverse}}" sorted: "{{ item.1.value.keys() | map('string') | community.general.version_sort }}" assert: that: actual == sorted quiet: true msg: "{{ actual | ansible.utils.fact_diff(sorted) }}" loop: "{{ _minimal_data_needed }}" loop_control: label: "{{ item.0 }}:{{ item.1.key }}" when: - item.1.value is not string - (item.1.value | dict2items)[0].value is string or (item.1.value | dict2items)[0].value is number # only do list, the others are checksums with a different structure - name: Include the packages list variable include_vars: ../roles/system_packages/vars/main.yml - name: Verify that the packages list is sorted loop: - pkgs_to_remove - pkgs vars: pkgs_lists: "{{ lookup('vars', item).keys() | list }}" ansible_distribution: irrelevant ansible_distribution_major_version: irrelevant ansible_distribution_minor_version: irrelevant ansible_distribution_version: 1.0 ansible_os_family: irrelevant assert: that: "pkgs_lists | sort == pkgs_lists" fail_msg: "{{ item }} is not sorted: {{ pkgs_lists | ansible.utils.fact_diff(pkgs_lists | sort) }}" ================================================ FILE: scripts/collect-info.yaml ================================================ --- - name: Collect debug info hosts: all become: true gather_facts: false vars: docker_bin_dir: /usr/bin bin_dir: /usr/local/bin ansible_ssh_pipelining: true etcd_cert_dir: /etc/ssl/etcd/ssl kube_network_plugin: calico archive_dirname: collect-info commands: - name: timedate_info cmd: timedatectl status - name: kernel_info cmd: uname -r - name: docker_info cmd: "{{ docker_bin_dir }}/docker info" - name: ip_info cmd: ip -4 -o a - name: route_info cmd: ip ro - name: proc_info cmd: ps auxf | grep -v ]$ - name: systemctl_failed_info cmd: systemctl --state=failed --no-pager - name: k8s_info cmd: "{{ bin_dir }}/kubectl get all --all-namespaces -o wide" - name: errors_info cmd: journalctl -p err --no-pager - name: etcd_info cmd: "{{ bin_dir }}/etcdctl endpoint --cluster health" - name: calico_info cmd: "{{ bin_dir }}/calicoctl node status" when: '{{ kube_network_plugin == "calico" }}' - name: calico_workload_info cmd: "{{ bin_dir }}/calicoctl get workloadEndpoint -o wide" when: '{{ kube_network_plugin == "calico" }}' - name: calico_pool_info cmd: "{{ bin_dir }}/calicoctl get ippool -o wide" when: '{{ kube_network_plugin == "calico" }}' - name: kube_describe_all cmd: "{{ bin_dir }}/kubectl describe all --all-namespaces" - name: kube_describe_nodes cmd: "{{ bin_dir }}/kubectl describe nodes" - name: kubelet_logs cmd: journalctl -u kubelet --no-pager - name: coredns_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=coredns -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done" - name: apiserver_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-apiserver -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done" - name: controller_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-controller-manager -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done" - name: scheduler_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-scheduler -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done" - name: proxy_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=kube-proxy -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done" - name: nginx_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=kube-nginx -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done" - name: flannel_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l app=flannel -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system flannel-container; done" when: '{{ kube_network_plugin == "flannel" }}' - name: canal_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=canal-node -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system flannel; done" when: '{{ kube_network_plugin == "canal" }}' - name: calico_policy_logs cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=calico-kube-controllers -o jsonpath={.items..metadata.name}`; do {{ bin_dir }}/kubectl logs ${i} -n kube-system ; done" when: '{{ kube_network_plugin in ["canal", "calico"] }}' - name: helm_show_releases_history cmd: "for i in `{{ bin_dir }}/helm list -q`; do {{ bin_dir }}/helm history ${i} --col-width=0; done" when: "{{ helm_enabled | default(true) }}" logs: - /var/log/syslog - /var/log/daemon.log - /var/log/kern.log - /var/log/dpkg.log - /var/log/apt/history.log - /var/log/yum.log - /var/log/messages - /var/log/dmesg environment: ETCDCTL_API: "3" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" tasks: - name: Set etcd_access_addresses set_fact: etcd_access_addresses: |- {% for item in groups['etcd'] -%} https://{{ item }}:2379{% if not loop.last %},{% endif %} {%- endfor %} when: "'etcd' in groups" - name: Storing commands output shell: "{{ item.cmd }} &> {{ item.name }}" failed_when: false with_items: "{{ commands }}" when: item.when | default(True) no_log: true - name: Fetch results fetch: src: "{{ item.name }}" dest: "/tmp/{{ archive_dirname }}/commands" with_items: "{{ commands }}" when: item.when | default(True) failed_when: false - name: Fetch logs fetch: src: "{{ item }}" dest: "/tmp/{{ archive_dirname }}/logs" with_items: "{{ logs }}" failed_when: false - name: Pack results and logs community.general.archive: path: "/tmp/{{ archive_dirname }}" dest: "{{ dir | default('.') }}/logs.tar.gz" remove: true mode: "0640" delegate_to: localhost connection: local become: false run_once: true - name: Clean up collected command outputs file: path: "{{ item.name }}" state: absent with_items: "{{ commands }}" ================================================ FILE: scripts/component_hash_update/pyproject.toml ================================================ [build-system] requires = ["setuptools >= 61.0", "setuptools_scm >= 8.0", ] build-backend = "setuptools.build_meta" [project] name = "kubespray_component_hash_update" version = "1.0.1" dependencies = [ "more_itertools", "ruamel.yaml", "requests", "packaging", ] requires-python = ">= 3.10" authors = [ { name = "Craig Rodrigues", email = "rodrigc@crodrigues.org" }, { name = "Simon Wessel" }, { name = "Max Gautier", email = "mg@max.gautier.name" }, ] maintainers = [ { name = "The Kubespray maintainers" }, ] description = "Download or compute hashes for new versions of components deployed by Kubespray" classifiers = [ "License :: OSI Approved :: Apache-2.0", ] [project.scripts] update-hashes = "component_hash_update.download:main" ================================================ FILE: scripts/component_hash_update/src/component_hash_update/__init__.py ================================================ ================================================ FILE: scripts/component_hash_update/src/component_hash_update/components.py ================================================ """ Static download metadata for components updated by the update-hashes command. """ infos = { "calicoctl_binary": { "url": "https://github.com/projectcalico/calico/releases/download/v{version}/SHA256SUMS", "graphql_id": "R_kgDOA87D0g", }, "calico_crds": { "url": "https://github.com/projectcalico/calico/raw/v{version}/manifests/crds.yaml", "graphql_id": "R_kgDOA87D0g", "binary": True, }, "ciliumcli_binary": { "url": "https://github.com/cilium/cilium-cli/releases/download/v{version}/cilium-{os}-{arch}.tar.gz.sha256sum", "graphql_id": "R_kgDOE0nmLg", }, "cni_binary": { "url": "https://github.com/containernetworking/plugins/releases/download/v{version}/cni-plugins-{os}-{arch}-v{version}.tgz.sha256", "graphql_id": "R_kgDOBQqEpg", }, "containerd_archive": { "url": "https://github.com/containerd/containerd/releases/download/v{version}/containerd-{version}-{os}-{arch}.tar.gz.sha256sum", "graphql_id": "R_kgDOAr9FWA", }, "containerd_static_archive": { "url": "https://github.com/containerd/containerd/releases/download/v{version}/containerd-static-{version}-{os}-{arch}.tar.gz.sha256sum", "graphql_id": "R_kgDOAr9FWA", }, "cri_dockerd_archive": { "binary": True, "url": "https://github.com/Mirantis/cri-dockerd/releases/download/v{version}/cri-dockerd-{version}.{arch}.tgz", "graphql_id": "R_kgDOEvvLcQ", }, "crictl": { "url": "https://github.com/kubernetes-sigs/cri-tools/releases/download/v{version}/crictl-v{version}-{os}-{arch}.tar.gz.sha256", "graphql_id": "R_kgDOBMdURA", }, "crio_archive": { "url": "https://storage.googleapis.com/cri-o/artifacts/cri-o.{arch}.v{version}.tar.gz.sha256sum", "graphql_id": "R_kgDOBAr5pg", }, "crun": { "url": "https://github.com/containers/crun/releases/download/{version}/crun-{version}-linux-{arch}", "binary": True, "graphql_id": "R_kgDOBip3vA", }, "etcd_binary": { "url": "https://github.com/etcd-io/etcd/releases/download/v{version}/SHA256SUMS", "graphql_id": "R_kgDOAKtHtg", }, "gvisor_containerd_shim_binary": { "url": "https://storage.googleapis.com/gvisor/releases/release/{version}/{alt_arch}/containerd-shim-runsc-v1.sha512", "hashtype": "sha512", "tags": True, "graphql_id": "R_kgDOB9IlXg", }, "gvisor_runsc_binary": { "url": "https://storage.googleapis.com/gvisor/releases/release/{version}/{alt_arch}/runsc.sha512", "hashtype": "sha512", "tags": True, "graphql_id": "R_kgDOB9IlXg", }, "kata_containers_binary": { "url": "https://github.com/kata-containers/kata-containers/releases/download/{version}/kata-static-{version}-{arch}.tar.xz", "binary": True, "graphql_id": "R_kgDOBsJsHQ", }, "kubeadm": { "url": "https://dl.k8s.io/release/v{version}/bin/linux/{arch}/kubeadm.sha256", "graphql_id": "R_kgDOAToIkg", }, "kubectl": { "url": "https://dl.k8s.io/release/v{version}/bin/linux/{arch}/kubectl.sha256", "graphql_id": "R_kgDOAToIkg", }, "kubelet": { "url": "https://dl.k8s.io/release/v{version}/bin/linux/{arch}/kubelet.sha256", "graphql_id": "R_kgDOAToIkg", }, "nerdctl_archive": { "url": "https://github.com/containerd/nerdctl/releases/download/v{version}/SHA256SUMS", "graphql_id": "R_kgDOEvuRnQ", }, "runc": { "url": "https://github.com/opencontainers/runc/releases/download/v{version}/runc.sha256sum", "graphql_id": "R_kgDOAjP4QQ", }, "skopeo_binary": { "url": "https://github.com/lework/skopeo-binary/releases/download/v{version}/skopeo-{os}-{arch}.sha256", "graphql_id": "R_kgDOHQ6J9w", }, "youki": { "url": "https://github.com/youki-dev/youki/releases/download/v{version}/youki-{version}-{alt_arch}-gnu.tar.gz", "binary": True, "graphql_id": "R_kgDOFPvgPg", }, "yq": { "url": "https://github.com/mikefarah/yq/releases/download/v{version}/checksums-bsd", # see https://github.com/mikefarah/yq/pull/1691 for why we use this url "graphql_id": "R_kgDOApOQGQ", }, "argocd_install": { "url": "https://raw.githubusercontent.com/argoproj/argo-cd/v{version}/manifests/install.yaml", "graphql_id": "R_kgDOBzS60g", "binary": True, "hashtype": "sha256", }, "gateway_api_standard_crds": { "url": "https://github.com/kubernetes-sigs/gateway-api/releases/download/v{version}/standard-install.yaml", "graphql_id": "R_kgDODQ6RZw", "binary": True, }, "gateway_api_experimental_crds": { "url": "https://github.com/kubernetes-sigs/gateway-api/releases/download/v{version}/experimental-install.yaml", "graphql_id": "R_kgDODQ6RZw", "binary": True, }, "prometheus_operator_crds": { "url": "https://github.com/prometheus-operator/prometheus-operator/releases/download/v{version}/stripped-down-crds.yaml", "graphql_id": "R_kgDOBBxPpw", "binary": True, }, } ================================================ FILE: scripts/component_hash_update/src/component_hash_update/download.py ================================================ #!/usr/bin/env python3 # After a new version of Kubernetes has been released, # run this script to update roles/kubespray_defaults/defaults/main/download.yml # with new hashes. import sys import os import logging import subprocess from itertools import groupby, chain from more_itertools import partition from functools import cache import argparse import requests import hashlib from datetime import datetime from ruamel.yaml import YAML from packaging.version import Version, InvalidVersion from importlib.resources import files from pathlib import Path from typing import Optional, Any from . import components CHECKSUMS_YML = Path("roles/kubespray_defaults/vars/main/checksums.yml") logger = logging.getLogger(__name__) def open_yaml(file: Path): yaml = YAML() yaml.explicit_start = True yaml.preserve_quotes = True yaml.width = 4096 with open(file, "r") as checksums_yml: data = yaml.load(checksums_yml) return data, yaml arch_alt_name = { "amd64": "x86_64", "arm64": "aarch64", "ppc64le": None, "arm": None, "no_arch": None, } # TODO: downloads not supported # helm_archive: PGP signatures # TODO: # different verification methods (gpg, cosign) ( needs download role changes) (or verify the sig in this script and only use the checksum in the playbook) # perf improvements (async) def download_hash(downloads: {str: {str: Any}}) -> None: # Handle file with multiples hashes, with various formats. # the lambda is expected to produce a dictionary of hashes indexed by arch name download_hash_extract = { "calicoctl_binary": lambda hashes: { line.split("-")[-1]: line.split()[0] for line in hashes.strip().split("\n") if line.count("-") == 2 and line.split("-")[-2] == "linux" }, "etcd_binary": lambda hashes: { line.split("-")[-1].removesuffix(".tar.gz"): line.split()[0] for line in hashes.strip().split("\n") if line.split("-")[-2] == "linux" }, "nerdctl_archive": lambda hashes: { line.split()[1].removesuffix(".tar.gz").split("-")[3]: line.split()[0] for line in hashes.strip().split("\n") if [x for x in line.split(" ") if x][1].split("-")[2] == "linux" }, "runc": lambda hashes: { parts[1].split(".")[1]: parts[0] for parts in (line.split() for line in hashes.split("\n")[3:9]) }, "yq": lambda rhashes_bsd: { pair[0].split("_")[-1]: pair[1] # pair = (yq__, ) for pair in ( (line.split()[1][1:-1], line.split()[3]) for line in rhashes_bsd.splitlines() if line.startswith("SHA256") ) if pair[0].startswith("yq") and pair[0].split("_")[1] == "linux" and not pair[0].endswith(".tar.gz") }, } checksums_file = ( Path( subprocess.Popen( ["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE ) .communicate()[0] .rstrip() .decode("utf-8") ) / CHECKSUMS_YML ) logger.info("Opening checksums file %s...", checksums_file) data, yaml = open_yaml(checksums_file) s = requests.Session() @cache def _get_hash_by_arch(download: str, version: str) -> {str: str}: hash_file = s.get( downloads[download]["url"].format( version=version, os="linux", ), allow_redirects=True, ) hash_file.raise_for_status() return download_hash_extract[download](hash_file.content.decode()) releases, tags = map( dict, partition(lambda r: r[1].get("tags", False), downloads.items()) ) unique_release_ids = list(dict.fromkeys( r["graphql_id"] for r in releases.values() )) unique_tag_ids = list(dict.fromkeys( t["graphql_id"] for t in tags.values() )) response = s.post( "https://api.github.com/graphql", json={ "query": files(__package__).joinpath("list_releases.graphql").read_text(), "variables": { "with_releases": unique_release_ids, "with_tags": unique_tag_ids, }, }, headers={ "Authorization": f"Bearer {os.environ['API_KEY']}", }, ) if "x-ratelimit-used" in response.headers._store: logger.info( "Github graphQL API ratelimit status: used %s of %s. Next reset at %s", response.headers["X-RateLimit-Used"], response.headers["X-RateLimit-Limit"], datetime.fromtimestamp(int(response.headers["X-RateLimit-Reset"])), ) response.raise_for_status() def valid_version(possible_version: str) -> Optional[Version]: try: return Version(possible_version) except InvalidVersion: return None resp_data = response.json()["data"] release_versions_by_id = { gql_id: { v for r in repo["releases"]["nodes"] if not r["isPrerelease"] and (v := valid_version(r["tagName"])) is not None } for gql_id, repo in zip(unique_release_ids, resp_data["with_releases"]) } tag_versions_by_id = { gql_id: { v for t in repo["refs"]["nodes"] if (v := valid_version(t["name"].removeprefix("release-"))) is not None } for gql_id, repo in zip(unique_tag_ids, resp_data["with_tags"]) } github_versions = {} for name, info in releases.items(): github_versions[name] = release_versions_by_id[info["graphql_id"]] for name, info in tags.items(): github_versions[name] = tag_versions_by_id[info["graphql_id"]] components_supported_arch = { component.removesuffix("_checksums"): [a for a in archs.keys()] for component, archs in data.items() } new_versions = { c: { v for v in github_versions[c] if any( v > version and ( (v.major, v.minor) == (version.major, version.minor) or c.startswith("gvisor") ) for version in [ max(minors) for _, minors in groupby(cur_v, lambda v: (v.minor, v.major)) ] ) # only get: # - patch versions (no minor or major bump) (exception for gvisor which does not have a major.minor.patch scheme # - newer ones (don't get old patch version) } - set(cur_v) for component, archs in data.items() if (c := component.removesuffix("_checksums")) in downloads.keys() # this is only to bound cur_v in the scope and ( cur_v := sorted( Version(str(k)) for k in next(archs.values().__iter__()).keys() ) ) } hash_set_to_0 = { c: { Version(str(v)) for v, h in chain.from_iterable(a.items() for a in archs.values()) if h == 0 } for component, archs in data.items() if (c := component.removesuffix("_checksums")) in downloads.keys() } def get_hash(component: str, version: Version, arch: str): if component in download_hash_extract: hashes = _get_hash_by_arch(component, version) return hashes[arch] else: hash_file = s.get( downloads[component]["url"].format( version=version, os="linux", arch=arch, alt_arch=arch_alt_name[arch], ), allow_redirects=True, ) hash_file.raise_for_status() if downloads[component].get("binary", False): return hashlib.new( downloads[component].get("hashtype", "sha256"), hash_file.content ).hexdigest() return hash_file.content.decode().split()[0] for component, versions in chain(new_versions.items(), hash_set_to_0.items()): c = component + "_checksums" for arch in components_supported_arch[component]: for version in versions: data[c][arch][ str(version) ] = f"{downloads[component].get('hashtype', 'sha256')}:{get_hash(component, version, arch)}" data[c] = { arch: { v: versions[v] for v in sorted( versions.keys(), key=lambda v: Version(str(v)), reverse=True ) } for arch, versions in data[c].items() } with open(checksums_file, "w") as checksums_yml: yaml.dump(data, checksums_yml) logger.info("Updated %s", checksums_file) def main(): logging.basicConfig(stream=sys.stdout, level=logging.INFO) parser = argparse.ArgumentParser( description=f"Add new patch versions hashes in {CHECKSUMS_YML}", formatter_class=argparse.RawTextHelpFormatter, epilog=f""" This script only lookup new patch versions relative to those already existing in the data in {CHECKSUMS_YML}, which means it won't add new major or minor versions. In order to add one of these, edit {CHECKSUMS_YML} by hand, adding the new versions with a patch number of 0 (or the lowest relevant patch versions) and a hash value of 0. ; then run this script. Note that the script will try to add the versions on all architecture keys already present for a given download target. EXAMPLES: crictl_checksums: ... amd64: + 1.30.0: 0 1.29.0: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb 1.28.0: 8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508""", ) # Workaround for https://github.com/python/cpython/issues/53834#issuecomment-2060825835 # Fixed in python 3.14 class Choices(tuple): def __init__(self, _iterable=None, default=None): self.default = default or [] def __contains__(self, item): return super().__contains__(item) or item == self.default choices = Choices(components.infos.keys(), default=list(components.infos.keys())) parser.add_argument( "only", nargs="*", choices=choices, help="if provided, only obtain hashes for these compoments", default=choices.default, ) parser.add_argument( "-e", "--exclude", action="append", choices=components.infos.keys(), help="do not obtain hashes for this component", default=[], ) args = parser.parse_args() download_hash( {k: components.infos[k] for k in (set(args.only) - set(args.exclude))} ) ================================================ FILE: scripts/component_hash_update/src/component_hash_update/list_releases.graphql ================================================ query($with_releases: [ID!]!, $with_tags: [ID!]!) { with_releases: nodes(ids: $with_releases) { ... on Repository { releases(first: 100) { nodes { tagName isPrerelease } } } } with_tags: nodes(ids: $with_tags) { ... on Repository { refs(refPrefix: "refs/tags/", last: 25) { nodes { name } } } } } ================================================ FILE: scripts/galaxy_version.py ================================================ #!/usr/bin/env python import subprocess import ruamel.yaml import os last_tag = ( subprocess.Popen( ["git", "describe", "--tags", "--abbrev=0"], stdout=subprocess.PIPE ) .communicate()[0] .rstrip() .decode("utf-8") .removeprefix("v") .split(".") ) # Use CI provided base ref if available, else use HEAD to guess git_branch = os.getenv( "GITHUB_BASE_REF", ( subprocess.Popen( ["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE ) .communicate()[0] .rstrip() .decode("utf-8") ), ) if git_branch.startswith("release"): version_comp_index = 2 else: version_comp_index = 1 last_tag[version_comp_index] = str(int(last_tag[version_comp_index]) + 1) new_tag = ".".join(last_tag) yaml = ruamel.yaml.YAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.explicit_start = True with open( "galaxy.yml", ) as galaxy_yml: config = yaml.load(galaxy_yml) config["version"] = new_tag with open("galaxy.yml", "w") as galaxy_yml: yaml.dump(config, galaxy_yml) ================================================ FILE: scripts/gen_docs_sidebar.sh ================================================ #!/usr/bin/env bash # Generate documentation # This script generates a list of all the markdown files in the docs folder # and prints them in a markdown list format. # The script will print the name of the folder and the files inside it. # The script will also convert the folder and file names to a more human-readable format. # The script will ignore any files that are not markdown files. # Usage: bash scripts/gen_docs_sidebar.sh > docs/_sidebar.md export LANG=C { echo "* [Readme](/)" for folder in $(find docs/*/ | sort -f); do # Check if it is a directory if [ -d "$folder" ]; then subdir=$(basename "$folder") subdir=${subdir//_/ } # Replace "_" with empty string subdir=$(echo "$subdir" | awk '{for(i=1;i<=NF;i++)sub(/./,toupper(substr($i,1,1)),$i)}1') # Convert first letter of each word to uppercase if [ -n "$(find "$folder" -name '*.md' -type f)" ]; then echo "* $subdir" fi for file in $(find docs/"$(basename "$folder")"/*.md | sort -f); do if [ -f "$file" ]; then FILE=$(basename "$file" .md) FILE=${FILE//_/ } # Replace "_" with empty string FILE=$(echo "$FILE" | awk '{for(i=1;i<=NF;i++)sub(/./,toupper(substr($i,1,1)),$i)}1') # Convert first letter of each word to uppercase echo " * [$FILE](/$file)" fi done fi done } > docs/_sidebar.md ================================================ FILE: scripts/get_node_ids.sh ================================================ #!/bin/sh gh api graphql -H "X-Github-Next-Global-ID: 1" -f query='{ calicoctl_binary: repository(owner: "projectcalico", name: "calico") { id } ciliumcli_binary: repository(owner: "cilium", name: "cilium-cli") { id } crictl: repository(owner: "kubernetes-sigs", name: "cri-tools") { id } crio_archive: repository(owner: "cri-o", name: "cri-o") { id } etcd_binary: repository(owner: "etcd-io", name: "etcd") { id } kubectl: repository(owner: "kubernetes", name: "kubernetes") { id } nerdctl_archive: repository(owner: "containerd", name: "nerdctl") { id } runc: repository(owner: "opencontainers", name: "runc") { id } skopeo_binary: repository(owner: "lework", name: "skopeo-binary") { id } yq: repository(owner: "mikefarah", name: "yq") { id } youki: repository(owner: "youki-dev", name: "youki") { id } kubernetes: repository(owner: "kubernetes", name: "kubernetes") { id } cri_dockerd: repository(owner: "Mirantis", name: "cri-dockerd") { id } kata: repository(owner: "kata-containers", name: "kata-containers") { id } crun: repository(owner: "containers", name: "crun") { id } gvisor: repository(owner: "google", name: "gvisor") { id } argocd: repository(owner: "argoproj", name: "argo-cd") { id } }' ================================================ FILE: scripts/gitlab-runner.sh ================================================ #!/bin/sh docker run -d --name gitlab-runner --restart always -v /srv/gitlab-runner/cache:/srv/gitlab-runner/cache -v /srv/gitlab-runner/config:/etc/gitlab-runner -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:v1.10.0 # #/srv/gitlab-runner/config# cat config.toml #concurrent = 10 #check_interval = 1 #[[runners]] # name = "2edf3d71fe19" # url = "https://gitlab.com" # token = "THE TOKEN-CHANGEME" # executor = "docker" # [runners.docker] # tls_verify = false # image = "docker:latest" # privileged = true # disable_cache = false # cache_dir = "/srv/gitlab-runner/cache" # volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/srv/gitlab-runner/cache:/cache:rw"] # [runners.cache] ================================================ FILE: scripts/openstack-cleanup/.gitignore ================================================ openrc ================================================ FILE: scripts/openstack-cleanup/README.md ================================================ # openstack-cleanup Tool to deletes openstack servers older than a specific age (default 4h). Useful to cleanup orphan servers that are left behind when CI is manually cancelled or fails unexpectedly. ## Installation ```shell pip install -r requirements.txt python main.py --help ``` ## Usage ```console $ python main.py This will delete VMs... (ctrl+c to cancel) Will delete server example1 Will delete server example2 ``` ================================================ FILE: scripts/openstack-cleanup/main.py ================================================ #!/usr/bin/env python import argparse import openstack import logging import datetime import time log = logging.getLogger(__name__) parser = argparse.ArgumentParser(description="Cleanup OpenStack resources") parser.add_argument( "--hours", type=int, default=4, help="Age (in hours) of VMs to cleanup (default: 4h)", ) parser.add_argument("--dry-run", action="store_true", help="Do not delete anything") args = parser.parse_args() oldest_allowed = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( hours=args.hours ) def main(): logging.basicConfig(level=logging.INFO) if args.dry_run: log.info("Running in dry-run mode") conn = openstack.connect() log.info("Deleting servers...") map_if_old(conn.compute.delete_server, conn.compute.servers()) log.info("Deleting ports...") try: map_if_old(conn.network.delete_port, conn.network.ports()) except openstack.exceptions.ConflictException as ex: # Need to find subnet-id which should be removed from a router for sn in conn.network.subnets(): try: fn_if_old(conn.network.delete_subnet, sn) except openstack.exceptions.ConflictException: for r in conn.network.routers(): log.info("Deleting subnet %s from router %s", sn, r) try: conn.network.remove_interface_from_router(r, subnet_id=sn.id) except Exception as ex: log.error("Failed to delete subnet from router", exc_info=True) for ip in conn.network.ips(): fn_if_old(conn.network.delete_ip, ip) # After removing unnecessary subnet from router, retry to delete ports map_if_old(conn.network.delete_port, conn.network.ports()) log.info("Deleting security groups...") try: map_if_old(conn.network.delete_security_group, conn.network.security_groups()) except openstack.exceptions.ConflictException as ex: # Need to delete port when security groups is in used map_if_old(conn.network.delete_port, conn.network.ports()) map_if_old(conn.network.delete_security_group, conn.network.security_groups()) log.info("Deleting Subnets...") map_if_old(conn.network.delete_subnet, conn.network.subnets()) log.info("Deleting networks...") for n in conn.network.networks(): if not n.is_router_external: fn_if_old(conn.network.delete_network, n) log.info("Deleting keypairs...") map_if_old( conn.compute.delete_keypair, (conn.compute.get_keypair(x.name) for x in conn.compute.keypairs()), # LIST API for keypairs does not give us a created_at (WTF) ) # runs the given fn to all elements of the that are older than allowed def map_if_old(fn, items): for item in items: fn_if_old(fn, item) # run the given fn function only if the passed item is older than allowed def fn_if_old(fn, item): created_at = datetime.datetime.fromisoformat(item.created_at) if created_at.tzinfo is None: created_at = created_at.replace(tzinfo=datetime.timezone.utc) # Handle TZ unaware object by assuming UTC # Can't compare to oldest_allowed otherwise if item.name == "default": # skip default security group return if created_at < oldest_allowed: log.info("Will delete %s %s)", item.name, item.id) if not args.dry_run: fn(item) if __name__ == "__main__": # execute only if run as a script main() ================================================ FILE: scripts/openstack-cleanup/requirements.txt ================================================ openstacksdk>=0.43.0 six ================================================ FILE: scripts/pipeline.Dockerfile.j2 ================================================ # Use immutable image tags rather than mutable tags (like ubuntu:24.04) FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b # Some tools like yamllint need this # Pip needs this as well at the moment to install ansible # (and potentially other packages) # See: https://github.com/pypa/pip/issues/10219 ENV VAGRANT_VERSION=2.4.1 \ VAGRANT_DEFAULT_PROVIDER=libvirt \ VAGRANT_ANSIBLE_TAGS=facts \ LANG=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 RUN apt update -q \ && apt install -yq \ libssl-dev \ python3-dev \ python3-pip \ sshpass \ apt-transport-https \ jq \ moreutils \ libvirt-dev \ openssh-client \ rsync \ git \ ca-certificates \ curl \ gnupg2 \ unzip \ libvirt-clients \ qemu-utils \ qemu-kvm \ dnsmasq \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | tee /etc/apt/sources.list.d/docker.list \ && apt update -q \ && apt install --no-install-recommends -yq docker-ce \ && apt autoremove -yqq --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* WORKDIR /kubespray ADD ./requirements.txt /kubespray/requirements.txt ADD ./tests/requirements.txt /kubespray/tests/requirements.txt RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \ && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \ && curl -L https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ # Install Vagrant && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \ && vagrant plugin install vagrant-libvirt \ # Install Kubernetes collections && pip install --break-system-packages --no-compile --no-cache-dir kubernetes \ && ansible-galaxy collection install kubernetes.core ================================================ FILE: scripts/propagate_ansible_variables.yml ================================================ #!/usr/bin/env ansible-playbook --- - name: Update README.md versions hosts: localhost connection: local gather_facts: false vars: fallback_ip: 'bypass tasks in kubespray_defaults' roles: - kubespray_defaults tasks: - name: Include versions not in kubespray_defaults include_vars: "{{ item }}" loop: - ../roles/container-engine/docker/defaults/main.yml - ../roles/kubernetes/node/defaults/main.yml - ../roles/kubernetes-apps/argocd/defaults/main.yml - name: Render versions in README.md blockinfile: marker: '' block: "\n{{ lookup('ansible.builtin.template', 'readme_versions.md.j2') }}\n\n" path: ../README.md - name: Render Dockerfiles template: src: "{{ item }}.j2" dest: "../{{ item }}" mode: "0644" loop: - 'pipeline.Dockerfile' - 'Dockerfile' ================================================ FILE: scripts/readme_versions.md.j2 ================================================ - Core - [kubernetes](https://github.com/kubernetes/kubernetes) {{ kube_version }} - [etcd](https://github.com/etcd-io/etcd) {{ etcd_version }} - [docker](https://www.docker.com/) {{ docker_version }} - [containerd](https://containerd.io/) {{ containerd_version }} - [cri-o](http://cri-o.io/) {{ crio_version }} (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) {{ cni_version }} - [calico](https://github.com/projectcalico/calico) {{ calico_version }} - [cilium](https://github.com/cilium/cilium) {{ cilium_version }} - [flannel](https://github.com/flannel-io/flannel) {{ flannel_version }} - [kube-ovn](https://github.com/alauda/kube-ovn) {{ kube_ovn_version }} - [kube-router](https://github.com/cloudnativelabs/kube-router) {{ kube_router_version }} - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) {{ multus_version }} - [kube-vip](https://github.com/kube-vip/kube-vip) {{ kube_vip_version }} - Application - [cert-manager](https://github.com/jetstack/cert-manager) {{ cert_manager_version }} - [coredns](https://github.com/coredns/coredns) {{ coredns_version }} - [argocd](https://argoproj.github.io/) {{ argocd_version }} - [helm](https://helm.sh/) {{ helm_version }} - [metallb](https://metallb.universe.tf/) {{ metallb_version }} - [registry](https://github.com/distribution/distribution) {{ registry_version }} - Storage Plugin - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) {{ aws_ebs_csi_plugin_version }} - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) {{ azure_csi_plugin_version }} - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) {{ cinder_csi_plugin_version }} - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) {{ gcp_pd_csi_plugin_version }} - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) {{ local_path_provisioner_version }} - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) {{ local_volume_provisioner_version }} - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) {{ node_feature_discovery_version }} ================================================ FILE: test-infra/image-builder/Makefile ================================================ deploy: ansible-playbook -i hosts.ini -e docker_password=$(docker_password) cluster.yml ================================================ FILE: test-infra/image-builder/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners approvers: - yankay - ant31 reviewers: - yankay - ant31 ================================================ FILE: test-infra/image-builder/README.md ================================================ # KubeVirt Image Builder Build and push KubeVirt VM disk images to quay.io for Kubespray CI testing. ## How It Works The Ansible playbook downloads upstream cloud images, converts them to qcow2, resizes (+8G), wraps each in a Docker image based on `kubevirt/registry-disk-v1alpha`, and pushes to `quay.io/kubespray/vm-:`. ## Prerequisites - Docker, `qemu-img`, Ansible - Push access to [quay.io/kubespray](https://quay.io/organization/kubespray) (robot account `kubespray+buildvmimages`) ## Image Definitions All OS images are defined in [`roles/kubevirt-images/defaults/main.yml`](roles/kubevirt-images/defaults/main.yml). Each entry specifies: | Field | Description | |-------|-------------| | `filename` | Downloaded file name | | `url` | Upstream cloud image URL | | `checksum` | Checksum for download verification | | `converted` | `true` if the source is already qcow2, `false` if conversion is needed | | `tag` | Docker image tag (usually `latest`) | ## Usage ### Build and push all images ```bash cd test-infra/image-builder/ make docker_password= ``` ### Add a new OS image 1. Add a new entry to `roles/kubevirt-images/defaults/main.yml`: ```yaml new-os-name: filename: cloud-image-file.qcow2 url: https://example.com/cloud-image-file.qcow2 checksum: sha256: converted: true tag: "latest" ``` 2. Build and push the image: ```bash make docker_password= ``` 3. Submit a PR with the `defaults/main.yml` change so CI can use the new image. See [#12379](https://github.com/kubernetes-sigs/kubespray/pull/12379) for an example. ================================================ FILE: test-infra/image-builder/cluster.yml ================================================ --- - name: Build kubevirt images hosts: image-builder gather_facts: false roles: - kubevirt-images ================================================ FILE: test-infra/image-builder/hosts.ini ================================================ image-builder-1 ansible_ssh_host=xxx.xxx.xxx.xxx [image-builder] image-builder-1 ================================================ FILE: test-infra/image-builder/roles/kubevirt-images/defaults/main.yml ================================================ --- images_dir: /images/base docker_user: kubespray+buildvmimages docker_host: quay.io registry: quay.io/kubespray images: ubuntu-2004: filename: focal-server-cloudimg-amd64.img url: https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64-disk-kvm.img checksum: sha256:8faf1f5a27c956ad0c49dac3114a355fbaf1b2d21709e10a18e67213fbb95b81 converted: false tag: "latest" ubuntu-2204: filename: jammy-server-cloudimg-amd64.img url: https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img checksum: sha256:d3f3f446bf35b2e58b82c10c8fa65525264efe5b0e398238f00ab670f49528ab converted: false tag: "latest" ubuntu-2404: filename: noble-server-cloudimg-amd64.img url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img checksum: sha256:0cf56a2b23b430c350311dbcb9221b64823a5f7a401b5cf6ab4821f2ffdabe76 converted: false tag: "latest" fedora-37: filename: Fedora-Cloud-Base-37-1.7.x86_64.qcow2 url: https://download.fedoraproject.org/pub/fedora/linux/releases/37/Cloud/x86_64/images/Fedora-Cloud-Base-37-1.7.x86_64.qcow2 checksum: sha256:b5b9bec91eee65489a5745f6ee620573b23337cbb1eb4501ce200b157a01f3a0 converted: true tag: "latest" fedora-38: filename: Fedora-Cloud-Base-38-1.6.x86_64.qcow2 url: https://download.fedoraproject.org/pub/fedora/linux/releases/38/Cloud/x86_64/images/Fedora-Cloud-Base-38-1.6.x86_64.qcow2 checksum: sha256:d334670401ff3d5b4129fcc662cf64f5a6e568228af59076cc449a4945318482 converted: true tag: "latest" fedora-39: filename: Fedora-Cloud-Base-39-1.5.x86_64.qcow2 url: https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2 checksum: sha256:ab5be5058c5c839528a7d6373934e0ce5ad6c8f80bd71ed3390032027da52f37 converted: true tag: "latest" fedora-40: filename: Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2 url: https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2 checksum: sha256:ac58f3c35b73272d5986fa6d3bc44fd246b45df4c334e99a07b3bbd00684adee converted: true tag: "latest" fedora-41: filename: Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2 url: https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2 checksum: sha256:6205ae0c524b4d1816dbd3573ce29b5c44ed26c9fbc874fbe48c41c89dd0bac2 converted: true tag: "latest" fedora-42: filename: Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2 url: https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2 checksum: sha256:e401a4db2e5e04d1967b6729774faa96da629bcf3ba90b67d8d9cce9906bec0f converted: true tag: "latest" fedora-coreos: filename: fedora-coreos-32.20200601.3.0-openstack.x86_64.qcow2.xz url: https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200601.3.0/x86_64/fedora-coreos-32.20200601.3.0-openstack.x86_64.qcow2.xz checksum: sha256:fe78c348189d745eb5f6f80ff9eb2af67da8e84880d264f4301faaf7c2a72646 converted: true tag: "latest" centos-8: filename: CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2 url: http://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2 checksum: sha256:7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0 converted: true tag: "latest" almalinux-8: filename: AlmaLinux-8-GenericCloud-latest.x86_64.qcow2 url: https://repo.almalinux.org/almalinux/8.9/cloud/x86_64/images/AlmaLinux-8-GenericCloud-8.9-20231128.x86_64.qcow2 checksum: sha256:a1686bc537bce699b512e3233666f5b8f69ed797ff1ce0af52c17fdc52942621 converted: true tag: "latest" almalinux-9: filename: AlmaLinux-9-GenericCloud-9.5-20241120.x86_64.qcow2 url: https://repo.almalinux.org/almalinux/9.5/cloud/x86_64/images/AlmaLinux-9-GenericCloud-9.5-20241120.x86_64.qcow2 checksum: sha256:abddf01589d46c841f718cec239392924a03b34c4fe84929af5d543c50e37e37 converted: true tag: "latest" rockylinux-8: filename: Rocky-8-GenericCloud-8.6-20220515.x86_64.qcow2 url: https://download.rockylinux.org/pub/rocky/8.6/images/Rocky-8-GenericCloud-8.6-20220515.x86_64.qcow2 checksum: sha256:77e79f487c70f6bfa5655d8084e02cb8d31900a2c2a22b2334c3401b40a1231c converted: true tag: "latest" rockylinux-9: filename: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2 url: https://download.rockylinux.org/pub/rocky/9.5/images/x86_64/Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2 checksum: sha256:069493fdc807300a22176540e9171fcff2227a92b40a7985a0c1c9e21aeebf57 converted: true tag: "latest" rockylinux-10: filename: Rocky-10-GenericCloud-Base-10.0-20250609.1.x86_64.qcow2 url: https://download.rockylinux.org/pub/rocky/10.0/images/x86_64/Rocky-10-GenericCloud-Base-10.0-20250609.1.x86_64.qcow2 checksum: sha256:20e771c654724e002c32fb92a05fdfdd7ac878c192f50e2fc21f53e8f098b8f9 converted: true tag: "latest" # rockylinux-10-extra: # default cloud image doesn't included `kernel-modules-extra`. How to build RockyLinux 10 + `kernel-module-extra` with dib # https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093 debian-10: filename: debian-10-openstack-amd64.qcow2 url: https://cdimage.debian.org/cdimage/openstack/current-10/debian-10-openstack-amd64.qcow2 checksum: sha512:296ad8345cb49e52464a0cb8bf4365eb0b9e4220c47ebdd73d134d51effc756d5554aee15027fffd038fef4ad5fa984c94208bce60572d58b2ab26f74bb2a5de converted: true tag: "latest" debian-11: filename: debian-11-generic-amd64-20210814-734.qcow2 url: https://cdimage.debian.org/cdimage/cloud/bullseye/20210814-734/debian-11-generic-amd64-20210814-734.qcow2 checksum: sha512:ed680265ce925e3e02336b052bb476883e2d3b023f7b7d39d064d58ba5f1856869f75dca637c27c0303b731d082ff23a7e45ea2e3f9bcb8f3c4ce0c24332885d converted: true tag: "latest" debian-12: filename: debian-12-generic-amd64-20230612-1409.qcow2 url: https://cdimage.debian.org/cdimage/cloud/bookworm/20230612-1409/debian-12-generic-amd64-20230612-1409.qcow2 checksum: sha512:61358292dbec302446a272d5011019091ca78e3fe8878b2d67d31b32e0661306c53a72f793f109394daf937a3db7b2db34422d504e07fdbb300a7bf87109fcf1 converted: true tag: "latest" debian-13: filename: debian-13-generic-amd64-20250806-2196.qcow2 url: https://cdimage.debian.org/cdimage/cloud/trixie/20250806-2196/debian-13-generic-amd64-20250806-2196.qcow2 checksum: sha512:a7dfe434afc40afb0a791c777f3edba6b1a5c4b7315a61073fe5e34752d3bc5fd44ff67ef054eb4263687a97f7ce4896bf5bad5f216ef8b9d4a84541759e743d converted: true tag: "latest" oracle-7: filename: oracle-linux-76.qcow2 url: https://storage.googleapis.com/born-images/oracle76/oracle-linux-76.qcow2 checksum: sha256:f396c03e907fa2a0c94d6807b9f62622f23ee3499df4456ae2a15da381fbdca5 converted: true tag: "latest" opensuse-leap-15: filename: openSUSE-Leap-15.3.x86_64-1.0.1-NoCloud-Build2.63.qcow2 url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.3/images/openSUSE-Leap-15.3.x86_64-1.0.1-NoCloud-Build2.63.qcow2 checksum: sha256:289248945e2d058551c71c1bdcb31a361cefe7136c7fd88a09b524eedfaf5215 converted: true tag: "latest" opensuse-leap-15-6: filename: openSUSE-Leap-15.6.x86_64-1.0.1-NoCloud-Build1.177.qcow2 url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.6/images/openSUSE-Leap-15.6.x86_64-1.0.1-NoCloud-Build1.177.qcow2 checksum: sha256:9ecd197b34faf1b43627946d0c26e38b5c3058207d1c86c4784b8f765c3289f3 converted: true tag: "latest" openeuler-2203: filename: openEuler-22.03-LTS-SP4-x86_64.qcow2.xz url: https://mirrors.ocf.berkeley.edu/openeuler/openEuler-22.03-LTS-SP4/virtual_machine_img/x86_64/openEuler-22.03-LTS-SP4-x86_64.qcow2.xz checksum: sha256:2dc9c9d73d172af0443d1a17786ee2e387006b9912fad24b5497ef103ecd7afb converted: true tag: "latest" openeuler-2403: filename: openEuler-24.03-LTS-x86_64.qcow2.xz url: https://mirrors.ocf.berkeley.edu/openeuler/openEuler-24.03-LTS/virtual_machine_img/x86_64/openEuler-24.03-LTS-x86_64.qcow2.xz checksum: sha256:c6af522d36d659b66da668cc4eb86b032a9cff05a95a0e37505a63e70ed585dc converted: true tag: "latest" flatcar-4081: filename: flatcar_production_kubevirt_image.qcow2 url: https://stable.release.flatcar-linux.net/amd64-usr/4081.2.1/flatcar_production_kubevirt_image.qcow2 checksum: sha512:6999ef068380c9842e4caf7afc2a1c66d4d03309f7bfa2f5f500757c36d1f935961f5662cc69376aa3d701e4c2d264f4356d4daadbb68e55becb710067e22c5d converted: true tag: latest ================================================ FILE: test-infra/image-builder/roles/kubevirt-images/tasks/main.yml ================================================ --- - name: Create image directory file: state: directory path: "{{ images_dir }}" mode: "0755" - name: Download images files get_url: url: "{{ item.value.url }}" dest: "{{ images_dir }}/{{ item.value.filename }}" checksum: "{{ item.value.checksum }}" mode: "0644" loop: "{{ images | dict2items }}" - name: Unxz compressed images command: unxz --force {{ images_dir }}/{{ item.value.filename }} loop: "{{ images | dict2items }}" when: - item.value.filename.endswith('.xz') - name: Convert images which is not in qcow2 format command: qemu-img convert -O qcow2 {{ images_dir }}/{{ item.value.filename.rstrip('.xz') }} {{ images_dir }}/{{ item.key }}.qcow2 loop: "{{ images | dict2items }}" when: - not (item.value.converted | bool) - name: Make sure all images are ending with qcow2 command: cp {{ images_dir }}/{{ item.value.filename.rstrip('.xz') }} {{ images_dir }}/{{ item.key }}.qcow2 loop: "{{ images | dict2items }}" when: - item.value.converted | bool - name: Resize images command: qemu-img resize {{ images_dir }}/{{ item.key }}.qcow2 +8G loop: "{{ images | dict2items }}" # STEP 2: Include the images inside a container - name: Template default Dockerfile template: src: Dockerfile dest: "{{ images_dir }}/Dockerfile" mode: "0644" - name: Create docker images for each OS command: docker build -t {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} --build-arg cloud_image="{{ item.key }}.qcow2" {{ images_dir }} loop: "{{ images | dict2items }}" - name: Docker login command: docker login -u="{{ docker_user }}" -p="{{ docker_password }}" "{{ docker_host }}" - name: Docker push image command: docker push {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} loop: "{{ images | dict2items }}" - name: Docker logout command: docker logout -u="{{ docker_user }}" "{{ docker_host }}" ================================================ FILE: test-infra/image-builder/roles/kubevirt-images/templates/Dockerfile ================================================ FROM kubevirt/registry-disk-v1alpha ARG cloud_image LABEL org.opencontainers.image.authors="The Kubespray Project " COPY $cloud_image /disk ================================================ FILE: test-infra/vagrant-docker/Dockerfile ================================================ # Docker image published at quay.io/kubespray/vagrant ARG KUBESPRAY_VERSION FROM quay.io/kubespray/kubespray:${KUBESPRAY_VERSION} ENV VAGRANT_VERSION=2.3.7 ENV VAGRANT_DEFAULT_PROVIDER=libvirt ENV VAGRANT_ANSIBLE_TAGS=facts RUN apt-get update && apt-get install -y wget libvirt-dev openssh-client rsync git build-essential # Install Vagrant RUN wget https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_amd64.deb && \ dpkg -i vagrant_${VAGRANT_VERSION}-1_amd64.deb && \ rm vagrant_${VAGRANT_VERSION}-1_amd64.deb && \ vagrant plugin install vagrant-libvirt ================================================ FILE: test-infra/vagrant-docker/README.md ================================================ # vagrant docker image This image is used for the vagrant CI jobs. It is using the libvirt driver. ## Usage ```console $ docker run --net host --rm -it -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock quay.io/kubespray/vagrant $ vagrant up Bringing machine 'k8s-1' up with 'libvirt' provider... Bringing machine 'k8s-2' up with 'libvirt' provider... Bringing machine 'k8s-3' up with 'libvirt' provider... [...] ``` ## Cache You can set `/root/kubespray_cache` as a volume to keep cache between runs. ## Building ```shell ./build.sh v2.12.5 ``` ================================================ FILE: test-infra/vagrant-docker/build.sh ================================================ #!/bin/sh set -euo pipefail if [ "$#" -ne 1 ]; then echo "Usage: $0 tag" >&2 exit 1 fi VERSION="$1" IMG="quay.io/kubespray/vagrant:${VERSION}" docker build . --build-arg "KUBESPRAY_VERSION=${VERSION}" --tag "$IMG" docker push "$IMG" ================================================ FILE: tests/Makefile ================================================ create-tf: ./scripts/create-tf.sh delete-tf: ./scripts/delete-tf.sh $(ANSIBLE_INVENTORY): mkdir $@ create-packet: | $(ANSIBLE_INVENTORY) ansible-playbook cloud_playbooks/create-kubevirt.yml -c local \ -e @"files/${CI_JOB_NAME}.yml" delete-packet: ; create-vagrant: | $(ANSIBLE_INVENTORY) vagrant up cp $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $| delete-vagrant: vagrant destroy -f ================================================ FILE: tests/ansible.cfg ================================================ [ssh_connection] pipelining=True ansible_ssh_common_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 retries=2 [defaults] forks = 20 host_key_checking=False gathering = smart fact_caching = jsonfile fact_caching_connection = /tmp stdout_callback = default display_skipped_hosts = no library = ./library:../library callbacks_enabled = profile_tasks jinja2_extensions = jinja2.ext.do roles_path = ../roles ================================================ FILE: tests/cloud_playbooks/create-kubevirt.yml ================================================ --- - name: Provision Packet VMs hosts: localhost gather_facts: false become: true tasks: - name: Create Kubevirt VMs import_role: name: packet-ci - name: Update inventory for Molecule meta: refresh_inventory - name: Wait until SSH is available hosts: all become: false gather_facts: false tasks: # Check ssh access without relying on python - this is an horrible hack # but wait_for_connection does not work without python # and 'until' is incompatible with unreachable errors # https://github.com/ansible/ansible/issues/78358 - name: Wait until SSH is available command: > ssh -i "{{ lookup('env', 'ANSIBLE_PRIVATE_KEY_FILE') }}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=3 "{{ lookup('env', 'ANSIBLE_REMOTE_USER') }}@{{ ansible_host }}" register: ssh_command delay: 0 until: ssh_command.rc != 255 retries: 60 delegate_to: localhost ================================================ FILE: tests/cloud_playbooks/roles/packet-ci/defaults/main.yml ================================================ --- # VM sizing vm_cpu_cores: 2 vm_cpu_sockets: 1 vm_cpu_threads: 2 vm_memory: 4096 releases_disk_size: 2Gi # Request/Limit allocation settings cpu_allocation_ratio: 0.25 memory_allocation_ratio: 1 # Deployment mode mode: default node_groups: - 'all' cluster_layout: "{{ molecule_yml.platforms | d(scenarios[mode]) }}" ================================================ FILE: tests/cloud_playbooks/roles/packet-ci/tasks/main.yml ================================================ --- - name: Generate SSH keypair community.crypto.openssh_keypair: size: 2048 path: "{{ lookup('env', 'ANSIBLE_PRIVATE_KEY_FILE') }}" mode: '400' register: ssh_key - name: Start vms for CI job kubernetes.core.k8s: definition: "{{ lookup('template', 'vm.yml.j2', template_vars=item) }}" loop: "{{ cluster_layout }}" loop_control: index_var: index - name: Wait for vms to have IP addresses kubernetes.core.k8s_info: api_version: kubevirt.io/v1 kind: VirtualMachineInstance label_selectors: - "ci_job_id={{ ci_job_id }}" namespace: "{{ pod_namespace }}" register: vmis until: vmis.resources | map(attribute='status.interfaces.0') | rejectattr('ipAddress', 'defined') == [] retries: 30 delay: 10 - name: Massage VirtualMachineInstance data into an Ansible inventory structure vars: ips: "{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}" names: "{{ vmis.resources | map(attribute='metadata.annotations.inventory_name') }}" _groups: "{{ (vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ','))}}" vm_hosts: "{{ ips | zip(_groups, names) | map('zip', ['ansible_host', 'ansible_groups', 'inventory_name']) | map('map', 'reverse') | map('community.general.dict') }}" loop: "{{ vm_hosts | map(attribute='ansible_groups') | flatten | unique }}" set_fact: ci_inventory: "{{ ci_inventory|d({}) | combine({ item: { 'hosts': vm_hosts | selectattr('ansible_groups', 'contains', item) | rekey_on_member('inventory_name') } }) }}" - name: Create inventory for CI tests copy: content: "{{ ci_inventory | to_yaml }}" dest: "{{ ansible_inventory_sources[0] }}/ci_inventory.yml" mode: "0644" ================================================ FILE: tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 ================================================ --- apiVersion: kubevirt.io/v1 kind: VirtualMachineInstance metadata: generateName: {{ cloud_image }}- namespace: {{ pod_namespace }} annotations: kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2" ansible_groups: "{{ node_groups | join(',') }}" inventory_name: "{{ name | d(cloud_image ~ '-' ~ index) }}" # This does not use a dns prefix because dots are hard to escape with map(attribute=) in Jinja labels: kubevirt.io/os: {{ cloud_image }} kubevirt.io/size: small ci_job_id: "{{ ci_job_id }}" ci_job_name: "{{ lookup('ansible.builtin.env', 'CI_JOB_NAME_SLUG') }}" ci_pipeline_id: "{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}" ci_pr_id: "{{ lookup('ansible.builtin.env', 'PR_ID') }}" # leverage the Kubernetes GC for resources cleanup ownerReferences: - apiVersion: v1 kind: Pod name: "{{ pod_name }}" uid: "{{ pod_uid }}" spec: domain: devices: blockMultiQueue: true disks: - disk: bus: virtio name: containervolume cache: writethrough - disk: bus: virtio name: cloudinitvolume - disk: bus: virtio name: releases serial: '2825A83CBDC8A32D5E' interfaces: - name: default bridge: {} cpu: cores: {{ vm_cpu_cores }} sockets: {{ vm_cpu_sockets }} threads: {{ vm_cpu_threads }} resources: requests: memory: "{{ vm_memory * memory_allocation_ratio }}Mi" cpu: {{ vm_cpu_cores * cpu_allocation_ratio }} limits: memory: "{{ vm_memory }}Mi" cpu: {{ vm_cpu_cores }} networks: - name: default pod: {} terminationGracePeriodSeconds: 0 volumes: - name: containervolume containerDisk: image: quay.io/kubespray/vm-{{ cloud_image }} - name: cloudinitvolume cloudInit{{ 'ConfigDrive' if cloud_image.startswith('flatcar') else 'NoCloud' }}: userDataBase64: '{{ ((ignition_config | to_json) if cloud_image.startswith('flatcar') else cloudinit_config) | b64encode }}' - name: releases emptyDisk: capacity: '{{ releases_disk_size }}' ================================================ FILE: tests/cloud_playbooks/roles/packet-ci/vars/main.yml ================================================ --- # This is a list of nodes with groups for each scenario/cluster layouts scenarios: separate: - node_groups: ['kube_control_plane'] - node_groups: ['kube_node'] - node_groups: ['etcd'] ha: - node_groups: ['kube_control_plane', 'etcd'] - node_groups: ['kube_control_plane', 'etcd'] - node_groups: ['kube_node', 'etcd'] default: - node_groups: ['kube_control_plane', 'etcd'] - node_groups: ['kube_node'] all-in-one: - node_groups: ['kube_control_plane', 'etcd', 'kube_node'] ha-recover: - node_groups: ['kube_control_plane', 'etcd'] - node_groups: ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd'] - node_groups: ['kube_node', 'etcd'] ha-recover-noquorum: - node_groups: ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd'] - node_groups: ['kube_control_plane', 'etcd', 'broken_kube_control_plane', 'broken_etcd'] - node_groups: ['kube_node', 'etcd'] node-etcd-client: - node_groups: ['kube_node', 'kube_control_plane', 'etcd'] - node_groups: ['kube_node', 'etcd'] - node_groups: ['kube_node', 'etcd'] - node_groups: ['kube_node'] # Get pod metadata / CI vars from environment ci_job_id: "{{ lookup('ansible.builtin.env', 'CI_JOB_ID', default=undefined) }}" pod_name: "{{ lookup('ansible.builtin.env', 'POD_NAME', default=undefined) }}" pod_uid: "{{ lookup('ansible.builtin.env', 'POD_UID', default=undefined) }}" pod_namespace: "{{ lookup('ansible.builtin.env', 'POD_NAMESPACE', default=undefined) }}" cloudinit_config: | #cloud-config users: - name: {{ lookup('env', 'ANSIBLE_REMOTE_USER') }} sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash lock_passwd: False ssh_authorized_keys: - {{ ssh_key.public_key }} fs_setup: - device: '/dev/disk/by-id/virtio-2825A83CBDC8A32D5E' filesystem: 'ext4' partition: 'none' mounts: - ['/dev/disk/by-id/virtio-2825A83CBDC8A32D5E', '/tmp/releases'] runcmd: - chmod 777 /tmp/releases ignition_config: ignition: version: "3.2.0" passwd: users: - name: "{{ lookup('env', 'ANSIBLE_REMOTE_USER') }}" groups: - sudo - wheel sshAuthorizedKeys: - "{{ ssh_key.public_key }}" storage: filesystems: - device: '/dev/disk/by-id/virtio-2825A83CBDC8A32D5E' format: ext4 path: /tmp/releases wipeFilesystem: true directories: - path: /tmp/releases # ignition require a integer, so using the octal notation is easier # than noting it in decimal form # yamllint disable-line rule:octal-values mode: 0777 ================================================ FILE: tests/common_vars.yml ================================================ --- # Kubespray settings for tests dns_min_replicas: 1 unsafe_show_logs: true bin_dir: "{{ '/opt/bin' if ansible_os_family == 'Flatcar' else '/usr/local/bin' }}" # Registry mirrors settings docker_registry_mirrors: - "https://mirror.gcr.io" containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - host: https://registry-1.docker.io capabilities: ["pull", "resolve"] skip_verify: false crio_registries: - prefix: docker.io insecure: false blocked: false unqualified: false location: registry-1.docker.io mirrors: - location: mirror.gcr.io insecure: false nginx_image_repo: "{{ quay_image_repo }}/kubespray/nginx" flannel_image_repo: "{{ quay_image_repo }}/kubespray/flannel" flannel_init_image_repo: "{{ quay_image_repo }}/kubespray/flannel-cni-plugin" local_release_dir: "{{ '/tmp/releases' if inventory_hostname != 'localhost' else (lookup('env', 'PWD') + '/downloads') }}" hydrophone_version: "0.7.0" hydrophone_arch: "x86_64" hydrophone_checksum: "sha256:15a6c09962f9bd4a1587af068b5edef1072327a77012d8fbb84992c7c87c0475" hydrophone_parallel: 1 hydrophone_path: "{{ bin_dir }}/hydrophone" ================================================ FILE: tests/files/almalinux9-calico-ha-ebpf.yml ================================================ --- # Instance settings cloud_image: almalinux-9 mode: ha vm_memory: 3072 # Kubespray settings calico_bpf_enabled: true loadbalancer_apiserver_localhost: true auto_renew_certificates: true ================================================ FILE: tests/files/almalinux9-calico-nodelocaldns-secondary.yml ================================================ --- # Instance settings cloud_image: almalinux-9 vm_memory: 3072 # Kubespray settings enable_nodelocaldns_secondary: true loadbalancer_apiserver_type: haproxy ================================================ FILE: tests/files/almalinux9-calico-remove-node ================================================ REMOVE_NODE_CHECK=true REMOVE_NODE_NAME=instance-3 ================================================ FILE: tests/files/almalinux9-calico-remove-node.yml ================================================ --- # Instance settings cloud_image: almalinux-9 mode: ha # Kubespray settings auto_renew_certificates: true ================================================ FILE: tests/files/almalinux9-calico.yml ================================================ --- # Instance settings cloud_image: almalinux-9 vm_memory: 3072 # Kubespray settings metrics_server_enabled: true loadbalancer_apiserver_type: haproxy local_path_provisioner_enabled: true kube_proxy_mode: nftables # NTP management ntp_enabled: true ntp_package: chrony ntp_timezone: Etc/UTC ntp_manage_config: true ntp_tinker_panic: true ntp_force_sync_immediately: true # Scheduler plugins scheduler_plugins_enabled: true ================================================ FILE: tests/files/almalinux9-crio.yml ================================================ --- # Instance settings cloud_image: almalinux-9 # Kubespray settings container_manager: crio auto_renew_certificates: true ================================================ FILE: tests/files/almalinux9-docker.yml ================================================ --- # Instance settings cloud_image: almalinux-9 vm_memory: 3072 # Use docker container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns ================================================ FILE: tests/files/almalinux9-kube-ovn.yml ================================================ --- # Instance settings cloud_image: almalinux-9 vm_memory: 3072 # Kubespray settings kube_network_plugin: kube-ovn ================================================ FILE: tests/files/amazon-linux-2-all-in-one.yml ================================================ --- # Instance settings cloud_image: amazon-linux-2 mode: all-in-one # Workaround for RHEL8: kernel version 4.18 is lower than Kubernetes system verification. kubeadm_ignore_preflight_errors: - SystemVerification ================================================ FILE: tests/files/custom_cni/README.md ================================================ # Custom CNI manifest generation As an example we are using Cilium for testing the network_plugins/custom_cni. To update the generated manifests to the latest version do the following: ```sh helm repo add cilium https://helm.cilium.io/ helm repo update helm template cilium/cilium -n kube-system -f values.yaml > cilium.yaml ``` ================================================ FILE: tests/files/custom_cni/cilium.yaml ================================================ --- # Source: cilium/templates/cilium-secrets-namespace.yaml apiVersion: v1 kind: Namespace metadata: name: "cilium-secrets" labels: app.kubernetes.io/part-of: cilium annotations: --- # Source: cilium/templates/cilium-agent/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: "cilium" namespace: kube-system --- # Source: cilium/templates/cilium-envoy/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: "cilium-envoy" namespace: kube-system --- # Source: cilium/templates/cilium-operator/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: "cilium-operator" namespace: kube-system --- # Source: cilium/templates/cilium-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: cilium-config namespace: kube-system data: # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd", "kvstore" or # "doublewrite-readkvstore" / "doublewrite-readcrd". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). # These can be queried with: # kubectl get ciliumid # - "kvstore" stores identities in an etcd kvstore, that is # configured below. Cilium versions before 1.6 supported only the kvstore # backend. Upgrades from these older cilium versions should continue using # the kvstore by commenting out the identity-allocation-mode below, or # setting it to "kvstore". # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful # for seamless migrations from the kvstore mode to the crd mode. Consult the # documentation for more information on how to perform the migration. identity-allocation-mode: crd identity-heartbeat-timeout: "30m0s" identity-gc-interval: "15m0s" cilium-endpoint-gc-interval: "5m0s" nodes-gc-interval: "5m0s" # If you want to run cilium in debug mode change this value to true debug: "false" debug-verbose: "" metrics-sampling-interval: "5m" # The agent can be put into the following three policy enforcement modes # default, always and never. # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes enable-policy: "default" policy-cidr-match-mode: "" # If you want metrics enabled in cilium-operator, set the port for # which the Cilium Operator will have their metrics exposed. # NOTE that this will open the port on the nodes where Cilium operator pod # is scheduled. operator-prometheus-serve-addr: ":9963" enable-metrics: "true" enable-policy-secrets-sync: "true" policy-secrets-only-from-secrets-namespace: "true" policy-secrets-namespace: "cilium-secrets" # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "true" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. enable-ipv6: "false" # Users who wish to specify their own custom CNI configuration file must set # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. custom-cni-conf: "false" enable-bpf-clock-probe: "false" # If you want cilium monitor to aggregate tracing for packets, set this level # to "low", "medium", or "maximum". The higher the level, the less packets # that will be seen in monitor output. monitor-aggregation: medium # The monitor aggregation interval governs the typical time between monitor # notification events for each allowed connection. # # Only effective when monitor aggregation is set to "medium" or higher. monitor-aggregation-interval: "5s" # The monitor aggregation flags determine which TCP flags which, upon the # first observation, cause monitor notifications to be generated. # # Only effective when monitor aggregation is set to "medium" or higher. monitor-aggregation-flags: all # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. bpf-map-dynamic-size-ratio: "0.0025" # bpf-policy-map-max specifies the maximum number of entries in endpoint # policy map (per endpoint) bpf-policy-map-max: "16384" # bpf-policy-stats-map-max specifies the maximum number of entries in global # policy stats map bpf-policy-stats-map-max: "65536" # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, # backend and affinity maps. bpf-lb-map-max: "65536" bpf-lb-external-clusterip: "false" bpf-lb-source-range-all-types: "false" bpf-lb-algorithm-annotation: "false" bpf-lb-mode-annotation: "false" bpf-distributed-lru: "false" bpf-events-drop-enabled: "true" bpf-events-policy-verdict-enabled: "true" bpf-events-trace-enabled: "true" # Pre-allocation of map entries allows per-packet latency to be reduced, at # the expense of up-front memory allocation for the entries in the maps. The # default value below will minimize memory usage in the default installation; # users who are sensitive to latency may consider setting this to "true". # # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore # this option and behave as though it is set to "true". # # If this value is modified, then during the next Cilium startup the restore # of existing endpoints and tracking of ongoing connections may be disrupted. # As a result, reply packets may be dropped and the load-balancing decisions # for established connections may change. # # If this option is set to "false" during an upgrade from 1.3 or earlier to # 1.4 or later, then it may cause one-time disruptions during the upgrade. preallocate-bpf-maps: "false" # Name of the cluster. Only relevant when building a mesh of clusters. cluster-name: "default" # Unique ID of the cluster. Must be unique across all conneted clusters and # in the range of 1 and 255. Only relevant when building a mesh of clusters. cluster-id: "0" # Encapsulation mode for communication between nodes # Possible values: # - disabled # - vxlan (default) # - geneve routing-mode: "tunnel" tunnel-protocol: "vxlan" tunnel-source-port-range: "0-0" service-no-backend-response: "reject" # Enables L7 proxy for L7 policy enforcement and visibility enable-l7-proxy: "true" enable-ipv4-masquerade: "true" enable-ipv4-big-tcp: "false" enable-ipv6-big-tcp: "false" enable-ipv6-masquerade: "true" enable-tcx: "true" datapath-mode: "veth" enable-masquerade-to-route-source: "false" enable-xt-socket-fallback: "true" install-no-conntrack-iptables-rules: "false" iptables-random-fully: "false" auto-direct-node-routes: "false" direct-routing-skip-unreachable: "false" kube-proxy-replacement: "false" bpf-lb-sock: "false" enable-node-port: "false" nodeport-addresses: "" enable-health-check-nodeport: "true" enable-health-check-loadbalancer-ip: "false" node-port-bind-protection: "true" enable-auto-protect-node-port-range: "true" bpf-lb-acceleration: "disabled" enable-svc-source-range-check: "true" enable-l2-neigh-discovery: "false" k8s-require-ipv4-pod-cidr: "false" k8s-require-ipv6-pod-cidr: "false" enable-k8s-networkpolicy: "true" enable-endpoint-lockdown-on-policy-overflow: "false" # Tell the agent to generate and write a CNI configuration file write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist cni-exclusive: "true" cni-log-file: "/var/run/cilium/cilium-cni.log" enable-endpoint-health-checking: "true" enable-health-checking: "true" health-check-icmp-failure-threshold: "3" enable-well-known-identities: "false" enable-node-selector-labels: "false" synchronize-k8s-nodes: "true" operator-api-serve-addr: "127.0.0.1:9234" enable-hubble: "false" ipam: "cluster-pool" ipam-cilium-node-update-rate: "15s" cluster-pool-ipv4-cidr: "{{ kube_pods_subnet }}" cluster-pool-ipv4-mask-size: "24" default-lb-service-ipam: "lbipam" egress-gateway-reconciliation-trigger-interval: "1s" enable-vtep: "false" vtep-endpoint: "" vtep-cidr: "" vtep-mask: "" vtep-mac: "" procfs: "/host/proc" bpf-root: "/sys/fs/bpf" cgroup-root: "/run/cilium/cgroupv2" identity-management-mode: "agent" enable-sctp: "false" remove-cilium-node-taints: "true" set-cilium-node-taints: "true" set-cilium-is-up-condition: "true" unmanaged-pod-watcher-interval: "15" # default DNS proxy to transparent mode in non-chaining modes dnsproxy-enable-transparent-mode: "true" dnsproxy-socket-linger-timeout: "10" tofqdns-dns-reject-response-code: "refused" tofqdns-enable-dns-compression: "true" tofqdns-endpoint-max-ip-per-hostname: "1000" tofqdns-idle-connection-grace-period: "0s" tofqdns-max-deferred-connection-deletes: "10000" tofqdns-proxy-response-max-delay: "100ms" tofqdns-preallocate-identities: "true" agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" mesh-auth-enabled: "true" mesh-auth-queue-size: "1024" mesh-auth-rotated-identities-queue-size: "1024" mesh-auth-gc-interval: "5m0s" proxy-xff-num-trusted-hops-ingress: "0" proxy-xff-num-trusted-hops-egress: "0" proxy-connect-timeout: "2" proxy-initial-fetch-timeout: "30" proxy-max-requests-per-connection: "0" proxy-max-connection-duration-seconds: "0" proxy-idle-timeout-seconds: "60" proxy-max-concurrent-retries: "128" http-retry-count: "3" http-stream-idle-timeout: "300" external-envoy-proxy: "true" envoy-base-id: "0" envoy-access-log-buffer-size: "4096" envoy-keep-cap-netbindservice: "false" max-connected-clusters: "255" clustermesh-enable-endpoint-sync: "false" clustermesh-enable-mcs-api: "false" policy-default-local-cluster: "false" nat-map-stats-entries: "32" nat-map-stats-interval: "30s" enable-internal-traffic-policy: "true" enable-lb-ipam: "true" enable-non-default-deny-policies: "true" enable-source-ip-verification: "true" # Extra config allows adding arbitrary properties to the cilium config. # By putting it at the end of the ConfigMap, it's also possible to override existing properties. --- # Source: cilium/templates/cilium-envoy/configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: cilium-envoy-config namespace: kube-system data: # Keep the key name as bootstrap-config.json to avoid breaking changes bootstrap-config.json: | {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} --- # Source: cilium/templates/cilium-agent/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - networking.k8s.io resources: - networkpolicies verbs: - get - list - watch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: - namespaces - services - pods - endpoints - nodes verbs: - get - list - watch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - list - watch # This is used when validating policies in preflight. This will need to stay # until we figure out how to avoid "get" inside the preflight, and then # should be removed ideally. - get - apiGroups: - cilium.io resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies - ciliumbgpnodeconfigs - ciliumbgpadvertisements - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies - ciliumendpoints - ciliumendpointslices - ciliumenvoyconfigs - ciliumidentities - ciliumlocalredirectpolicies - ciliumnetworkpolicies - ciliumnodes - ciliumnodeconfigs - ciliumcidrgroups - ciliuml2announcementpolicies - ciliumpodippools verbs: - list - watch - apiGroups: - cilium.io resources: - ciliumidentities - ciliumendpoints - ciliumnodes verbs: - create - apiGroups: - cilium.io # To synchronize garbage collection of such resources resources: - ciliumidentities verbs: - update - apiGroups: - cilium.io resources: - ciliumendpoints verbs: - delete - get - apiGroups: - cilium.io resources: - ciliumnodes - ciliumnodes/status verbs: - get - update - apiGroups: - cilium.io resources: - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status - ciliumbgpnodeconfigs/status verbs: - patch --- # Source: cilium/templates/cilium-operator/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium-operator labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - pods verbs: - get - list - watch # to automatically delete [core|kube]dns pods so that are starting to being # managed by Cilium - delete - apiGroups: - "" resources: - configmaps resourceNames: - cilium-config verbs: # allow patching of the configmap to set annotations - patch - apiGroups: - "" resources: - nodes verbs: - list - watch - apiGroups: - "" resources: # To remove node taints - nodes # To set NetworkUnavailable false on startup - nodes/status verbs: - patch - apiGroups: - discovery.k8s.io resources: - endpointslices verbs: - get - list - watch - apiGroups: - "" resources: # to perform LB IP allocation for BGP - services/status verbs: - update - patch - apiGroups: - "" resources: # to check apiserver connectivity - namespaces - secrets verbs: - get - list - watch - apiGroups: - "" resources: # to perform the translation of a CNP that contains `ToGroup` to its endpoints - services - endpoints verbs: - get - list - watch - apiGroups: - cilium.io resources: - ciliumnetworkpolicies - ciliumclusterwidenetworkpolicies verbs: # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' - create - update - deletecollection # To update the status of the CNPs and CCNPs - patch - get - list - watch - apiGroups: - cilium.io resources: - ciliumnetworkpolicies/status - ciliumclusterwidenetworkpolicies/status verbs: # Update the auto-generated CNPs and CCNPs status. - patch - update - apiGroups: - cilium.io resources: - ciliumendpoints - ciliumidentities verbs: # To perform garbage collection of such resources - delete - list - watch - apiGroups: - cilium.io resources: - ciliumidentities verbs: # To synchronize garbage collection of such resources - update - apiGroups: - cilium.io resources: - ciliumnodes verbs: - create - update - get - list - watch # To perform CiliumNode garbage collector - delete - apiGroups: - cilium.io resources: - ciliumnodes/status verbs: - update - apiGroups: - cilium.io resources: - ciliumendpointslices - ciliumenvoyconfigs - ciliumbgppeerconfigs - ciliumbgpadvertisements - ciliumbgpnodeconfigs verbs: - create - update - get - list - watch - delete - patch - apiGroups: - cilium.io resources: - ciliumbgpclusterconfigs/status - ciliumbgppeerconfigs/status verbs: - update - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - create - get - list - watch - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions verbs: - update resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io - ciliumbgpclusterconfigs.cilium.io - ciliumbgppeerconfigs.cilium.io - ciliumbgpadvertisements.cilium.io - ciliumbgpnodeconfigs.cilium.io - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io - ciliumendpoints.cilium.io - ciliumendpointslices.cilium.io - ciliumenvoyconfigs.cilium.io - ciliumidentities.cilium.io - ciliumlocalredirectpolicies.cilium.io - ciliumnetworkpolicies.cilium.io - ciliumnodes.cilium.io - ciliumnodeconfigs.cilium.io - ciliumcidrgroups.cilium.io - ciliuml2announcementpolicies.cilium.io - ciliumpodippools.cilium.io - ciliumgatewayclassconfigs.cilium.io - apiGroups: - cilium.io resources: - ciliumloadbalancerippools - ciliumpodippools - ciliumbgppeeringpolicies - ciliumbgpclusterconfigs - ciliumbgpnodeconfigoverrides - ciliumbgppeerconfigs verbs: - get - list - watch - apiGroups: - cilium.io resources: - ciliumpodippools verbs: - create - apiGroups: - cilium.io resources: - ciliumloadbalancerippools/status verbs: - patch # For cilium-operator running in HA mode. # # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election # between multiple running instances. # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less # common and fewer objects in the cluster watch "all Leases". - apiGroups: - coordination.k8s.io resources: - leases verbs: - create - get - update --- # Source: cilium/templates/cilium-agent/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cilium subjects: - kind: ServiceAccount name: "cilium" namespace: kube-system --- # Source: cilium/templates/cilium-operator/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium-operator labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cilium-operator subjects: - kind: ServiceAccount name: "cilium-operator" namespace: kube-system --- # Source: cilium/templates/cilium-agent/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cilium-config-agent namespace: kube-system labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - configmaps verbs: - get - list - watch --- # Source: cilium/templates/cilium-agent/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cilium-tlsinterception-secrets namespace: "cilium-secrets" labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - secrets verbs: - get - list - watch --- # Source: cilium/templates/cilium-operator/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cilium-operator-tlsinterception-secrets namespace: "cilium-secrets" labels: app.kubernetes.io/part-of: cilium rules: - apiGroups: - "" resources: - secrets verbs: - create - delete - update - patch --- # Source: cilium/templates/cilium-agent/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cilium-config-agent namespace: kube-system labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cilium-config-agent subjects: - kind: ServiceAccount name: "cilium" namespace: kube-system --- # Source: cilium/templates/cilium-agent/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cilium-tlsinterception-secrets namespace: "cilium-secrets" labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cilium-tlsinterception-secrets subjects: - kind: ServiceAccount name: "cilium" namespace: kube-system --- # Source: cilium/templates/cilium-operator/rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cilium-operator-tlsinterception-secrets namespace: "cilium-secrets" labels: app.kubernetes.io/part-of: cilium roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cilium-operator-tlsinterception-secrets subjects: - kind: ServiceAccount name: "cilium-operator" namespace: kube-system --- # Source: cilium/templates/cilium-envoy/service.yaml apiVersion: v1 kind: Service metadata: name: cilium-envoy namespace: kube-system annotations: prometheus.io/scrape: "true" prometheus.io/port: "9964" labels: k8s-app: cilium-envoy app.kubernetes.io/name: cilium-envoy app.kubernetes.io/part-of: cilium io.cilium/app: proxy spec: clusterIP: None type: ClusterIP selector: k8s-app: cilium-envoy ports: - name: envoy-metrics port: 9964 protocol: TCP targetPort: envoy-metrics --- # Source: cilium/templates/cilium-agent/daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: name: cilium namespace: kube-system labels: k8s-app: cilium app.kubernetes.io/part-of: cilium app.kubernetes.io/name: cilium-agent spec: selector: matchLabels: k8s-app: cilium updateStrategy: rollingUpdate: maxUnavailable: 2 type: RollingUpdate template: metadata: annotations: kubectl.kubernetes.io/default-container: cilium-agent labels: k8s-app: cilium app.kubernetes.io/name: cilium-agent app.kubernetes.io/part-of: cilium spec: securityContext: appArmorProfile: type: Unconfined seccompProfile: type: Unconfined containers: - name: cilium-agent image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent command: - cilium-agent args: - --config-dir=/tmp/cilium/config-map startupProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9879 scheme: HTTP httpHeaders: - name: "brief" value: "true" failureThreshold: 300 periodSeconds: 2 successThreshold: 1 initialDelaySeconds: 5 livenessProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9879 scheme: HTTP httpHeaders: - name: "brief" value: "true" - name: "require-k8s-connectivity" value: "false" periodSeconds: 30 successThreshold: 1 failureThreshold: 10 timeoutSeconds: 5 readinessProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9879 scheme: HTTP httpHeaders: - name: "brief" value: "true" periodSeconds: 30 successThreshold: 1 failureThreshold: 3 timeoutSeconds: 5 env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ - name: GOMEMLIMIT valueFrom: resourceFieldRef: resource: limits.memory divisor: '1' - name: KUBE_CLIENT_BACKOFF_BASE value: "1" - name: KUBE_CLIENT_BACKOFF_DURATION value: "120" lifecycle: postStart: exec: command: - "bash" - "-c" - | set -o errexit set -o pipefail set -o nounset # When running in AWS ENI mode, it's likely that 'aws-node' has # had a chance to install SNAT iptables rules. These can result # in dropped traffic, so we should attempt to remove them. # We do it using a 'postStart' hook since this may need to run # for nodes which might have already been init'ed but may still # have dangling rules. This is safe because there are no # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore fi echo 'Done!' preStop: exec: command: - /cni-uninstall.sh securityContext: seLinuxOptions: level: s0 type: spc_t capabilities: add: - CHOWN - KILL - NET_ADMIN - NET_RAW - IPC_LOCK - SYS_MODULE - SYS_ADMIN - SYS_RESOURCE - DAC_OVERRIDE - FOWNER - SETGID - SETUID drop: - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: envoy-sockets mountPath: /var/run/cilium/envoy/sockets readOnly: false # Unprivileged containers need to mount /proc/sys/net from the host # to have write access - mountPath: /host/proc/sys/net name: host-proc-sys-net # Unprivileged containers need to mount /proc/sys/kernel from the host # to have write access - mountPath: /host/proc/sys/kernel name: host-proc-sys-kernel - name: bpf-maps mountPath: /sys/fs/bpf # Unprivileged containers can't set mount propagation to bidirectional # in this case we will mount the bpf fs from an init container that # is privileged and set the mount propagation from host to container # in Cilium. mountPropagation: HostToContainer - name: cilium-run mountPath: /var/run/cilium - name: cilium-netns mountPath: /var/run/cilium/netns mountPropagation: HostToContainer - name: etc-cni-netd mountPath: /host/etc/cni/net.d - name: clustermesh-secrets mountPath: /var/lib/cilium/clustermesh readOnly: true # Needed to be able to load kernel modules - name: lib-modules mountPath: /lib/modules readOnly: true - name: xtables-lock mountPath: /run/xtables.lock - name: tmp mountPath: /tmp initContainers: - name: config image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent command: - cilium-dbg - build-config env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace volumeMounts: - name: tmp mountPath: /tmp terminationMessagePolicy: FallbackToLogsOnError # Required to mount cgroup2 filesystem on the underlying Kubernetes node. # We use nsenter command with host's cgroup and mount namespaces enabled. - name: mount-cgroup image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent env: - name: CGROUP_ROOT value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin command: - sh - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh and mount that can be missing on certain # distros installed on the underlying host. Copy the binary to the # same directory where we install cilium cni plugin so that exec permissions # are available. - | cp /usr/bin/cilium-mount /hostbin/cilium-mount; nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; rm /hostbin/cilium-mount volumeMounts: - name: hostproc mountPath: /hostproc - name: cni-path mountPath: /hostbin terminationMessagePolicy: FallbackToLogsOnError securityContext: seLinuxOptions: level: s0 type: spc_t capabilities: add: - SYS_ADMIN - SYS_CHROOT - SYS_PTRACE drop: - ALL - name: apply-sysctl-overwrites image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent env: - name: BIN_PATH value: /opt/cni/bin command: - sh - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh that can be missing on certain # distros installed on the underlying host. Copy the binary to the # same directory where we install cilium cni plugin so that exec permissions # are available. - | cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; rm /hostbin/cilium-sysctlfix volumeMounts: - name: hostproc mountPath: /hostproc - name: cni-path mountPath: /hostbin terminationMessagePolicy: FallbackToLogsOnError securityContext: seLinuxOptions: level: s0 type: spc_t capabilities: add: - SYS_ADMIN - SYS_CHROOT - SYS_PTRACE drop: - ALL # Mount the bpf fs if it is not mounted. We will perform this task # from a privileged container because the mount propagation bidirectional # only works from privileged containers. - name: mount-bpf-fs image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent args: - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' command: - /bin/bash - -c - -- terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf mountPropagation: Bidirectional - name: clean-cilium-state image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent command: - /init-container.sh env: - name: CILIUM_ALL_STATE valueFrom: configMapKeyRef: name: cilium-config key: clean-cilium-state optional: true - name: CILIUM_BPF_STATE valueFrom: configMapKeyRef: name: cilium-config key: clean-cilium-bpf-state optional: true - name: WRITE_CNI_CONF_WHEN_READY valueFrom: configMapKeyRef: name: cilium-config key: write-cni-conf-when-ready optional: true terminationMessagePolicy: FallbackToLogsOnError securityContext: seLinuxOptions: level: s0 type: spc_t capabilities: add: - NET_ADMIN - SYS_MODULE - SYS_ADMIN - SYS_RESOURCE drop: - ALL volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf # Required to mount cgroup filesystem from the host to cilium agent pod - name: cilium-cgroup mountPath: /run/cilium/cgroupv2 mountPropagation: HostToContainer - name: cilium-run mountPath: /var/run/cilium # wait-for-kube-proxy # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - name: install-cni-binaries image: "quay.io/cilium/cilium:v1.18.6@sha256:42ec562a5ff6c8a860c0639f5a7611685e253fd9eb2d2fcdade693724c9166a4" imagePullPolicy: IfNotPresent command: - "/install-plugin.sh" resources: requests: cpu: 100m memory: 10Mi securityContext: seLinuxOptions: level: s0 type: spc_t capabilities: drop: - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: cni-path mountPath: /host/opt/cni/bin # .Values.cni.install restartPolicy: Always priorityClassName: system-node-critical serviceAccountName: "cilium" automountServiceAccountToken: true terminationGracePeriodSeconds: 1 hostNetwork: true affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: k8s-app: cilium topologyKey: kubernetes.io/hostname nodeSelector: kubernetes.io/os: linux tolerations: - operator: Exists volumes: # For sharing configuration between the "config" initContainer and the agent - name: tmp emptyDir: {} # To keep state between restarts / upgrades - name: cilium-run hostPath: path: /var/run/cilium type: DirectoryOrCreate # To exec into pod network namespaces - name: cilium-netns hostPath: path: /var/run/netns type: DirectoryOrCreate # To keep state between restarts / upgrades for bpf maps - name: bpf-maps hostPath: path: /sys/fs/bpf type: DirectoryOrCreate # To mount cgroup2 filesystem on the host or apply sysctlfix - name: hostproc hostPath: path: /proc type: Directory # To keep state between restarts / upgrades for cgroup2 filesystem - name: cilium-cgroup hostPath: path: /run/cilium/cgroupv2 type: DirectoryOrCreate # To install cilium cni plugin in the host - name: cni-path hostPath: path: /opt/cni/bin type: DirectoryOrCreate # To install cilium cni configuration in the host - name: etc-cni-netd hostPath: path: /etc/cni/net.d type: DirectoryOrCreate # To be able to load kernel modules - name: lib-modules hostPath: path: /lib/modules # To access iptables concurrently with other processes (e.g. kube-proxy) - name: xtables-lock hostPath: path: /run/xtables.lock type: FileOrCreate # Sharing socket with Cilium Envoy on the same node by using a host path - name: envoy-sockets hostPath: path: "/var/run/cilium/envoy/sockets" type: DirectoryOrCreate # To read the clustermesh configuration - name: clustermesh-secrets projected: # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 sources: - secret: name: cilium-clustermesh optional: true # note: items are not explicitly listed here, since the entries of this secret # depend on the peers configured, and that would cause a restart of all agents # at every addition/removal. Leaving the field empty makes each secret entry # to be automatically projected into the volume as a file whose name is the key. - secret: name: clustermesh-apiserver-remote-cert optional: true items: - key: tls.key path: common-etcd-client.key - key: tls.crt path: common-etcd-client.crt - key: ca.crt path: common-etcd-client-ca.crt # note: we configure the volume for the kvstoremesh-specific certificate # regardless of whether KVStoreMesh is enabled or not, so that it can be # automatically mounted in case KVStoreMesh gets subsequently enabled, # without requiring an agent restart. - secret: name: clustermesh-apiserver-local-cert optional: true items: - key: tls.key path: local-etcd-client.key - key: tls.crt path: local-etcd-client.crt - key: ca.crt path: local-etcd-client-ca.crt - name: host-proc-sys-net hostPath: path: /proc/sys/net type: Directory - name: host-proc-sys-kernel hostPath: path: /proc/sys/kernel type: Directory --- # Source: cilium/templates/cilium-envoy/daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: name: cilium-envoy namespace: kube-system labels: k8s-app: cilium-envoy app.kubernetes.io/part-of: cilium app.kubernetes.io/name: cilium-envoy name: cilium-envoy spec: selector: matchLabels: k8s-app: cilium-envoy updateStrategy: rollingUpdate: maxUnavailable: 2 type: RollingUpdate template: metadata: annotations: labels: k8s-app: cilium-envoy name: cilium-envoy app.kubernetes.io/name: cilium-envoy app.kubernetes.io/part-of: cilium spec: securityContext: appArmorProfile: type: Unconfined containers: - name: cilium-envoy image: "quay.io/cilium/cilium-envoy:v1.35.9-1767794330-db497dd19e346b39d81d7b5c0dedf6c812bcc5c9@sha256:81398e449f2d3d0a6a70527e4f641aaa685d3156bea0bb30712fae3fd8822b86" imagePullPolicy: IfNotPresent command: - /usr/bin/cilium-envoy-starter args: - '--' - '-c /var/run/cilium/envoy/bootstrap-config.json' - '--base-id 0' - '--log-level info' startupProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9878 scheme: HTTP failureThreshold: 105 periodSeconds: 2 successThreshold: 1 initialDelaySeconds: 5 livenessProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9878 scheme: HTTP periodSeconds: 30 successThreshold: 1 failureThreshold: 10 timeoutSeconds: 5 readinessProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9878 scheme: HTTP periodSeconds: 30 successThreshold: 1 failureThreshold: 3 timeoutSeconds: 5 env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace ports: - name: envoy-metrics containerPort: 9964 hostPort: 9964 protocol: TCP securityContext: seLinuxOptions: level: s0 type: spc_t capabilities: add: - NET_ADMIN - SYS_ADMIN drop: - ALL terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: envoy-sockets mountPath: /var/run/cilium/envoy/sockets readOnly: false - name: envoy-artifacts mountPath: /var/run/cilium/envoy/artifacts readOnly: true - name: envoy-config mountPath: /var/run/cilium/envoy/ readOnly: true - name: bpf-maps mountPath: /sys/fs/bpf mountPropagation: HostToContainer restartPolicy: Always priorityClassName: system-node-critical serviceAccountName: "cilium-envoy" automountServiceAccountToken: true terminationGracePeriodSeconds: 1 hostNetwork: true affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: cilium.io/no-schedule operator: NotIn values: - "true" podAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: k8s-app: cilium topologyKey: kubernetes.io/hostname podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: k8s-app: cilium-envoy topologyKey: kubernetes.io/hostname nodeSelector: kubernetes.io/os: linux tolerations: - operator: Exists volumes: - name: envoy-sockets hostPath: path: "/var/run/cilium/envoy/sockets" type: DirectoryOrCreate - name: envoy-artifacts hostPath: path: "/var/run/cilium/envoy/artifacts" type: DirectoryOrCreate - name: envoy-config configMap: name: "cilium-envoy-config" # note: the leading zero means this number is in octal representation: do not remove it defaultMode: 0400 items: - key: bootstrap-config.json path: bootstrap-config.json # To keep state between restarts / upgrades # To keep state between restarts / upgrades for bpf maps - name: bpf-maps hostPath: path: /sys/fs/bpf type: DirectoryOrCreate --- # Source: cilium/templates/cilium-operator/deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: name: cilium-operator namespace: kube-system labels: io.cilium/app: operator name: cilium-operator app.kubernetes.io/part-of: cilium app.kubernetes.io/name: cilium-operator spec: # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go # for more details. replicas: 2 selector: matchLabels: io.cilium/app: operator name: cilium-operator # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case # of one replica and no user configured Recreate strategy. # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 50% type: RollingUpdate template: metadata: annotations: prometheus.io/port: "9963" prometheus.io/scrape: "true" labels: io.cilium/app: operator name: cilium-operator app.kubernetes.io/part-of: cilium app.kubernetes.io/name: cilium-operator spec: securityContext: seccompProfile: type: RuntimeDefault containers: - name: cilium-operator image: "quay.io/cilium/operator-generic:v1.18.6@sha256:34a827ce9ed021c8adf8f0feca131f53b3c54a3ef529053d871d0347ec4d69af" imagePullPolicy: IfNotPresent command: - cilium-operator-generic args: - --config-dir=/tmp/cilium/config-map - --debug=$(CILIUM_DEBUG) env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: CILIUM_K8S_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: CILIUM_DEBUG valueFrom: configMapKeyRef: key: debug name: cilium-config optional: true ports: - name: prometheus containerPort: 9963 hostPort: 9963 protocol: TCP livenessProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9234 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 readinessProbe: httpGet: host: "127.0.0.1" path: /healthz port: 9234 scheme: HTTP initialDelaySeconds: 0 periodSeconds: 5 timeoutSeconds: 3 failureThreshold: 5 volumeMounts: - name: cilium-config-path mountPath: /tmp/cilium/config-map readOnly: true securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL terminationMessagePolicy: FallbackToLogsOnError hostNetwork: true restartPolicy: Always priorityClassName: system-cluster-critical serviceAccountName: "cilium-operator" automountServiceAccountToken: true # In HA mode, cilium-operator pods must not be scheduled on the same # node as they will clash with each other. affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: io.cilium/app: operator topologyKey: kubernetes.io/hostname nodeSelector: kubernetes.io/os: linux tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists - key: node-role.kubernetes.io/master operator: Exists - key: node.kubernetes.io/not-ready operator: Exists - key: node.cloudprovider.kubernetes.io/uninitialized operator: Exists - key: node.cilium.io/agent-not-ready operator: Exists volumes: # To read the configuration from the config map - name: cilium-config-path configMap: name: cilium-config ================================================ FILE: tests/files/custom_cni/values.yaml ================================================ --- # We disable hubble so that helm doesn't try to generate any certificate. # This is not needed to test network_plugin/custom_cni anyway. hubble: enabled: false ipam: operator: # Set the appropriate pods subnet clusterPoolIPv4PodCIDRList: ["{{ kube_pods_subnet }}"] ================================================ FILE: tests/files/debian11-calico-collection.yml ================================================ --- # Instance settings cloud_image: debian-11 # Use static containerd binary for older distributions like Debian 11. containerd_static_binary: true ================================================ FILE: tests/files/debian11-calico-upgrade ================================================ UPGRADE_TEST=graceful ================================================ FILE: tests/files/debian11-calico-upgrade-once ================================================ UPGRADE_TEST=graceful ================================================ FILE: tests/files/debian11-calico-upgrade-once.yml ================================================ --- # Instance settings cloud_image: debian-11 # Kubespray settings download_run_once: true # Pin disabling ipip mode to ensure proper upgrade ipip: false calico_pool_blocksize: 26 calico_vxlan_mode: Always calico_network_backend: bird # Needed to bypass deprecation check ignore_assert_errors: true # Use static containerd binary for older distributions like Debian 11. containerd_static_binary: true ================================================ FILE: tests/files/debian11-calico-upgrade.yml ================================================ --- # Instance settings cloud_image: debian-11 # Pin disabling ipip mode to ensure proper upgrade ipip: false calico_pool_blocksize: 26 calico_vxlan_mode: Always calico_network_backend: bird # Needed to bypass deprecation check ignore_assert_errors: true # Remove anonymous access to cluster remove_anonymous_access: true # Use static containerd binary for older distributions like Debian 11. containerd_static_binary: true ================================================ FILE: tests/files/debian11-custom-cni.yml ================================================ --- # Instance settings cloud_image: debian-11 # Kubespray settings kube_owner: root kube_network_plugin: custom_cni custom_cni_manifests: - "{{ playbook_dir }}/../tests/files/custom_cni/cilium.yaml" # Use static containerd binary for older distributions like Debian 11. containerd_static_binary: true ================================================ FILE: tests/files/debian11-docker.yml ================================================ --- # Instance settings cloud_image: debian-11 # Use docker container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns ================================================ FILE: tests/files/debian11-kubelet-csr-approver.yml ================================================ --- # Instance settings cloud_image: debian-11 # Kubespray settings kubelet_rotate_server_certificates: true kubelet_csr_approver_enabled: true kubelet_csr_approver_values: # Do not check DNS resolution in testing (not recommended in production) bypassDnsResolution: true # Use static containerd binary for older distributions like Debian 11. containerd_static_binary: true ================================================ FILE: tests/files/debian11-macvlan.yml ================================================ --- # Instance settings cloud_image: debian-11 # Kubespray settings kube_network_plugin: macvlan enable_nodelocaldns: false kube_proxy_masquerade_all: true macvlan_interface: "eth0" auto_renew_certificates: true # Use static containerd binary for older distributions like Debian 11. containerd_static_binary: true ================================================ FILE: tests/files/debian12-calico.yml ================================================ --- # Instance settings cloud_image: debian-12 # Kubespray settings dns_mode: coredns_dual kube_asymmetric_encryption_algorithm: "RSA-3072" ================================================ FILE: tests/files/debian12-cilium-svc-proxy.yml ================================================ --- # Instance settings cloud_image: debian-12 mode: ha # Kubespray settings kube_network_plugin: cilium enable_network_policy: true cilium_kube_proxy_replacement: true kube_owner: root ================================================ FILE: tests/files/debian12-cilium.yml ================================================ --- # Instance settings cloud_image: debian-12 # Kubespray settings kube_network_plugin: cilium # ntp settings ntp_enabled: true ntp_package: ntp kube_owner: root ================================================ FILE: tests/files/debian12-custom-cni-helm.yml ================================================ --- # Instance settings cloud_image: debian-12 # Kubespray settings kube_owner: root kube_network_plugin: custom_cni custom_cni_chart_namespace: kube-system custom_cni_chart_release_name: cilium custom_cni_chart_repository_name: cilium custom_cni_chart_repository_url: https://helm.cilium.io custom_cni_chart_ref: cilium/cilium custom_cni_chart_version: 1.18.6 custom_cni_chart_values: cluster: name: kubespray hubble: enabled: false ipam: operator: clusterPoolIPv4PodCIDRList: - "{{ kube_pods_subnet }}" ================================================ FILE: tests/files/debian12-docker.yml ================================================ --- # Instance settings cloud_image: debian-12 # Use docker container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg ================================================ FILE: tests/files/debian13-calico.yml ================================================ --- # Instance settings cloud_image: debian-13 # Kubespray settings gateway_api_enabled: true dns_mode: coredns_dual kube_asymmetric_encryption_algorithm: "RSA-3072" ================================================ FILE: tests/files/debian13-cilium.yml ================================================ --- # Instance settings cloud_image: debian-13 # Kubespray settings kube_network_plugin: cilium kube_owner: root prometheus_operator_crds_enabled: true ================================================ FILE: tests/files/fedora39-calico-selinux.yml ================================================ --- # Instance settings cloud_image: fedora-39 # Kubespray settings auto_renew_certificates: true # Test with SELinux in enforcing mode preinstall_selinux_state: enforcing ================================================ FILE: tests/files/fedora39-calico-swap-selinux.yml ================================================ --- # Instance settings cloud_image: fedora-39 # Kubespray settings auto_renew_certificates: true # Test with SELinux in enforcing mode preinstall_selinux_state: enforcing # Test Alpha swap feature by leveraging zswap default config in Fedora 35 kubelet_fail_swap_on: false kube_feature_gates: - "NodeSwap=True" ================================================ FILE: tests/files/fedora39-crio.yml ================================================ --- # Instance settings cloud_image: fedora-39 # Kubespray settings container_manager: crio auto_renew_certificates: true # Test with SELinux in enforcing mode preinstall_selinux_state: enforcing ================================================ FILE: tests/files/fedora39-kube-router.yml ================================================ --- cloud_image: fedora-39 cluster_layout: - node_groups: ['kube_control_plane', 'etcd', 'kube_node'] - node_groups: ['kube_node'] kube_network_plugin: "kube-router" ================================================ FILE: tests/files/fedora40-docker-calico.yml ================================================ --- # Instance settings cloud_image: fedora-40 # Kubespray settings auto_renew_certificates: true # Docker specific settings: container_manager: docker etcd_deployment_type: docker ================================================ FILE: tests/files/fedora40-docker.calico ================================================ RESET_CHECK=true ================================================ FILE: tests/files/fedora40-flannel-crio-collection-scale.yml ================================================ --- cloud_image: fedora-40 network_plugin: flannel container_manager: crio cluster_layout: - node_groups: ['kube_control_plane', 'etcd'] - node_groups: ['kube_node'] - node_groups: ['kube_node', 'for_scale'] ================================================ FILE: tests/files/fedora41-calico-selinux.yml ================================================ --- # Instance settings cloud_image: fedora-41 # Kubespray settings auto_renew_certificates: true # Test with SELinux in enforcing mode preinstall_selinux_state: enforcing ================================================ FILE: tests/files/fedora41-calico-swap-selinux.yml ================================================ --- # Instance settings cloud_image: fedora-41 # Kubespray settings auto_renew_certificates: true # Test with SELinux in enforcing mode preinstall_selinux_state: enforcing # Test Alpha swap feature by leveraging zswap default config in Fedora 35 kubelet_fail_swap_on: false kube_feature_gates: - "NodeSwap=True" ================================================ FILE: tests/files/fedora41-crio.yml ================================================ --- # Instance settings cloud_image: fedora-41 # Kubespray settings container_manager: crio auto_renew_certificates: true ================================================ FILE: tests/files/fedora41-kube-router.yml ================================================ --- cloud_image: fedora-41 cluster_layout: - node_groups: ['kube_control_plane', 'etcd', 'kube_node'] - node_groups: ['kube_node'] kube_network_plugin: "kube-router" ================================================ FILE: tests/files/fedora42-calico.yml ================================================ --- # Instance settings cloud_image: fedora-42 # Kubespray settings auto_renew_certificates: true # Test with SELinux in enforcing mode preinstall_selinux_state: enforcing ================================================ FILE: tests/files/flatcar4081-calico.yml ================================================ --- # Instance settings cloud_image: flatcar-4081 mode: default vm_memory: 3072 # Kubespray settings metrics_server_enabled: true loadbalancer_apiserver_type: haproxy ================================================ FILE: tests/files/openeuler24-calico.yml ================================================ --- # Instance settings cloud_image: openeuler-2403 vm_memory: 3072 # Use metalink for faster package downloads (auto-selects closest mirror) openeuler_metalink_enabled: true # CI package installation takes ~7min; default 5min is too tight, use 15min for margin pkg_install_timeout: "{{ 15 * 60 }}" # Work around so the Kubernetes 1.35 tests can pass. We will discuss the openeuler support later. kubeadm_ignore_preflight_errors: - SystemVerification kubelet_fail_cgroup_v1: false ================================================ FILE: tests/files/rockylinux10-calico.yml ================================================ --- # Instance settings cloud_image: rockylinux-10-extra vm_memory: 3072 # Kubespray settings metrics_server_enabled: true loadbalancer_apiserver_type: haproxy ================================================ FILE: tests/files/rockylinux10-cilium ================================================ RESET_CHECK=true ================================================ FILE: tests/files/rockylinux10-cilium.yml ================================================ --- # Instance settings cloud_image: rockylinux-10-extra vm_memory: 3072 # Kubespray settings kube_network_plugin: cilium cilium_kube_proxy_replacement: true kube_owner: root # Node Feature Discovery node_feature_discovery_enabled: true kube_asymmetric_encryption_algorithm: "ECDSA-P256" # Testing no_proxy setup # The proxy is not intended to be accessed at all, we're only testing # the no_proxy construction https_proxy: "http://some-proxy.invalid" http_proxy: "http://some-proxy.invalid" additional_no_proxy_list: - github.com - githubusercontent.com - k8s.io - rockylinux.org - docker.io - googleapis.com - quay.io - pkg.dev - amazonaws.com - cilium.io skip_http_proxy_on_os_packages: true ================================================ FILE: tests/files/rockylinux9-calico.yml ================================================ --- # Instance settings cloud_image: rockylinux-9 vm_memory: 3072 # Kubespray settings metrics_server_enabled: true loadbalancer_apiserver_type: haproxy ================================================ FILE: tests/files/rockylinux9-cilium ================================================ RESET_CHECK=true ================================================ FILE: tests/files/rockylinux9-cilium.yml ================================================ --- # Instance settings cloud_image: rockylinux-9 vm_memory: 3072 # Kubespray settings kube_network_plugin: cilium cilium_kube_proxy_replacement: true kube_owner: root # Node Feature Discovery node_feature_discovery_enabled: true kube_asymmetric_encryption_algorithm: "ECDSA-P256" loadbalancer_apiserver_localhost: false ================================================ FILE: tests/files/tf-elastx_ubuntu24-calico.yml ================================================ --- sonobuoy_enabled: true # Ignore ping errors ignore_assert_errors: true ================================================ FILE: tests/files/ubuntu22-all-in-one-docker.yml ================================================ --- # Instance settings cloud_image: ubuntu-2204 mode: all-in-one vm_memory: 3072 # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # Use docker container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg ================================================ FILE: tests/files/ubuntu22-calico-all-in-one-upgrade ================================================ UPGRADE_TEST=graceful ================================================ FILE: tests/files/ubuntu22-calico-all-in-one-upgrade.yml ================================================ --- # Instance settings cloud_image: ubuntu-2204 mode: all-in-one vm_memory: 3072 # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # Single node don't need the DNS autoscaler enable_dns_autoscaler: false containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - prefix: 172.19.16.11:5000 mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve", "push"] skip_verify: true ================================================ FILE: tests/files/ubuntu22-calico-all-in-one.yml ================================================ --- # Instance settings cloud_image: ubuntu-2204 mode: all-in-one vm_memory: 3072 # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - prefix: 172.19.16.11:5000 mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve", "push"] skip_verify: true ================================================ FILE: tests/files/ubuntu22-crio.yml ================================================ --- # Instance settings cloud_image: ubuntu-2204 # Kubespray settings container_manager: crio download_localhost: false download_run_once: true ================================================ FILE: tests/files/ubuntu24-all-in-one-docker.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: all-in-one vm_memory: 3072 # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # Use docker container_manager: docker etcd_deployment_type: docker resolvconf_mode: docker_dns docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg ================================================ FILE: tests/files/ubuntu24-calico-all-in-one ================================================ RESET_CHECK=true ================================================ FILE: tests/files/ubuntu24-calico-all-in-one-hardening.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: all-in-one # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # The followings are for hardening ## kube-apiserver authorization_modes: ["Node", "RBAC"] kube_apiserver_request_timeout: 120s kube_apiserver_service_account_lookup: true # enable kubernetes audit kubernetes_audit: true audit_log_path: "/var/log/kube-apiserver-log.json" audit_log_maxage: 30 audit_log_maxbackups: 10 audit_log_maxsize: 100 tls_min_version: VersionTLS12 tls_cipher_suites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # enable encryption at rest kube_encrypt_secret_data: true kube_encryption_resources: [secrets] kube_encryption_algorithm: "secretbox" kube_apiserver_enable_admission_plugins: - EventRateLimit - AlwaysPullImages - ServiceAccount - NamespaceLifecycle - NodeRestriction - LimitRanger - ResourceQuota - MutatingAdmissionWebhook - ValidatingAdmissionWebhook - PodNodeSelector - PodSecurity kube_apiserver_admission_control_config_file: true # EventRateLimit plugin configuration kube_apiserver_admission_event_rate_limits: limit_1: type: Namespace qps: 50 burst: 100 cache_size: 2000 limit_2: type: User qps: 50 burst: 100 kube_profiling: false ## kube-controller-manager kube_controller_manager_bind_address: 127.0.0.1 kube_controller_terminated_pod_gc_threshold: 50 kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"] ## kube-scheduler kube_scheduler_bind_address: 127.0.0.1 ## etcd etcd_deployment_type: kubeadm ## kubelet kubelet_authentication_token_webhook: true kube_read_only_port: 0 kubelet_rotate_server_certificates: true kubelet_csr_approver_enabled: true # For hydrophone kubelet_csr_approver_values: # Do not check DNS resolution in testing (not recommended in production) bypassDnsResolution: true kubelet_protect_kernel_defaults: true kubelet_event_record_qps: 1 kubelet_rotate_certificates: true kubelet_streaming_connection_idle_timeout: "5m" kubelet_make_iptables_util_chains: true kubelet_feature_gates: ["RotateKubeletServerCertificate=true"] kubelet_seccomp_default: true kubelet_systemd_hardening: true # In case you have multiple interfaces in your # control plane nodes and you want to specify the right # IP addresses, kubelet_secure_addresses allows you # to specify the IP from which the kubelet # will receive the packets. # kubelet_secure_addresses: "192.168.10.110 192.168.10.111 192.168.10.112" # additional configurations kube_owner: root kube_cert_group: root # create a default Pod Security Configuration and deny running of insecure pods # kube-system namespace is exempted by default kube_pod_security_use_default: true kube_pod_security_default_enforce: restricted # Remove anonymous access to cluster remove_anonymous_access: true ================================================ FILE: tests/files/ubuntu24-calico-all-in-one.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: all-in-one vm_memory: 3072 # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - prefix: 172.19.16.11:5000 mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve", "push"] skip_verify: true kube_reserved: true system_reserved: true ================================================ FILE: tests/files/ubuntu24-calico-dual-stack.rb ================================================ $os = "ubuntu2404" $vm_cpus = 2 $libvirt_volume_cache = "unsafe" # Checking for box update can trigger API rate limiting # https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html $box_check_update = false $network_plugin = "calico" ================================================ FILE: tests/files/ubuntu24-calico-dual-stack.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 # Kubespray settings ipv4_stack: true ipv6_stack: true ================================================ FILE: tests/files/ubuntu24-calico-etcd-datastore.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: node-etcd-client vm_memory: 3072 # Kubespray settings auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: nftables enable_nodelocaldns: false containerd_registries_mirrors: - prefix: docker.io mirrors: - host: https://mirror.gcr.io capabilities: ["pull", "resolve"] skip_verify: false - prefix: 172.19.16.11:5000 mirrors: - host: http://172.19.16.11:5000 capabilities: ["pull", "resolve", "push"] skip_verify: true calico_datastore: "etcd" # Test kubeadm patches kubeadm_patches: - target: kube-apiserver patch: metadata: annotations: example.com/test: "true" labels: example.com/prod_level: "prep" - target: kube-controller-manager patch: metadata: annotations: example.com/test: "false" labels: example.com/prod_level: "prep" # ntp settings ntp_enabled: true ntp_package: ntpsec ================================================ FILE: tests/files/ubuntu24-calico-etcd-kubeadm-upgrade-ha ================================================ UPGRADE_TEST=basic ================================================ FILE: tests/files/ubuntu24-calico-etcd-kubeadm-upgrade-ha.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: ha # use the kubeadm etcd setting to test the upgrade etcd_deployment_type: kubeadm upgrade_cluster_setup: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # Pin disabling ipip mode to ensure proper upgrade ipip: false calico_vxlan_mode: Always calico_network_backend: bird # Needed to bypass deprecation check ignore_assert_errors: true ### FIXME FLORYUT Needed for upgrade job, will be removed when releasing kubespray 2.20 calico_pool_blocksize: 24 ### /FIXME ================================================ FILE: tests/files/ubuntu24-calico-etcd-kubeadm.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 # use the kubeadm etcd setting to test the upgrade etcd_deployment_type: kubeadm # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # Remove anonymous access to cluster remove_anonymous_access: true ================================================ FILE: tests/files/ubuntu24-calico-ha-recover ================================================ RECOVER_CONTROL_PLANE_TEST=true RECOVER_CONTROL_PLANE_TEST_GROUPS="etcd[2:]:kube_control_plane[1:]" ================================================ FILE: tests/files/ubuntu24-calico-ha-recover-noquorum ================================================ RECOVER_CONTROL_PLANE_TEST=true RECOVER_CONTROL_PLANE_TEST_GROUPS="etcd[1:]:kube_control_plane[1:]" ================================================ FILE: tests/files/ubuntu24-calico-ha-recover-noquorum.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: ha-recover-noquorum ================================================ FILE: tests/files/ubuntu24-calico-ha-recover.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: ha-recover ================================================ FILE: tests/files/ubuntu24-calico-ha-wireguard.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: ha # Kubespray settings calico_wireguard_enabled: true auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables # KVM kernel used by packet instances is missing the dummy.ko kernel module so it cannot enable nodelocaldns enable_nodelocaldns: false ================================================ FILE: tests/files/ubuntu24-calico-ipv6only-stack.rb ================================================ $os = "ubuntu2404" $vm_cpus = 2 $libvirt_volume_cache = "unsafe" # Checking for box update can trigger API rate limiting # https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html $box_check_update = false $network_plugin = "calico" ================================================ FILE: tests/files/ubuntu24-calico-ipv6only-stack.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 # Kubespray settings ipv4_stack: false ipv6_stack: true kube_network_plugin: calico etcd_deployment_type: kubeadm kube_proxy_mode: iptables enable_nodelocaldns: false ================================================ FILE: tests/files/ubuntu24-cilium-sep.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: separate # Kubespray settings kube_network_plugin: cilium enable_network_policy: true auto_renew_certificates: true kube_owner: root ================================================ FILE: tests/files/ubuntu24-crio-scale.yml ================================================ --- cloud_image: ubuntu-2404 container_manager: crio cluster_layout: - node_groups: ["kube_control_plane", "etcd"] - node_groups: ["kube_node"] - node_groups: ["kube_node", "for_scale"] ================================================ FILE: tests/files/ubuntu24-crio-upgrade ================================================ UPGRADE_TEST=graceful ================================================ FILE: tests/files/ubuntu24-crio-upgrade.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: all-in-one vm_memory: 3072 # Kubespray settings container_manager: crio auto_renew_certificates: true # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=noble&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: false # Single node don't need the DNS autoscaler enable_dns_autoscaler: false ================================================ FILE: tests/files/ubuntu24-flannel-collection.yml ================================================ --- cloud_image: ubuntu-2404 cluster_layout: - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["etcd", "kube_node"] kube_network_plugin: flannel ================================================ FILE: tests/files/ubuntu24-flannel-ha-once.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: ha # Kubespray settings kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085 kube_proxy_mode: iptables kube_network_plugin: flannel helm_enabled: true kubernetes_audit: true etcd_events_cluster_enabled: true local_volume_provisioner_enabled: true kube_encrypt_secret_data: true ingress_nginx_enabled: true cert_manager_enabled: true # Disable as health checks are still unstable and slow to respond. metrics_server_enabled: false metrics_server_kubelet_insecure_tls: true kube_token_auth: true enable_nodelocaldns: false ================================================ FILE: tests/files/ubuntu24-flannel-ha.yml ================================================ --- # Instance settings cloud_image: ubuntu-2404 mode: ha # Kubespray settings kube_network_plugin: flannel etcd_deployment_type: kubeadm kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085 skip_non_kubeadm_warning: true kube_asymmetric_encryption_algorithm: "RSA-4096" # This test the variable usage, it is not a prerequisite of the test itself kubeadm_ignore_preflight_errors: - all ================================================ FILE: tests/files/ubuntu24-flannel.yml ================================================ --- cloud_image: ubuntu-2404 cluster_layout: - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["etcd", "kube_node"] kube_network_plugin: flannel ================================================ FILE: tests/files/ubuntu24-ha-separate-etcd ================================================ REMOVE_NODE_CHECK=true REMOVE_NODE_NAME=etcd[2] ================================================ FILE: tests/files/ubuntu24-ha-separate-etcd.yml ================================================ --- cloud_image: ubuntu-2404 cluster_layout: - node_groups: ['kube_control_plane'] - node_groups: ['kube_control_plane'] - node_groups: ['kube_control_plane'] - node_groups: ['kube_node'] - node_groups: ['etcd'] - node_groups: ['etcd'] - node_groups: ['etcd'] kube_network_plugin: calico calico_datastore: etcd ================================================ FILE: tests/files/ubuntu24-kube-router-sep.yml ================================================ --- cloud_image: ubuntu-2404 cluster_layout: - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["kube_node"] kube_network_plugin: "kube-router" ================================================ FILE: tests/files/ubuntu24-kube-router-svc-proxy.yml ================================================ --- cloud_image: ubuntu-2404 cluster_layout: - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["kube_control_plane", "etcd", "kube_node"] - node_groups: ["etcd", "kube_node"] kube_network_plugin: "kube-router" kube_router_run_service_proxy: true ================================================ FILE: tests/requirements.txt ================================================ -r ../requirements.txt distlib==0.4.0 # required for building collections molecule==26.3.0 pytest-testinfra==10.2.2 ================================================ FILE: tests/scripts/check-templates.py ================================================ #!/usr/bin/env python import sys import traceback from jinja2 import Environment from jinja2.exceptions import TemplateSyntaxError env = Environment() errors = False for template in sys.argv[1:]: try: with open(template) as t: env.parse(t.read()) except TemplateSyntaxError as e: print (template) traceback.print_exc() errors = True if errors: exit (1) ================================================ FILE: tests/scripts/collection-build-install.sh ================================================ #!/bin/sh -e export ANSIBLE_COLLECTIONS_PATH="./ansible_collections" ansible-galaxy collection build --force ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray" test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml ================================================ FILE: tests/scripts/md-table/main.py ================================================ #!/usr/bin/env python import argparse import sys import glob from pathlib import Path import yaml import re import jinja2 import sys from pprint import pprint parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage') parser.add_argument('--dir', default='tests/files/', help='folder with test yml files') parser.add_argument('--output', default='docs/developers/ci.md', help='output file') args = parser.parse_args() p = Path(args.dir) env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0])) # Data represents CI coverage data matrix class Data: def __init__(self): self.container_managers = set() self.network_plugins = set() self.os = set() self.combination = set() def set(self, container_manager, network_plugin, os): self.container_managers.add(container_manager) self.network_plugins.add(network_plugin) self.os.add(os) self.combination.add(container_manager+network_plugin+os) def exists(self, container_manager, network_plugin, os): return (container_manager+network_plugin+os) in self.combination def jinja(self): template = env.get_template('table.md.j2') container_engines = sorted(self.container_managers) network_plugins = sorted(self.network_plugins) operating_systems = sorted(self.os) return template.render( container_engines=container_engines, network_plugins=network_plugins, operating_systems=operating_systems, exists=self.exists ) def markdown(self): out = '' for container_manager in self.db.get_unique_ids('container_manager'): # Prepare the headers out += "# " + container_manager + "\n" headers = '|OS / CNI| ' underline = '|----|' for network_plugin in self.db.get_unique_ids("network_plugin"): headers += network_plugin + ' | ' underline += '----|' out += headers + "\n" + underline + "\n" for operating_system in self.db.get_unique_ids("operating_system"): out += '| ' + operating_system + ' | ' for network_plugin in self.db.get_unique_ids("network_plugin"): if self.exists(container_manager, network_plugin, operating_system): emoji = ':white_check_mark:' else: emoji = ':x:' out += emoji + ' | ' out += "\n" pprint(self.db.get_unique_ids('operating_system')) pprint(self.db.get_unique_ids('network_plugin')) return out if not p.is_dir(): print("Path is not a directory") sys.exit(2) data = Data() files = p.glob('*.yml') for f in files: y = yaml.load(f.open(), Loader=yaml.FullLoader) container_manager = y.get('container_manager', 'containerd') network_plugin = y.get('kube_network_plugin', 'calico') x = re.match(r"^([a-z-]+_)?([a-z0-9]+).*", f.name) operating_system = x.group(2) data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system) print(data.jinja(), file=open(args.output, 'w')) ================================================ FILE: tests/scripts/md-table/table.md.j2 ================================================ # CI test coverage To generate this Matrix run `./tests/scripts/md-table/main.py` {%- for container_engine in container_engines %} ## {{ container_engine }} | OS / CNI |{% for cni in network_plugins %} {{ cni }} |{% endfor %} |---|{% for cni in network_plugins %} --- |{% endfor %} {%- for os in operating_systems %} {{ os }} | {% for cni in network_plugins %} {{ ':white_check_mark:' if exists(container_engine, cni, os) else ':x:' }} |{% endfor %} {%- endfor %} {%- endfor %} ================================================ FILE: tests/scripts/molecule_run.sh ================================================ #!/bin/bash set -euxo pipefail -o noglob export LC_ALL=C.UTF-8 export LANG=C.UTF-8 _PATH='roles' _EXCLUDE="" while [[ $# -gt 0 ]] ; do case $1 in -e|--exclude) _EXCLUDE="${_EXCLUDE} -not -path ${_PATH}/$2/*" shift shift ;; -i|--include) _PATH="${_PATH}/$2" shift shift ;; -h|--help) echo "Usage: molecule_run.sh [-h|--help] [-e|--exclude] [-i|--include]" exit 0 ;; esac done for d in $(find ${_PATH} ${_EXCLUDE} -name molecule -type d) do pushd $(dirname $d) molecule test --all popd done ================================================ FILE: tests/scripts/opentofu_install.sh ================================================ #!/bin/bash set -euxo pipefail curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh chmod +x install-opentofu.sh ./install-opentofu.sh --install-method standalone rm -f install-opentofu.sh tofu --version ================================================ FILE: tests/scripts/rebase.sh ================================================ #!/bin/sh set -ex if [ "${GITHUB_BASE_REF}" ]; then git pull --rebase origin $GITHUB_BASE_REF fi ================================================ FILE: tests/scripts/testcases_run.sh ================================================ #!/bin/bash set -euxo pipefail if [[ -v TESTCASE ]]; then TESTCASE_FILE=files/${TESTCASE}.yml else TESTCASE_FILE=common_vars.yml TESTCASE=default fi echo "TESTCASE is $TESTCASE" source tests/files/$TESTCASE || true # Check out latest tag if testing upgrade if [ "${UPGRADE_TEST}" != "false" ]; then git fetch --all && git checkout $(git describe --tags --abbrev=0) # Checkout the current tests/ directory ; even when testing old version, # we want the up-to-date test setup/provisionning git checkout "${CI_COMMIT_SHA}" -- tests/ pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt fi export ANSIBLE_BECOME=true export ANSIBLE_BECOME_USER=root run_playbook () { if [[ "${TESTCASE}" =~ "collection" ]]; then playbook=kubernetes_sigs.kubespray.$1 # Handle upgrade case properly rm -f kubernetes_sigs-kubespray-*.tar.gz ansible-galaxy collection build ansible-galaxy collection install kubernetes_sigs-kubespray-*.tar.gz else playbook=$1.yml fi shift ansible-playbook \ -e @tests/common_vars.yml \ -e @tests/${TESTCASE_FILE} \ "$@" \ ${playbook} } ## START KUBESPRAY # Create cluster if [[ "${TESTCASE}" =~ "scale" ]]; then run_playbook cluster --limit '!for_scale' run_playbook scale --limit 'for_scale' else run_playbook cluster fi # Repeat deployment if testing upgrade if [ "${UPGRADE_TEST}" != "false" ]; then git checkout "${CI_COMMIT_SHA}" pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt case "${UPGRADE_TEST}" in "basic") run_playbook cluster ;; "graceful") run_playbook upgrade_cluster ;; *) ;; esac fi # Test control plane recovery if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then run_playbook reset --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}" -e reset_confirmation=yes run_playbook recover-control-plane -e etcd_retries=10 --limit "etcd:kube_control_plane" fi # Run tests ansible-playbook \ -e @tests/common_vars.yml \ -e @tests/${TESTCASE_FILE} \ -e local_release_dir=${PWD}/downloads \ tests/testcases/tests.yml # Test node removal procedure if [ "${REMOVE_NODE_CHECK}" = "true" ]; then run_playbook remove-node -e skip_confirmation=yes -e node="${REMOVE_NODE_NAME}" fi # Clean up at the end, this is to allow stage1 tests to include cleanup test if [ "${RESET_CHECK}" = "true" ]; then run_playbook reset -e reset_confirmation=yes fi ================================================ FILE: tests/scripts/vagrant-install.sh ================================================ #!/bin/bash # install_vagrant() { # sudo apt install vagrant-libvirt vagrant -y # sudo vagrant plugin install vagrant-libvirt # } # prep(){ # sudo apt-get update -y # sudo apt-get install ca-certificates curl libvirt-daemon-system\ # libvirt-clients qemu-utils qemu-kvm htop atop -y # sudo install -m 0755 -d /etc/apt/keyrings # } # install_docker() { # VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble # sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc # sudo chmod a+r /etc/apt/keyrings/docker.asc # # Add the repository to Apt sources: # echo \ # "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ # $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ # sudo tee /etc/apt/sources.list.d/docker.list > /dev/null # sudo apt-get update -y # sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y # } # install_docker_auto () { # curl -fsSL https://get.docker.com -o get-docker.sh # sudo sh ./get-docker.sh --dry-run # } VAGRANT_VERSION=2.4.1 VAGRANT_DEFAULT_PROVIDER=libvirt VAGRANT_ANSIBLE_TAGS=facts LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive PYTHONDONTWRITEBYTECODE=1 KUBE_VERSION=1.29.5 pipeline_install() { cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")" sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources apt update # libssl-dev \ # python3-dev \ # # jq \ # moreutils \ # libvirt-dev \ # # rsync \ # git \ # # htop \ # gpg \ # atop # gnupg2 \ # software-properties-common # apt install --no-install-recommends -y \ git \ make \ python3-pip \ sshpass \ apt-transport-https \ openssh-client \ ca-certificates \ curl \ libfuse2 \ unzip \ qemu-utils \ libvirt-daemon-system \ libvirt-clients \ qemu-kvm \ ebtables libguestfs-tools \ ruby-fog-libvirt \ libvirt-dev \ gcc \ build-essential \ ruby-libvirt \ libxslt-dev libxml2-dev zlib1g-dev \ python3-venv python3-full \ dnsmasq apt-get build-dep -y ruby-libvirt ruby-dev ### VAGRANT ### # apt-get install -y unzip curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip mv vagrant /usr/local/bin/vagrant chmod a+x /usr/local/bin/vagrant # ls -la /usr/local/bin/vagrant /usr/local/bin/vagrant plugin install vagrant-libvirt usermod -aG kvm kubespray usermod -aG libvirt kubespray ### DOCKER ### curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" apt update apt install --no-install-recommends -y docker-ce apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/* ### KUBECTL ### curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" mv kubectl /usr/local/bin/kubectl chmod a+x /usr/local/bin/kubectl systemctl restart libvirtd # Install Vagrant # apt update -y # echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt # apt --fix-broken install -y # dpkg --configure -a -y } # wrapped up in a function so that we have some protection against only getting # half the file during "curl | sh" pipeline_install ================================================ FILE: tests/scripts/vagrant-validate.sh ================================================ #!/bin/bash set -euxo pipefail curl -sL "https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_amd64.deb" -o "/tmp/vagrant_${VAGRANT_VERSION}-1_amd64.deb" dpkg -i "/tmp/vagrant_${VAGRANT_VERSION}-1_amd64.deb" vagrant validate --ignore-provider ================================================ FILE: tests/scripts/vagrant_clean.sh ================================================ #!/bin/bash set -euxo pipefail # Cleanup vagrant VMs to avoid name conflicts for i in $(virsh list --name) do virsh destroy "$i" virsh undefine "$i" done # Cleanup domain volumes for i in $(virsh vol-list default|grep \.img |grep -v VAGRANTSLASH | cut -f 2 -d ' ') do virsh vol-delete "$i" --pool default done ================================================ FILE: tests/testcases/000_install-hydrophone.yml ================================================ --- - name: Download hydrophone get_url: url: "https://github.com/kubernetes-sigs/hydrophone/releases/download/v{{ hydrophone_version }}/hydrophone_Linux_{{ hydrophone_arch }}.tar.gz" dest: /tmp/hydrophone.tar.gz checksum: "{{ hydrophone_checksum }}" mode: "0644" - name: Extract hydrophone unarchive: src: /tmp/hydrophone.tar.gz dest: "{{ bin_dir }}" copy: false ================================================ FILE: tests/testcases/010_check-apiserver.yml ================================================ --- - name: Check the API servers are responding uri: url: "https://{{ (access_ip if (ipv4_stack | default(true)) else access_ip6) | default(ansible_default_ipv4.address if (ipv4_stack | default(true)) else ansible_default_ipv6.address) | ansible.utils.ipwrap }}:{{ kube_apiserver_port | default(6443) }}/version" validate_certs: false status_code: 200 register: apiserver_response retries: 12 delay: 5 until: apiserver_response is success - name: Check API servers version assert: that: - apiserver_response.json.gitVersion == ('v' + kube_version) fail_msg: "apiserver is {{ apiserver_response.json.gitVersion }}, expected {{ kube_version }}" ================================================ FILE: tests/testcases/015_check-nodes-ready.yml ================================================ --- - import_role: # noqa name[missing] name: cluster-dump - name: Check kubectl output command: "{{ bin_dir }}/kubectl get nodes" changed_when: false register: get_nodes - name: Check that all nodes are running and ready command: "{{ bin_dir }}/kubectl get nodes --no-headers -o yaml" changed_when: false register: get_nodes_yaml until: # Check that all nodes are Status=Ready - '(get_nodes_yaml.stdout | from_yaml)["items"] | map(attribute = "status.conditions") | map("items2dict", key_name="type", value_name="status") | map(attribute="Ready") | list | min' retries: 30 delay: 10 ================================================ FILE: tests/testcases/020_check-pods-running.yml ================================================ --- - import_role: # noqa name[missing] name: cluster-dump - name: Check kubectl output command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide" changed_when: false - name: Check pods vars: query_pods_not_running: "items[?status.phase != 'Running']" query_pods_not_ready: "items[?(status.conditions[?type == 'Ready'])[0].status != 'True']" pods_not_running: "{{ run_pods_log.stdout | from_json | json_query(query_pods_not_running + '.metadata') }}" pods_not_ready: "{{ run_pods_log.stdout | from_json | json_query(query_pods_not_ready + '.metadata') }}" block: - name: Check that all pods are running command: "{{ bin_dir }}/kubectl get pods --all-namespaces -o json" register: run_pods_log changed_when: false until: # Check that all pods are running - run_pods_log.stdout | from_json | json_query(query_pods_not_running) == [] # Check that all pods are ready - run_pods_log.stdout | from_json | json_query(query_pods_not_ready) == [] retries: 30 delay: 10 rescue: - name: Describe broken pods command: "{{ bin_dir }}/kubectl describe pod -n {{ item.namespace }} {{ item.name }}" loop: "{{ pods_not_running + pods_not_ready }}" loop_control: label: "{{ item.namespace }}/{{ item.name }}" - name: Get logs from broken pods command: "{{ bin_dir }}/kubectl logs -n {{ item.namespace }} {{ item.name }}" loop: "{{ pods_not_running + pods_not_ready }}" loop_control: label: "{{ item.namespace }}/{{ item.name }}" - name: Fail CI fail: {} ================================================ FILE: tests/testcases/025_check-csr-request.yml ================================================ --- - name: Check kubelet serving certificates approved with kubelet_csr_approver when: - kubelet_rotate_server_certificates | default(false) - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)) vars: csrs: "{{ csr_json.stdout | from_json }}" block: - name: Get certificate signing requests command: "{{ bin_dir }}/kubectl get csr -o jsonpath-as-json={.items[*]}" register: csr_json changed_when: false - name: Check there are csrs assert: that: csrs | length > 0 fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found - name: Check there are Denied/Pending csrs assert: that: - csrs | rejectattr('status') | length == 0 # Pending == no status - csrs | map(attribute='status.conditions') | flatten | selectattr('type', 'equalto', 'Denied') | length == 0 # Denied fail_msg: kubelet_csr_approver is enabled but CSRs are not approved - name: Approve kubelet serving certificates when: - kubelet_rotate_server_certificates | default(false) - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))) block: - name: Get certificate signing requests command: "{{ bin_dir }}/kubectl get csr -o name" register: get_csr changed_when: false - name: Check there are csrs assert: that: get_csr.stdout_lines | length > 0 fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found - name: Approve certificates command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}" register: certificate_approve when: get_csr.stdout_lines | length > 0 changed_when: certificate_approve.stdout ================================================ FILE: tests/testcases/030_check-network.yml ================================================ --- - name: Run the hydrophone checks vars: networking_check: "\\[sig-network\\] Networking Granular Checks.+\\[Conformance\\]" block: - name: Run the networking granular checks command: "{{ hydrophone_path }} --focus=\"{{ networking_check }}\" --parallel {{ hydrophone_parallel }}" rescue: - name: List pods cluster-wide command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide" changed_when: false - import_role: # noqa name[missing] name: cluster-dump - fail: # noqa name[missing] ================================================ FILE: tests/testcases/040_check-network-adv.yml ================================================ --- - name: Test tunl0 routes command: "/sbin/ip route" register: routes failed_when: routes.stdout_lines | select('contains', '/' ~ calico_pool_blocksize|d(26)) | select('contains', 'tunl0') | length == 0 when: - ('kube_node' in group_names) - (calico_ipip_mode is defined and calico_ipip_mode != 'Never') - kube_network_plugin | default('calico') == 'calico' - import_role: # noqa name[missing] name: cluster-dump - name: Create macvlan network conf command: cmd: "{{ bin_dir }}/kubectl create -f -" stdin: | apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: name: macvlan-conf spec: config: '{ "cniVersion": "0.4.0", "type": "macvlan", "master": "eth0", "mode": "bridge", "ipam": { "type": "host-local", "subnet": "192.168.1.0/24", "rangeStart": "192.168.1.200", "rangeEnd": "192.168.1.216", "routes": [ { "dst": "0.0.0.0/0" } ], "gateway": "192.168.1.1" } }' --- apiVersion: v1 kind: Pod metadata: name: samplepod annotations: k8s.v1.cni.cncf.io/networks: macvlan-conf spec: containers: - name: samplepod command: ["/bin/bash", "-c", "sleep 2000000000000"] image: dougbtv/centos-network delegate_to: groups['kube_control_plane'][0] run_once: true when: - kube_network_plugin_multus | default(false) | bool - name: Check secondary macvlan interface command: "{{ bin_dir }}/kubectl exec samplepod -- ip addr show dev net1" register: output until: output.rc == 0 retries: 90 changed_when: false delegate_to: groups['kube_control_plane'][0] run_once: true when: - kube_network_plugin_multus | default(false) | bool ================================================ FILE: tests/testcases/100_check-k8s-conformance.yml ================================================ --- - name: Download sonobuoy get_url: url: "https://github.com/vmware-tanzu/sonobuoy/releases/download/v{{ sonobuoy_version }}/sonobuoy_{{ sonobuoy_version }}_linux_{{ sonobuoy_arch }}.tar.gz" dest: /tmp/sonobuoy.tar.gz mode: "0644" - name: Extract sonobuoy unarchive: src: /tmp/sonobuoy.tar.gz dest: /usr/local/bin/ copy: false - name: Run sonobuoy command: "{{ sonobuoy_path }} run --mode {{ sonobuoy_mode }} --e2e-parallel {{ sonobuoy_parallel }} --wait" - name: Run sonobuoy retrieve command: "{{ sonobuoy_path }} retrieve" register: sonobuoy_retrieve - name: Run inspect results command: "{{ sonobuoy_path }} results {{ sonobuoy_retrieve.stdout }} --plugin e2e --mode report" ================================================ FILE: tests/testcases/roles/cluster-dump/tasks/main.yml ================================================ --- - name: Generate dump folder command: "{{ bin_dir }}/kubectl cluster-info dump --all-namespaces --output-directory /tmp/cluster-dump" when: inventory_hostname in groups['kube_control_plane'] - name: Compress directory cluster-dump community.general.archive: path: /tmp/cluster-dump dest: /tmp/cluster-dump.tgz mode: "0644" when: inventory_hostname in groups['kube_control_plane'] - name: Fetch dump file fetch: src: /tmp/cluster-dump.tgz dest: "{{ lookup('env', 'CI_PROJECT_DIR') }}/cluster-dump/{{ inventory_hostname }}.tgz" flat: true when: inventory_hostname in groups['kube_control_plane'] ================================================ FILE: tests/testcases/tests.yml ================================================ --- - name: Define dynamic groups import_playbook: ../../playbooks/boilerplate.yml - name: Kubespray CI tests hosts: k8s_cluster gather_facts: false vars: testcase: "{{ lookup('env', 'TESTCASE') }}" tasks: - name: Import Kubespray variables import_role: name: ../../roles/kubespray_defaults - name: Install the Hydrophone for tests import_tasks: 000_install-hydrophone.yml - name: Testcases for apiserver import_tasks: 010_check-apiserver.yml when: - ('kube_control_plane') in group_names - name: Test using API delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true block: - name: Testcases checking nodes import_tasks: 015_check-nodes-ready.yml - name: Testcases checking pods import_tasks: 020_check-pods-running.yml when: ('macvlan' not in testcase) - name: Checking CSR approver import_tasks: 025_check-csr-request.yml - name: Testcases for network import_tasks: 030_check-network.yml when: ('macvlan' not in testcase) - name: Testcases for calico / advanced network import_tasks: 040_check-network-adv.yml when: - ('macvlan' not in testcase) - ('hardening' not in testcase) - name: Testcases for kubernetes conformance import_tasks: 100_check-k8s-conformance.yml delegate_to: "{{ groups['kube_control_plane'][0] }}" run_once: true when: - sonobuoy_enabled is defined - sonobuoy_enabled vars: sonobuoy_version: 0.57.3 sonobuoy_arch: amd64 sonobuoy_parallel: 30 sonobuoy_path: /usr/local/bin/sonobuoy sonobuoy_mode: Quick ================================================ FILE: upgrade-cluster.yml ================================================ --- - name: Upgrade cluster ansible.builtin.import_playbook: playbooks/upgrade_cluster.yml ================================================ FILE: upgrade_cluster.yml ================================================ --- - name: Upgrade cluster ansible.builtin.import_playbook: playbooks/upgrade_cluster.yml