Repository: oracle-terraform-modules/terraform-oci-oke Branch: main Commit: 25ef9a02ada3 Files: 197 Total size: 734.7 KB Directory structure: gitextract_s2i9ymxq/ ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── Bug_Report.md │ │ ├── Feature_Request.md │ │ └── Question.md │ └── ISSUE_TEMPLATE.md ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── THIRD_PARTY_LICENSES.txt ├── data-common.tf ├── data-images.tf ├── docs/ │ ├── diagrams.md │ ├── prerequisites.md │ ├── quickstart.md │ └── terraformoptions.md ├── examples/ │ ├── bastion/ │ │ └── README.md │ ├── cluster/ │ │ └── README.md │ ├── cluster-addons/ │ │ └── README.md │ ├── extensions/ │ │ └── README.md │ ├── iam/ │ │ └── README.md │ ├── istio-mc/ │ │ ├── README.md │ │ ├── c1.tf │ │ ├── c2.tf │ │ ├── contexts.tf │ │ ├── istio.tf │ │ ├── locals.tf │ │ ├── outputs.tf │ │ ├── providers.tf │ │ ├── resources/ │ │ │ └── istio.template.yaml │ │ ├── scripts/ │ │ │ ├── cloud-init.sh │ │ │ ├── generate_kubeconfig.template.sh │ │ │ ├── istioctl.template.sh │ │ │ ├── kubeconfig_set_credentials.template.sh │ │ │ ├── set_alias.template.sh │ │ │ └── token_helper.template.sh │ │ ├── templates.tf │ │ ├── terraform.tfvars.example │ │ ├── variables.tf │ │ └── versions.tf │ ├── network/ │ │ ├── README.md │ │ └── vars-network-drg-create.auto.tfvars.example │ ├── operator/ │ │ └── README.md │ ├── profiles/ │ │ ├── README.md │ │ ├── cluster-workers-only/ │ │ │ ├── main.tf │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ ├── network-cluster-workers/ │ │ │ ├── main.tf │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ ├── network-only/ │ │ │ ├── main.tf │ │ │ ├── variables.tf │ │ │ └── versions.tf │ │ └── workers-only/ │ │ ├── main.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── provider-basic.tf │ ├── rms/ │ │ ├── README.md │ │ ├── oke-cluster-only/ │ │ │ ├── data.tf │ │ │ ├── main.tf │ │ │ ├── output.tf │ │ │ ├── schema.yaml │ │ │ ├── variables-cluster.tf │ │ │ ├── variables-extensions.tf │ │ │ ├── variables-iam.tf │ │ │ ├── variables-network.tf │ │ │ ├── variables-operator.tf │ │ │ └── versions.tf │ │ ├── oke-network-only/ │ │ │ ├── data.tf │ │ │ ├── main.tf │ │ │ ├── output.tf │ │ │ ├── schema.yaml │ │ │ ├── variables-bastion.tf │ │ │ ├── variables-iam.tf │ │ │ ├── variables-network.tf │ │ │ ├── variables-subnets.tf │ │ │ └── versions.tf │ │ └── oke-workers-only/ │ │ ├── data.tf │ │ ├── main.tf │ │ ├── output.tf │ │ ├── schema.yaml │ │ ├── variables.tf │ │ └── versions.tf │ ├── utilities/ │ │ └── README.md │ └── workers/ │ └── README.md ├── migration.tf ├── module-bastion.tf ├── module-cluster-addons.tf ├── module-cluster.tf ├── module-extensions.tf ├── module-iam.tf ├── module-network.tf ├── module-operator.tf ├── module-utilities.tf ├── module-workers.tf ├── modules/ │ ├── bastion/ │ │ ├── README.md │ │ ├── cloudinit.tf │ │ ├── compute.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── cluster/ │ │ ├── README.md │ │ ├── cluster.tf │ │ ├── outputs.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── cluster-addons/ │ │ ├── README.md │ │ ├── addons.tf │ │ ├── delete_addons.tf │ │ ├── outputs.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── extensions/ │ │ ├── README.md │ │ ├── argocd.tf │ │ ├── autoscaler.tf │ │ ├── cilium.tf │ │ ├── dcgm_exporter.tf │ │ ├── gatekeeper.tf │ │ ├── locals.tf │ │ ├── metricserver.tf │ │ ├── mpi_operator.tf │ │ ├── multus.tf │ │ ├── prometheus.tf │ │ ├── rdma_cni_plugin.tf │ │ ├── service_account.tf │ │ ├── sriov_cni_plugin.tf │ │ ├── sriov_device_plugin.tf │ │ ├── variables.tf │ │ ├── versions.tf │ │ └── whereabouts.tf │ ├── iam/ │ │ ├── README.md │ │ ├── await.tf │ │ ├── group-autoscaling.tf │ │ ├── group-cluster.tf │ │ ├── group-operator.tf │ │ ├── group-workers.tf │ │ ├── outputs.tf │ │ ├── policy.tf │ │ ├── tagging.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── network/ │ │ ├── README.md │ │ ├── datasources.tf │ │ ├── drgs.tf │ │ ├── locals.tf │ │ ├── nsg-bastion.tf │ │ ├── nsg-controlplane.tf │ │ ├── nsg-fss.tf │ │ ├── nsg-loadbalancers-int.tf │ │ ├── nsg-loadbalancers-pub.tf │ │ ├── nsg-operator.tf │ │ ├── nsg-pods.tf │ │ ├── nsg-workers.tf │ │ ├── rules.tf │ │ ├── subnets.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── operator/ │ │ ├── README.md │ │ ├── cloudinit.tf │ │ ├── compute.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── utilities/ │ │ ├── README.md │ │ ├── drain.tf │ │ ├── nodeready.tf │ │ ├── ocir.tf │ │ ├── resources/ │ │ │ └── await_node_readiness.tpl.sh │ │ ├── variables.tf │ │ └── versions.tf │ └── workers/ │ ├── README.md │ ├── cloudinit-oke.sh │ ├── cloudinit-ubuntu.sh.tftpl │ ├── cloudinit.tf │ ├── clusternetworks.tf │ ├── computecluster.tf │ ├── data-faultdomains.tf │ ├── data-shapes.tf │ ├── gpumemorycluster.tf │ ├── instance.tf │ ├── instanceconfig.tf │ ├── instancepools.tf │ ├── locals.tf │ ├── nodepools.tf │ ├── outputs.tf │ ├── variables.tf │ ├── versions.tf │ └── virtualnodepools.tf ├── variables-bastion.tf ├── variables-cluster-addons.tf ├── variables-cluster.tf ├── variables-common.tf ├── variables-extensions.tf ├── variables-iam.tf ├── variables-network.tf ├── variables-operator.tf ├── variables-utilities.tf ├── variables-workers.tf └── versions.tf ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/Bug_Report.md ================================================ --- name: 🐛 Bug Report about: If something isn't working as expected 🤔. labels: bug --- ### Community Note * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request * Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request * If you are interested in working on this issue or have submitted a pull request, please leave a comment ### Terraform Version and Provider Version ### Affected Resource(s) ### Terraform Configuration Files ```hcl # Copy-paste your Terraform configurations here - for large Terraform configs, # please use a service like Dropbox and share a link to the ZIP file. # Please remove any sensitive information from configuration files before sharing them. ``` ### Debug Output ### Panic Output ### Expected Behavior ### Actual Behavior ### Steps to Reproduce 1. `terraform apply` ### Important Factoids ### References ================================================ FILE: .github/ISSUE_TEMPLATE/Feature_Request.md ================================================ --- name: 🚀 Feature Request about: I have a suggestion (and might want to implement myself 🙂)! labels: enhancement --- ### Community Note * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request * Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request * If you are interested in working on this issue or have submitted a pull request, please leave a comment ### Description ### New or Affected Resource(s) ### Potential Terraform Configuration ```hcl # Copy-paste any Terraform configurations for how the requested feature may be used. ``` ### References ================================================ FILE: .github/ISSUE_TEMPLATE/Question.md ================================================ --- name: 💬 Question about: Questions regarding the Terraform Oracle Cloud Infrastructure OKE module labels: question --- This Github template is intended for questions regarding the Terraform **Oracle Cloud Infrastructure** OKE module. If you have a support request or question related to core Terraform functionality or the OCI provider, please submit them to one of these resources: * [Terraform OCI provider](https://github.com/terraform-providers/terraform-provider-oci) * [Terraform community resources](https://www.terraform.io/docs/extend/community/index.html) * [HashiCorp support](https://support.hashicorp.com) (Terraform Enterprise customers) ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ ================================================ FILE: .gitignore ================================================ # Local .terraform directories **/.terraform/* provider.tf # .tfstate files *.tfstate *.tfstate.* # .tfvars files *.tfvars generated/** # visual code **/.vscode/* **/.metals* .terraform.lock.hcl # Generated documentation docs/book .idea* .oca* ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing Oracle welcomes contributions to this repository from anyone. If you want to submit a pull request to fix a bug or enhance an existing feature, please first open an issue and link to that issue when you submit your pull request. If you have any questions about a possible submission, feel free to open an issue too. ## Contributing to the terraform-oci-oke repository Pull requests can be made under [The Oracle Contributor Agreement](https://oca.opensource.oracle.com/). For pull requests to be accepted, the bottom of your commit message must have the following line using your name and e-mail address as it appears in the OCA Signatories list. ``` Signed-off-by: Your Name ``` This can be automatically added to pull requests by committing with: ``` git commit --signoff ``` Only pull requests from committers that can be verified as having signed the OCA can be accepted. ### Pull request process 1. Fork this repository 1. Create a branch in your fork to implement the changes. We recommend using the issue number as part of your branch name, e.g. `1234-fixes` 1. Ensure that any documentation is updated with the changes that are required by your fix. 1. Ensure that any samples are updated if the base image has been changed. 1. Submit the pull request. *Do not leave the pull request blank*. Explain exactly what your changes are meant to do and provide simple steps on how to validate your changes. Ensure that you reference the issue you created as well. We will assign the pull request to 2-3 people for review before it is merged. ================================================ FILE: LICENSE ================================================ Copyright (c) 2019 Oracle and/or its affiliates. The Universal Permissive License (UPL), Version 1.0 Subject to the condition set forth below, permission is hereby granted to any person obtaining a copy of this software, associated documentation and/or data (collectively the "Software"), free of charge and under any and all copyright rights in the Software, and any and all patent rights owned or freely licensable by each licensor hereunder covering either (i) the unmodified Software as contributed to or provided by such licensor, or (ii) the Larger Works (as defined below), to deal in both (a) the Software, and (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if one is included with the Software (each a "Larger Work" to which the Software is contributed by such licensors), without restriction, including without limitation the rights to copy, create derivative works of, display, perform, and distribute the Software and make, use, sell, offer for sale, import, export, have made, and have sold the Software and the Larger Work(s), and to sublicense the foregoing rights on either these or other terms. This license is subject to the following condition: The above copyright notice and either this complete permission notice or at a minimum a reference to the UPL must be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ PROJECT_NAME := "terraform-oci-oke" SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec .PHONY: all all: build ##@ General .PHONY: help help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) .terraform: terraform init ##@ Usage .PHONY: plan plan: .terraform ## Run terraform plan terraform plan .PHONY: apply apply: .terraform ## Run terraform apply terraform apply .PHONY: ssh ssh: ## Print SSH command terraform output -json | jq -rcM '.output.value.ssh_to_operator' .PHONY: clean clean: ## Clear Terraform module cache rm -rf ./.terraform ##@ Hygiene .PHONY: fmt fmt: ## Run terraform fmt terraform fmt -recursive . .PHONY: validate validate: ## Run terraform validate terraform validate .PHONY: tflint tflint: ## Run tflint tflint --recursive . ================================================ FILE: README.md ================================================ # Terraform OKE for Oracle Cloud Infrastructure [changelog]: https://github.com/oracle-terraform-modules/terraform-oci-oke/releases [contributing]: https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/CONTRIBUTING.md [license]: https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/LICENSE [canonical_license]: https://oss.oracle.com/licenses/upl/ [oci]: https://cloud.oracle.com/cloud-infrastructure [oci_documentation]: https://docs.oracle.com/iaas/Content/services.htm [oke]: https://docs.oracle.com/iaas/Content/ContEng/Concepts/contengoverview.htm [docs]: https://github.com/oracle-terraform-modules/terraform-oci-oke/tree/main/docs [prerequisites]: https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/docs/prerequisites.md [quickstart]: https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/docs/quickstart.md [diagrams]: https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/docs/diagrams.md [terraform_options]: https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/docs/terraformoptions.md [examples]: https://github.com/oracle-terraform-modules/terraform-oci-oke/tree/main/examples [repo]: https://github.com/oracle-terraform-modules/terraform-oci-oke [releases]: https://github.com/oracle-terraform-modules/terraform-oci-oke/releases [terraform]: https://www.terraform.io [terraform_oci]: https://registry.terraform.io/providers/oracle/oci/latest [terraform_oci_examples]: https://github.com/oracle/terraform-provider-oci/tree/master/examples [terraform_guides_examples]: https://github.com/hashicorp/terraform-guides/tree/master/infrastructure-as-code/terraform-0.12-examples [terraform_oci_bastion]: https://github.com/oracle-terraform-modules/terraform-oci-bastion [terraform_oci_operator]: https://github.com/oracle-terraform-modules/terraform-oci-operator [terraform_oci_vcn]: https://github.com/oracle-terraform-modules/terraform-oci-vcn The [Terraform OKE Module][repo] for [Oracle Cloud Infrastructure][oci] (OCI) provides a [Terraform][terraform] module that provisions an [OCI Kubernetes Engine (OKE)][oke] cluster with supporting infrastructure. It creates the following resources: * A Virtual Cloud Network (VCN) with public and private subnets, network security groups, and gateways (internet, NAT, service, DRG) * An OKE cluster (basic or enhanced) with configurable CNI, Kubernetes version, and OIDC authentication * Worker node pools in various modes: OKE-managed node pools, virtual node pools, self-managed instances, instance pools, cluster networks, and compute clusters * A bastion host for SSH access into the VCN * An operator host for cluster management with kubectl, Helm, and optional tools (k9s, istioctl, stern, k8sgpt) * IAM dynamic groups, policies, and optional tag namespaces * Kubernetes extensions deployed via Helm or YAML manifests The module outputs the OKE cluster ID, endpoints, bastion and operator SSH commands, and network resource IDs. Detailed outputs such as kubeconfig are available when `output_detail = true`. ## Topology The default deployment creates a VCN with the following subnets: | Subnet | Purpose | Access | |--------|---------|--------| | bastion | Bastion host | Public | | operator | Operator host | Private | | cp | Kubernetes control plane | Private (or public) | | workers | Worker nodes | Private | | pods | Pod network (NPN CNI) | Private | | int_lb | Internal load balancers | Private | | pub_lb | Public load balancers | Public | ![Default Multi-AD topology](docs/images/defaultmad-large.svg) ## Worker Modes The module supports multiple worker management modes: | Mode | Description | Use Case | |------|-------------|----------| | `node-pool` | OKE-managed node pools | General purpose workloads | | `virtual-node-pool` | OKE-managed virtual nodes | Serverless, burstable workloads | | `instance` | Self-managed compute instances | Custom node configuration | | `instance-pool` | Self-managed instance pools | Scalable self-managed nodes | | `cluster-network` | Self-managed cluster networks | HPC/GPU with RDMA networking | | `compute-cluster` | Shared compute clusters | Multi-nodepool HPC clusters | ## Extensions The module can deploy the following Kubernetes extensions: | Extension | Method | Purpose | |-----------|--------|---------| | Cilium | Helm | eBPF-based networking, security, and observability | | Multus | Daemonset | Multi-network pod interfaces | | SR-IOV Device Plugin | Daemonset | SR-IOV network device advertisement | | SR-IOV CNI Plugin | Daemonset | SR-IOV network connections | | RDMA CNI Plugin | Daemonset | RDMA network connections | | Whereabouts | Daemonset | IP address management for Multus | | Metrics Server | Helm | Kubernetes metrics API | | Cluster Autoscaler | Helm | Automatic node pool scaling | | Prometheus | Helm | Monitoring and alerting | | DCGM Exporter | Helm | GPU metrics for NVIDIA GPUs | | Gatekeeper | Helm | OPA policy enforcement | | MPI Operator | Manifest | MPI/NCCL distributed training jobs | | ArgoCD | Helm | GitOps continuous delivery | ## [Documentation][docs] - [Prerequisites][prerequisites] - [Quickstart][quickstart] - [Diagrams][diagrams] - [Terraform Options][terraform_options] - [Examples][examples] ## Related Documentation - [Oracle Cloud Infrastructure Documentation][oci_documentation] - [Terraform OCI Provider Documentation][terraform_oci] - [OCI Kubernetes Engine Documentation][oke] - [Terraform OCI Bastion Module][terraform_oci_bastion] ## Acknowledgement Code derived and adapted from [Terraform OCI Examples][terraform_oci_examples] and HashiCorp's [Terraform 0.12 examples][terraform_guides_examples]. ## Contributing Learn how to [contribute][contributing]. ## License Copyright (c) 2017, 2025 Oracle Corporation and/or its affiliates. Licensed under the [Universal Permissive License 1.0][license] as shown at [https://oss.oracle.com/licenses/upl][canonical_license]. ================================================ FILE: THIRD_PARTY_LICENSES.txt ================================================ github.com/open-policy-agent/gatekeeper -------- Copyrights Copyright 2018-2020 The Gatekeeper Authors Copyright 2018 The Kubernetes Authors. Copyright (c) 2015 Paxton Hare Copyright 2019 The Kubernetes Authors. -------- Notices Gatekeeper Copyright 2018-2020 The Gatekeeper Authors This product includes modified code from the following projects, as denoted by notifications in the source code headers: controller-runtime https://sigs.k8s.io/controller-runtime Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 -------- License SPDX:Apache-2.0 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/PuerkitoBio/purell -------- Copyrights Copyright (c) 2012, Martin Angers -------- Dependencies Summary github.com/PuerkitoBio/purell -------- License used by Dependencies Copyright (c) 2012, Martin Angers All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/evanphx/json-patch -------- Copyrights Copyright (c) 2014, Evan Phoenix -------- Dependencies Summary github.com/evanphx/json-patch -------- License used by Dependencies Copyright (c) 2014, Evan Phoenix All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the Evan Phoenix nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/pkg/errors -------- Copyrights Copyright (c) 2015, Dave Cheney -------- Dependencies Summary github.com/pkg/errors -------- License used by Dependencies Copyright (c) 2015, Dave Cheney All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/rcrowley/go-metrics -------- Copyrights Copyright 2012 Richard Crowley. All rights reserved. -------- Dependencies Summary github.com/rcrowley/go-metrics -------- License used by Dependencies Copyright 2012 Richard Crowley. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of Richard Crowley. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/hashicorp/golang-lru -------- Copyrights -------- Dependencies Summary github.com/hashicorp/golang-lru -------- License used by Dependencies Mozilla Public License, version 2.0 1. Definitions 1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. "Contribution" means Covered Software of a particular Contributor. 1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. "Incompatible With Secondary Licenses" means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. "Executable Form" means any form of the work other than Source Code Form. 1.7. "Larger Work" means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. "License" means this document. 1.9. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. "Modifications" means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. "Patent Claims" of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. "Source Code Form" means the form of the work preferred for making modifications. 1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an "as is" basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - "Incompatible With Secondary Licenses" Notice This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. ----------------------- Dependencies Grouped by License ------------ -------- Dependency cloud.google.com/go -------- Copyrights Copyright 2020 Google LLC Copyright 2019 Google LLC Copyright 2016 Google LLC Copyright (c) 1996-1998 John D. Polstra. All rights reserved. Copyright (c) 2001 David E. O'Brien Portions Copyright 2018 Google LLC. Copyright 2018 Google LLC Copyright 2014 Google LLC Copyright 2017 Google LLC Copyright 2018 Google Inc. All Rights Reserved. Copyright 2020, Google LLC Copyright 2017, Google LLC -------- Dependency contrib.go.opencensus.io/exporter/prometheus -------- Copyrights Copyright 2017, OpenCensus Authors Copyright 2018, OpenCensus Authors -------- Dependency github.com/OneOfOne/xxhash -------- Copyrights The C implementation is ([Copyright](https://github.com/Cyan4973/xxHash/blob/master/LICENSE) (c) 2012-2014, Yann Collet) -------- Dependency github.com/go-logr/logr -------- Copyrights Copyright 2020 The logr Authors. Copyright 2019 The logr Authors. Copyright 2021 The logr Authors. -------- Dependency github.com/go-logr/zapr -------- Copyrights Copyright 2019 The logr Authors. Copyright 2018 Solly Ross -------- Dependency github.com/go-openapi/jsonpointer -------- Copyrights Copyright 2013 sigu-399 ( https://github.com/sigu-399 ) -------- Dependency github.com/go-openapi/jsonreference -------- Copyrights Copyright 2013 sigu-399 ( https://github.com/sigu-399 ) -------- Dependency github.com/go-openapi/spec -------- Copyrights Copyright 2015 go-swagger maintainers Copyright 2017 go-swagger maintainers -------- Dependency github.com/go-openapi/swag -------- Copyrights Copyright 2015 go-swagger maintainers -------- Dependency github.com/golang/glog -------- Copyrights Copyright 2013 Google Inc. All Rights Reserved. -------- Dependency github.com/golang/groupcache -------- Copyrights Copyright 2012 Google Inc. Copyright 2013 Google Inc. -------- Dependency github.com/google/gofuzz -------- Copyrights Copyright 2014 Google Inc. All rights reserved. -------- Dependency github.com/googleapis/gnostic -------- Copyrights Copyright 2017-2020, Google LLC. Copyright 2019 Google LLC. All Rights Reserved. Copyright 2020 Google LLC. All Rights Reserved. Copyright 2017 Google LLC. All Rights Reserved. Copyright 2018 Google LLC. All Rights Reserved. Copyright 2020 Google LLC. All Rights Reserved.\n" + -------- Dependency github.com/matttproud/golang_protobuf_extensions -------- Copyrights Copyright 2012 Matt T. Proud (matt.proud@gmail.com) Copyright 2013 Matt T. Proud Copyright 2016 Matt T. Proud -------- Notices Copyright 2012 Matt T. Proud (matt.proud@gmail.com) -------- Dependency github.com/modern-go/concurrent -------- Copyrights -------- Dependency github.com/modern-go/reflect2 -------- Copyrights -------- Dependency github.com/open-policy-agent/cert-controller -------- Copyrights Copyright 2018-2020 The Gatekeeper Authors -------- Notices cert-controller Copyright 2018-2020 The Gatekeeper Authors -------- Dependency github.com/open-policy-agent/frameworks/constraint -------- Copyrights Copyright 2017 The OPA Authors. All rights reserved. Copyright 2018 The OPA Authors. All rights reserved. -------- Dependency github.com/open-policy-agent/opa -------- Copyrights Copyright 2016 The OPA Authors. All rights reserved. Copyright 2017 The OPA Authors. All rights reserved. Copyright 2020 The OPA Authors. All rights reserved. Copyright 2019 The OPA Authors. All rights reserved. Copyright 2018 The OPA Authors. All rights reserved. Copyright 2021 The OPA Authors. All rights reserved. Copyright (c) 2019 Jeremy Thomas Code copyright 2019 Jeremy Thomas. Code released under [the MIT license](https://github.com/jgthms/bulma/blob/master/LICENSE). Copyright 2017-2020 Authors of Cilium _, err := out.Write([]byte(`// Copyright 2018 The OPA Authors. All rights reserved. Copyright 2015 xeipuuv Copyright 2018 johandorland ( https://github.com/johandorland ) Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) Copyright 2013 MongoDB, Inc. Copyright 2017 johandorland ( https://github.com/johandorland ) Copyright (c) 2015 lestrrat Copyright 2011 The Go Authors. All rights reserved. Copyright 2013-2015 CoreOS, Inc. Copyright The Helm Authors. Copyright 2012 The Gorilla Authors. All rights reserved. Copyright (c) 2012 Rodrigo Moraes. All rights reserved. Copyright 2021 icza -------- Dependency github.com/prometheus/client_golang -------- Copyrights Copyright 2018 The Prometheus Authors Copyright 2012-2015 The Prometheus Authors Copyright 2013-2015 Blake Mizerany, Björn Rabenstein Copyright 2010 The Go Authors Copyright 2013 Matt T. Proud Copyright 2015 The Prometheus Authors Copyright 2017 The Prometheus Authors Copyright 2019 The Prometheus Authors Copyright 2014 The Prometheus Authors Copyright 2021 The Prometheus Authors Copyright 2016 The Prometheus Authors Copyright 2020 The Prometheus Authors Copyright (c) 2013, The Prometheus Authors -------- Notices Prometheus instrumentation library for Go applications Copyright 2012-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). The following components are included in this product: perks - a fork of https://github.com/bmizerany/perks https://github.com/beorn7/perks Copyright 2013-2015 Blake Mizerany, Björn Rabenstein See https://github.com/beorn7/perks/blob/master/README.md for license details. Go support for Protocol Buffers - Google's data interchange format http://github.com/golang/protobuf/ Copyright 2010 The Go Authors See source code for license details. Support for streaming Protocol Buffer messages for the Go language (golang). https://github.com/matttproud/golang_protobuf_extensions Copyright 2013 Matt T. Proud Licensed under the Apache License, Version 2.0 -------- Dependency github.com/prometheus/client_model -------- Copyrights Copyright 2013 Prometheus Team Copyright 2012-2015 The Prometheus Authors -------- Notices Data model artifacts for Prometheus. Copyright 2012-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). -------- Dependency github.com/prometheus/common -------- Copyrights Copyright 2018 The Prometheus Authors Copyright 2015 The Prometheus Authors Copyright 2016 The Prometheus Authors Copyright 2021 The Prometheus Authors Copyright 2014 The Prometheus Authors Copyright 2020 The Prometheus Authors Copyright (c) 2011, Open Knowledge Foundation Ltd. Copyright 2013 The Prometheus Authors Copyright 2019 The Prometheus Authors Copyright 2017 The Prometheus Authors -------- Notices Common libraries shared by Prometheus Go components. Copyright 2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). -------- Dependency github.com/prometheus/procfs -------- Copyrights Copyright 2018 The Prometheus Authors Copyright 2014-2015 The Prometheus Authors Copyright 2019 The Prometheus Authors Copyright 2017 The Prometheus Authors Copyright 2021 The Prometheus Authors Copyright 2020 The Prometheus Authors Copyright 2014 Prometheus Team Copyright 2017 Prometheus Team -------- Notices procfs provides functions to retrieve system, kernel and process metrics from the pseudo-filesystem proc. Copyright 2014-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). -------- Dependency github.com/prometheus/statsd_exporter -------- Copyrights Copyright 2013 The Prometheus Authors Copyright 2018 The Prometheus Authors Copyright 2013-2015 The Prometheus Authors Copyright 2020 The Prometheus Authors Copyright 2019 The Prometheus Authors Copyright 2021 The Prometheus Authors -------- Notices StatsD-to-Prometheus exporter Copyright 2013-2015 The Prometheus Authors This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). -------- Dependency github.com/spf13/cobra -------- Copyrights Copyright © 2020 Steve Francia Copyright © 2015 Steve Francia . Copyright: copyrightLine(), Copyright (C) 2007 Free Software Foundation, Inc. 11 of the WIPO copyright treaty adopted on 20 December 1996, or Copyright (C) 1989, 1991 Free Software Foundation, Inc., Copyright string copyright": copyrightLine(), Copyright © 2013 Steve Francia . Copyright 2015 Red Hat Inc. All rights reserved. Copyright 2016 French Ben. All rights reserved. -------- Dependency github.com/xeipuuv/gojsonpointer -------- Copyrights Copyright 2015 xeipuuv Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) -------- Dependency github.com/xeipuuv/gojsonreference -------- Copyrights Copyright 2015 xeipuuv Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) -------- Dependency github.com/yashtewari/glob-intersection -------- Copyrights -------- Dependency go.opencensus.io -------- Copyrights Copyright 2019, OpenCensus Authors Copyright 2017, OpenCensus Authors Copyright 2018, OpenCensus Authors Copyright 2020, OpenCensus Authors -------- Dependency gomodules.xyz/jsonpatch/v2 -------- Copyrights -------- Dependency google.golang.org/genproto -------- Copyrights Copyright 2020 Google LLC. Copyright 2020 Google LLC Copyright 2019 Google LLC. Copyright (c) 2015, Google Inc. Copyright 2016 Google Inc. Copyright 2017 Google Inc. Copyright 2018 Google Inc. Copyright 2018 Google LLC Copyright 2018 The Grafeas Authors. All rights reserved. Copyright 2019 Google LLC -------- Dependency google.golang.org/grpc -------- Copyrights Copyright 2019 gRPC authors. Copyright 2017 gRPC authors. Copyright 2015 The gRPC Authors Copyright 2016 gRPC authors. Copyright 2018 gRPC authors. Copyright 2020 gRPC authors. Copyright 2020 The gRPC Authors Copyright 2014 gRPC authors. Copyright 2018 The gRPC Authors Copyright 2015 gRPC authors. Copyright 2019 Istio Authors. All Rights Reserved. Copyright 2016 The gRPC Authors Copyright 2015-2016 gRPC authors. -------- Dependency gopkg.in/yaml.v2 -------- Copyrights Copyright (c) 2006 Kirill Simonov Copyright 2011-2016 Canonical Ltd. -------- Notices Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------- Dependency k8s.io/api -------- Copyrights Copyright 2019 The Kubernetes Authors. Copyright The Kubernetes Authors. Copyright 2017 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2015 The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. -------- Dependency k8s.io/apiextensions-apiserver -------- Copyrights Copyright 2019 The Kubernetes Authors. Copyright 2017 The Kubernetes Authors. Copyright The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. -------- Dependency k8s.io/apimachinery -------- Copyrights Copyright 2017 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright 2014 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. Copyright 2015 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright The Kubernetes Authors. Copyright 2013 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. -------- Dependency k8s.io/apiserver -------- Copyrights Copyright 2014 The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright 2015 The Kubernetes Authors. Copyright 2017 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright The Kubernetes Authors. -------- Dependency k8s.io/client-go -------- Copyrights Copyright 2016 The Kubernetes Authors. Copyright 2017 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright 2015 The Kubernetes Authors. Copyright 2014 The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. -------- Dependency k8s.io/component-base -------- Copyrights Copyright 2017 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. Copyright 2014 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright The Kubernetes Authors. Copyright 2015 The Kubernetes Authors. -------- Dependency k8s.io/klog/v2 -------- Copyrights Copyright 2013 Google Inc. All Rights Reserved. Copyright 2020 The Kubernetes Authors. -------- Dependency k8s.io/kube-openapi -------- Copyrights Copyright The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright 2017 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. Copyright 2015 go-swagger maintainers Copyright (C) MongoDB, Inc. 2017-present. Copyright 2017 go-swagger maintainers -------- Dependency k8s.io/utils -------- Copyrights Copyright 2018 The Kubernetes Authors. Copyright 2017 The Kubernetes Authors. Copyright 2014 The Kubernetes Authors. Copyright 2015 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. -------- Dependency sigs.k8s.io/apiserver-network-proxy/konnectivity-client -------- Copyrights Copyright 2019 The Kubernetes Authors. Copyright The Kubernetes Authors. -------- Dependency sigs.k8s.io/controller-runtime -------- Copyrights Copyright 2020 The Kubernetes Authors. Copyright 2018 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright 2018 The Kubernetes authors. Copyright 2017 The Kubernetes Authors. Copyright 2016 The Kubernetes Authors. Copyright 2014 The Kubernetes Authors. -------- Dependency sigs.k8s.io/structured-merge-diff/v4 -------- Copyrights Copyright 2018 The Kubernetes Authors. Copyright 2019 The Kubernetes Authors. Copyright 2020 The Kubernetes Authors. -------- Dependencies Summary cloud.google.com/go contrib.go.opencensus.io/exporter/prometheus github.com/OneOfOne/xxhash github.com/go-logr/logr github.com/go-logr/zapr github.com/go-openapi/jsonpointer github.com/go-openapi/jsonreference github.com/go-openapi/spec github.com/go-openapi/swag github.com/golang/glog github.com/golang/groupcache github.com/google/gofuzz github.com/googleapis/gnostic github.com/matttproud/golang_protobuf_extensions github.com/modern-go/concurrent github.com/modern-go/reflect2 github.com/open-policy-agent/cert-controller github.com/open-policy-agent/frameworks/constraint github.com/open-policy-agent/opa github.com/prometheus/client_golang github.com/prometheus/client_model github.com/prometheus/common github.com/prometheus/procfs github.com/prometheus/statsd_exporter github.com/spf13/cobra github.com/xeipuuv/gojsonpointer github.com/xeipuuv/gojsonreference github.com/yashtewari/glob-intersection go.opencensus.io gomodules.xyz/jsonpatch/v2 google.golang.org/genproto google.golang.org/grpc gopkg.in/yaml.v2 k8s.io/api k8s.io/apiextensions-apiserver k8s.io/apimachinery k8s.io/apiserver k8s.io/client-go k8s.io/component-base k8s.io/klog/v2 k8s.io/kube-openapi k8s.io/utils sigs.k8s.io/apiserver-network-proxy/konnectivity-client sigs.k8s.io/controller-runtime sigs.k8s.io/structured-merge-diff/v4 -------- License used by Dependencies SPDX:Apache-2.0 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/PuerkitoBio/urlesc -------- Copyrights Copyright (c) 2012 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. -------- Dependency github.com/fsnotify/fsnotify -------- Copyrights Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. -------- Dependency github.com/gogo/protobuf -------- Copyrights Copyright (c) 2013, The GoGo Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. Copyright (c) 2015, The GoGo Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. Copyright (c) 2018, The GoGo Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright (c) 2016, The GoGo Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright (c) 2019, The GoGo Authors. All rights reserved. Copyright (c) 2017, The GoGo Authors. All rights reserved. Copyright (c) 2015, The GoGo Authors. rights reserved. -------- Dependency github.com/golang/protobuf -------- Copyrights Copyright 2010 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. -------- Dependency github.com/google/go-cmp -------- Copyrights Copyright (c) 2017 The Go Authors. All rights reserved. Copyright 2017, The Go Authors. All rights reserved. Copyright 2021, The Go Authors. All rights reserved. Copyright 2020, The Go Authors. All rights reserved. Copyright 2018, The Go Authors. All rights reserved. Copyright 2019, The Go Authors. All rights reserved. -------- Dependency github.com/google/uuid -------- Copyrights Copyright (c) 2009,2014 Google Inc. All rights reserved. Copyright 2016 Google Inc. All rights reserved. Copyright 2017 Google Inc. All rights reserved. Copyright 2018 Google Inc. All rights reserved. -------- Dependency github.com/imdario/mergo -------- Copyrights Copyright (c) 2013 Dario Castañé. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. Copyright 2013 Dario Castañé. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. Copyright 2014 Dario Castañé. All rights reserved. -------- Dependency github.com/spf13/pflag -------- Copyrights Copyright (c) 2012 Alex Ogier. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. -------- Dependency golang.org/x/crypto -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency golang.org/x/net -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. Copyright (C) 2009 Apple Inc. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2021 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency golang.org/x/oauth2 -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2017 The oauth2 Authors. All rights reserved. Copyright 2021 The Go Authors. All rights reserved. Copyright 2015 The oauth2 Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2018 The oauth2 Authors. All rights reserved. -------- Dependency golang.org/x/sync -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency golang.org/x/sys -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2010 The Go Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2021 The Go Authors. All rights reserved. Copyright 2009,2010 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All right reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency golang.org/x/term -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency golang.org/x/text -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2014 The Go Authors. All rights reserved. Copyright 2016 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. Copyright 2017 The Go Authors. All rights reserved. Copyright 2012 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2009 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency golang.org/x/time -------- Copyrights Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2015 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency google.golang.org/protobuf -------- Copyrights Copyright (c) 2018 The Go Authors. All rights reserved. Copyright 2018 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved. Copyright 2020 The Go Authors. All rights reserved. Copyright 2019 The Go Authors. All rights reserved.", Copyright 2018 The Go Authors. All rights reserved.", Copyright 2008 Google Inc. All rights reserved. Copyright 2021 The Go Authors. All rights reserved. -------- Patents Additional IP Rights Grant (Patents) "This implementation" means the copyrightable works distributed by Google as part of the Go project. Google hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, transfer and otherwise run, modify and propagate the contents of this implementation of Go, where such license applies only to those patent claims, both currently owned or controlled by Google and acquired in the future, licensable by Google that are necessarily infringed by this implementation of Go. This grant does not include claims that would be infringed only as a consequence of further modification of this implementation. If you or your agent or exclusive licensee institute or order or agree to the institution of patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that this implementation of Go or any code incorporated within this implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. -------- Dependency gopkg.in/inf.v0 -------- Copyrights Copyright (c) 2012 Péter Surányi. Portions Copyright (c) 2009 The Go -------- Dependencies Summary github.com/PuerkitoBio/urlesc github.com/fsnotify/fsnotify github.com/gogo/protobuf github.com/golang/protobuf github.com/google/go-cmp github.com/google/uuid github.com/imdario/mergo github.com/spf13/pflag golang.org/x/crypto golang.org/x/net golang.org/x/oauth2 golang.org/x/sync golang.org/x/sys golang.org/x/term golang.org/x/text golang.org/x/time google.golang.org/protobuf gopkg.in/inf.v0 -------- License used by Dependencies SPDX:BSD-3-Clause--modified-by-Google Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/davecgh/go-spew -------- Copyrights Copyright (c) 2012-2016 Dave Collins Copyright (c) 2015-2016 Dave Collins Copyright (c) 2013-2016 Dave Collins Copyright (c) 2013 Dave Collins -------- Dependencies Summary github.com/davecgh/go-spew -------- License used by Dependencies SPDX:ISC Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/asaskevich/govalidator -------- Copyrights Copyright (c) 2014-2020 Alex Saskevich -------- Dependency github.com/beorn7/perks -------- Copyrights Copyright (C) 2013 Blake Mizerany -------- Dependency github.com/blang/semver -------- Copyrights Copyright (c) 2014 Benedikt Lang -------- Dependency github.com/cespare/xxhash/v2 -------- Copyrights Copyright (c) 2016 Caleb Spare -------- Dependency github.com/go-kit/log -------- Copyrights Copyright (c) 2021 Go kit Copyright (c) 2014 Simon Eskildsen Copyright 2013 The Go Authors. All rights reserved. Copyright 2011 The Go Authors. All rights reserved. -------- Dependency github.com/go-logfmt/logfmt -------- Copyrights Copyright (c) 2015 go-logfmt Copyright 2010 The Go Authors. All rights reserved. -------- Dependency github.com/gobwas/glob -------- Copyrights Copyright (c) 2016 Sergey Kamardin -------- Dependency github.com/josharian/intern -------- Copyrights Copyright (c) 2019 Josh Bleecher Snyder -------- Dependency github.com/json-iterator/go -------- Copyrights Copyright (c) 2016 json-iterator -------- Dependency github.com/mailru/easyjson -------- Copyrights Copyright (c) 2016 Mail.Ru Group Copyright (c) 2009 The Go Authors. All rights reserved. -------- Dependency github.com/mitchellh/mapstructure -------- Copyrights Copyright (c) 2013 Mitchell Hashimoto -------- Dependency go.uber.org/atomic -------- Copyrights Copyright (c) 2016 Uber Technologies, Inc. Copyright (c) 2019 Uber Technologies, Inc. -------- Dependency go.uber.org/multierr -------- Copyrights Copyright (c) 2017 Uber Technologies, Inc. Copyright (c) 2019 Uber Technologies, Inc. -------- Dependency go.uber.org/zap -------- Copyrights Copyright (c) 2016-2017 Uber Technologies, Inc. Copyright (c) 2016 Uber Technologies, Inc. Copyright (c) "*" Uber Technologies, Inc.") Copyright (c) 2017 Uber Technologies, Inc. Copyright (c) 2019 Uber Technologies, Inc. Copyright (c) 2020 Uber Technologies, Inc. Copyright (c) 2016, 2017 Uber Technologies, Inc. Copyright (c) 2018 Uber Technologies, Inc. -------- Dependency gopkg.in/yaml.v3 -------- Copyrights copyright staring in 2011 when the project was ported over: Copyright (c) 2006-2010 Kirill Simonov Copyright (c) 2006-2011 Kirill Simonov Copyright (c) 2011-2019 Canonical Ltd Copyright 2011-2016 Canonical Ltd. -------- Notices Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------- Dependencies Summary github.com/asaskevich/govalidator github.com/beorn7/perks github.com/blang/semver github.com/cespare/xxhash/v2 github.com/go-kit/log github.com/go-logfmt/logfmt github.com/gobwas/glob github.com/josharian/intern github.com/json-iterator/go github.com/mailru/easyjson github.com/mitchellh/mapstructure go.uber.org/atomic go.uber.org/multierr go.uber.org/zap gopkg.in/yaml.v3 -------- License used by Dependencies SPDX:MIT Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ----------------------- Dependencies Grouped by License ------------ -------- Dependency github.com/ghodss/yaml -------- Copyrights Copyright (c) 2014 Sam Ghods Copyright (c) 2012 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. -------- Dependency sigs.k8s.io/yaml -------- Copyrights Copyright (c) 2014 Sam Ghods Copyright (c) 2012 The Go Authors. All rights reserved. Copyright 2013 The Go Authors. All rights reserved. -------- Dependencies Summary github.com/ghodss/yaml sigs.k8s.io/yaml -------- License used by Dependencies The MIT License (MIT) Copyright (c) 2014 Sam Ghods Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Copyright (c) 2012 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ATTRIBUTION-HELPER-GENERATED: License file based on go.mod with md5 sum: 8e6c0582e13472356e90672b8842731d ================================================ FILE: data-common.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { state_id = coalesce(var.state_id, random_string.state_id.id) } resource "random_string" "state_id" { length = 6 lower = true numeric = false special = false upper = false } output "state_id" { value = local.state_id } ================================================ FILE: data-images.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Used to retrieve available worker node images, k8s versions, shapes... data "oci_containerengine_node_pool_option" "oke" { count = local.cluster_enabled ? 1 : 0 node_pool_option_id = "all" compartment_id = local.compartment_id } locals { k8s_versions = toset(concat([var.kubernetes_version], [for k, v in var.worker_pools : lookup(v, "kubernetes_version", "") if lookup(v, "kubernetes_version", "") != ""])) k8s_versions_only = [for k8_version in local.k8s_versions : trimprefix(lower(k8_version), "v")] # OKE managed node pool images node_pool_images = try(one(data.oci_containerengine_node_pool_option.oke[*].sources), []) # Parse platform/operating system information from node pool image names indexed_images = try({ for k, v in local.node_pool_images : v.image_id => merge( try(element(regexall("OKE-(?P[0-9\\.]+)-(?P[0-9]+)", v.source_name), 0), { k8s_version = "none" }), { arch = length(regexall("aarch64", v.source_name)) > 0 ? "aarch64" : "x86_64" image_type = length(regexall("OKE", v.source_name)) > 0 ? "oke" : "platform" is_gpu = length(regexall("GPU", v.source_name)) > 0 os = trimspace(replace(element(regexall("^[a-zA-Z-]+", v.source_name), 0), "-", " ")) os_version = element(regexall("[0-9\\.]+", v.source_name), 0) sort_key = replace(try(join(".", regex("-([0-9]{4}\\.[01][0-9].[0-9]{1,2}).*?-([0-9]+)$", v.source_name)), v.source_name), ".", "") source_name = v.source_name }, ) }, {}) # Create non-exclusive groupings of image IDs for intersection when selecting based on config and instance shape image_ids = try(merge({ x86_64 = [for k, v in local.indexed_images : k if v.arch == "x86_64"] aarch64 = [for k, v in local.indexed_images : k if v.arch == "aarch64"] oke = [for k, v in local.indexed_images : k if v.image_type == "oke" && contains(local.k8s_versions_only, v.k8s_version)] platform = [for k, v in local.indexed_images : k if v.image_type == "platform"] gpu = [for k, v in local.indexed_images : k if v.is_gpu] nongpu = [for k, v in local.indexed_images : k if !v.is_gpu] }, { # Include groups for OS name and major version # https://developer.hashicorp.com/terraform/language/expressions/for#grouping-results for k, v in local.indexed_images : format("%v %v", v.os, split(".", v.os_version)[0]) => k... }, { # Include groups for referenced Kubernetes versions for k, v in local.indexed_images : format("%v", v.k8s_version) => k... if contains(local.k8s_versions_only, v.k8s_version) }), {}) } ================================================ FILE: docs/diagrams.md ================================================ # Diagrams This page collects the currently relevant architecture diagrams for the module. ## Default topologies ### Default Multi-AD topology ![Default Multi-AD topology](./images/defaultmad-large.svg) Shows the default regional deployment spread across multiple availability domains. The control plane, worker nodes, bastion, operator, and load balancer subnets are separated so the module can provide both public entry points and private east-west traffic paths. ### Default Single-AD topology ![Default Single-AD topology](./images/defaultsad-large.svg) Shows the same baseline layout constrained to a single availability domain. This is the simpler topology when multi-AD placement is not required or not available in the target region. ## Network and access ### Network layout ![Network layout](./images/networking-large.svg) Illustrates how the module divides the VCN into functional subnets and NSG boundaries. Use it to understand where the control plane, workers, pods, and load balancers live and how traffic is expected to flow between them. ### Load balancer layout ![Load balancer layout](./images/loadbalancers-large.svg) Highlights the public and internal load balancer subnet choices. This is the diagram to consult when deciding how to set `load_balancers`, `preferred_load_balancer`, and the related service exposure model. ### Bastion access layout ![Bastion access layout](./images/bastion-large.svg) Shows the administrative access path into the VCN through the bastion host. It is useful when validating SSH reachability to private resources such as the operator or worker nodes. ## Exposure variants ### Public control plane topology ![Public control plane topology](./images/publiccluster-large.svg) Shows the variant where the Kubernetes API endpoint is reachable through a public address. This is the most direct management model, but it also requires tighter control of the allowed CIDR ranges. ### Private control plane topology ![Private control plane topology](./images/privatecluster-large.svg) Shows the variant where the Kubernetes API endpoint stays private inside the VCN. This is the preferred layout when cluster administration should happen from the bastion, operator, or connected private networks only. ### Public workers topology ![Public workers topology](./images/publicworkers-large.svg) Shows worker nodes with public IPs and direct outbound reachability. This can simplify bootstrap and troubleshooting, but it expands the exposed surface compared with private workers. ### Private workers topology ![Private workers topology](./images/privateworkers-large.svg) Shows worker nodes kept on private addresses behind the VCN gateways. This is the more typical production posture when outbound access is routed through NAT or service gateways instead of direct public addressing. ## Identity ### OIDC discovery flow ![OIDC discovery flow](./images/oidc-discovery.png) Explains the OIDC discovery integration exposed by the cluster. Use it when enabling `oidc_discovery_enabled` or documenting how external identity providers and token validation interact with the OKE API server. ================================================ FILE: docs/prerequisites.md ================================================ # Pre-requisites [Quick Start guide](https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/docs/quickstart.md) This section will guide you through the pre-requisites before you can use this project. You can proceed to the [Quick Start guide](https://github.com/oracle-terraform-modules/terraform-oci-oke/blob/main/docs/quickstart.md) if you have already done these. 1. [Install Terraform](#install-terraform) 2. [Generate and upload your OCI API keys](#generate-and-upload-your-oci-api-keys) 3. [Create an OCI compartment](#create-an-oci-compartment) 4. [Obtain the necessary OCIDs](#obtain-the-necessary-ocids) 5. [Generate an SSH key pair](#generate-an-ssh-key-pair) 6. [Identity and Access Management Rights](#identity-and-access-management-rights) ### Install Terraform Start by installing Terraform and configuring your path. You need version 1.3.0+. #### Installing Terraform on Oracle Linux ```bash yum -y install oraclelinux-developer-release-el7 && yum -y install terraform ``` #### Installing Terraform on macOS ```bash brew install terraform ``` #### Manual Installation 1. Open your browser and navigate to the [Terraform download page](https://www.terraform.io/downloads.html). You need version 1.3.0+. 2. Download the appropriate version for your operating system. 3. Extract the contents of the compressed file and copy the `terraform` binary to a location that is in your path. ##### Configure path on Linux/macOS ```bash sudo mv /path/to/terraform /usr/local/bin ``` ##### Configure path on Windows 1. Click on `Start`, type `Control Panel` and open it. 2. Select `System > Advanced System Settings > Environment Variables`. 3. Select `System variables > PATH` and click `Edit`. 4. Click `New` and paste the location of the directory where you extracted `terraform.exe`. 5. Close all open windows by clicking `OK`. 6. Open a new terminal and verify Terraform has been properly installed. #### Testing Terraform installation ```bash terraform -v Terraform v1.x.x ``` ### Generate and upload your OCI API keys Follow the documentation for [generating and uploading your API keys](https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm#two). Note the key fingerprint. ### Create an OCI compartment Follow the documentation for [creating a compartment](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcompartments.htm#two). ### Obtain the necessary OCIDs The following OCIDs are required: 1. Compartment OCID 2. Tenancy OCID 3. User OCID Follow the documentation for [obtaining the tenancy and user OCIDs](https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm#five). To obtain the compartment OCID: 1. Navigate to `Identity > Compartments`. 2. Click on your compartment. 3. Locate `OCID` on the page and click `Copy`. ### Generate an SSH key pair An SSH key pair is required for access to the bastion and operator hosts. Generate one if you don't have one: ```bash ssh-keygen -t rsa -b 4096 -f ~/.ssh/oke_key ``` This creates `~/.ssh/oke_key` (private key) and `~/.ssh/oke_key.pub` (public key). ### Identity and Access Management Rights The user or group running Terraform needs the following permissions: | Permission | Purpose | |------------|---------| | `manage all-resources in compartment` | Full management of all OKE resources | | `manage instance-family in compartment` | Create and manage compute instances | | `manage virtual-network-family in compartment` | Create and manage VCN, subnets, NSGs | | `manage cluster-family in compartment` | Create and manage OKE clusters | | `manage volume-family in compartment` | Create and manage block volumes | | `manage dynamic-groups in tenancy` | Create IAM dynamic groups (if `create_iam_resources = true`) | | `manage policies in tenancy` | Create IAM policies (if `create_iam_resources = true`) | For a least-privilege setup, set `create_iam_resources = true` and the module will create the required dynamic groups and policies automatically. ================================================ FILE: docs/quickstart.md ================================================ # Quickstart 1. [Assumptions](#assumptions) 2. [Pre-requisites](#pre-requisites) 3. [Instructions](#instructions) 4. [Connect to the cluster](#connect-to-the-cluster) 5. [Update the cluster](#update-the-cluster) 6. [Destroy the cluster](#destroy-the-cluster) 7. [Related documentation](#related-documentation) ### Assumptions 1. You have set up the [required API keys](https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm). 2. You know the [required OCIDs](https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm#five). 3. You have the necessary [permissions](./prerequisites.md#identity-and-access-management-rights). 4. You have an SSH key pair available. ### Pre-requisites 1. `git` is installed. 2. An SSH client is installed. 3. Terraform 1.3.0+ is installed. See [Pre-requisites](./prerequisites.md) for detailed setup instructions. ### Instructions #### Provisioning using this git repo 1. Clone the repo: ```bash git clone https://github.com/oracle-terraform-modules/terraform-oci-oke.git tfoke cd tfoke ``` Create a `terraform.tfvars` file for your environment. This repository does not ship a generic root `terraform.tfvars.example`. 2. Create a `provider.tf` file and add the following: ```hcl terraform { required_providers { oci = { source = "oracle/oci" version = ">= 7.30.0" } } } provider "oci" { tenancy_ocid = var.tenancy_id user_ocid = var.user_id fingerprint = var.api_fingerprint private_key_path = var.api_private_key_path region = var.region } provider "oci" { alias = "home" tenancy_ocid = var.tenancy_id user_ocid = var.user_id fingerprint = var.api_fingerprint private_key_path = var.api_private_key_path region = coalesce(var.home_region, var.region) } ``` Provider credentials are intentionally configured in `provider.tf`, not in `terraform.tfvars`. 3. Set mandatory provider parameters: - `api_fingerprint` - `api_private_key_path` - `region` - `tenancy_id` - `user_id` 4. Set other required parameters: - `compartment_id` - One of `ssh_public_key` or `ssh_public_key_path` 5. Set cluster and worker parameters. At minimum, configure: ```hcl # Cluster create_cluster = true cluster_name = "oke-cluster" kubernetes_version = "v1.34.2" # Workers worker_pool_mode = "node-pool" worker_pool_size = 1 worker_pools = { np1 = { size = 1 } } ``` 6. Optional parameters to override (see [Terraform Options](./terraformoptions.md) for the full list): - Cluster: `cluster_type`, `cni_type`, `control_plane_is_public`, `pods_cidr`, `services_cidr` - Workers: `worker_shape`, `worker_image_type`, `worker_image_os`, `worker_image_os_version` - Network: `vcn_cidrs`, `subnets`, `nsgs`, `load_balancers` - Bastion: `create_bastion`, `bastion_shape`, `bastion_allowed_cidrs` - Operator: `create_operator`, `operator_shape`, `operator_upgrade` 7. Run Terraform: ```bash terraform init terraform plan terraform apply ``` 8. Retrieve the cluster and access information: ```bash terraform output cluster_id terraform output cluster_endpoints terraform output ssh_to_bastion terraform output ssh_to_operator ``` If you want Terraform to emit `cluster_kubeconfig`, also set: ```hcl output_detail = true ``` ### Connect to the cluster #### Via the operator host 1. SSH to the operator through the bastion: ```bash # Use the output from terraform output ssh_to_operator ssh -o ProxyCommand='ssh -W %h:%p -i ~/.ssh/oke_key opc@' -i ~/.ssh/oke_key opc@ ``` 2. Verify connectivity: ```bash kubectl get nodes ``` #### Via kubeconfig 1. Retrieve the kubeconfig: ```bash terraform output -raw cluster_kubeconfig > ~/.kube/config-oke export KUBECONFIG=~/.kube/config-oke ``` `cluster_kubeconfig` is only populated when `output_detail = true`. 2. Verify connectivity: ```bash kubectl get nodes ``` ### Update the cluster To update the infrastructure: ```bash # Modify terraform.tfvars as needed terraform plan terraform apply ``` Common updates: - **Kubernetes version**: Change `kubernetes_version` and run `terraform apply` - **Worker pool size**: Adjust `worker_pool_size` or individual pool `size` - **Add worker pools**: Add entries to the `worker_pools` map - **Extensions**: Enable extensions by setting `_install = true` ### Destroy the cluster ```bash terraform destroy ``` ### Related documentation - [All Terraform configuration options](./terraformoptions.md) for this module - [Example configurations](https://github.com/oracle-terraform-modules/terraform-oci-oke/tree/main/examples) - [Pre-requisites](./prerequisites.md) ================================================ FILE: docs/terraformoptions.md ================================================ # Terraform Options Configuration Terraform Options: 1. [General](#general) 2. [Identity and Access Management](#identity-and-access-management) 3. [Network](#network) 4. [Cluster](#cluster) 5. [Cluster Add-ons](#cluster-add-ons) 6. [Workers](#workers) 7. [Bastion](#bastion) 8. [Operator](#operator) 9. [Extensions](#extensions) - [Cilium](#cilium) - [Multus](#multus) - [SR-IOV Device Plugin](#sr-iov-device-plugin) - [SR-IOV CNI Plugin](#sr-iov-cni-plugin) - [RDMA CNI Plugin](#rdma-cni-plugin) - [Whereabouts](#whereabouts) - [Metrics Server](#metrics-server) - [Cluster Autoscaler](#cluster-autoscaler) - [Prometheus](#prometheus) - [DCGM Exporter](#dcgm-exporter) - [Gatekeeper](#gatekeeper) - [MPI Operator](#mpi-operator) - [ArgoCD](#argocd) - [Service Accounts](#service-accounts) 10. [Utilities](#utilities) 11. [Tagging](#tagging) 12. [Validation Rules](#validation-rules) ## General | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `state_id` | Optional Terraform state_id from an existing deployment for resource reuse. | string | `null` | | `output_detail` | Whether to include detailed output in the Terraform state. | `true` / `false` | `false` | | `timezone` | Preferred timezone for worker, operator, and bastion instances. | string (IANA timezone) | `"Etc/UTC"` | | `ssh_private_key` | SSH private key contents, optionally base64-encoded. Sensitive. | string | `null` | | `ssh_private_key_path` | Path to SSH private key on the machine running Terraform. | string | `null` | | `ssh_public_key` | SSH public key contents, optionally base64-encoded. | string | `null` | | `ssh_public_key_path` | Path to SSH public key. | string | `null` | ## Identity and Access Management | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `tenancy_id` | Tenancy OCID. Required unless using `config_file_profile` or Resource Manager. | OCID string | `null` | | `tenancy_ocid` | Tenancy OCID for Resource Manager. Used as alias for `tenancy_id` in RMS. | OCID string | `null` | | `user_id` | User OCID for API key authentication. | OCID string | `null` | | `current_user_ocid` | User OCID for Resource Manager. | OCID string | `null` | | `compartment_id` | Compartment OCID where resources are created. Required. | OCID string | `null` | | `compartment_ocid` | Compartment OCID for Resource Manager. | OCID string | `null` | | `worker_compartment_id` | Compartment for worker resources. Defaults to `compartment_id`. | OCID string | `null` | | `network_compartment_id` | Compartment for network resources. Defaults to `compartment_id`. | OCID string | `null` | | `region` | OCI region for resource provisioning. | [OCI region identifier](https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm) | `"us-ashburn-1"` | | `home_region` | Tenancy home region. Required when `create_iam_resources = true`. | OCI region identifier | `null` | | `api_fingerprint` | Fingerprint of the OCI API public key. | string | `null` | | `api_private_key` | OCI API private key contents. Sensitive. | string | `null` | | `api_private_key_password` | Password for the OCI API private key. Sensitive. | string | `null` | | `api_private_key_path` | Path to the OCI API private key file. | string | `null` | | `config_file_profile` | OCI CLI config file profile name for authentication. | string | `"DEFAULT"` | | `create_iam_resources` | Whether to create IAM dynamic groups and policies. | `true` / `false` | `false` | | `create_iam_autoscaler_policy` | Create IAM policy for cluster autoscaler. | `"never"` / `"auto"` / `"always"` | `"auto"` | | `create_iam_kms_policy` | Create IAM policy for KMS encryption. | `"never"` / `"auto"` / `"always"` | `"auto"` | | `create_iam_operator_policy` | Create IAM policy for operator instance principal. | `"never"` / `"auto"` / `"always"` | `"auto"` | | `create_iam_worker_policy` | Create IAM policy for worker nodes. | `"never"` / `"auto"` / `"always"` | `"auto"` | | `create_iam_tag_namespace` | Create IAM tag namespace and tags. | `true` / `false` | `false` | | `create_iam_defined_tags` | Create IAM defined tags in the tag namespace. | `true` / `false` | `false` | | `use_defined_tags` | Apply defined tags to created resources. | `true` / `false` | `false` | | `tag_namespace` | Tag namespace name for OKE defined tags. | string | `"oke"` | ## Network Relevant diagrams: - [Network layout](./diagrams.md#network-layout) - [Load balancer layout](./diagrams.md#load-balancer-layout) - [Bastion access layout](./diagrams.md#bastion-access-layout) ### VCN | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `create_vcn` | Whether to create a VCN. Set to `false` to use an existing VCN. | `true` / `false` | `true` | | `vcn_name` | Display name for the VCN. | string | `null` | | `vcn_id` | OCID of an existing VCN. Required when `create_vcn = false`. | OCID string | `null` | | `vcn_cidrs` | IPv4 CIDR blocks for the VCN. | list(string) | `["10.0.0.0/16"]` | | `vcn_dns_label` | DNS label for the VCN. | string | `null` | | `vcn_enable_ipv6_gua` | Enable IPv6 Global Unicast Address. | `true` / `false` | `true` | | `vcn_ipv6_ula_cidrs` | IPv6 ULA CIDR blocks for the VCN. | list(string) | `[]` | | `assign_dns` | Whether to assign DNS records to created instances and subnet hostname labels. | `true` / `false` | `true` | | `lockdown_default_seclist` | Remove all default rules from the VCN default security list. | `true` / `false` | `true` | ### Gateways | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `vcn_create_internet_gateway` | Create an internet gateway. | `"auto"` / `"always"` / `"never"` | `"auto"` | | `vcn_create_nat_gateway` | Create a NAT gateway. | `"auto"` / `"always"` / `"never"` | `"auto"` | | `vcn_create_service_gateway` | Create a service gateway. | `"auto"` / `"always"` / `"never"` | `"always"` | | `internet_gateway_id` | OCID of an existing internet gateway. | OCID string | `null` | | `nat_gateway_id` | OCID of an existing NAT gateway. | OCID string | `null` | | `nat_gateway_public_ip_id` | Reserved public IP OCID for the NAT gateway. | OCID string | `null` | ### Routing | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `ig_route_table_id` | OCID of an existing internet gateway route table. | OCID string | `null` | | `nat_route_table_id` | OCID of an existing NAT gateway route table. | OCID string | `null` | | `igw_ngw_mixed_route_id` | OCID of a mixed route table (NAT GW for IPv4, IGW for IPv6). | OCID string | `null` | | `internet_gateway_route_rules` | Additional route rules for the internet gateway route table. | list(map(string)) | `null` | | `nat_gateway_route_rules` | Additional route rules for the NAT gateway route table. | list(map(string)) | `null` | ### DRG | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `create_drg` | Whether to create a Dynamic Routing Gateway. | `true` / `false` | `false` | | `drg_display_name` | Display name for the DRG. | string | `null` | | `drg_id` | OCID of an existing DRG. | OCID string | `null` | | `drg_compartment_id` | Compartment for the DRG. Defaults to `network_compartment_id`. | OCID string | `null` | | `drg_attachments` | DRG attachment configurations. | map(any) | `{}` | | `remote_peering_connections` | Remote peering connection configurations. | map(any) | `{}` | | `local_peering_gateways` | Local peering gateway configurations. | map(any) | `null` | ### Subnets See [Network layout](./diagrams.md#network-layout) for the default subnet split used by the module. | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `subnets` | Configuration for standard subnets (bastion, operator, cp, int_lb, pub_lb, workers, pods). Each entry supports `create`, `id`, `cidr`, `netnum`, `newbits`, `display_name`, `dns_label`, and `ipv6_cidr`. | map(object) | Module-defined defaults for all standard subnets | Example with automatic subnet creation: ```hcl subnets = { bastion = { newbits = 13 } operator = { newbits = 13 } cp = { newbits = 13 } int_lb = { newbits = 11 } pub_lb = { newbits = 11 } workers = { newbits = 4 } pods = { newbits = 2 } } ``` Example with explicit CIDRs: ```hcl subnets = { bastion = { cidr = "10.0.0.0/29" } operator = { cidr = "10.0.0.64/29" } cp = { cidr = "10.0.0.8/29" } int_lb = { cidr = "10.0.0.32/27" } pub_lb = { cidr = "10.0.128.0/27" } workers = { cidr = "10.0.144.0/20" } pods = { cidr = "10.0.64.0/18" } } ``` Example with existing subnets: ```hcl subnets = { operator = { id = "ocid1.subnet..." } cp = { id = "ocid1.subnet..." } workers = { id = "ocid1.subnet..." } } ``` ### Network Security Groups See [Network layout](./diagrams.md#network-layout) for how the NSG-backed subnet boundaries fit together. | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `nsgs` | Configuration for NSGs (bastion, operator, cp, int_lb, pub_lb, workers, pods, optional `fss`). Each entry supports `create` and `id`. | map(object) | Module-defined defaults for standard NSGs | | `allow_node_port_access` | Allow NodePort access to load balancers. | `true` / `false` | `false` | | `allow_worker_internet_access` | Allow worker nodes outbound internet access. | `true` / `false` | `true` | | `allow_pod_internet_access` | Allow pod outbound internet access. | `true` / `false` | `true` | | `allow_worker_ssh_access` | Allow SSH access to worker nodes. | `true` / `false` | `false` | | `allow_bastion_cluster_access` | Allow bastion to cluster endpoint access. | `true` / `false` | `false` | | `allow_rules_cp` | Additional NSG rules for the control plane. | map(any) | `{}` | | `allow_rules_internal_lb` | Additional NSG rules for internal load balancers. | map(any) | `{}` | | `allow_rules_pods` | Additional NSG rules for pods. | map(any) | `{}` | | `allow_rules_public_lb` | Additional NSG rules for public load balancers. | map(any) | `{}` | | `allow_rules_workers` | Additional NSG rules for workers. | map(any) | `{}` | | `control_plane_allowed_cidrs` | CIDR blocks allowed to access the control plane. | list(string) | `[]` | | `enable_waf` | Enable WAF monitoring for load balancers. | `true` / `false` | `false` | | `use_stateless_rules` | Use stateless NSG rules instead of stateful. | `true` / `false` | `false` | Additional NSG rule example: ```hcl allow_rules_workers = { "Allow TCP 8080 from VCN" = { protocol = 6, port = 8080, source = "10.0.0.0/16", source_type = "CIDR_BLOCK", }, } ``` ## Cluster Relevant diagrams: - [Public control plane topology](./diagrams.md#public-control-plane-topology) - [Private control plane topology](./diagrams.md#private-control-plane-topology) - [OIDC discovery flow](./diagrams.md#oidc-discovery-flow) | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `create_cluster` | Whether to create an OKE cluster. | `true` / `false` | `true` | | `cluster_name` | Name of the OKE cluster. | string | `"oke"` | | `cluster_type` | Cluster type. Enhanced clusters support additional features like virtual node pools and workload identity. | `"basic"` / `"enhanced"` | `"basic"` | | `control_plane_is_public` | Whether the control plane has a public IP. | `true` / `false` | `false` | | `assign_public_ip_to_control_plane` | Assign a public IP to the API endpoint. | `true` / `false` | `false` | | `control_plane_nsg_ids` | Additional NSG IDs for the cluster endpoint. | set(string) | `[]` | | `backend_nsg_ids` | Additional NSG IDs for load balancer backends. Workers and pods NSGs are always included. | set(string) | `[]` | | `cni_type` | Container Network Interface type. | `"flannel"` / `"npn"` | `"flannel"` | | `enable_ipv6` | Create a dual-stack (IPv4 and IPv6) cluster. | `true` / `false` | `false` | | `oke_ip_families` | Override the `ip_families` cluster attribute. | list(string) | `[]` | | `pods_cidr` | CIDR range for Kubernetes pods. Must not overlap with VCN, worker, or LB subnets. | CIDR string | `"10.244.0.0/16"` | | `services_cidr` | CIDR range for Kubernetes services. Must not overlap with the VCN CIDR. | CIDR string | `"10.96.0.0/16"` | | `kubernetes_version` | Kubernetes version for the cluster. | string (e.g. `"v1.34.2"`) | `"v1.34.2"` | | `cluster_kms_key_id` | KMS key OCID for Kubernetes secrets encryption. | OCID string | `""` | | `use_signed_images` | Enforce that only signed container images can be deployed. | `true` / `false` | `false` | | `image_signing_keys` | KMS key IDs used to verify signed images. | set(string) | `[]` | | `load_balancers` | Type of subnets created for load balancers. | `"public"` / `"internal"` / `"both"` | `"both"` | | `preferred_load_balancer` | Preferred load balancer subnet type. | `"public"` / `"internal"` | `"public"` | | `oidc_discovery_enabled` | Enable OIDC discovery for third-party token validation. Requires enhanced cluster. | `true` / `false` | `false` | | `oidc_token_auth_enabled` | Enable OIDC token authentication via API server flags. Requires enhanced cluster. | `true` / `false` | `false` | | `oidc_token_authentication_config` | OIDC token authentication configuration (client_id, issuer_url, username_claim, required_claims). | any | `{}` | Basic cluster example: ```hcl cluster_name = "oke-example" kubernetes_version = "v1.34.2" ``` Enhanced cluster example: ```hcl cluster_name = "oke" cluster_type = "enhanced" cni_type = "flannel" kubernetes_version = "v1.34.2" assign_public_ip_to_control_plane = true ``` OIDC authentication example for GitHub Actions: ```hcl cluster_type = "enhanced" oidc_token_auth_enabled = true oidc_token_authentication_config = { client_id = "oke-kubernetes-cluster" issuer_url = "https://token.actions.githubusercontent.com" username_claim = "sub" required_claims = [ { key = "repository", value = "GITHUB_ACCOUNT/GITHUB_REPOSITORY" }, { key = "workflow", value = "oke-oidc" }, { key = "ref", value = "refs/heads/main" }, ] } ``` ## Cluster Add-ons | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `cluster_addons` | Map of cluster addons to enable. Each addon supports `remove_addon_resources_on_delete`, `override_existing`, and `configurations`. | any | `{}` | | `cluster_addons_to_remove` | Map of cluster addons to remove. Each entry supports `remove_k8s_resources`. | any | `{}` | Example: ```hcl cluster_addons = { "CertManager" = { remove_addon_resources_on_delete = true override_existing = true configurations = [ { key = "numOfReplicas", value = "1" } ] } "NvidiaGpuPlugin" = { remove_addon_resources_on_delete = true } } cluster_addons_to_remove = { Flannel = { remove_k8s_resources = true } } ``` ## Workers Relevant diagrams: - [Public workers topology](./diagrams.md#public-workers-topology) - [Private workers topology](./diagrams.md#private-workers-topology) ### Default Pool Configuration | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `cluster_id` | Existing OKE cluster OCID. Required when `create_cluster = false`. | OCID string | `null` | | `cluster_ca_cert` | Base64+PEM-encoded cluster CA certificate. Required when `create_cluster = false`. | string | `null` | | `cluster_dns` | Cluster DNS resolver IP address. Required when `create_cluster = false`. | string | `null` | | `worker_pools` | Map of worker pool definitions. Key is the pool name, value is the pool configuration. | any | `{}` | | `worker_pool_mode` | Default management mode for worker pools. | `"node-pool"` / `"virtual-node-pool"` / `"instance"` / `"instance-pool"` / `"cluster-network"` / `"compute-cluster"` | `"node-pool"` | | `worker_pool_size` | Default size for worker pools. | number | `0` | | `worker_compute_clusters` | Shared compute cluster definitions for use by multiple pools. | map(any) | `{}` | ### Worker Pool Defaults These parameters set defaults for all worker pools. Individual pools can override these. | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `worker_is_public` | Provision workers with public IPs. | `true` / `false` | `false` | | `worker_nsg_ids` | Additional NSG IDs for all worker nodes. | list(string) | `[]` | | `pod_nsg_ids` | Additional NSG IDs for pods (NPN CNI). | list(string) | `[]` | | `kubeproxy_mode` | Kube-proxy mode. | `"iptables"` / `"ipvs"` | `"iptables"` | | `worker_block_volume_type` | Block volume attachment type for self-managed workers. | `"paravirtualized"` / `"iscsi"` | `"paravirtualized"` | | `worker_node_labels` | Default Kubernetes node labels. | map(string) | `{}` | | `worker_node_metadata` | Additional worker node metadata. | map(string) | `{}` | | `worker_image_id` | Default image OCID for worker pools. | OCID string | `null` | | `worker_image_type` | Default image type. `"oke"` uses OKE Oracle Linux images. | `"oke"` / `"custom"` / `"platform"` | `"oke"` | | `worker_image_os` | Default OS for platform/OKE images. | string | `"Oracle Linux"` | | `worker_image_os_version` | Default OS version for platform/OKE images. | string | `"8"` | | `worker_shape` | Default shape for worker instances. | map(any) | `{shape = "VM.Standard.E4.Flex", ocpus = 2, memory = 16, boot_volume_size = 50, boot_volume_vpus_per_gb = 10}` | | `worker_capacity_reservation_id` | Capacity reservation OCID for worker instances. | OCID string | `null` | | `worker_preemptible_config` | Preemptible compute configuration. | map(any) | `{}` | | `worker_cloud_init` | Default cloud-init MIME parts for all pools. | list(map(string)) | `[]` | | `worker_disable_default_cloud_init` | Disable the default OKE cloud-init. | `true` / `false` | `false` | | `worker_volume_kms_key_id` | KMS key OCID for boot volume encryption. | OCID string | `null` | | `worker_pv_transit_encryption` | Enable in-transit encryption for paravirtualized volumes. | `true` / `false` | `false` | | `worker_legacy_imds_endpoints_disabled` | Disable IMDSv1 endpoint on workers. | `true` / `false` | `false` | | `max_pods_per_node` | Maximum pods per node (1-110). Only applies with NPN CNI. | number | `31` | | `platform_config` | Platform configuration for self-managed pools (shielded instances). | object | `null` | | `agent_config` | Management agent configuration for self-managed pools. | object | `null` | | `allow_short_container_image_names` | Allow short container image names without full registry path. Requires Kubernetes >= 1.34.0. | `true` / `false` | `false` | ### Worker Pool Entry Configuration Each entry in the `worker_pools` map supports the following attributes: | Attribute | Description | Values | |-----------|-------------|--------| | `mode` | Worker management mode. Overrides `worker_pool_mode`. | `"node-pool"` / `"virtual-node-pool"` / `"instance"` / `"instance-pool"` / `"cluster-network"` / `"compute-cluster"` | | `size` | Number of nodes in the pool. | number | | `shape` | Instance shape name. | string | | `ocpus` | Number of OCPUs (Flex shapes). | number | | `memory` | Memory in GB (Flex shapes). | number | | `boot_volume_size` | Boot volume size in GB. | number | | `boot_volume_vpus_per_gb` | Boot volume performance (10/20/30-120). Self-managed modes only. | number | | `description` | Pool description. | string | | `create` | Whether to create this pool. | `true` / `false` | | `image_type` | Image type for this pool. | `"oke"` / `"custom"` / `"platform"` | | `image_id` | Custom image OCID. | OCID string | | `os` | OS name. | string | | `os_version` | OS version. | string | | `node_labels` | Kubernetes node labels. | map(string) | | `subnet_id` | Custom subnet OCID for this pool. | OCID string | | `pod_subnet_id` | Custom pod subnet OCID (NPN CNI). | OCID string | | `nsg_ids` | Additional NSG IDs for this pool. | list(string) | | `pod_nsg_ids` | Additional pod NSG IDs for this pool (NPN CNI). | list(string) | | `assign_public_ip` | Assign a public IP to nodes. | `true` / `false` | | `cloud_init` | Pool-specific cloud-init MIME parts. | list(map(string)) | | `secondary_vnics` | Secondary VNIC configurations. | map(any) | | `autoscale` | Enable cluster autoscaler for this pool. | `true` / `false` | | `min_size` | Minimum pool size for autoscaling. | number | | `max_size` | Maximum pool size for autoscaling. | number | | `allow_autoscaler` | Allow cluster autoscaler to manage this pool. | `true` / `false` | | `ignore_initial_pool_size` | Ignore initial pool size when autoscaling. | `true` / `false` | | `drain` | Mark pool for draining (disables scheduling, drains through operator). | `true` / `false` | | `placement_ads` | List of AD numbers for placement. | list(number) | | `compute_cluster` | Name of a shared compute cluster (compute-cluster mode). | string | | `instance_ids` | Instance IDs in compute cluster. | list(string) | | `platform_config` | Platform configuration (shielded instances). | object | | `agent_config` | Management agent configuration. | object | | `burst` | CPU bursting configuration for Flex shapes. | `"BASELINE_1_8"` / `"BASELINE_1_2"` | | `node_cycling_enabled` | Enable node cycling for updates. | `true` / `false` | | `node_cycling_max_surge` | Max surge during cycling (percentage or number). | string | | `node_cycling_max_unavailable` | Max unavailable during cycling. | number | | `node_cycling_mode` | Cycling mode. | `["instance"]` / `["boot_volume"]` | | `eviction_grace_duration` | Grace duration for eviction in seconds. | number | | `is_force_delete_after_grace_duration` | Force delete after grace duration. | `true` / `false` | Basic node pool example: ```hcl worker_pool_mode = "node-pool" worker_pool_size = 1 worker_pools = { oke-vm-standard = {} oke-vm-standard-large = { size = 1 shape = "VM.Standard.E4.Flex" ocpus = 8 memory = 128 boot_volume_size = 200 } } ``` Autoscaled node pool example: ```hcl worker_pools = { np-autoscaled = { size = 2 min_size = 1 max_size = 3 autoscale = true ignore_initial_pool_size = true } } ``` Cluster network (HPC/GPU) example: ```hcl worker_pools = { oke-bm-gpu-rdma = { mode = "cluster-network" size = 1 shape = "BM.GPU.B4.8" placement_ads = [1] image_id = "ocid1.image..." secondary_vnics = { "vnic-display-name" = { nic_index = 1 subnet_id = "ocid1.subnet..." } } } } ``` ## Bastion The bastion instance provides a public SSH entrypoint into the VCN. See [Bastion access layout](./diagrams.md#bastion-access-layout) for the administrative access path. | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `create_bastion` | Whether to create a bastion host. | `true` / `false` | `true` | | `bastion_public_ip` | IP address of an existing bastion. Ignored when `create_bastion = true`. | string | `null` | | `bastion_allowed_cidrs` | List of CIDR blocks allowed SSH access to the bastion. Set to `["0.0.0.0/0"]` to allow from anywhere. | list(string) | `[]` | | `bastion_availability_domain` | Availability domain number for the bastion. Defaults to first available. | string | `null` | | `bastion_nsg_ids` | Additional NSG IDs for the bastion. Combined with the created NSG. | list(string) | `[]` | | `bastion_user` | SSH user for the bastion host. | string | `"opc"` | | `bastion_image_id` | Custom image OCID for the bastion. Ignored when `bastion_image_type = "platform"`. | OCID string | `null` | | `bastion_image_type` | Image type for the bastion. | `"platform"` / `"custom"` | `"platform"` | | `bastion_image_os` | Platform image OS name. | string | `"Oracle Autonomous Linux"` | | `bastion_image_os_version` | Platform image OS version. | string | `"8"` | | `bastion_shape` | Shape of the bastion instance. | map(any) | `{shape = "VM.Standard.E4.Flex", ocpus = 1, memory = 4, boot_volume_size = 50, baseline_ocpu_utilization = 100}` | | `bastion_is_public` | Whether the bastion is provisioned with a public IP. | `true` / `false` | `true` | | `bastion_upgrade` | Whether to upgrade bastion packages after provisioning. | `true` / `false` | `false` | | `bastion_await_cloudinit` | Block Terraform until cloud-init completes on the bastion. | `true` / `false` | `true` | | `bastion_volume_kms_key_id` | KMS key OCID for bastion boot volume encryption. | OCID string | `null` | | `bastion_legacy_imds_endpoints_disabled` | Disable IMDSv1 endpoint on the bastion. | `true` / `false` | `true` | Example: ```hcl create_bastion = true bastion_allowed_cidrs = ["0.0.0.0/0"] bastion_image_type = "platform" bastion_upgrade = false bastion_user = "opc" bastion_shape = { shape = "VM.Standard.E4.Flex" ocpus = 1 memory = 4 boot_volume_size = 50 baseline_ocpu_utilization = 100 } ``` ## Operator The operator instance provides an environment within the VCN from which the OKE cluster can be managed. It comes pre-installed with kubectl, Helm, and optional tools. | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `create_operator` | Whether to create an operator host. | `true` / `false` | `true` | | `operator_availability_domain` | Availability domain for the operator. Defaults to first available. | string | `null` | | `operator_cloud_init` | Cloud-init MIME parts for custom operator initialization. | list(map(string)) | `[]` | | `operator_nsg_ids` | Additional NSG IDs for the operator. | list(string) | `[]` | | `operator_user` | SSH user for the operator host. | string | `"opc"` | | `operator_image_id` | Custom image OCID for the operator. Ignored when `operator_image_type = "platform"`. | OCID string | `null` | | `operator_image_os` | Platform image OS name. | string | `"Oracle Linux"` | | `operator_image_os_version` | Platform image OS version. | string | `"8"` | | `operator_image_type` | Image type for the operator. | `"platform"` / `"custom"` | `"platform"` | | `operator_install_helm` | Whether to install Helm on the operator. | `true` / `false` | `true` | | `operator_install_helm_from_repo` | Install Helm from the package repository. | `true` / `false` | `false` | | `operator_install_oci_cli_from_repo` | Install OCI CLI from the package repository. | `true` / `false` | `false` | | `operator_install_istioctl` | Whether to install istioctl on the operator. | `true` / `false` | `false` | | `operator_install_k8sgpt` | Whether to install k8sgpt on the operator. | `true` / `false` | `false` | | `operator_install_k9s` | Whether to install k9s on the operator. | `true` / `false` | `false` | | `operator_install_kubectl_from_repo` | Install kubectl from the package repository. | `true` / `false` | `true` | | `operator_install_kubectx` | Whether to install kubectx/kubens on the operator. | `true` / `false` | `true` | | `operator_install_stern` | Whether to install stern on the operator. | `true` / `false` | `false` | | `operator_shape` | Shape of the operator instance. | map(any) | `{shape = "VM.Standard.E4.Flex", ocpus = 1, memory = 4, boot_volume_size = 50, baseline_ocpu_utilization = 100}` | | `operator_volume_kms_key_id` | KMS key OCID for operator boot volume encryption. | OCID string | `null` | | `operator_pv_transit_encryption` | Enable in-transit encryption for paravirtualized volumes. | `true` / `false` | `false` | | `operator_upgrade` | Whether to upgrade operator packages after provisioning. | `true` / `false` | `false` | | `operator_private_ip` | IP address of an existing operator. Ignored when `create_operator = true`. | string | `null` | | `operator_await_cloudinit` | Block Terraform until cloud-init completes on the operator. | `true` / `false` | `true` | | `operator_legacy_imds_endpoints_disabled` | Disable IMDSv1 endpoint on the operator. | `true` / `false` | `true` | Example with cloud-init: ```hcl create_operator = true operator_upgrade = false operator_user = "opc" operator_cloud_init = [ { content = <<-EOT runcmd: - echo "Operator cloud_init using cloud-config" EOT content_type = "text/cloud-config" } ] operator_shape = { shape = "VM.Standard.E4.Flex" ocpus = 1 memory = 4 boot_volume_size = 50 baseline_ocpu_utilization = 100 } ``` ## Extensions ### Cilium | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `cilium_install` | Whether to install Cilium. | `true` / `false` | `false` | | `cilium_reapply` | Reapply Cilium Helm release on every Terraform apply. | `true` / `false` | `false` | | `cilium_namespace` | Kubernetes namespace for Cilium. | string | `"kube-system"` | | `cilium_helm_version` | Cilium Helm chart version. | string | `"1.16.3"` | | `cilium_helm_values` | Helm values for Cilium. | any | `{}` | | `cilium_helm_values_files` | List of Helm values files for Cilium. | list(string) | `[]` | ### Multus | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `multus_install` | Whether to install Multus CNI. | `true` / `false` | `false` | | `multus_namespace` | Kubernetes namespace for Multus. | string | `"network"` | | `multus_daemonset_url` | URL to the Multus daemonset manifest. Determined automatically by default. | string | `null` | | `multus_version` | Multus version. | string | `"3.9.3"` | ### SR-IOV Device Plugin | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `sriov_device_plugin_install` | Whether to install the SR-IOV device plugin. | `true` / `false` | `false` | | `sriov_device_plugin_namespace` | Kubernetes namespace. | string | `"network"` | | `sriov_device_plugin_daemonset_url` | URL to the daemonset manifest. Determined automatically by default. | string | `null` | | `sriov_device_plugin_version` | SR-IOV device plugin version. | string | `"master"` | ### SR-IOV CNI Plugin | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `sriov_cni_plugin_install` | Whether to install the SR-IOV CNI plugin. | `true` / `false` | `false` | | `sriov_cni_plugin_namespace` | Kubernetes namespace. | string | `"network"` | | `sriov_cni_plugin_daemonset_url` | URL to the daemonset manifest. Determined automatically by default. | string | `null` | | `sriov_cni_plugin_version` | SR-IOV CNI plugin version. | string | `"master"` | ### RDMA CNI Plugin | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `rdma_cni_plugin_install` | Whether to install the RDMA CNI plugin. | `true` / `false` | `false` | | `rdma_cni_plugin_namespace` | Kubernetes namespace. | string | `"network"` | | `rdma_cni_plugin_daemonset_url` | URL to the daemonset manifest. Determined automatically by default. | string | `null` | | `rdma_cni_plugin_version` | RDMA CNI plugin version. | string | `"master"` | ### Whereabouts | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `whereabouts_install` | Whether to install Whereabouts IPAM. | `true` / `false` | `false` | | `whereabouts_namespace` | Kubernetes namespace. | string | `"default"` | | `whereabouts_daemonset_url` | URL to the daemonset manifest. Determined automatically by default. | string | `null` | | `whereabouts_version` | Whereabouts version. | string | `"master"` | ### Metrics Server | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `metrics_server_install` | Whether to install Metrics Server. | `true` / `false` | `false` | | `metrics_server_namespace` | Kubernetes namespace. | string | `"metrics"` | | `metrics_server_helm_version` | Helm chart version. | string | `"3.8.3"` | | `metrics_server_helm_values` | Helm values. | map(string) | `{}` | | `metrics_server_helm_values_files` | List of Helm values files. | list(string) | `[]` | ### Cluster Autoscaler | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `cluster_autoscaler_install` | Whether to install the standalone Cluster Autoscaler. | `true` / `false` | `false` | | `cluster_autoscaler_namespace` | Kubernetes namespace. | string | `"kube-system"` | | `cluster_autoscaler_helm_version` | Helm chart version. | string | `"9.24.0"` | | `cluster_autoscaler_helm_values` | Helm values. | map(string) | `{}` | | `cluster_autoscaler_helm_values_files` | List of Helm values files. | list(string) | `[]` | ### Prometheus | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `prometheus_install` | Whether to install Prometheus. | `true` / `false` | `false` | | `prometheus_reapply` | Reapply Prometheus Helm release on every apply. | `true` / `false` | `false` | | `prometheus_namespace` | Kubernetes namespace. | string | `"metrics"` | | `prometheus_helm_version` | Helm chart version. | string | `"45.2.0"` | | `prometheus_helm_values` | Helm values. | map(string) | `{}` | | `prometheus_helm_values_files` | List of Helm values files. | list(string) | `[]` | ### DCGM Exporter | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `dcgm_exporter_install` | Whether to install the DCGM Exporter for GPU metrics. | `true` / `false` | `false` | | `dcgm_exporter_reapply` | Reapply DCGM Exporter Helm release on every apply. | `true` / `false` | `false` | | `dcgm_exporter_namespace` | Kubernetes namespace. | string | `"metrics"` | | `dcgm_exporter_helm_version` | Helm chart version. | string | `"3.1.5"` | | `dcgm_exporter_helm_values` | Helm values. | map(string) | `{}` | | `dcgm_exporter_helm_values_files` | List of Helm values files. | list(string) | `[]` | ### Gatekeeper | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `gatekeeper_install` | Whether to install Gatekeeper (OPA). | `true` / `false` | `false` | | `gatekeeper_namespace` | Kubernetes namespace. | string | `"kube-system"` | | `gatekeeper_helm_version` | Helm chart version. | string | `"3.11.0"` | | `gatekeeper_helm_values` | Helm values. | map(string) | `{}` | | `gatekeeper_helm_values_files` | List of Helm values files. | list(string) | `[]` | ### MPI Operator | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `mpi_operator_install` | Whether to install the MPI Operator. | `true` / `false` | `false` | | `mpi_operator_namespace` | Kubernetes namespace. | string | `"default"` | | `mpi_operator_deployment_url` | URL to the deployment manifest. Determined automatically by default. | string | `null` | | `mpi_operator_version` | MPI Operator version. | string | `"0.4.0"` | ### ArgoCD | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `argocd_install` | Whether to install ArgoCD. | `true` / `false` | `false` | | `argocd_namespace` | Kubernetes namespace. | string | `"argocd"` | | `argocd_helm_version` | Helm chart version. | string | `"8.1.2"` | | `argocd_helm_values` | Helm values. | map(string) | `{}` | | `argocd_helm_values_files` | List of Helm values files. | list(string) | `[]` | ### Service Accounts | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `create_service_account` | Whether to create Kubernetes service accounts with RBAC. | `true` / `false` | `false` | | `service_accounts` | Map of service account definitions. Each supports `sa_name`, `sa_namespace`, `sa_cluster_role`, `sa_cluster_role_binding`, `sa_role`, `sa_role_binding`. | map(any) | Seeded with a default `kubeconfigsa` entry | Example: ```hcl create_service_account = true service_accounts = { example_cluster_role_binding = { sa_name = "sa1" sa_namespace = "kube-system" sa_cluster_role = "cluster-admin" sa_cluster_role_binding = "sa1-crb" } } ``` ## Utilities | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `await_node_readiness` | Block Terraform until nodes are ready. | `"none"` / `"one"` / `"all"` | `"none"` | | `ocir_email_address` | Email address for OCIR secret. | string | `null` | | `ocir_secret_id` | OCIR secret OCID from OCI Vault. | OCID string | `null` | | `ocir_secret_name` | Name of the Kubernetes Docker registry secret. | string | `"ocirsecret"` | | `ocir_secret_namespace` | Kubernetes namespace for the OCIR secret. | string | `"default"` | | `ocir_username` | Username for OCIR secret access. | string | `null` | | `worker_drain_ignore_daemonsets` | Ignore DaemonSet pods when draining workers. | `true` / `false` | `true` | | `worker_drain_delete_local_data` | Delete local data when draining workers. | `true` / `false` | `true` | | `worker_drain_timeout_seconds` | Timeout for worker draining in seconds. | number | `900` | ## Tagging | Parameter | Description | Values | Default | | --- | --- | --- | --- | | `freeform_tags` | Freeform tags applied to all resources. | any | `{access = "private", environment = "dev", role = "oke", version = "5"}` | | `defined_tags` | Defined tags applied to all resources. Requires `use_defined_tags = true`. | any | `{}` | | `bastion_defined_tags` | Defined tags for bastion resources only. | any | `{}` | | `bastion_freeform_tags` | Freeform tags for bastion resources only. | any | `{}` | | `cluster_defined_tags` | Defined tags for cluster resources only. | any | `{}` | | `cluster_freeform_tags` | Freeform tags for cluster resources only. | any | `{}` | | `iam_defined_tags` | Defined tags for IAM resources only. | any | `{}` | | `iam_freeform_tags` | Freeform tags for IAM resources only. | any | `{}` | | `network_defined_tags` | Defined tags for network resources only. | any | `{}` | | `network_freeform_tags` | Freeform tags for network resources only. | any | `{}` | | `operator_defined_tags` | Defined tags for operator resources only. | any | `{}` | | `operator_freeform_tags` | Freeform tags for operator resources only. | any | `{}` | | `persistent_volume_defined_tags` | Defined tags for persistent volume resources only. | any | `{}` | | `persistent_volume_freeform_tags` | Freeform tags for persistent volume resources only. | any | `{}` | | `service_lb_defined_tags` | Defined tags for service load balancer resources only. | any | `{}` | | `service_lb_freeform_tags` | Freeform tags for service load balancer resources only. | any | `{}` | | `workers_defined_tags` | Defined tags for worker resources only. | any | `{}` | | `workers_freeform_tags` | Freeform tags for worker resources only. | any | `{}` | ## Validation Rules - `compartment_id` is required. - Either `ssh_public_key` or `ssh_public_key_path` must be provided when creating bastion or operator. - `bastion_image_type = "custom"` requires `bastion_image_id`. - `operator_image_type = "custom"` requires `operator_image_id`. - `cni_type = "npn"` requires `cluster_type = "enhanced"`. - `oidc_discovery_enabled = true` requires `cluster_type = "enhanced"`. - `oidc_token_auth_enabled = true` requires `cluster_type = "enhanced"`. - `worker_pool_mode = "node-pool"` is the only mode that supports OKE-managed node pools. - `worker_pool_mode = "cluster-network"` or `"instance-pool"` or `"instance"` are self-managed modes. - Pods CIDR must not overlap with VCN, worker, or load balancer subnets. - Services CIDR must not overlap with the VCN CIDR. ## Outputs | Output | Description | |--------|-------------| | `state_id` | Generated state identifier. | | `cluster_id` | OKE cluster OCID. | | `cluster_endpoints` | Cluster endpoints (public and private). | | `cluster_oidc_discovery_endpoint` | OIDC discovery endpoint URL. | | `cluster_kubeconfig` | Kubernetes kubeconfig YAML (requires `output_detail = true`). | | `cluster_ca_cert` | Base64-encoded cluster CA certificate. | | `apiserver_private_host` | Private API server hostname. | | `bastion_id` | Bastion instance OCID. | | `bastion_public_ip` | Bastion public IP address. | | `ssh_to_bastion` | SSH command to connect to the bastion. | | `operator_id` | Operator instance OCID. | | `operator_private_ip` | Operator private IP address. | | `ssh_to_operator` | SSH command to connect to the operator (via bastion). | | `vcn_id` | VCN OCID. | | `ig_route_table_id` | Internet gateway route table OCID. | | `nat_route_table_id` | NAT gateway route table OCID. | | `drg_id` | Dynamic Routing Gateway OCID (when created). | | `lpg_all_attributes` | Local Peering Gateway attributes. | | `bastion_subnet_id` | Bastion subnet OCID. | | `bastion_subnet_cidr` | Bastion subnet CIDR. | | `operator_subnet_id` | Operator subnet OCID. | | `operator_subnet_cidr` | Operator subnet CIDR. | | `control_plane_subnet_id` | Control plane subnet OCID. | | `control_plane_subnet_cidr` | Control plane subnet CIDR. | | `worker_subnet_id` | Worker subnet OCID. | | `worker_subnet_cidr` | Worker subnet CIDR. | | `pod_subnet_id` | Pod subnet OCID. | | `pod_subnet_cidr` | Pod subnet CIDR. | | `int_lb_subnet_id` | Internal load balancer subnet OCID. | | `int_lb_subnet_cidr` | Internal load balancer subnet CIDR. | | `pub_lb_subnet_id` | Public load balancer subnet OCID. | | `pub_lb_subnet_cidr` | Public load balancer subnet CIDR. | | `fss_subnet_id` | FSS subnet OCID. | | `fss_subnet_cidr` | FSS subnet CIDR. | | `bastion_nsg_id` | Bastion NSG OCID. | | `operator_nsg_id` | Operator NSG OCID. | | `control_plane_nsg_id` | Control plane NSG OCID. | | `int_lb_nsg_id` | Internal load balancer NSG OCID. | | `pub_lb_nsg_id` | Public load balancer NSG OCID. | | `worker_nsg_id` | Worker NSG OCID. | | `pod_nsg_id` | Pod NSG OCID. | | `fss_nsg_id` | FSS NSG OCID. | | `network_security_rules` | Map of all NSG security rules (requires `output_detail = true`). | | `availability_domains` | Map of availability domains. | | `dynamic_group_ids` | IAM dynamic group OCIDs. | | `policy_statements` | IAM policy statements. | | `worker_pools` | Worker pool details. | | `worker_instances` | Worker instance details. | | `worker_pool_ids` | Worker pool OCIDs. | | `worker_pool_ips` | Worker pool IP addresses. | ================================================ FILE: examples/bastion/README.md ================================================ # Bastion Example Enables the bastion host with a public IP for SSH access into the VCN. ## Usage Copy `vars-bastion.auto.tfvars` to your root module and adjust the values as needed. ================================================ FILE: examples/cluster/README.md ================================================ # Cluster Examples Example configurations for OKE cluster creation: | File | Description | |------|-------------| | `vars-cluster-basic.auto.tfvars` | Basic cluster with default settings | | `vars-cluster-enhanced.auto.tfvars` | Enhanced cluster with additional features | | `vars-cluster-oidc-discovery.auto.tfvars` | Cluster with OIDC discovery enabled | | `vars-cluster-oidc-auth-single.auto.tfvars` | Cluster with single OIDC token authentication | | `vars-cluster-oidc-auth-multiple.auto.tfvars` | Cluster with multiple OIDC token authentication configurations | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: examples/cluster-addons/README.md ================================================ # Cluster Add-ons Example Example configuration for enabling and configuring OKE cluster add-ons such as CertManager and NvidiaGpuPlugin. ## Usage Copy `vars-cluster-addons.auto.tfvars` to your root module and adjust the values as needed. ================================================ FILE: examples/extensions/README.md ================================================ # Extension Examples Example configurations for deploying Kubernetes extensions: | File | Extension | Description | |------|-----------|-------------| | `vars-extensions-argocd.auto.tfvars` | ArgoCD | GitOps continuous delivery | | `vars-extensions-cilium.auto.tfvars` | Cilium | eBPF-based networking and security | | `vars-extensions-cluster-autoscaler.auto.tfvars` | Cluster Autoscaler | Automatic node pool scaling | | `vars-extensions-dcgm-exporter.auto.tfvars` | DCGM Exporter | GPU metrics for NVIDIA GPUs | | `vars-extensions-gatekeeper.auto.tfvars` | Gatekeeper | OPA policy enforcement | | `vars-extensions-metrics-server.auto.tfvars` | Metrics Server | Kubernetes metrics API | | `vars-extensions-mpi-operator.auto.tfvars` | MPI Operator | MPI/NCCL distributed training jobs | | `vars-extensions-multus.auto.tfvars` | Multus | Multi-network pod interfaces | | `vars-extensions-prometheus.auto.tfvars` | Prometheus | Monitoring and alerting | | `vars-extensions-rdma-cni.auto.tfvars` | RDMA CNI | RDMA network connections | | `vars-extensions-service-account.auto.tfvars` | Service Accounts | Kubernetes service accounts with RBAC | | `vars-extensions-sriov-cni.auto.tfvars` | SR-IOV CNI | SR-IOV network connections | | `vars-extensions-sriov-device.auto.tfvars` | SR-IOV Device Plugin | SR-IOV network device advertisement | | `vars-extensions-whereabouts.auto.tfvars` | Whereabouts | IP address management for Multus | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: examples/iam/README.md ================================================ # IAM Examples Example configurations for IAM resources: | File | Description | |------|-------------| | `vars-iam-policies.auto.tfvars` | IAM dynamic groups and policies | | `vars-iam-tags.auto.tfvars` | IAM tag namespaces and defined tags | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: examples/istio-mc/README.md ================================================ # Multi-region service mesh with Istio and OKE ## Assumptions 1. A pair of OKE clusters in 2 different OCI regions will be used. 2. The OKE clusters will use private control planes. 3. The topology model used is [Multi-Primary on different networks](https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/). ![Multi-primary on multiple networks](docs/assets/multi-primary%20multi-networks.png) 4. This example uses self-signed certificates. ## Create the OKE Clusters 1. Copy the terraform.tfvars.example to terraform.tfvars and provide the necessary values as detailed in steps 2-6. 2. Configure the provider parameters: ``` # provider api_fingerprint = "" api_private_key_path = "~/.oci/oci_rsa.pem" home_region = "ashburn" tenancy_id = "ocid1.tenancy.oc1.." user_id = "ocid1.user.oc1.." compartment_id = "ocid1.compartment.oc1.." ``` 3. Configure an ssh key pair: ``` # ssh ssh_private_key_path = "~/.ssh/id_rsa" ssh_public_key_path = "~/.ssh/id_rsa.pub" ``` 4. Configure your clusters' regions. ``` # clusters clusters = { c1 = { region = "sydney", vcn = "10.1.0.0/16", pods = "10.201.0.0/16", services = "10.101.0.0/16", enabled = true } c2 = { region = "melbourne", vcn = "10.2.0.0/16", pods = "10.202.0.0/16", services = "10.102.0.0/16", enabled = true } } ``` 5. Configure additional parameters if necessary: ``` kubernetes_version = "v1.32.1" cluster_type = "basic" oke_control_plane = "private" ``` 6. Configure your node pools: ``` nodepools = { np1 = { shape = "VM.Standard.E4.Flex", ocpus = 2, memory = 64, size = 2, boot_volume_size = 150, } } ``` 7. Run terraform to create your clusters: ``` terraform apply --auto-approve ``` 8. Once the Dynamic Routing Gateways (DRGs) and Remote Peering Connections (RPCs) have been created, use the OCI console to establish a connection between them. ## Install Istio 1. Terraform will output an ssh convenience command. Use it to ssh to the operator host: ``` ssh_to_operator = "ssh -o ProxyCommand='ssh -W %h:%p -i ~/.ssh/id_rsa opc@' -i ~/.ssh/id_rsa opc@" ``` 2. Verify connectivity to both clusters: ``` for cluster in c1 c2; do ktx $cluster k get nodes done ``` 3. Generate certs for each cluster: ``` export ISTIO_HOME=/home/opc/istio-1.20.2 cd $ISTIO_HOME/tools/certs make -f Makefile.selfsigned.mk c1-cacerts make -f Makefile.selfsigned.mk c2-cacerts ``` 4. Create and label istio-system namespace in each cluster: ``` for cluster in c1 c2; do ktx $cluster k create ns istio-system k label namespace istio-system topology.istio.io/network=$cluster done ``` 5. Create a secret containing the certificates in istio-system namespace for both clusters: ``` for cluster in c1 c2; do ktx $cluster kubectl create secret generic cacerts -n istio-system \ --from-file=$cluster/ca-cert.pem \ --from-file=$cluster/ca-key.pem \ --from-file=$cluster/root-cert.pem \ --from-file=$cluster/cert-chain.pem done ``` 6. Install Istio in both clusters: ``` for cluster in c1 c2; do ktx $cluster istioctl install --set profile=default -f $HOME/$cluster.yaml done ``` 7. Verify the Istio installation in both clusters: ``` for cluster in c1 c2; do ktx $cluster istioctl verify-install done ``` 8. Check if the load balancers have been properly provisioned: ``` for cluster in c1 c2; do ktx $cluster k -n istio-system get svc done ``` 9. Check if Istio pods are running: ``` for cluster in c1 c2; do ktx $cluster k -n istio-system get pods done ``` 10. Create an Gateway to expose all services through the eastwest ingress gateway: ``` cd $ISTIO_HOME for cluster in c1 c2; do ktx $cluster k apply -f samples/multicluster/expose-services.yaml done ``` 11. Set the environment variables to verify multi-cluster connectivity: ``` export CTX_CLUSTER1=c1 export CTX_CLUSTER2=c2 ``` 12. Enable endpoint discovery in each cluster by creating a remote secret: ``` istioctl create-remote-secret \ --context="${CTX_CLUSTER1}" \ --name="${CTX_CLUSTER1}" | \ kubectl apply -f - --context="${CTX_CLUSTER2}" istioctl create-remote-secret \ --context="${CTX_CLUSTER2}" \ --name="${CTX_CLUSTER2}" | \ kubectl apply -f - --context="${CTX_CLUSTER1}" ``` ## Verify cross-cluster connectivity 1. Deploy the HelloWorld Service in both clusters: ``` for cluster in c1 c2; do kubectl create --context="${cluster}" namespace sample kubectl label --context="${cluster}" namespace sample istio-injection=enabled kubectl apply --context="${cluster}" -f samples/helloworld/helloworld.yaml -l service=helloworld -n sample done ``` 2. Deploy v1 to cluster c1: ``` kubectl apply --context="${CTX_CLUSTER1}" \ -f samples/helloworld/helloworld.yaml \ -l version=v1 -n sample kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l app=helloworld ``` 3. Deploy v2 to cluster c2: ``` kubectl apply --context="${CTX_CLUSTER2}" \ -f samples/helloworld/helloworld.yaml \ -l version=v2 -n sample kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l app=helloworld ``` 4. Deploy Sleep client pod in both clusters: ``` kubectl apply --context="${CTX_CLUSTER1}" \ -f samples/sleep/sleep.yaml -n sample kubectl apply --context="${CTX_CLUSTER2}" \ -f samples/sleep/sleep.yaml -n sample ``` 5. Generate traffic from c1. The response should alternate between c1 (v1) and c2 (v2) regions: ``` for i in $(seq 1 100); do kubectl exec --context="${CTX_CLUSTER1}" -n sample -c sleep \ "$(kubectl get pod --context="${CTX_CLUSTER1}" -n sample -l \ app=sleep -o jsonpath='{.items[0].metadata.name}')" \ -- curl -sS helloworld.sample:5000/hello done ``` 6. Generate traffic from c2. The response should alternate between c1 (v1) and c2 (v2) regions: ``` for i in $(seq 1 100); do kubectl exec --context="${CTX_CLUSTER2}" -n sample -c sleep \ "$(kubectl get pod --context="${CTX_CLUSTER2}" -n sample -l \ app=sleep -o jsonpath='{.items[0].metadata.name}')" \ -- curl -sS helloworld.sample:5000/hello done ``` 7. Cross-cluster connectivity has been verified. ================================================ FILE: examples/istio-mc/c1.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl module "c1" { source = "oracle-terraform-modules/oke/oci" version = "5.2.2" count = lookup(lookup(var.clusters, "c1"), "enabled") ? 1 : 0 home_region = lookup(local.regions, var.home_region) region = lookup(local.regions, lookup(lookup(var.clusters, "c1"), "region")) tenancy_id = var.tenancy_id # general oci parameters compartment_id = var.compartment_id # ssh keys ssh_private_key_path = var.ssh_private_key_path ssh_public_key_path = var.ssh_public_key_path # networking create_drg = var.oke_control_plane == "private" ? true : false drg_display_name = "c1-drg" remote_peering_connections = var.oke_control_plane == "private" ? { for k, v in var.clusters : "rpc-to-${k}" => {} if k != "c1" } : {} nat_gateway_route_rules = var.oke_control_plane == "private" ? [ for k, v in var.clusters : { destination = lookup(v, "vcn") destination_type = "CIDR_BLOCK" network_entity_id = "drg" description = "Routing to allow connectivity to ${title(k)} cluster" } if k != "c1" ] : [] vcn_cidrs = [lookup(lookup(var.clusters, "c1"), "vcn")] vcn_dns_label = "c1" vcn_name = "c1" #subnets subnets = { bastion = { newbits = 13, netnum = 0, dns_label = "bastion" } operator = { newbits = 13, netnum = 1, dns_label = "operator" } cp = { newbits = 13, netnum = 2, dns_label = "cp" } int_lb = { newbits = 11, netnum = 16, dns_label = "ilb" } pub_lb = { newbits = 11, netnum = 17, dns_label = "plb" } workers = { newbits = 2, netnum = 1, dns_label = "workers" } } # bastion host create_bastion = true bastion_allowed_cidrs = ["0.0.0.0/0"] bastion_upgrade = false # operator host create_operator = true operator_upgrade = false create_iam_resources = true create_iam_operator_policy = "always" operator_install_k9s = true # oke cluster options cluster_name = "c1" cluster_type = var.cluster_type cni_type = var.preferred_cni control_plane_is_public = var.oke_control_plane == "public" control_plane_allowed_cidrs = [local.anywhere] kubernetes_version = var.kubernetes_version pods_cidr = lookup(lookup(var.clusters, "c1"), "pods") services_cidr = lookup(lookup(var.clusters, "c1"), "services") # node pools allow_worker_ssh_access = true kubeproxy_mode = "iptables" worker_pool_mode = "node-pool" worker_pools = var.nodepools worker_cloud_init = local.worker_cloud_init worker_image_type = "oke" # oke load balancers load_balancers = "both" preferred_load_balancer = "public" allow_rules_internal_lb = merge({ for p in local.service_mesh_ports : format("Allow ingress to port %v from cluster c2 for Istio", p) => { protocol = local.tcp_protocol, port = p, source = lookup(lookup(var.clusters, "c2"), "vcn"), source_type = local.rule_type_cidr, } }, { for c in var.clusters : format("Allow TCP ingress from cluster %v for Cilium clustermesh", lookup(c, "name")) => { protocol = local.tcp_protocol, port = 2379, source = lookup(c, "vcn"), source_type = local.rule_type_cidr, } if lookup(c, "name") != "c1" }, { for c in var.clusters : format("Allow UDP ingress from cluster %v for cross-cluster DNS lookup via NLB for Coherence WKA", lookup(c, "name")) => { protocol = local.udp_protocol, port = 53, source = lookup(c, "vcn"), source_type = local.rule_type_cidr, } if lookup(c, "name") != "c1" }, ) allow_rules_public_lb = merge({ for p in local.public_lb_allowed_ports : format("Allow ingress to port %v", p) => { protocol = local.tcp_protocol, port = p, source = "0.0.0.0/0", source_type = local.rule_type_cidr, } }, ) allow_rules_workers = merge( { for c in var.clusters : format("Allow UDP ingress to workers from cluster %v for default VXLAN", lookup(c, "name")) => { protocol = local.udp_protocol, port = 8472, source = lookup(c, "vcn"), source_type = local.rule_type_cidr, } if lookup(c, "name") != "c1" }, ) user_id = var.user_id providers = { oci = oci.c1 oci.home = oci.home } } ================================================ FILE: examples/istio-mc/c2.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl module "c2" { source = "oracle-terraform-modules/oke/oci" version = "5.2.2" count = lookup(lookup(var.clusters, "c2"), "enabled") ? 1 : 0 home_region = lookup(local.regions, var.home_region) region = lookup(local.regions, lookup(lookup(var.clusters, "c2"), "region")) tenancy_id = var.tenancy_id # general oci parameters compartment_id = var.compartment_id # ssh keys ssh_private_key_path = var.ssh_private_key_path ssh_public_key_path = var.ssh_public_key_path # networking create_drg = var.oke_control_plane == "private" ? true : false drg_display_name = "c2" remote_peering_connections = var.oke_control_plane == "private" ? { for k, v in var.clusters : "rpc-to-${k}" => {} if k != "c2" } : {} nat_gateway_route_rules = var.oke_control_plane == "private" ? [ for k, v in var.clusters : { destination = lookup(v, "vcn") destination_type = "CIDR_BLOCK" network_entity_id = "drg" description = "Routing to allow connectivity to ${title(k)} cluster" } if k != "c2" ] : [] vcn_cidrs = [lookup(lookup(var.clusters, "c2"), "vcn")] vcn_dns_label = "c2" vcn_name = "c2" #subnets subnets = { cp = { newbits = 13, netnum = 2, dns_label = "cp" } int_lb = { newbits = 11, netnum = 16, dns_label = "ilb" } pub_lb = { newbits = 11, netnum = 17, dns_label = "plb" } workers = { newbits = 2, netnum = 1, dns_label = "workers" } } # bastion host create_bastion = false bastion_allowed_cidrs = ["0.0.0.0/0"] bastion_upgrade = false # operator host create_operator = false operator_upgrade = false create_iam_resources = true create_iam_operator_policy = "always" operator_install_k9s = true # oke cluster options cluster_name = "c2" cluster_type = var.cluster_type cni_type = var.preferred_cni control_plane_is_public = var.oke_control_plane == "public" control_plane_allowed_cidrs = [local.anywhere] kubernetes_version = var.kubernetes_version pods_cidr = lookup(lookup(var.clusters, "c2"), "pods") services_cidr = lookup(lookup(var.clusters, "c2"), "services") # node pools kubeproxy_mode = "iptables" worker_pool_mode = "node-pool" worker_pools = var.nodepools worker_cloud_init = local.worker_cloud_init worker_image_type = "oke" # oke load balancers load_balancers = "both" preferred_load_balancer = "public" allow_rules_internal_lb = merge({ for p in local.service_mesh_ports : format("Allow ingress to port %v from cluster c1", p) => { protocol = local.tcp_protocol, port = p, source = lookup(lookup(var.clusters, "c1"), "vcn"), source_type = local.rule_type_cidr, } }, { for c in var.clusters : format("Allow TCP ingress from cluster %v for Cilium clustermesh", lookup(c, "name")) => { protocol = local.tcp_protocol, port = 2379, source = lookup(c, "vcn"), source_type = local.rule_type_cidr, } if lookup(c, "name") != "c2" }, { for c in var.clusters : format("Allow UDP ingress from cluster %v for cross-cluster DNS lookup via NLB for Coherence WKA", lookup(c, "name")) => { protocol = local.udp_protocol, port = 53, source = lookup(c, "vcn"), source_type = local.rule_type_cidr, } if lookup(c, "name") != "c2" }, ) allow_rules_public_lb = merge({ for p in local.public_lb_allowed_ports : format("Allow ingress to port %v", p) => { protocol = local.tcp_protocol, port = p, source = "0.0.0.0/0", source_type = local.rule_type_cidr, } }, ) allow_rules_workers = merge( { for c in var.clusters : format("Allow UDP ingress to workers from cluster %v for default VXLAN", lookup(c, "name")) => { protocol = local.udp_protocol, port = 8472, source = lookup(c, "vcn"), source_type = local.rule_type_cidr } if lookup(c, "name") != "c2" }, ) user_id = var.user_id providers = { oci = oci.c2 oci.home = oci.home } } ================================================ FILE: examples/istio-mc/contexts.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl resource "null_resource" "tools" { depends_on = [module.c1] connection { host = local.operator_ip private_key = file(var.ssh_private_key_path) timeout = "40m" type = "ssh" user = "opc" bastion_host = local.bastion_ip bastion_user = "opc" bastion_private_key = file(var.ssh_private_key_path) } provisioner "file" { content = local.token_helper_template destination = "/home/opc/token_helper.sh" } provisioner "file" { content = local.istioctl_template destination = "/home/opc/install_istioctl.sh" } provisioner "remote-exec" { inline = [ "mkdir /home/opc/bin; mv token_helper.sh /home/opc/bin; chmod +x /home/opc/bin/token_helper.sh", "if [ -f \"$HOME/install_istioctl.sh\" ]; then bash \"$HOME/install_istioctl.sh\";fi", ] } } resource "null_resource" "set_contexts" { depends_on = [module.c1, module.c2] for_each = local.all_cluster_ids connection { host = local.operator_ip private_key = file(var.ssh_private_key_path) timeout = "40m" type = "ssh" user = "opc" bastion_host = local.bastion_ip bastion_user = "opc" bastion_private_key = file(var.ssh_private_key_path) } provisioner "file" { content = lookup(local.kubeconfig_templates, each.key) destination = "/home/opc/generate_kubeconfig_${each.key}.sh" } provisioner "file" { content = lookup(local.set_credentials_templates, each.key) destination = "/home/opc/kubeconfig_set_credentials_${each.key}.sh" } provisioner "file" { content = lookup(local.set_alias_templates, each.key) destination = "/home/opc/set_alias_${each.key}.sh" } provisioner "remote-exec" { inline = [ "if [ -f \"$HOME/generate_kubeconfig_${each.key}.sh\" ]; then bash \"$HOME/generate_kubeconfig_${each.key}.sh\";fi", "if [ -f \"$HOME/kubeconfig_set_credentials_${each.key}.sh\" ]; then bash \"$HOME/kubeconfig_set_credentials_${each.key}.sh\";fi", "if [ -f \"$HOME/set_alias_${each.key}.sh\" ]; then bash \"$HOME/set_alias_${each.key}.sh\";fi", ] } triggers = { clusters = length(var.clusters) } lifecycle { create_before_destroy = true } } ================================================ FILE: examples/istio-mc/istio.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { istio_c1 = templatefile("${path.module}/resources/istio.template.yaml", { mesh_id = var.istio_mesh_id cluster = "c1" mesh_network = "c1" pub_nsg_id = one(element([module.c1[*].pub_lb_nsg_id], 0)) int_lb_subnet_id = one(element([module.c1[*].int_lb_subnet_id], 0)) int_nsg_id = one(element([module.c1[*].int_lb_nsg_id], 0)) } ) istio_c2 = templatefile("${path.module}/resources/istio.template.yaml", { mesh_id = var.istio_mesh_id cluster = "c2" mesh_network = "c2" pub_nsg_id = one(element([module.c2[*].pub_lb_nsg_id], 0)) int_lb_subnet_id = one(element([module.c2[*].int_lb_subnet_id], 0)) int_nsg_id = one(element([module.c2[*].int_lb_nsg_id], 0)) } ) } resource "null_resource" "istio" { depends_on = [module.c1, module.c2] connection { host = local.operator_ip private_key = file(var.ssh_private_key_path) timeout = "40m" type = "ssh" user = "opc" bastion_host = local.bastion_ip bastion_user = "opc" bastion_private_key = file(var.ssh_private_key_path) } provisioner "file" { content = local.istio_c1 destination = "/home/opc/c1.yaml" } provisioner "file" { content = local.istio_c2 destination = "/home/opc/c2.yaml" } } ================================================ FILE: examples/istio-mc/locals.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { all_ports = -1 # Protocols # See https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml all_protocols = "all" icmp_protocol = 1 tcp_protocol = 6 udp_protocol = 17 anywhere = "0.0.0.0/0" rule_type_nsg = "NETWORK_SECURITY_GROUP" rule_type_cidr = "CIDR_BLOCK" rule_type_service = "SERVICE_CIDR_BLOCK" bastion_ip = one(element([module.c1[*].bastion_public_ip], 0)) operator_ip = one(element([module.c1[*].operator_private_ip], 0)) # TODO: check when is 15021 required for public public_lb_allowed_ports = [80, 443, 15021] # ports required to be opened for inter-cluster communication between for Istio service_mesh_ports = [15012, 15017, 15021, 15443] regions = { # Africa johannesburg = "af-johannesburg-1" # Asia chuncheon = "ap-chuncheon-1" hyderabad = "ap-hyderabad-1" mumbai = "ap-mumbai-1" osaka = "ap-osaka-1" seoul = "ap-seoul-1" singapore = "ap-singapore-1" tokyo = "ap-tokyo-1" # Europe amsterdam = "eu-amsterdam-1" frankfurt = "eu-frankfurt-1" london = "uk-london-1" madrid = "eu-madrid-1" marseille = "eu-marseille-1" milan = "eu-milan-1" newport = "uk-cardiff-1" paris = "eu-paris-1" stockholm = "eu-stockholm-1" zurich = "eu-zurich-1" # Middle East abudhabi = "me-abudhabi-1" dubai = "me-dubai-1" jeddah = "me-jeddah-1" jerusalem = "il-jerusalem-1" # Oceania melbourne = "ap-melbourne-1" sydney = "ap-sydney-1" # South America bogota = "sa-bogota-1" santiago = "sa-santiago-1" saupaulo = "sa-saupaulo-1" valparaiso = "sa-valparaiso-1" vinhedo = "sa-vinhedo-1" # North America ashburn = "us-ashburn-1" chicago = "us-chicago-1" monterrey = "mx-monterrey-1" montreal = "ca-montreal-1" phoenix = "us-phoenix-1" queretaro = "mx-queretaro-1" sanjose = "us-sanjose-1" toronto = "ca-toronto-1" # US Gov FedRamp us-gov-ashburn = "us-langley-1" us-gov-phoenix = "us-luke-1" # US Gov DISA L5 us-dod-east = "us-gov-ashburn-1" us-dod-north = "us-gov-chicago-1" us-dod-west = "us-gov-phoenix-1" # UK Gov uk-gov-south = "uk-gov-london-1" uk-gov-west = "uk-gov-cardiff-1" # Australia Gov au-gov-cbr = "ap-dcc-canberra-1" } worker_cloud_init = [ { content = <<-EOT runcmd: - 'echo "Kernel module configuration for Istio and worker node initialization"' - 'modprobe br_netfilter' - 'modprobe nf_nat' - 'modprobe xt_REDIRECT' - 'modprobe xt_owner' - 'modprobe iptable_nat' - 'modprobe iptable_mangle' - 'modprobe iptable_filter' - '/usr/libexec/oci-growfs -y' - 'timedatectl set-timezone Australia/Sydney' - 'curl --fail -H "Authorization: Bearer Oracle" -L0 http://169.254.169.254/opc/v2/instance/metadata/oke_init_script | base64 --decode >/var/run/oke-init.sh' - 'bash -x /var/run/oke-init.sh' EOT content_type = "text/cloud-config", } ] } ================================================ FILE: examples/istio-mc/outputs.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl output "ssh_to_operator" { description = "convenient command to ssh to the Admin operator host" value = one(element([module.c1[*].ssh_to_operator], 0)) } ================================================ FILE: examples/istio-mc/providers.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl provider "oci" { fingerprint = var.api_fingerprint private_key_path = var.api_private_key_path region = lookup(local.regions, var.home_region) tenancy_ocid = var.tenancy_id user_ocid = var.user_id alias = "home" ignore_defined_tags = ["Oracle-Tags.CreatedBy", "Oracle-Tags.CreatedOn"] } provider "oci" { fingerprint = var.api_fingerprint private_key_path = var.api_private_key_path region = lookup(local.regions, lookup(lookup(var.clusters, "c1"), "region")) tenancy_ocid = var.tenancy_id user_ocid = var.user_id alias = "c1" ignore_defined_tags = ["Oracle-Tags.CreatedBy", "Oracle-Tags.CreatedOn"] } provider "oci" { fingerprint = var.api_fingerprint private_key_path = var.api_private_key_path region = lookup(local.regions, lookup(lookup(var.clusters, "c2"), "region")) tenancy_ocid = var.tenancy_id user_ocid = var.user_id alias = "c2" ignore_defined_tags = ["Oracle-Tags.CreatedBy", "Oracle-Tags.CreatedOn"] } ================================================ FILE: examples/istio-mc/resources/istio.template.yaml ================================================ apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: values: global: meshID: ${mesh_id} multiCluster: clusterName: ${cluster} network: ${mesh_network} components: egressGateways: - name: istio-egressgateway enabled: true ingressGateways: - name: istio-ingressgateway enabled: true k8s: serviceAnnotations: service.beta.kubernetes.io/oci-load-balancer-internal: "false" service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "50" service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" service.beta.kubernetes.io/oci-load-balancer-security-list-management-mode: "None" oci.oraclecloud.com/oci-network-security-groups: "${pub_nsg_id}" - name: istio-eastwestgateway enabled: true k8s: serviceAnnotations: service.beta.kubernetes.io/oci-load-balancer-internal: "true" service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "50" service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" service.beta.kubernetes.io/oci-load-balancer-security-list-management-mode: "None" service.beta.kubernetes.io/oci-load-balancer-subnet1: "${int_lb_subnet_id}" oci.oraclecloud.com/oci-network-security-groups: "${int_nsg_id}" env: - name: ISTIO_META_REQUESTED_NETWORK_VIEW value: ${mesh_network} - name: ISTIO_META_ROUTER_MODE value: "sni-dnat" service: ports: - name: status-port port: 15021 targetPort: 15021 - name: tls port: 15443 targetPort: 15443 - name: tls-istiod port: 15012 targetPort: 15012 - name: tls-webhook port: 15017 targetPort: 15017 label: app: istio-eastwestgateway istio: eastwestgateway topology.istio.io/network: ${mesh_network} ================================================ FILE: examples/istio-mc/scripts/cloud-init.sh ================================================ #!/bin/sh modprobe br_netfilter modprobe nf_nat modprobe xt_REDIRECT modprobe xt_owner modprobe iptable_nat modprobe iptable_mangle modprobe iptable_filter /usr/libexec/oci-growfs -y timedatectl set-timezone Australia/Sydney 'curl --fail -H "Authorization: Bearer Oracle" -L0 http://169.254.169.254/opc/v2/instance/metadata/oke_init_script | base64 --decode >/var/run/oke-init.sh' bash -x /var/run/oke-init.sh touch /var/log/oke.done ================================================ FILE: examples/istio-mc/scripts/generate_kubeconfig.template.sh ================================================ #!/usr/bin/bash # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl oci ce cluster create-kubeconfig --cluster-id ${cluster_id} --file $HOME/.kube/config --region ${region} --token-version 2.0.0 --kube-endpoint ${endpoint} ================================================ FILE: examples/istio-mc/scripts/istioctl.template.sh ================================================ #!/usr/bin/bash # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl echo "Installing istioctl" curl -L curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${version} TARGET_ARCH=x86_64 sh - ================================================ FILE: examples/istio-mc/scripts/kubeconfig_set_credentials.template.sh ================================================ #!/usr/bin/bash # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl kubectl config set-credentials "user-${cluster_id_11}" --exec-command="$HOME/bin/token_helper.sh" \ --exec-arg="ce" \ --exec-arg="cluster" \ --exec-arg="generate-token" \ --exec-arg="--cluster-id" \ --exec-arg="${cluster_id}" \ --exec-arg="--region" \ --exec-arg="${region}" ================================================ FILE: examples/istio-mc/scripts/set_alias.template.sh ================================================ #!/usr/bin/bash # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl kubectx ${cluster}=context-${cluster_id_11} ================================================ FILE: examples/istio-mc/scripts/token_helper.template.sh ================================================ #!/bin/bash # Copyright 2024 Oracle Corporation and/or affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl CLUSTER=$5 REGION=$7 TOKEN_FILE=$HOME/.kube/TOKEN-$CLUSTER if ! test -f "$TOKEN_FILE" || test $(( `date +%s` - `stat -L -c %Y $TOKEN_FILE` )) -gt 240; then umask 022 oci ce cluster generate-token --cluster-id "$CLUSTER" --region "$REGION" > $TOKEN_FILE fi cat $TOKEN_FILE ================================================ FILE: examples/istio-mc/templates.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { all_cluster_ids = merge( { c1 = one(element([module.c1[*].cluster_id], 0)) }, { c2 = one(element([module.c2[*].cluster_id], 0)) } ) kubeconfig_templates = { for cluster_name, cluster_id in local.all_cluster_ids : cluster_name => templatefile("${path.module}/scripts/generate_kubeconfig.template.sh", { cluster_id = cluster_id endpoint = var.oke_control_plane == "public" ? "PUBLIC_ENDPOINT" : "PRIVATE_ENDPOINT" region = lookup(local.regions, lookup(lookup(var.clusters, cluster_name), "region")) } ) } set_credentials_templates = { for cluster_name, cluster_id in local.all_cluster_ids : cluster_name => templatefile("${path.module}/scripts/kubeconfig_set_credentials.template.sh", { cluster_id = cluster_id cluster_id_11 = substr(cluster_id, (length(cluster_id) - 11), length(cluster_id)) region = lookup(local.regions, lookup(lookup(var.clusters, cluster_name), "region")) } ) } set_alias_templates = { for cluster_name, cluster_id in local.all_cluster_ids : cluster_name => templatefile("${path.module}/scripts/set_alias.template.sh", { cluster = cluster_name cluster_id_11 = substr(cluster_id, (length(cluster_id) - 11), length(cluster_id)) } ) } token_helper_template = templatefile("${path.module}/scripts/token_helper.template.sh", {}) istioctl_template = templatefile("${path.module}/scripts/istioctl.template.sh", { version = var.istio_version }) } ================================================ FILE: examples/istio-mc/terraform.tfvars.example ================================================ # provider api_fingerprint = "" api_private_key_path = "~/.oci/oci_rsa.pem" home_region = "ashburn" # Use short form e.g. ashburn from location column https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm tenancy_id = "ocid1.tenancy.oc1.." user_id = "ocid1.user.oc1.." compartment_id = "ocid1.compartment.oc1.." # ssh ssh_private_key_path = "~/.ssh/id_rsa" ssh_public_key_path = "~/.ssh/id_rsa.pub" # clusters ## For regions, # Use short form e.g. ashburn from location column https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm ## VCN, Pods and services clusters must not overlap with each other and with those of other clusters. clusters = { c1 = { region = "sydney", vcn = "10.1.0.0/16", pods = "10.201.0.0/16", services = "10.101.0.0/16", enabled = true } c2 = { region = "melbourne", vcn = "10.2.0.0/16", pods = "10.202.0.0/16", services = "10.102.0.0/16", enabled = true } } kubernetes_version = "v1.32.1" cluster_type = "basic" oke_control_plane = "private" nodepools = { np1 = { shape = "VM.Standard.E4.Flex", ocpus = 2, memory = 64, size = 2, boot_volume_size = 150, } } ================================================ FILE: examples/istio-mc/variables.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # OCI Provider parameters variable "api_fingerprint" { default = "" description = "Fingerprint of the API private key to use with OCI API." type = string } variable "api_private_key_path" { default = "" description = "The path to the OCI API private key." type = string } variable "home_region" { description = "The tenancy's home region. Use the short form in lower case e.g. phoenix." type = string } variable "tenancy_id" { description = "The tenancy id of the OCI Cloud Account in which to create the resources." type = string } variable "user_id" { description = "The id of the user that Terraform will use to create the resources." type = string default = "" } # General OCI parameters variable "compartment_id" { description = "The compartment id where to create all resources." type = string } # ssh keys variable "ssh_private_key_path" { default = "none" description = "The path to ssh private key." type = string } variable "ssh_public_key_path" { default = "none" description = "The path to ssh public key." type = string } # clusters variable "clusters" { description = "A map of cidrs for vcns, pods and services for each region" type = map(any) default = { c1 = { region = "sydney", vcn = "10.1.0.0/16", pods = "10.201.0.0/16", services = "10.101.0.0/16", enabled = true } c2 = { region = "melbourne", vcn = "10.2.0.0/16", pods = "10.202.0.0/16", services = "10.102.0.0/16", enabled = true } } } variable "kubernetes_version" { default = "v1.32.1" description = "The version of Kubernetes to use." type = string } variable "cluster_type" { default = "basic" description = "Whether to use basic or enhanced OKE clusters" type = string validation { condition = contains(["basic", "enhanced"], lower(var.cluster_type)) error_message = "Accepted values are 'basic' or 'enhanced'." } } variable "oke_control_plane" { default = "public" description = "Whether to keep all OKE control planes public or private." type = string validation { condition = contains(["public", "private"], lower(var.oke_control_plane)) error_message = "Accepted values are 'public' or 'private'." } } variable "preferred_cni" { default = "flannel" description = "Whether to use flannel or NPN" type = string validation { condition = contains(["flannel", "npn"], lower(var.preferred_cni)) error_message = "Accepted values are 'flannel' or 'npn'." } } # node pools variable "timezone" { type = string description = "Preferred timezone of computes" default = "Australia/Sydney" } variable "nodepools" { type = any description = "Node pools for all clusters" default = { np1 = { shape = "VM.Standard.E4.Flex", ocpus = 2, memory = 32, size = 3, boot_volume_size = 150, } } } # istio variable "istio_version" { default = "1.20.2" description = "Istio version to install" type = string } variable "istio_mesh_id" { description = "mesh id to be used" type = string default = "yggdrasil" } ================================================ FILE: examples/istio-mc/versions.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl #Terraform and provider version to use terraform { required_providers { oci = { source = "oracle/oci" } } required_version = ">= 1.0.0" } ================================================ FILE: examples/network/README.md ================================================ # Network Examples Example configurations for VCN networking: | File | Description | |------|-------------| | `vars-network.auto.tfvars` | Full network configuration with subnets, NSGs, and gateways | | `vars-network-subnets-create.auto.tfvars` | Automatic subnet creation with defaults | | `vars-network-subnets-create-cidr.auto.tfvars` | Subnet creation with explicit CIDRs | | `vars-network-subnets-create-cidr-ipv4-and-ipv6.tfvars` | Dual-stack (IPv4/IPv6) subnet creation | | `vars-network-subnets-create-force.auto.tfvars` | Force subnet creation regardless of component settings | | `vars-network-subnets-existing.auto.tfvars` | Using existing subnets | | `vars-network-nsgs-create.auto.tfvars` | Creating network security groups | | `vars-network-nsgs-existing.auto.tfvars` | Using existing NSGs | | `vars-network-drg-create.auto.tfvars.example` | Creating a Dynamic Routing Gateway (example) | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: examples/network/vars-network-drg-create.auto.tfvars.example ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # to reuse an existing drg, set to false create_drg = true drg_display_name = "drg" # to reuse an existing drg, provide the drg ocid drg_id = null # remote_peering_connections = { # unpeered rpc rpc1 = {} # peered rpc rpc2 : { "rpc_acceptor_id" : "ocid1.remotepeeringconnection.oc1.aaaaaa" "rpc_acceptor_region" : "us-ashburn-1" } } ================================================ FILE: examples/operator/README.md ================================================ # Operator Examples Example configurations for the operator host: | File | Description | |------|-------------| | `vars-operator.auto.tfvars` | Basic operator configuration | | `vars-operator-cloudinit.auto.tfvars` | Operator with custom cloud-init | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: examples/profiles/README.md ================================================ # Deployment Profiles Composable deployment profiles that enable only the components you need: | Profile | Description | |---------|-------------| | `network-only` | VCN, subnets, NSGs, and gateways only | | `cluster-workers-only` | OKE cluster and worker pools (requires existing network) | | `workers-only` | Worker pools only (requires existing cluster) | | `network-cluster-workers` | Full stack: network, cluster, and workers | ## Usage Each profile is a self-contained Terraform root module. Copy the profile directory and configure the variables. ================================================ FILE: examples/profiles/cluster-workers-only/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl provider "oci" { config_file_profile = var.config_file_profile tenancy_ocid = var.tenancy_id region = var.region } module "cluster_workers_only" { source = "../../../" providers = { oci.home = oci } tenancy_id = var.tenancy_id compartment_id = var.compartment_id ssh_public_key = var.ssh_public_key_path create_vcn = false // *true/false; vcn_id required if false vcn_id = var.vcn_id subnets = var.subnets nsgs = var.nsgs create_bastion = false // *true/false bastion_public_ip = var.bastion_public_ip create_operator = true // *true/false create_cluster = true // *true/false cluster_type = "enhanced" // *basic/enhanced cni_type = "flannel" // *flannel/npn worker_pool_size = 1 worker_pools = { oke-pool = {} } } ================================================ FILE: examples/profiles/cluster-workers-only/variables.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "tenancy_id" { type = string } variable "compartment_id" { type = string } variable "region" { type = string } variable "config_file_profile" { default = "DEFAULT" type = string } variable "ssh_public_key_path" { default = null type = string } variable "subnets" { type = map(object({ create = optional(string) id = optional(string) newbits = optional(string) netnum = optional(string) cidr = optional(string) dns_label = optional(string) })) } variable "nsgs" { type = map(object({ create = optional(string) id = optional(string) })) } ================================================ FILE: examples/profiles/cluster-workers-only/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.3.0" required_providers { oci = { source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/profiles/network-cluster-workers/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl provider "oci" { config_file_profile = var.config_file_profile tenancy_ocid = var.tenancy_id region = var.region } module "network_cluster_workers" { source = "../../../" providers = { oci.home = oci } tenancy_id = var.tenancy_id compartment_id = var.compartment_id ssh_public_key = var.ssh_public_key_path worker_pool_size = 1 worker_pools = { oke-pool = {} } } ================================================ FILE: examples/profiles/network-cluster-workers/variables.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "tenancy_id" { type = string } variable "compartment_id" { type = string } variable "region" { type = string } variable "config_file_profile" { default = "DEFAULT" type = string } variable "ssh_public_key_path" { default = null type = string } ================================================ FILE: examples/profiles/network-cluster-workers/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.3.0" required_providers { oci = { source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/profiles/network-only/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl provider "oci" { config_file_profile = var.config_file_profile tenancy_ocid = var.tenancy_id region = var.region } module "network_only" { source = "../../../" providers = { oci.home = oci } tenancy_id = var.tenancy_id compartment_id = var.compartment_id create_bastion = false // *true/false create_cluster = false // *true/false create_operator = false // *true/false # Force creation of NSGs with associated components disabled nsgs = { bastion = { create = "always" } operator = { create = "always" } cp = { create = "always" } int_lb = { create = "always" } pub_lb = { create = "always" } workers = { create = "always" } pods = { create = "always" } } # Force creation of subnets with associated components disabled subnets = { bastion = { create = "always", newbits = 13 } operator = { create = "always", newbits = 13 } cp = { create = "always", newbits = 13 } int_lb = { create = "always", newbits = 11 } pub_lb = { create = "always", newbits = 11 } workers = { create = "always", newbits = 4 } pods = { create = "always", newbits = 2 } } } ================================================ FILE: examples/profiles/network-only/variables.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "tenancy_id" { type = string } variable "compartment_id" { type = string } variable "region" { type = string } variable "config_file_profile" { default = "DEFAULT" type = string } variable "ssh_public_key_path" { default = null type = string } ================================================ FILE: examples/profiles/network-only/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.3.0" required_providers { oci = { source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/profiles/workers-only/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl provider "oci" { config_file_profile = var.config_file_profile tenancy_ocid = var.tenancy_id region = var.region } module "workers_only" { source = "../../../" providers = { oci.home = oci } tenancy_id = var.tenancy_id compartment_id = var.compartment_id vcn_id = var.vcn_id bastion_public_ip = var.bastion_public_ip cluster_id = var.cluster_id operator_private_ip = var.operator_private_ip ssh_public_key_path = var.ssh_public_key_path create_vcn = false create_bastion = false create_cluster = false create_operator = false subnets = { workers = { id = var.worker_subnet_id } } nsgs = {} worker_nsg_ids = var.worker_nsg_ids worker_pool_size = 1 worker_pools = { oke-pool = {} } } ================================================ FILE: examples/profiles/workers-only/variables.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "tenancy_id" { type = string } variable "compartment_id" { type = string } variable "region" { type = string } variable "vcn_id" { type = string } variable "bastion_public_ip" { type = string } variable "cluster_id" { type = string } variable "worker_subnet_id" { type = string } variable "config_file_profile" { default = "DEFAULT" type = string } variable "operator_private_ip" { default = null type = string } variable "worker_nsg_ids" { default = [] type = list(string) } variable "ssh_public_key_path" { default = null type = string } ================================================ FILE: examples/profiles/workers-only/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.3.0" required_providers { oci = { source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/provider-basic.tf ================================================ # Copyright 2017, 2023 Oracle Corporation and/or affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl provider "oci" { alias = "home" region = "us-ashburn-1" tenancy_ocid = "ocid1.tenancy..." } provider "oci" { region = "ap-osaka-1" tenancy_ocid = "ocid1.tenancy..." } ================================================ FILE: examples/rms/README.md ================================================ # Oracle Resource Manager Stack Examples Pre-built configurations for deploying via [OCI Resource Manager (ORM)](https://docs.oracle.com/en-us/iaas/Content/ResourceManager/home.htm): | Stack | Description | |-------|-------------| | `oke-network-only` | Network infrastructure (VCN, subnets, NSGs, gateways, bastion, operator) | | `oke-cluster-only` | Full OKE cluster deployment | | `oke-workers-only` | Worker pools added to an existing cluster | Each stack includes a `schema.yaml` for the Resource Manager UI. ## Usage Upload the stack directory as a Terraform configuration in the OCI Console under Resource Manager > Stacks. ================================================ FILE: examples/rms/oke-cluster-only/data.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "ssh_public_key" { default = null type = string } variable "ssh_kms_vault_id" { default = null type = string } variable "ssh_kms_secret_id" { default = null type = string } data "oci_identity_region_subscriptions" "home" { tenancy_id = var.tenancy_ocid filter { name = "is_home_region" values = [true] } } data "oci_secrets_secretbundle" "ssh_key" { secret_id = var.ssh_kms_secret_id } locals { ssh_public_key = try(base64decode(var.ssh_public_key), var.ssh_public_key) ssh_key_bundle = sensitive(one(data.oci_secrets_secretbundle.ssh_key.secret_bundle_content)) ssh_key_bundle_content = sensitive(lookup(local.ssh_key_bundle, "content", null)) } ================================================ FILE: examples/rms/oke-cluster-only/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_secrets_secretbundle" "ocir" { count = var.ocir_kms_vault_id == null ? 0 : 1 secret_id = var.ocir_kms_secret_id } module "oke" { source = "github.com/oracle-terraform-modules/terraform-oci-oke.git?ref=5.x&depth=1" providers = { oci.home = oci.home } tenancy_id = var.tenancy_ocid compartment_id = var.compartment_ocid # General timezone = var.timezone output_detail = var.output_detail # Identity create_iam_resources = true # Network create_vcn = false vcn_id = var.vcn_id vcn_create_internet_gateway = "never" vcn_create_nat_gateway = "never" vcn_create_service_gateway = "never" assign_dns = var.assign_dns ig_route_table_id = var.ig_route_table_id subnets = { operator = { create = "never", id = var.operator_subnet_id } cp = { create = "never", id = var.control_plane_subnet_id } int_lb = { create = "never", id = var.int_lb_subnet_id } pub_lb = { create = "never", id = var.pub_lb_subnet_id } } nsgs = { operator = { create = "never", id = var.operator_nsg_id } cp = { create = "never", id = var.control_plane_nsg_id } } # Network Security control_plane_is_public = var.control_plane_is_public load_balancers = lower(var.load_balancers) create_bastion = false bastion_public_ip = var.bastion_public_ip # Operator create_operator = var.create_operator operator_availability_domain = var.operator_availability_domain operator_cloud_init = var.operator_cloud_init operator_image_id = var.operator_image_id operator_image_os = var.operator_image_os operator_image_os_version = var.operator_image_os_version operator_install_helm = var.operator_install_helm operator_install_k9s = var.operator_install_k9s operator_install_kubectx = var.operator_install_kubectx operator_private_ip = var.operator_private_ip operator_pv_transit_encryption = var.operator_pv_transit_encryption operator_shape = var.operator_shape operator_upgrade = var.operator_upgrade operator_user = var.operator_user operator_volume_kms_key_id = var.operator_volume_kms_key_id # SSH ssh_public_key = local.ssh_public_key ssh_private_key = sensitive(local.ssh_key_bundle_content) # Cluster cluster_kms_key_id = var.cluster_kms_key_id cluster_name = var.cluster_name cluster_type = lower(var.cluster_type) cni_type = lower(var.cni_type) create_cluster = true image_signing_keys = var.image_signing_keys kubernetes_version = var.kubernetes_version pods_cidr = var.pods_cidr preferred_load_balancer = lower(var.preferred_load_balancer) services_cidr = var.services_cidr use_signed_images = var.use_signed_images # CNI: Cilium cilium_install = var.cilium_install cilium_reapply = var.cilium_reapply cilium_namespace = var.cilium_namespace cilium_helm_version = var.cilium_helm_version cilium_helm_values = var.cilium_helm_values cilium_helm_values_files = var.cilium_helm_values_files # Metrics server metrics_server_install = var.metrics_server_install metrics_server_namespace = var.metrics_server_namespace metrics_server_helm_version = var.metrics_server_helm_version metrics_server_helm_values = var.metrics_server_helm_values metrics_server_helm_values_files = var.metrics_server_helm_values_files # Cluster autoscaler cluster_autoscaler_install = var.cluster_autoscaler_install cluster_autoscaler_namespace = var.cluster_autoscaler_namespace cluster_autoscaler_helm_version = var.cluster_autoscaler_helm_version cluster_autoscaler_helm_values = var.cluster_autoscaler_helm_values cluster_autoscaler_helm_values_files = var.cluster_autoscaler_helm_values_files # Gatekeeper gatekeeper_install = var.gatekeeper_install gatekeeper_namespace = var.gatekeeper_namespace gatekeeper_helm_version = var.gatekeeper_helm_version gatekeeper_helm_values = var.gatekeeper_helm_values gatekeeper_helm_values_files = var.gatekeeper_helm_values_files # Prometheus prometheus_install = var.prometheus_install prometheus_reapply = var.prometheus_reapply prometheus_namespace = var.prometheus_namespace prometheus_helm_version = var.prometheus_helm_version prometheus_helm_values = var.prometheus_helm_values prometheus_helm_values_files = var.prometheus_helm_values_files # DCGM exporter dcgm_exporter_install = var.dcgm_exporter_install dcgm_exporter_reapply = var.dcgm_exporter_reapply dcgm_exporter_namespace = var.dcgm_exporter_namespace dcgm_exporter_helm_version = var.dcgm_exporter_helm_version # Multus multus_install = var.multus_install multus_namespace = var.multus_namespace multus_daemonset_url = var.multus_daemonset_url multus_version = var.multus_version # SR-IOV device plugin sriov_device_plugin_install = var.sriov_device_plugin_install sriov_device_plugin_namespace = var.sriov_device_plugin_namespace sriov_device_plugin_daemonset_url = var.sriov_device_plugin_daemonset_url sriov_device_plugin_version = var.sriov_device_plugin_version # Whereabouts whereabouts_install = var.whereabouts_install whereabouts_namespace = var.whereabouts_namespace whereabouts_daemonset_url = var.whereabouts_daemonset_url whereabouts_version = var.whereabouts_version # MPI operator mpi_operator_install = var.mpi_operator_install mpi_operator_namespace = var.mpi_operator_namespace mpi_operator_deployment_url = var.mpi_operator_deployment_url mpi_operator_version = var.mpi_operator_version # Tags use_defined_tags = var.use_defined_tags tag_namespace = var.tag_namespace freeform_tags = { # TODO Remaining tags in schema cluster = lookup(var.cluster_tags, "freeformTags", {}) persistent_volume = {} service_lb = {} operator = lookup(var.operator_tags, "freeformTags", {}) } defined_tags = { # TODO Remaining tags in schema cluster = lookup(var.cluster_tags, "definedTags", {}) persistent_volume = {} service_lb = {} operator = lookup(var.operator_tags, "definedTags", {}) } } ================================================ FILE: examples/rms/oke-cluster-only/output.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Terraform output "state_id" { value = module.oke.state_id } # Identity output "dynamic_group_ids" { value = module.oke.dynamic_group_ids } output "policy_statements" { value = module.oke.policy_statements } # Cluster output "cluster_id" { value = module.oke.cluster_id } output "cluster_endpoints" { value = module.oke.cluster_endpoints } output "cluster_kubeconfig" { value = module.oke.cluster_kubeconfig } output "cluster_ca_cert" { value = module.oke.cluster_ca_cert } # Network output "vcn_id" { value = module.oke.vcn_id } output "bastion_public_ip" { value = module.oke.bastion_public_ip } output "bastion_ssh_command" { value = module.oke.ssh_to_bastion } output "bastion_ssh_secret_id" { value = var.ssh_kms_secret_id } # Operator output "operator_id" { value = module.oke.operator_id } output "operator_private_ip" { value = module.oke.operator_private_ip } output "operator_ssh_command" { value = module.oke.ssh_to_operator } output "operator_ssh_secret_id" { value = var.ssh_kms_secret_id } output "operator_subnet_id" { value = module.oke.operator_subnet_id } output "operator_nsg_id" { value = var.operator_nsg_id } # Cluster output "control_plane_subnet_id" { value = module.oke.control_plane_subnet_id } output "control_plane_nsg_id" { value = var.control_plane_nsg_id } output "int_lb_subnet_id" { value = module.oke.int_lb_subnet_id } output "pub_lb_subnet_id" { value = module.oke.pub_lb_subnet_id } ================================================ FILE: examples/rms/oke-cluster-only/schema.yaml ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl title: "OKE: Network & Cluster" description: OKE cluster with VCN network resources informationalText: "Connect to operator host using the SSH command below for Kubernetes endpoint access." schemaVersion: 1.1.0 version: "20230304" locale: "en" variableGroups: - title: "Hidden" visible: false variables: - api_fingerprint - current_user_ocid - tenancy_ocid - region - output_detail - timezone - subnets - await_node_readiness - title: "Identity" variables: - compartment_ocid - create_iam_resources - create_iam_tag_namespace - create_iam_defined_tags - create_iam_autoscaler_policy - create_iam_kms_policy - create_iam_operator_policy - create_iam_worker_policy - use_defined_tags - tag_namespace - title: "Network" variables: - assign_dns - vcn_id - ig_route_table_id - nat_gateway_id - nat_route_table_id - nat_gateway_public_ip_id - service_gateway_id - title: "SSH" variables: - bastion_public_ip - ssh_public_key - ssh_kms_vault_id - ssh_kms_secret_id - title: "Operator" variables: - operator_subnet_id - operator_nsg_id - ${create_operator} - operator_upgrade - operator_availability_domain - operator_private_ip - operator_image_os - operator_image_os_version - operator_image_id - operator_user - operator_shape_name - operator_shape_ocpus - operator_shape_memory - operator_shape_boot - operator_shape - operator_use_encryption - operator_volume_kms_vault_id - operator_volume_kms_key_id - operator_pv_transit_encryption - operator_install_helm - operator_install_k9s - operator_install_kubectx - operator_tags - title: "Cluster" variables: - control_plane_is_public - control_plane_subnet_id - control_plane_nsg_id - pods_cidr - services_cidr - int_lb_subnet_id - pub_lb_subnet_id - load_balancers - preferred_load_balancer - cluster_name - cluster_type - kubernetes_version - cluster_dns - cluster_ca_cert - cluster_use_encryption - cluster_kms_vault_id - cluster_kms_key_id - cni_type - use_signed_images - configure_ocir - ocir_username - ocir_email_address - ocir_kms_vault_id - ocir_secret_id - ocir_secret_namespace - ocir_secret_name - cluster_tags - title: "Extensions" variables: - cluster_autoscaler_install - metrics_server_install - prometheus_install - dcgm_exporter_install - gatekeeper_install - multus_install - cilium_install - whereabouts_install - sriov_device_plugin_install - sriov_cni_plugin_install - rdma_cni_plugin_install - mpi_operator_install - title: "Cluster Autoscaler" visible: cluster_autoscaler_install variables: - ${cluster_autoscaler_namespace} - ${cluster_autoscaler_helm_version} - ${cluster_autoscaler_helm_values} - ${cluster_autoscaler_helm_values_files} - title: "Metrics Server" visible: and: metrics_server_install variables: - metrics_server_namespace - metrics_server_helm_version - metrics_server_helm_values - metrics_server_helm_values_files - title: "Prometheus" visible: prometheus_install variables: - prometheus_namespace - prometheus_reapply - prometheus_helm_version - prometheus_helm_values - prometheus_helm_values_files - title: "DCGM exporter" visible: dcgm_exporter_install variables: - dcgm_exporter_namespace - dcgm_exporter_reapply - dcgm_exporter_helm_version - dcgm_exporter_helm_values - dcgm_exporter_helm_values_files - title: "Gatekeeper" visible: gatekeeper_install variables: - gatekeeper_namespace - gatekeeper_helm_version - gatekeeper_helm_values - gatekeeper_helm_values_files - title: "Multus" visible: multus_install variables: - multus_namespace - multus_daemonset_url - multus_version - title: "Cilium" visible: cilium_install variables: - cilium_namespace - cilium_reapply - cilium_helm_version - title: "Whereabouts" visible: whereabouts_install variables: - whereabouts_namespace - whereabouts_daemonset_url - whereabouts_version - title: "SR-IOV Device Plugin" visible: sriov_device_plugin_install variables: - sriov_device_plugin_namespace - sriov_device_plugin_daemonset_url - sriov_device_plugin_version - title: "SR-IOV CNI Plugin" visible: sriov_cni_plugin_install variables: - sriov_cni_plugin_namespace - sriov_cni_plugin_daemonset_url - sriov_cni_plugin_version - title: "MPI Operator" visible: mpi_operator_install variables: - mpi_operator_namespace - mpi_operator_deployment_url - mpi_operator_version variables: # Identity api_fingerprint: required: false visible: false current_user_ocid: title: User type: ocid required: true tenancy_ocid: title: Tenancy type: oci:identity:compartment:id required: true compartment_ocid: title: Compartment description: The default compartment for created resources. type: oci:identity:compartment:id required: true region: required: true title: Region type: oci:identity:region:name defined_tags: visible: false freeform_tags: visible: false create_iam_resources: default: false description: Whether to create any IAM dynamic groups, policies, and tags. required: true title: Create IAM resources type: boolean create_iam_tag_namespace: default: false required: true title: Create tag namespace type: boolean visible: ${create_iam_resources} create_iam_defined_tags: default: false required: true title: Create defined tags type: boolean visible: ${create_iam_resources} create_iam_autoscaler_policy: title: Create IAM policy for Cluster Autoscaler required: false visible: ${create_iam_resources} type: enum enum: [Never, Auto, Always] default: Auto create_iam_kms_policy: title: Create IAM policy for KMS required: false visible: ${create_iam_resources} type: enum enum: [Never, Auto, Always] default: Auto create_iam_operator_policy: title: Create IAM policy for operater cluster management required: false visible: ${create_iam_resources} type: enum enum: [Never, Auto, Always] default: Auto create_iam_worker_policy: title: Create IAM policy for self-managed worker nodes required: false visible: ${create_iam_resources} type: enum enum: [Never, Auto, Always] default: Auto use_defined_tags: title: Use defined tags default: false type: boolean tag_namespace: title: Tag namespace visible: or: - ${create_iam_tag_namespace} - ${create_iam_defined_tags} - ${use_defined_tags} timezone: title: Timezone type: string # type: oci:identity:region:name # required: true output_detail: title: Output detail type: boolean default: false required: true # VCN network_compartment_id: required: false visible: false vcn_id: title: Existing VCN type: oci:core:vcn:id required: true dependsOn: compartmentId: ${compartment_ocid} assign_dns: title: Assign DNS records type: boolean required: true ig_route_table_id: title: Internet Gateway Route Table type: ocid required: false service_gateway_id: title: Service gateway description: Existing service gateway in the VCN. type: oci:core:servicegateway:id required: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} nat_gateway_id: title: NAT gateway description: Existing NAT gateway in the VCN. type: oci:core:natgateway:id required: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} # NSGs operator_nsg_id: title: Additional Network Security Group type: oci:core:nsg:id # additionalProps: # allowMultiple: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} control_plane_nsg_id: title: Additional Network Security Group type: oci:core:nsg:id # additionalProps: # allowMultiple: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} control_plane_is_public: title: Public Kubernetes control plane type: boolean default: false required: false # Subnets control_plane_subnet_id: title: Control plane subnet type: oci:core:subnet:id required: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: true int_lb_subnet_id: title: Internal load balancer subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: true pub_lb_subnet_id: title: Public load balancer subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: true operator_subnet_id: title: Operator subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: false # SSH ssh_public_key: title: SSH Public Key type: oci:core:ssh:publickey pattern: "((^(ssh-rsa AAAAB3NzaC1yc2|ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNT|ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzOD|ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1Mj|ssh-ed25519 AAAAC3NzaC1lZDI1NTE5|ssh-dss AAAAB3NzaC1kc3)[0-9A-Za-z+\/]+[=]{0,3})( [^,]*)?)(,((ssh-rsa AAAAB3NzaC1yc2|ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNT|ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzOD|ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1Mj|ssh-ed25519 AAAAC3NzaC1lZDI1NTE5|ssh-dss AAAAB3NzaC1kc3)[0-9A-Za-z+\/]+[=]{0,3})( [^,]*)?)*$" required: false ssh_kms_vault_id: title: SSH Vault description: The OCI Vault used to encrypt the SSH key pair. type: oci:kms:vault:id required: true dependsOn: compartmentId: ${compartment_ocid} ssh_kms_secret_id: title: SSH Vault secret description: The OCI Vault secret containing the SSH private key. type: oci:kms:secret:id required: true dependsOn: compartmentId: ${compartment_ocid} vaultId: ${ssh_kms_vault_id} # Bastion bastion_public_ip: title: Bastion IP description: The address of an existing bastion host for SSH access to operator/cluster resources. type: string required: true # Operator variables create_operator: title: Create operator instance description: Provision an OCI Compute instance configured with IAM access to interact with the OKE Kubernetes endpoint, e.g. through a bastion host for private networks. Required for installation of extensions using the module. type: boolean default: true required: false operator_availability_domain: title: Availability domain type: oci:identity:availabilitydomain:name required: false dependsOn: compartmentId: ${compartment_ocid} visible: ${create_operator} operator_private_ip: title: Operator IP type: string required: false visible: not: - ${create_operator} operator_user: title: User type: string default: opc required: true operator_shape_name: title: Shape type: oci:core:instanceshape:name default: "VM.Standard.E4.Flex" required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} operator_shape_ocpus: title: OCPUs (Cores) type: number default: 4 required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} operator_shape_memory: title: Memory (GB) type: number default: 16 required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} operator_shape_boot: title: Boot volume size (GB) type: number default: 50 required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} operator_shape: visible: false operator_image_os: title: Image OS type: enum default: "Oracle Linux" enum: ["Oracle Linux"] required: true visible: and: - ${create_operator} - not: - eq: - ${operator_image_type} - custom operator_image_os_version: title: Image OS version type: enum default: "8" enum: ["7.9", "8"] required: true visible: and: - ${create_operator} - not: - eq: - ${operator_image_type} - custom operator_image_type: title: Image type type: enum default: platform enum: [platform, custom] required: true visible: false operator_image_id: title: Image ID type: oci:core:image:id required: false visible: ${create_operator} dependsOn: compartmentId: ${compartment_ocid} operatingSystem: ${operator_image_os} operatingSystemVersion: ${operator_image_os_version} shape: ${operator_shape_name} operator_pv_transit_encryption: title: In-transit volume encryption type: boolean visible: ${create_operator} operator_use_encryption: title: Use encryption type: boolean default: false required: true visible: ${create_operator} operator_volume_kms_vault_id: title: KMS volume encryption vault description: Vault containing operator volume encryption keys. type: oci:kms:vault:id required: false dependsOn: compartmentId: ${compartment_ocid} visible: and: - ${create_operator} - operator_use_encryption operator_volume_kms_key_id: title: KMS volume encryption key type: oci:kms:key:id dependsOn: compartmentId: ${compartment_ocid} vaultId: ${operator_volume_kms_vault_id} visible: and: - ${create_operator} - operator_use_encryption operator_upgrade: title: Upgrade OS description: Upgrade the operating system on boot. type: boolean default: false required: true visible: ${create_operator} operator_install_helm: title: Install Helm type: boolean visible: ${create_operator} operator_install_k9s: title: Install K9s type: boolean default: true visible: ${create_operator} operator_install_kubectx: title: Install Kubectx type: boolean visible: ${create_operator} operator_tags: type: oci:identity:tag:value required: false title: Tagging description: Tag values for created resources. dependsOn: compartmentId: ${compartment_ocid} visible: ${create_operator} # Cluster cluster_use_encryption: title: Use encryption type: boolean default: false required: true cluster_kms_vault_id: title: Cluster etcd KMS encryption vault type: oci:kms:vault:id required: true dependsOn: compartmentId: ${compartment_ocid} visible: cluster_use_encryption cluster_kms_key_id: title: Cluster etcd KMS encryption key type: oci:kms:key:id required: true dependsOn: compartmentId: ${compartment_ocid} vaultId: ${cluster_kms_vault_id} visible: cluster_use_encryption cluster_name: title: Cluster name description: Display name for the created cluster. Defaults to 'oke' suffixed with the generated Terraform 'state_id' value. type: string required: false cluster_type: title: Cluster Type type: enum default: Basic enum: [Basic, Enhanced] allowMultiple: false required: true cni_type: title: CNI Type type: enum default: Flannel enum: [Flannel, NPN] allowMultiple: false required: true kubernetes_version: type: oci:kubernetes:versions:id title: Kubernetes version description: The Kubernetes version for the created OKE cluster and managed nodes. required: true dependsOn: compartmentId: ${compartment_ocid} clusterOptionId: "all" pods_cidr: title: Pod CIDR services_cidr: title: Service CIDR preferred_load_balancer: title: Preferred load balancer type: enum default: Internal enum: [Internal, Public] allowMultiple: false required: true load_balancers: title: Load balancers type: enum enum: [Public, Internal, Both] default: Internal required: true use_signed_images: title: Use signed images type: boolean default: false required: true # OCIR configure_ocir: title: Configure container registry (OCIR) type: boolean default: false ocir_username: title: Username visible: configure_ocir ocir_email_address: title: Email address visible: configure_ocir ocir_kms_vault_id: title: Vault ID type: oci:kms:vault:id required: false dependsOn: compartmentId: ${compartment_ocid} visible: configure_ocir ocir_kms_secret_id: title: Vault secret ID type: oci:kms:secret:id required: false dependsOn: compartmentId: ${compartment_ocid} vaultId: ${ocir_kms_vault_id} visible: configure_ocir ocir_secret_namespace: title: Secret namespace visible: configure_ocir ocir_secret_name: title: Secret name visible: configure_ocir cluster_tags: type: oci:identity:tag:value required: false title: Tagging description: Tag values for created resources. dependsOn: compartmentId: ${compartment_ocid} # Metrics server metrics_server_install: title: Install Metrics Server description: Deploy the Kubernetes Metrics Server with Oracle Cloud configuration. See kubernetes-sigs/metrics-server for more information. type: boolean default: false required: true metrics_server_namespace: title: Kubernetes namespace type: string metrics_server_helm_version: title: Helm chart version type: string metrics_server_helm_values: title: Helm chart values visible: false metrics_server_helm_values_files: title: Helm chart values files visible: false # Cluster autoscaler cluster_autoscaler_install: title: Install Cluster Autoscaler description: Deploy the Kubernetes Cluster Autoscaler with Oracle Cloud configuration. See kubernetes/autoscaler for more information. type: boolean default: false required: true cluster_autoscaler_namespace: title: Kubernetes namespace type: string cluster_autoscaler_helm_version: title: Helm chart version type: string cluster_autoscaler_helm_values: title: Helm chart values visible: false cluster_autoscaler_helm_values_files: title: Helm chart values files visible: false # Gatekeeper gatekeeper_install: title: Install Gatekeeper description: Deploy Gatekeeper with Oracle Cloud configuration. See open-policy-agent/gatekeeper for more information. type: boolean default: false required: true gatekeeper_namespace: title: Kubernetes namespace type: string gatekeeper_helm_version: title: Helm chart version type: string gatekeeper_helm_values: title: Helm chart values visible: false gatekeeper_helm_values_files: title: Helm chart values files visible: false # Prometheus prometheus_install: title: Install Prometheus description: Deploy Prometheus with Oracle Cloud configuration. See prometheus-community/kube-prometheus-stack for more information. type: boolean default: false required: true prometheus_reapply: title: Re-apply type: boolean default: false required: true prometheus_namespace: title: Kubernetes namespace type: string prometheus_helm_version: title: Helm chart version type: string prometheus_helm_values: title: Helm chart values visible: false prometheus_helm_values_files: title: Helm chart values files visible: false # Multus multus_install: title: Install Multus description: Deploy Multus for extended CNI configuration. See k8snetworkplumbingwg/multus-cni for more information. Preempts existing CNI configuration, with the configured cluster CNI plugin used by default. type: boolean default: false required: true multus_namespace: title: Kubernetes namespace type: string multus_daemonset_url: title: Daemonset URL description: The URL path to the Multus manifest. Leave unset for tags of k8snetworkplumbingwg/multus-cni using the configured Multus version. type: string multus_version: title: Version type: string # SR-IOV Device Plugin sriov_device_plugin_install: title: Install SR-IOV device plugin description: Deploy SR-IOV device plugin to advertise Virtual Functions (VFs) to Kubernetes. See k8snetworkplumbingwg/sriov-network-device-plugin for more information. type: boolean default: false required: true sriov_device_plugin_namespace: title: Kubernetes namespace type: string sriov_device_plugin_daemonset_url: title: Daemonset URL description: The URL path to the manifest. Leave unset for tags of k8snetworkplumbingwg/sriov-network-device-plugin using the configured version. type: string sriov_device_plugin_version: title: Version type: string # SR-IOV CNI Plugin sriov_cni_plugin_install: title: Install SR-IOV CNI plugin description: Deploy SR-IOV CNI plugin to configure Virtual Functions (VFs) for Pod networking with Kubernetes. See openshift/sriov-cni for more information. type: boolean default: false required: true sriov_cni_plugin_namespace: title: Kubernetes namespace type: string sriov_cni_plugin_daemonset_url: title: Daemonset URL description: The URL path to the manifest. Leave unset for tags of openshift/sriov-cni using the configured version. type: string sriov_cni_plugin_version: title: Version type: string # Whereabouts whereabouts_install: title: Install Whereabouts description: Deploy Whereabouts for cluster-wide IP Address Management (IPAM). See k8snetworkplumbingwg/whereabouts for more information. type: boolean default: false required: true whereabouts_namespace: title: Kubernetes namespace type: string whereabouts_daemonset_url: title: Daemonset URL description: The URL path to the manifest. Leave unset for tags of k8snetworkplumbingwg/whereabouts using the configured version. type: string whereabouts_version: title: Version type: string # MPI Operator mpi_operator_install: title: Install MPI operator description: Deploy MPI operator to run MPI jobs. See kubeflow/mpi-operator for more information. type: boolean default: false required: true mpi_operator_namespace: title: Kubernetes namespace type: string mpi_operator_deployment_url: title: Deployment URL description: The URL path to the manifest. Leave unset for tags of kubeflow/mpi-operator using the configured version. type: string mpi_operator_version: title: Version type: string outputGroups: - title: Terraform outputs: - state_id - title: Identity outputs: - dynamic_group_ids - policy_statements - title: Network outputs: - vcn_id - bastion_public_ip - title: Operator outputs: - operator_id - operator_subnet_id - operator_nsg_id - operator_private_ip - operator_ssh_command - operator_ssh_secret_id - title: Cluster outputs: - cluster_id - cluster_endpoints - control_plane_subnet_id - control_plane_nsg_id - int_lb_subnet_id - int_lb_nsg_id - pub_lb_subnet_id - pub_lb_nsg_id outputs: # Terraform state_id: title: State ID type: copyableString # Identity dynamic_group_ids: title: Dynamic groups type: list policy_statements: title: Policy statements type: list # Network vcn_id: title: VCN type: ocid bastion_public_ip: title: Bastion IP type: copyableString bastion_ssh_command: title: Bastion SSH command type: copyableString bastion_ssh_secret_id: title: Bastion SSH Vault secret type: ocid # Operator operator_id: title: Operator instance type: ocid operator_subnet_id: title: Operator subnet type: ocid operator_nsg_id: title: Operator NSG type: ocid operator_private_ip: title: Operator IP type: copyableString operator_ssh_command: title: Operator SSH command type: copyableString operator_ssh_secret_id: title: Operator SSH Vault secret type: ocid # Cluster cluster_id: title: OKE cluster type: ocid cluster_endpoints: title: Cluster endpoints type: map control_plane_subnet_id: title: Control plane subnet type: ocid control_plane_nsg_id: title: Control plane NSG type: ocid int_lb_subnet_id: title: Internal load balancer subnet type: ocid int_lb_nsg_id: title: Internal load balancer NSG type: ocid pub_lb_subnet_id: title: Public load balancer subnet type: ocid pub_lb_nsg_id: title: Public load balancer NSG type: ocid ================================================ FILE: examples/rms/oke-cluster-only/variables-cluster.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # General variable "output_detail" { default = false } variable "timezone" { default = "Etc/UTC" } # Cluster variable "cluster_type" { description = "The cluster type. See Working with Enhanced Clusters and Basic Clusters for more information. NOTE: An Enhanced cluster is required for self-managed worker pools (mode != Node Pool)." type = string } variable "cluster_name" { default = null type = string } variable "cni_type" { type = string } variable "pods_cidr" { default = "10.244.0.0/16" type = string } variable "services_cidr" { default = "10.96.0.0/16" type = string } variable "kubernetes_version" { default = "v1.32.1" } variable "cluster_kms_vault_id" { default = null type = string } variable "cluster_kms_key_id" { default = "" type = string } variable "use_signed_images" { default = false type = bool } variable "image_signing_keys" { default = [] type = set(string) } variable "load_balancers" { default = "Public" type = string } variable "preferred_load_balancer" { default = "Public" type = string } variable "cluster_tags" { default = {} type = map(any) } # Oracle Container Image Registry (OCIR) variable "ocir_email_address" { default = null type = string } variable "ocir_secret_name" { default = "ocirsecret" } variable "ocir_secret_namespace" { default = "default" } variable "ocir_username" { default = null type = string } variable "ocir_kms_vault_id" { default = null type = string } variable "ocir_kms_secret_id" { default = null type = string } ================================================ FILE: examples/rms/oke-cluster-only/variables-extensions.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # CNI: Multus variable "multus_install" { default = false } variable "multus_namespace" { default = "network" } variable "multus_daemonset_url" { default = null type = string } variable "multus_version" { default = "master" } # Metrics server variable "metrics_server_install" { default = false } variable "metrics_server_namespace" { default = "metrics" } variable "metrics_server_helm_version" { default = "3.8.3" } variable "metrics_server_helm_values" { default = {} type = map(string) } variable "metrics_server_helm_values_files" { default = [] type = list(string) } # Cluster autoscaler variable "cluster_autoscaler_install" { default = false } variable "cluster_autoscaler_namespace" { default = "kube-system" } variable "cluster_autoscaler_helm_version" { default = "9.24.0" } variable "cluster_autoscaler_helm_values" { default = {} type = map(string) } variable "cluster_autoscaler_helm_values_files" { default = [] type = list(string) } # Prometheus variable "prometheus_install" { default = false } variable "prometheus_reapply" { default = false } variable "prometheus_namespace" { default = "metrics" } variable "prometheus_helm_version" { default = "45.2.0" } variable "prometheus_helm_values" { default = {} type = map(string) } variable "prometheus_helm_values_files" { default = [] type = list(string) } # DCGM exporter variable "dcgm_exporter_install" { default = false } variable "dcgm_exporter_reapply" { default = false } variable "dcgm_exporter_namespace" { default = "metrics" } variable "dcgm_exporter_helm_version" { default = "3.1.5" } # SR-IOV device plugin variable "sriov_device_plugin_install" { default = false } variable "sriov_device_plugin_install_config" { default = false } variable "sriov_device_plugin_namespace" { default = "network" } variable "sriov_device_plugin_daemonset_url" { default = null type = string } variable "sriov_device_plugin_version" { default = "master" } # SR-IOV CNI plugin variable "sriov_cni_plugin_install" { default = false } variable "sriov_cni_plugin_namespace" { default = "network" } variable "sriov_cni_plugin_daemonset_url" { default = null type = string } variable "sriov_cni_plugin_version" { default = "master" } # RDMA CNI plugin variable "rdma_cni_plugin_install" { default = false } variable "rdma_cni_plugin_namespace" { default = "network" } variable "rdma_cni_plugin_daemonset_url" { default = null type = string } variable "rdma_cni_plugin_version" { default = "master" } # MPI operator variable "mpi_operator_install" { default = false } variable "mpi_operator_namespace" { default = "default" } variable "mpi_operator_deployment_url" { default = null type = string } variable "mpi_operator_version" { default = "0.4.0" } # Whereabouts variable "whereabouts_install" { default = false } variable "whereabouts_namespace" { default = "network" } variable "whereabouts_daemonset_url" { default = null type = string } variable "whereabouts_version" { default = "master" } # Gatekeeper variable "gatekeeper_install" { default = false } variable "gatekeeper_namespace" { default = "kube-system" } variable "gatekeeper_helm_version" { default = "3.11.0" } variable "gatekeeper_helm_values" { default = {} type = map(string) } variable "gatekeeper_helm_values_files" { default = [] type = list(string) } ================================================ FILE: examples/rms/oke-cluster-only/variables-iam.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "tenancy_ocid" { default = null type = string } variable "current_user_ocid" { default = null type = string } variable "compartment_ocid" { default = null type = string } variable "region" { default = null type = string } variable "create_iam_autoscaler_policy" { default = "Auto" type = string } variable "create_iam_kms_policy" { default = "Auto" type = string } variable "create_iam_operator_policy" { default = "Auto" type = string } variable "create_iam_worker_policy" { default = "Auto" type = string } variable "create_iam_resources" { default = false } variable "create_iam_tag_namespace" { default = false } variable "create_iam_defined_tags" { default = false } variable "use_defined_tags" { default = false description = "Add existing tags in the configured namespace to created resources when applicable." type = bool } variable "tag_namespace" { default = "oke" description = "Tag namespace containing standard tags for resources created by the module: [state_id, role, pool, cluster_autoscaler]." type = string } variable "freeform_tags" { default = { cluster = {} persistent_volume = {} service_lb = {} workers = {} bastion = {} operator = {} vcn = {} } type = any } variable "defined_tags" { default = { cluster = {} persistent_volume = {} service_lb = {} workers = {} bastion = {} operator = {} vcn = {} } type = any } ================================================ FILE: examples/rms/oke-cluster-only/variables-network.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "vcn_id" { type = string } variable "ig_route_table_id" { default = null type = string } variable "service_gateway_id" { default = null type = string } variable "nat_gateway_id" { default = null type = string } variable "assign_dns" { default = true } variable "control_plane_is_public" { default = false } variable "control_plane_nsg_id" { default = "" } variable "operator_nsg_id" { default = "" } variable "control_plane_subnet_id" { type = string } variable "int_lb_subnet_id" { type = string default = null } variable "operator_subnet_id" { type = string default = null } variable "pub_lb_subnet_id" { type = string default = null } variable "bastion_public_ip" { default = null type = string } ================================================ FILE: examples/rms/oke-cluster-only/variables-operator.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_operator" { default = true } variable "operator_install_helm" { default = true } variable "operator_install_k9s" { default = false } variable "operator_install_kubectx" { default = true } variable "operator_pv_transit_encryption" { default = false } variable "operator_upgrade" { default = false } variable "operator_availability_domain" { default = null type = string } variable "operator_cloud_init" { default = [] type = list(map(string)) } variable "operator_user" { default = "opc" } variable "operator_image_id" { default = null type = string } variable "operator_image_os" { default = "Oracle Linux" } variable "operator_image_os_version" { default = "8" } variable "operator_image_type" { default = "Platform" type = string validation { condition = contains(["custom", "platform"], lower(var.operator_image_type)) error_message = "Accepted values are custom or platform" } } variable "operator_shape" { default = { shape = "VM.Standard.E4.Flex", ocpus = 1, memory = 4, boot_volume_size = 50 } type = map(any) } variable "operator_volume_kms_vault_id" { default = null type = string } variable "operator_volume_kms_key_id" { default = null type = string } variable "operator_private_ip" { default = null type = string } variable "operator_tags" { default = {} type = map(any) } ================================================ FILE: examples/rms/oke-cluster-only/versions.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { experiments = [module_variable_optional_attrs] required_version = ">= 1.2.0" required_providers { oci = { configuration_aliases = [oci.home] source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/rms/oke-network-only/data.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "ssh_public_key" { default = null type = string } variable "ssh_kms_vault_id" { default = null type = string } variable "ssh_kms_secret_id" { default = null type = string } data "oci_identity_region_subscriptions" "home" { tenancy_id = var.tenancy_ocid filter { name = "is_home_region" values = [true] } } data "oci_secrets_secretbundle" "ssh_key" { secret_id = var.ssh_kms_secret_id } locals { ssh_public_key = try(base64decode(var.ssh_public_key), var.ssh_public_key) ssh_key_bundle = sensitive(one(data.oci_secrets_secretbundle.ssh_key.secret_bundle_content)) ssh_key_bundle_content = sensitive(lookup(local.ssh_key_bundle, "content", null)) } ================================================ FILE: examples/rms/oke-network-only/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl module "oke" { source = "github.com/oracle-terraform-modules/terraform-oci-oke.git?ref=5.x&depth=1" providers = { oci.home = oci.home } tenancy_id = var.tenancy_ocid compartment_id = var.compartment_ocid # Identity create_iam_resources = true create_iam_tag_namespace = var.create_iam_tag_namespace create_iam_defined_tags = var.create_iam_tag_namespace || var.create_iam_defined_tags use_defined_tags = var.use_defined_tags tag_namespace = var.tag_namespace # Network create_vcn = var.create_vcn vcn_id = var.vcn_id vcn_cidrs = split(",", var.vcn_cidrs) vcn_create_internet_gateway = var.vcn_create_internet_gateway ? "always" : "never" vcn_create_nat_gateway = var.vcn_create_nat_gateway ? "always" : "never" vcn_create_service_gateway = var.vcn_create_service_gateway ? "always" : "never" vcn_name = var.vcn_name vcn_dns_label = var.vcn_dns_label assign_dns = var.assign_dns ig_route_table_id = var.ig_route_table_id local_peering_gateways = var.local_peering_gateways lockdown_default_seclist = var.lockdown_default_seclist create_drg = var.create_drg drg_id = var.drg_id drg_display_name = var.drg_display_name subnets = { bastion = { create = var.bastion_subnet_create ? "always" : "never", newbits = var.bastion_subnet_newbits, id = var.bastion_subnet_id } operator = { create = var.operator_subnet_create ? "always" : "never", newbits = var.operator_subnet_newbits, id = var.operator_subnet_id } cp = { create = var.control_plane_subnet_create ? "always" : "never", newbits = var.control_plane_subnet_newbits, id = var.control_plane_subnet_id } int_lb = { create = var.int_lb_subnet_create ? "always" : "never", newbits = var.int_lb_subnet_newbits, id = var.int_lb_subnet_id } pub_lb = { create = var.pub_lb_subnet_create ? "always" : "never", newbits = var.pub_lb_subnet_newbits, id = var.pub_lb_subnet_id } workers = { create = var.worker_subnet_create ? "always" : "never", newbits = var.worker_subnet_newbits, id = var.worker_subnet_id } pods = { create = var.pod_subnet_create ? "always" : "never", newbits = var.pod_subnet_newbits, id = var.pod_subnet_id } } # Network Security nsgs = { bastion = { create = var.create_nsgs ? "always" : "never" } operator = { create = var.create_nsgs ? "always" : "never" } cp = { create = var.create_nsgs ? "always" : "never" } int_lb = { create = var.create_nsgs ? "always" : "never" } pub_lb = { create = var.create_nsgs ? "always" : "never" } workers = { create = var.create_nsgs ? "always" : "never" } pods = { create = var.create_nsgs ? "always" : "never" } } allow_node_port_access = var.allow_node_port_access allow_pod_internet_access = var.allow_pod_internet_access allow_rules_internal_lb = var.allow_rules_internal_lb allow_rules_public_lb = var.allow_rules_public_lb allow_worker_internet_access = var.allow_worker_internet_access allow_worker_ssh_access = var.allow_worker_ssh_access enable_waf = var.enable_waf bastion_allowed_cidrs = compact(split(",", var.bastion_allowed_cidrs)) control_plane_allowed_cidrs = compact(split(",", var.control_plane_allowed_cidrs)) control_plane_is_public = var.control_plane_is_public load_balancers = lower(var.load_balancers) worker_is_public = var.worker_is_public # Bastion bastion_availability_domain = var.bastion_availability_domain bastion_image_id = var.bastion_image_id bastion_image_os = var.bastion_image_os bastion_image_os_version = var.bastion_image_os_version bastion_image_type = lower(var.bastion_image_type) bastion_is_public = var.bastion_is_public bastion_shape = var.bastion_shape bastion_upgrade = var.bastion_upgrade bastion_user = var.bastion_user create_bastion = var.create_bastion # SSH ssh_public_key = local.ssh_public_key ssh_private_key = sensitive(local.ssh_key_bundle_content) # Cluster create_cluster = false preferred_load_balancer = lower(var.preferred_load_balancer) create_operator = false freeform_tags = { # TODO Remaining tags in schema bastion = lookup(var.bastion_tags, "freeformTags", {}) vcn = {} } defined_tags = { # TODO Remaining tags in schema bastion = lookup(var.bastion_tags, "definedTags", {}) vcn = {} } } ================================================ FILE: examples/rms/oke-network-only/output.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Terraform output "state_id" { value = module.oke.state_id } # Network output "vcn_id" { value = module.oke.vcn_id } output "drg_id" { value = module.oke.drg_id } output "ig_route_table_id" { value = module.oke.ig_route_table_id } output "nat_route_table_id" { value = module.oke.nat_route_table_id } # Bastion output "bastion_id" { value = module.oke.bastion_id } output "bastion_public_ip" { value = module.oke.bastion_public_ip } output "bastion_subnet_id" { value = module.oke.bastion_subnet_id } output "bastion_subnet_cidr" { value = module.oke.bastion_subnet_cidr } output "bastion_nsg_id" { value = module.oke.bastion_nsg_id } output "bastion_ssh_command" { value = module.oke.ssh_to_bastion } output "bastion_ssh_secret_id" { value = var.ssh_kms_secret_id } # Operator output "operator_subnet_id" { value = module.oke.operator_subnet_id } output "operator_subnet_cidr" { value = module.oke.operator_subnet_cidr } output "operator_nsg_id" { value = module.oke.operator_nsg_id } # Cluster output "control_plane_subnet_id" { value = module.oke.control_plane_subnet_id } output "control_plane_subnet_cidr" { value = module.oke.control_plane_subnet_cidr } output "control_plane_nsg_id" { value = module.oke.control_plane_nsg_id } output "int_lb_subnet_id" { value = module.oke.int_lb_subnet_id } output "pub_lb_subnet_id" { value = module.oke.pub_lb_subnet_id } output "int_lb_nsg_id" { value = module.oke.int_lb_nsg_id } output "int_lb_subnet_cidr" { value = module.oke.int_lb_subnet_cidr } output "pub_lb_nsg_id" { value = module.oke.pub_lb_nsg_id } output "pub_lb_subnet_cidr" { value = module.oke.pub_lb_subnet_cidr } # Workers output "worker_subnet_id" { value = module.oke.worker_subnet_id } output "worker_subnet_cidr" { value = module.oke.worker_subnet_cidr } output "worker_nsg_id" { value = module.oke.worker_nsg_id } output "pod_subnet_id" { value = module.oke.pod_subnet_id } output "pod_subnet_cidr" { value = module.oke.pod_subnet_cidr } output "pod_nsg_id" { value = module.oke.pod_nsg_id } ================================================ FILE: examples/rms/oke-network-only/schema.yaml ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl title: "OKE: Network Only" description: Common Virtual Cloud Network resources for OKE cluster installations. schemaVersion: 1.1.0 version: "20230304" locale: "en" variableGroups: - title: "Hidden" visible: false variables: - api_fingerprint - current_user_ocid - tenancy_ocid - region - subnets - bastion_public_ip - title: "Identity" variables: - compartment_ocid - create_iam_tag_namespace - create_iam_defined_tags - use_defined_tags - tag_namespace - title: "VCN" variables: - ${create_vcn} - assign_dns - vcn_id - vcn_name - vcn_cidrs - vcn_dns_label - ig_route_table_id - vcn_create_internet_gateway - vcn_create_nat_gateway - vcn_create_service_gateway - nat_gateway_id - nat_route_table_id - nat_gateway_public_ip_id - service_gateway_id - create_drg - drg_id - local_peering_gateways - title: "Security" variables: - ${create_nsgs} - lockdown_default_seclist - enable_waf - title: "Bastion" variables: - bastion_is_public - bastion_subnet_create - bastion_subnet_newbits - bastion_subnet_id - bastion_allowed_cidrs - ${create_bastion} - bastion_upgrade - bastion_availability_domain - bastion_image_os - bastion_image_os_version - bastion_image_id - bastion_user - bastion_shape_name - bastion_shape_ocpus - bastion_shape_memory - bastion_shape_boot - bastion_shape - bastion_tags - title: "SSH" visible: ${create_bastion} variables: - ssh_public_key - ssh_kms_vault_id - ssh_kms_secret_id - title: "Operator" variables: - operator_subnet_create - operator_subnet_newbits - operator_subnet_id - title: "Control Plane" variables: - control_plane_is_public - control_plane_allowed_cidrs - control_plane_subnet_create - control_plane_subnet_newbits - control_plane_subnet_id - title: "Load Balancers" variables: - load_balancers - preferred_load_balancer - allow_rules_internal_lb - allow_rules_public_lb - int_lb_subnet_create - int_lb_subnet_newbits - int_lb_subnet_id - pub_lb_subnet_create - pub_lb_subnet_newbits - pub_lb_subnet_id - title: "Workers" variables: - worker_is_public - worker_subnet_create - worker_subnet_newbits - worker_subnet_id - allow_worker_ssh_access - allow_worker_internet_access - allow_node_port_access - pod_subnet_create - pod_subnet_newbits - pod_subnet_id - allow_pod_internet_access variables: # Identity api_fingerprint: required: false visible: false current_user_ocid: title: User type: ocid required: true tenancy_ocid: title: Tenancy type: oci:identity:compartment:id required: true compartment_ocid: title: Compartment description: The default compartment for created resources. type: oci:identity:compartment:id required: true region: required: true title: Region type: oci:identity:region:name defined_tags: visible: false freeform_tags: visible: false create_iam_tag_namespace: default: false required: true title: Create tag namespace type: boolean create_iam_defined_tags: default: false required: true title: Create defined tags type: boolean use_defined_tags: title: Use defined tags default: false type: boolean tag_namespace: title: Tag namespace visible: or: - ${create_iam_tag_namespace} - ${create_iam_defined_tags} - ${use_defined_tags} # VCN create_vcn: title: Create VCN type: boolean default: true vcn_id: title: Existing VCN type: oci:core:vcn:id required: true dependsOn: compartmentId: ${compartment_ocid} visible: { not: [create_vcn] } vcn_name: title: Virtual Cloud Network (VCN) name description: Display name for the created VCN. Defaults to 'oke' suffixed with the generated Terraform 'state_id' value. type: string visible: ${create_vcn} assign_dns: title: Assign DNS records type: boolean required: true visible: ${create_vcn} vcn_dns_label: title: DNS label description: DNS label for the created VCN. Defaults to the generated Terraform 'state_id' value. visible: and: - ${create_vcn} - ${assign_dns} vcn_cidrs: title: CIDR ranges description: Comma-separated list of CIDR blocks for the created VCN. type: string required: true visible: ${create_vcn} lockdown_default_seclist: title: Secure default security list type: boolean required: true visible: ${create_vcn} vcn_create_internet_gateway: title: Create internet gateway type: boolean required: true visible: ${create_vcn} vcn_create_nat_gateway: title: Create service gateway type: boolean required: true visible: ${create_vcn} vcn_create_service_gateway: title: Create NAT gateway type: boolean required: true visible: ${create_vcn} local_peering_gateways: title: Local peering gateways visible: false ig_route_table_id: title: Internet Gateway Route Table type: ocid required: false visible: not: - ${create_vcn} service_gateway_id: title: Service gateway description: Existing service gateway in the VCN for access to Oracle Cloud services. type: oci:core:servicegateway:id required: true visible: not: - ${create_vcn} dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} nat_gateway_id: title: NAT gateway description: Existing NAT gateway in the VCN for private network egress. type: oci:core:natgateway:id required: true visible: not: - ${create_vcn} dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} # NSGs create_nsgs: title: Create NSGs description: Create standard network security groups and traffic rules. Additional configuration may be required when disabled, e.g. providing existing NSGs, subnet security lists, etc. type: boolean default: true required: true bastion_allowed_cidrs: title: Comma-separated list of CIDR blocks allowed SSH access to the bastion host. type: string required: false visible: ${create_nsgs} control_plane_allowed_cidrs: title: Comma-separated list of CIDR blocks allowed access to the OKE control plane. type: string required: false visible: ${create_nsgs} control_plane_is_public: title: Public Kubernetes control plane type: boolean default: false required: true visible: ${create_nsgs} preferred_load_balancer: title: Preferred load balancer type: enum default: Internal enum: [Internal, Public] allowMultiple: false required: true worker_is_public: title: Public Kubernetes worker nodes type: boolean default: false required: true visible: ${create_nsgs} allow_node_port_access: title: Allow NodePort access type: boolean default: false required: true visible: ${create_nsgs} allow_worker_ssh_access: title: Allow worker SSH access type: boolean default: true required: true visible: ${create_nsgs} allow_worker_internet_access: title: Allow worker egress to internet type: boolean default: true required: true visible: ${create_nsgs} allow_pod_internet_access: title: Allow pod egress to internet type: boolean required: false default: true visible: ${create_nsgs} allow_rules_internal_lb: title: Additional internal load balancer rules additionalProps: allowMultiple: true required: false visible: ${create_nsgs} allow_rules_public_lb: title: Additional public load balancer rules additionalProps: allowMultiple: true required: false visible: ${create_nsgs} create_drg: title: Create DRG description: Create a Dynamic Routing Gateway and attach it to the VCN. type: boolean default: false required: false visible: false drg_id: title: Dynamic Routing Gateway description: Existing Dynamic Routing Gateway to attach to the VCN. type: ocid required: false visible: { not: [create_drg] } drg_display_name: title: DRG display name visible: ${create_drg} enable_waf: title: Enable WAF type: boolean default: false required: false visible: false load_balancers: title: Load balancers type: enum enum: [Public, Internal, Both] default: Internal required: true visible: ${create_nsgs} bastion_is_public: title: Public network description: Public network and address for bastion host. type: boolean default: true required: true visible: or: - ${bastion_subnet_create} - ${create_bastion} bastion_subnet_create: title: Create bastion subnet type: boolean default: true required: true bastion_subnet_newbits: title: Bastion subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${bastion_subnet_create} bastion_subnet_id: title: Bastion subnet type: oci:core:subnet:id required: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: { not: [bastion_is_public] } hidePrivateSubnet: ${bastion_is_public} hideRegionalSubnet: false hideAdSubnet: false visible: not: - or: - ${bastion_subnet_create} - not: - ${create_bastion} create_bastion: title: Create bastion instance type: boolean default: true required: true bastion_availability_domain: title: Availability domain type: oci:identity:availabilitydomain:name required: false dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} bastion_user: title: User type: string default: opc required: true visible: ${create_bastion} bastion_shape_name: title: Shape type: oci:core:instanceshape:name default: "VM.Standard.E4.Flex" required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} bastion_shape_ocpus: title: OCPUs (Cores) type: number default: 4 required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} bastion_shape_memory: title: Memory (GB) type: number default: 16 required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} bastion_shape_boot: title: Boot volume size (GB) type: number default: 50 required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} bastion_shape: visible: false bastion_image_os: title: Image OS description: Image operating system. type: enum default: "Oracle Autonomous Linux" enum: ["Oracle Autonomous Linux", "Oracle Linux"] required: true visible: and: - ${create_bastion} - not: - eq: - ${bastion_image_type} - custom bastion_image_os_version: title: Image OS version description: Image operating system version. type: enum default: "8.7" enum: ["7.9", "8.7"] required: true visible: and: - ${create_bastion} - not: - eq: - ${bastion_image_type} - custom bastion_image_type: title: Image type type: enum default: platform enum: [platform, custom] required: true visible: false bastion_image_id: title: Image ID type: oci:core:image:id required: false dependsOn: compartmentId: ${compartment_ocid} operatingSystem: ${bastion_image_os} operatingSystemVersion: ${bastion_image_os_version} shape: ${bastion_shape_name} visible: ${create_bastion} bastion_upgrade: title: Upgrade OS description: Upgrade the operating system on boot. type: boolean default: false required: true visible: ${create_bastion} bastion_tags: type: oci:identity:tag:value required: false title: Tagging description: Tag values for created resources. dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} # SSH ssh_public_key: title: SSH Public Key type: oci:core:ssh:publickey pattern: "((^(ssh-rsa AAAAB3NzaC1yc2|ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNT|ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzOD|ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1Mj|ssh-ed25519 AAAAC3NzaC1lZDI1NTE5|ssh-dss AAAAB3NzaC1kc3)[0-9A-Za-z+\/]+[=]{0,3})( [^,]*)?)(,((ssh-rsa AAAAB3NzaC1yc2|ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNT|ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzOD|ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1Mj|ssh-ed25519 AAAAC3NzaC1lZDI1NTE5|ssh-dss AAAAB3NzaC1kc3)[0-9A-Za-z+\/]+[=]{0,3})( [^,]*)?)*$" required: false ssh_kms_vault_id: title: SSH Vault description: The OCI Vault used to encrypt the SSH key pair. type: oci:kms:vault:id required: true dependsOn: compartmentId: ${compartment_ocid} visible: ${create_bastion} ssh_kms_secret_id: title: SSH Vault secret description: The OCI Vault secret containing the SSH private key. type: oci:kms:secret:id required: true dependsOn: compartmentId: ${compartment_ocid} vaultId: ${ssh_kms_vault_id} visible: ${create_bastion} operator_subnet_create: title: Create operator subnet type: boolean required: true operator_subnet_newbits: title: Operator subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${operator_subnet_create} operator_subnet_id: title: Operator subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: true hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: false visible: not: ${operator_subnet_create} control_plane_subnet_create: title: Create control plane subnet type: boolean required: true control_plane_subnet_newbits: title: Control plane subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${control_plane_subnet_create} control_plane_subnet_id: title: Control plane subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: { not: [control_plane_is_public] } hidePrivateSubnet: ${control_plane_is_public} hideRegionalSubnet: false hideAdSubnet: true visible: not: ${control_plane_subnet_create} int_lb_subnet_create: title: Create internal load balancer subnet type: boolean required: true int_lb_subnet_newbits: title: Internal load balancer subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${int_lb_subnet_create} int_lb_subnet_id: title: Internal load balancer subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: true hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: true visible: not: ${int_lb_subnet_create} pub_lb_subnet_create: title: Create public load balancer subnet type: boolean required: true pub_lb_subnet_newbits: title: Public load balancer subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${pub_lb_subnet_create} pub_lb_subnet_id: title: Public load balancer subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: true hideRegionalSubnet: false hideAdSubnet: true visible: not: ${pub_lb_subnet_create} worker_subnet_create: title: Create worker subnet type: boolean required: true worker_subnet_newbits: title: Worker subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${worker_subnet_create} worker_subnet_id: title: Worker subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: { not: [worker_is_public] } hidePrivateSubnet: ${worker_is_public} hideRegionalSubnet: false hideAdSubnet: false visible: not: ${worker_subnet_create} pod_subnet_create: title: Create pod subnet type: boolean required: true pod_subnet_newbits: title: Pod subnet size description: The number of network prefix bits added to the VCN CIDR block for the created subnet address range (i.e. a higher value is smaller). Refer to the Terraform cidrsubnets function for more information. type: number visible: ${pod_subnet_create} pod_subnet_id: title: Pod subnet type: oci:core:subnet:id required: false dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: true hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: true visible: not: ${pod_subnet_create} outputGroups: - title: Terraform outputs: - state_id - title: Network outputs: - vcn_id - ig_route_table_id - nat_route_table_id - drg_id - title: Bastion outputs: - bastion_id - bastion_subnet_id - bastion_subnet_cidr - bastion_public_ip - bastion_ssh_command - bastion_ssh_secret_id - title: Operator outputs: - operator_subnet_id - operator_subnet_cidr - title: Cluster outputs: - control_plane_subnet_id - control_plane_subnet_cidr - int_lb_subnet_id - int_lb_subnet_cidr - pub_lb_subnet_id - pub_lb_subnet_cidr - title: Workers outputs: - worker_subnet_id - worker_subnet_cidr - pod_subnet_id - pod_subnet_cidr outputs: # Terraform state_id: title: State ID type: copyableString # Network vcn_id: title: VCN type: ocid drg_id: title: Dynamic routing gateway type: ocid ig_route_table_id: title: Internet gateway route table type: ocid nat_route_table_id: title: NAT gateway route table type: ocid subnet_cidrs: visible: false subnet_ids: visible: false network_security_rules: visible: false # Bastion bastion_id: title: Bastion instance type: ocid bastion_subnet_id: title: Bastion subnet type: ocid bastion_subnet_cidr: title: Bastion CIDR type: string bastion_public_ip: title: Bastion IP type: copyableString bastion_ssh_command: title: Bastion SSH command type: copyableString bastion_ssh_secret_id: title: Bastion SSH Vault secret type: ocid # Operator operator_subnet_id: title: Operator subnet type: ocid operator_subnet_cidr: title: Operator CIDR type: string # Cluster control_plane_subnet_id: title: Control plane subnet type: ocid control_plane_subnet_cidr: title: Control plane CIDR type: string int_lb_subnet_id: title: Internal load balancer subnet type: ocid int_lb_subnet_cidr: title: Internal load balancer CIDR type: string pub_lb_subnet_id: title: Public load balancer subnet type: ocid pub_lb_subnet_cidr: title: Public load balancer CIDR type: string # Workers worker_subnet_id: title: Worker subnet type: ocid worker_subnet_cidr: title: Worker CIDR type: string pod_subnet_id: title: Pod subnet type: ocid pod_subnet_cidr: title: Pod CIDR type: string ================================================ FILE: examples/rms/oke-network-only/variables-bastion.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_bastion" { default = true } variable "bastion_is_public" { default = true } variable "bastion_upgrade" { default = false } variable "bastion_allowed_cidrs" { default = "0.0.0.0/0" type = string } variable "bastion_availability_domain" { default = null type = string } variable "bastion_user" { default = "opc" type = string } variable "bastion_image_id" { default = null type = string } variable "bastion_image_type" { default = "platform" type = string validation { condition = contains(["custom", "platform"], lower(var.bastion_image_type)) error_message = "Accepted values are custom or platform" } } variable "bastion_image_os" { default = "Oracle Autonomous Linux" type = string } variable "bastion_image_os_version" { default = "8.7" type = string } variable "bastion_shape" { default = { shape = "VM.Standard.E4.Flex", ocpus = 1, memory = 4, boot_volume_size = 50 } type = map(any) } variable "bastion_tags" { default = {} type = map(any) } ================================================ FILE: examples/rms/oke-network-only/variables-iam.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "tenancy_ocid" { default = null type = string } variable "current_user_ocid" { default = null type = string } variable "compartment_ocid" { default = null type = string } variable "region" { default = null type = string } variable "api_fingerprint" { default = null type = string } variable "create_iam_tag_namespace" { default = false } variable "create_iam_defined_tags" { default = false } variable "use_defined_tags" { default = false description = "Add existing tags in the configured namespace to created resources when applicable." type = bool } variable "tag_namespace" { default = "oke" description = "Tag namespace containing standard tags for resources created by the module: [state_id, role, pool, cluster_autoscaler]." type = string } variable "freeform_tags" { default = { cluster = {} persistent_volume = {} service_lb = {} workers = {} bastion = {} operator = {} vcn = {} } type = any } variable "defined_tags" { default = { cluster = {} persistent_volume = {} service_lb = {} workers = {} bastion = {} operator = {} vcn = {} } type = any } ================================================ FILE: examples/rms/oke-network-only/variables-network.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_vcn" { default = true } variable "assign_dns" { default = true } variable "control_plane_is_public" { default = false } variable "enable_waf" { default = false } variable "worker_is_public" { default = false } variable "vcn_name" { default = null type = string } variable "vcn_id" { default = null type = string } variable "vcn_create_nat_gateway" { default = true type = bool } variable "vcn_create_internet_gateway" { default = true type = bool } variable "vcn_create_service_gateway" { default = true type = bool } variable "ig_route_table_id" { default = null type = string } variable "create_drg" { default = false } variable "drg_display_name" { default = null type = string } variable "drg_id" { default = null type = string } variable "internet_gateway_route_rules" { default = null type = list(map(string)) } variable "local_peering_gateways" { default = null type = map(any) } variable "lockdown_default_seclist" { default = true } variable "nat_gateway_route_rules" { default = null type = list(map(string)) } variable "vcn_cidrs" { default = "10.0.0.0/16" type = string } variable "vcn_dns_label" { default = null type = string } variable "load_balancers" { default = "Internal" type = string } variable "preferred_load_balancer" { default = "Internal" type = string } variable "create_nsgs" { default = true } variable "allow_node_port_access" { default = false } variable "allow_pod_internet_access" { default = true } variable "allow_worker_internet_access" { default = true } variable "allow_worker_ssh_access" { default = false } variable "allow_rules_internal_lb" { default = {} type = any } variable "allow_rules_public_lb" { default = {} type = any } variable "control_plane_allowed_cidrs" { default = "" type = string } ================================================ FILE: examples/rms/oke-network-only/variables-subnets.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "bastion_subnet_create" { default = true } variable "control_plane_subnet_create" { default = true } variable "int_lb_subnet_create" { default = true } variable "operator_subnet_create" { default = true } variable "pod_subnet_create" { default = true } variable "pub_lb_subnet_create" { default = true } variable "worker_subnet_create" { default = true } variable "bastion_subnet_newbits" { default = 13 } variable "control_plane_subnet_newbits" { default = 13 } variable "int_lb_subnet_newbits" { default = 11 } variable "operator_subnet_newbits" { default = 13 } variable "pod_subnet_newbits" { default = 2 } variable "pub_lb_subnet_newbits" { default = 11 } variable "worker_subnet_newbits" { default = 2 } variable "bastion_subnet_id" { type = string default = null } variable "control_plane_subnet_id" { type = string default = null } variable "int_lb_subnet_id" { type = string default = null } variable "operator_subnet_id" { type = string default = null } variable "pod_subnet_id" { type = string default = null } variable "pub_lb_subnet_id" { type = string default = null } variable "worker_subnet_id" { type = string default = null } ================================================ FILE: examples/rms/oke-network-only/versions.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { experiments = [module_variable_optional_attrs] required_version = ">= 1.2.0" required_providers { oci = { configuration_aliases = [oci.home] source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/rms/oke-workers-only/data.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "ssh_public_key" { default = null type = string } variable "ssh_kms_vault_id" { default = null type = string } variable "ssh_kms_secret_id" { default = null type = string } data "oci_identity_region_subscriptions" "home" { tenancy_id = var.tenancy_ocid filter { name = "is_home_region" values = [true] } } data "oci_secrets_secretbundle" "ssh_key" { secret_id = var.ssh_kms_secret_id } locals { ssh_public_key = try(base64decode(var.ssh_public_key), var.ssh_public_key) ssh_key_bundle = sensitive(one(data.oci_secrets_secretbundle.ssh_key.secret_bundle_content)) ssh_key_bundle_content = sensitive(lookup(local.ssh_key_bundle, "content", null)) } ================================================ FILE: examples/rms/oke-workers-only/main.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { worker_image_id = coalesce(var.worker_image_custom_id, var.worker_image_platform_id, "none") worker_image_type = contains(["platform", "custom"], lower(var.worker_image_type)) ? "custom" : "oke" worker_cloud_init = var.worker_cloud_init_configure ? [{ content_type = "text/x-shellscript", content = var.worker_pool_mode == "Node Pool" ? var.worker_cloud_init_oke : var.worker_cloud_init_byon }] : [] } module "oke" { source = "github.com/oracle-terraform-modules/terraform-oci-oke.git?ref=5.x&depth=1" providers = { oci.home = oci.home } # Identity tenancy_id = var.tenancy_ocid compartment_id = var.compartment_ocid create_iam_resources = true create_iam_autoscaler_policy = var.create_iam_autoscaler_policy ? "always" : "never" create_iam_worker_policy = var.create_iam_worker_policy ? "always" : "never" create_bastion = false create_operator = false create_cluster = false # Network create_vcn = false vcn_id = var.vcn_id assign_dns = var.assign_dns worker_nsg_ids = compact([var.worker_nsg_id]) pod_nsg_ids = compact([var.pod_nsg_id]) subnets = { workers = { create = "never", id = var.worker_subnet_id } pods = { create = "never", id = var.pod_subnet_id } } nsgs = { workers = { create = "never", id = var.worker_nsg_id } pods = { create = "never", id = var.pod_nsg_id } } # Cluster cluster_id = var.cluster_id cni_type = lower(var.cni_type) control_plane_is_public = false # workers only need private # Workers ssh_public_key = local.ssh_public_key worker_pool_size = var.worker_pool_size worker_pool_mode = lookup({ "Node Pool" = "node-pool" "Instances" = "instances" "Instance Pool" = "instance-pool", "Cluster Network" = "cluster-network", }, var.worker_pool_mode, "node-pool") worker_image_type = lower(local.worker_image_type) worker_image_id = local.worker_image_id worker_image_os = var.worker_image_os worker_image_os_version = var.worker_image_os_version worker_cloud_init = local.worker_cloud_init worker_shape = { shape = var.worker_shape ocpus = var.worker_ocpus memory = var.worker_memory boot_volume_size = var.worker_boot_volume_size } worker_pools = { format("%v", var.worker_pool_name) = { description = lookup({ "Node Pool" = "OKE-managed Node Pool" "Instances" = "Self-managed Instances" "Instance Pool" = "Self-managed Instance Pool" "Cluster Network" = "Self-managed Cluster Network" }, var.worker_pool_mode, "") } } freeform_tags = { workers = lookup(var.worker_tags, "freeformTags", {}) } defined_tags = { workers = lookup(var.worker_tags, "definedTags", {}) } } ================================================ FILE: examples/rms/oke-workers-only/output.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Terraform output "state_id" { value = module.oke.state_id } # Network output "worker_subnet_id" { value = var.worker_subnet_id } output "worker_nsg_id" { value = var.worker_nsg_id } # Identity output "dynamic_group_ids" { value = module.oke.dynamic_group_ids } output "policy_statements" { value = module.oke.policy_statements } output "create_iam_autoscaler_policy" { value = var.create_iam_autoscaler_policy } output "create_iam_worker_policy" { value = var.create_iam_worker_policy } # Cluster output "cluster_id" { value = var.cluster_id } output "apiserver_private_host" { value = module.oke.apiserver_private_host } # Workers output "worker_pool_name" { value = var.worker_pool_name } output "worker_pool_mode" { value = var.worker_pool_mode } output "worker_shape" { value = var.worker_shape } output "worker_pool_size" { value = var.worker_pool_size } output "worker_image_id" { value = local.worker_image_id } output "autoscale" { value = var.autoscale } output "worker_pool_ids" { value = concat( values(coalesce(module.oke.worker_pool_ids, {})), values(coalesce(module.oke.worker_instance_ids, {})), ) } ================================================ FILE: examples/rms/oke-workers-only/schema.yaml ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl title: "OKE: Worker Pool" description: Kubernetes worker nodes for an OKE cluster schemaVersion: 1.1.0 version: "20230304" locale: "en" variableGroups: - title: "Hidden" visible: false variables: - api_fingerprint - config_file_profile - current_user_ocid - user_id - tenancy_id - tenancy_ocid - region - kubernetes_version - apiserver_private_host - cluster_ca_cert - title: "Identity" variables: - compartment_ocid - create_iam_autoscaler_policy - create_iam_worker_policy - use_defined_tags - tag_namespace - title: "Cluster" variables: - cluster_id - cni_type - kubeproxy_mode - worker_node_labels - title: "Network" variables: - vcn_id - assign_dns - vcn_dns_label - worker_subnet_id - worker_nsg_id - pod_subnet_id - pod_nsg_id - title: "SSH" variables: - ssh_public_key - ssh_kms_vault_id - ssh_kms_secret_id - title: "Image" variables: - worker_image_type - worker_image_os - worker_image_os_version - worker_image_platform_id - worker_image_custom_id - title: "Instances" variables: - worker_pool_name - worker_pool_mode - worker_shape - worker_pool_size - worker_ocpus - worker_memory - worker_boot_volume_size - worker_block_volume_type - worker_use_encryption - worker_volume_kms_vault_id - worker_volume_kms_key_id - worker_pv_transit_encryption - worker_cloud_init_configure - worker_cloud_init - worker_cloud_init_byon - worker_cloud_init_oke - autoscale - drain - worker_tags variables: # Identity current_user_ocid: title: User type: ocid required: true tenancy_ocid: title: Tenancy type: oci:identity:compartment:id required: true compartment_ocid: title: Compartment description: The default compartment for created resources. type: oci:identity:compartment:id required: true region: required: true title: Region type: oci:identity:region:name create_iam_autoscaler_policy: title: Authorize instance(s) to manage pools with Cluster Autoscaler description: | Create the required Identity policy with a dynamic group to authorize pool management by worker nodes for Cluster Autoscaler. See Using the Kubernetes Cluster Autoscaler for more information. type: boolean default: false required: true visible: { not: [autoscale] } create_iam_worker_policy: title: Authorize instance(s) to join the target cluster description: | Create the required Identity policy with a dynamic group to authorize self-managed worker node membership for an OKE cluster, e.g. `Allow dynamic-group ... to {CLUSTER_JOIN} in compartment id ... where { target.cluster.id = '...' }`. See OKE Self-managed nodes for more information. type: boolean default: false required: true visible: not: - eq: - worker_pool_mode - Node Pool use_defined_tags: title: Use defined tags default: false type: boolean tag_namespace: title: Tag namespace visible: ${use_defined_tags} # VCN vcn_id: title: Virtual Cloud Network type: oci:core:vcn:id required: true dependsOn: compartmentId: ${compartment_ocid} assign_dns: title: Assign DNS records type: boolean default: false required: true worker_subnet_id: title: Worker subnet description: VCN subnet for the primary network interface of created worker node(s). type: oci:core:subnet:id required: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: false worker_nsg_id: title: Worker Network Security Group description: Network Security Groups for the created worker node(s), used to configure network access to instances. See Security Rule Configuration in ... Network Security Groups for more information. type: oci:core:nsg:id # additionalProps: # allowMultiple: true required: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} pod_subnet_id: title: Pod subnet description: VCN subnet for the primary network interface of created worker node(s). type: oci:core:subnet:id dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} hidePublicSubnet: false hidePrivateSubnet: false hideRegionalSubnet: false hideAdSubnet: false pod_nsg_id: title: Pod Network Security Group description: Network Security Groups for the created worker node(s), used to configure network access to instances. See Security Rule Configuration in ... Network Security Groups for more information. type: oci:core:nsg:id # additionalProps: # allowMultiple: true dependsOn: compartmentId: ${compartment_ocid} vcnId: ${vcn_id} # SSH ssh_public_key: title: SSH Public Key type: oci:core:ssh:publickey pattern: "((^(ssh-rsa AAAAB3NzaC1yc2|ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNT|ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzOD|ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1Mj|ssh-ed25519 AAAAC3NzaC1lZDI1NTE5|ssh-dss AAAAB3NzaC1kc3)[0-9A-Za-z+\/]+[=]{0,3})( [^,]*)?)(,((ssh-rsa AAAAB3NzaC1yc2|ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNT|ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzOD|ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1Mj|ssh-ed25519 AAAAC3NzaC1lZDI1NTE5|ssh-dss AAAAB3NzaC1kc3)[0-9A-Za-z+\/]+[=]{0,3})( [^,]*)?)*$" required: false ssh_kms_vault_id: title: SSH Vault description: The OCI Vault used to encrypt the SSH key pair. type: oci:kms:vault:id required: true dependsOn: compartmentId: ${compartment_ocid} ssh_kms_secret_id: title: SSH Vault secret description: The OCI Vault secret containing the SSH private key. type: oci:kms:secret:id required: true dependsOn: compartmentId: ${compartment_ocid} vaultId: ${ssh_kms_vault_id} # Cluster cluster_id: title: Cluster type: oci:container:cluster:id required: true dependsOn: compartmentId: ${compartment_ocid} cni_type: title: CNI Type description: The CNI type ('Flannel' or 'NPN') of the target OKE cluster. type: enum default: Flannel enum: [Flannel, NPN] allowMultiple: false required: true visible: eq: [worker_pool_mode, Node Pool] apiserver_private_host: title: Cluster private endpoint description: | Private OKE endpoint IP address only - no protocol/port. Retrieve from an existing kubeconfig with: `kubectl config view --raw -o json | jq -rcM '.clusters[0].cluster.server' | cut -d: -f2 | tr -d '/'` type: string required: false cluster_ca_cert: title: Cluster CA certificate description: | Base64+PEM-encoded cluster CA certificate for a trusted connection to the OKE managed control plane. Retrieve from an existing kubeconfig with: `kubectl config view --raw -o json | jq -rcM '.clusters[0].cluster["certificate-authority-data"]' type: text multiline: true # Worker worker_pool_name: title: Pool name description: Display name for created worker node resources. type: string default: "oke-worker" required: true worker_pool_mode: title: Resource type description: Type of Oracle Cloud Compute resources for the created worker nodes. type: enum default: Instance Pool enum: [Node Pool, Instances, Instance Pool, Cluster Network] required: true worker_pool_size: title: Number of worker nodes type: number default: 1 required: true worker_shape: title: Shape # type: oci:core:instanceshapewithflex:name type: oci:core:instanceshape:name default: "VM.Standard.E4.Flex" required: true dependsOn: compartmentId: ${compartment_ocid} worker_ocpus: title: OCPUs type: number default: 2 required: false worker_memory: title: Memory (GB) type: number default: 16 required: false worker_boot_volume_size: title: Boot volume size (GB) type: number required: false worker_image_type: title: Image type description: Whether to use a Platform, OKE, or Custom image for worker nodes. type: enum default: OKE enum: [OKE, Platform, Custom] required: true worker_image_os: title: Operating system type: enum default: "Oracle Linux" enum: ["Oracle Linux"] required: true visible: not: - eq: - worker_image_type - Custom worker_image_os_version: title: Operating system version type: enum default: "8" enum: ["7.9", "8"] required: true visible: not: - eq: - worker_image_type - Custom worker_image_id: visible: false worker_image_custom_id: title: Image ID type: ocid required: true visible: eq: - worker_image_type - Custom worker_image_platform_id: title: Image type: oci:core:image:id required: true dependsOn: compartmentId: ${compartment_ocid} operatingSystem: ${worker_image_os} operatingSystemVersion: ${worker_image_os_version} shape: ${worker_shape_name} visible: eq: - worker_image_type - Platform worker_cloud_init: visible: false worker_cloud_init_configure: type: boolean title: Custom cloud-init worker_cloud_init_byon: title: Cloud-init script type: text multiline: true required: true description: | Custom cloud-init script for worker node startup configuration. Must include the default OKE initialization call to join the cluster and begin operation. visible: and: - worker_cloud_init_configure - not: - eq: - worker_pool_mode - Node Pool worker_cloud_init_oke: title: Cloud-init script type: text multiline: true required: true description: | Custom cloud-init script for self-managed worker node startup configuration. Must uncomment and define the two required parameters and include the default OKE initialization call to join the cluster and begin operation. visible: and: - worker_cloud_init_configure - eq: - worker_pool_mode - Node Pool worker_block_volume_type: title: Block volume type type: enum enum: [Paravirtualized, iSCSI] default: Paravirtualized required: true worker_node_labels: title: Node labels type: map required: false worker_use_encryption: title: KMS volume encryption type: boolean default: false required: true worker_volume_kms_vault_id: title: KMS volume encryption vault description: Vault containing operator volume encryption keys. type: oci:kms:vault:id required: false dependsOn: compartmentId: ${compartment_ocid} visible: worker_use_encryption worker_volume_kms_key_id: title: KMS volume encryption key type: oci:kms:key:id dependsOn: compartmentId: ${compartment_ocid} vaultId: ${worker_volume_kms_vault_id} required: false visible: worker_use_encryption worker_pv_transit_encryption: title: In-transit volume encryption type: boolean default: false visible: eq: - worker_block_volume_type - Paravirtualized kubeproxy_mode: title: Kubeproxy mode type: enum enum: [IPTables, IPVS] default: IPTables required: true worker_tags: title: Tagging type: oci:identity:tag:value required: false description: Tag values for created resources. dependsOn: compartmentId: ${compartment_ocid} autoscale: title: Manage with Cluster Autoscaler description: | Include pool in Dynamic Group for policies required by the Cluster Autoscaler. See Using the Kubernetes Cluster Autoscaler for more information. type: boolean default: false required: true visible: and: - eq: [worker_pool_mode, Node Pool] - not: [create_iam_autoscaler_policy] drain: title: Cordon & Drain description: | Move non-Daemonset workloads off of the pool. See also: Safely Drain a Node, Protecting an Application with a PodDisruptionBudget. type: boolean default: false required: true outputGroups: - title: Identity outputs: - state_id - create_iam_autoscaler_policy - create_iam_worker_policy - dynamic_group_ids - policy_statements - title: Network outputs: - worker_subnet_id - worker_nsg_id - title: Cluster outputs: - cluster_id - apiserver_private_host - title: Workers outputs: - worker_pool_name - worker_pool_ids - worker_pool_mode - worker_pool_size - worker_shape - worker_image_id - autoscale outputs: state_id: title: Terraform state ID type: copyableString worker_pool_mode: title: Resource type type: string worker_pool_name: title: Pool name type: string worker_pool_size: title: Size type: number cluster_id: title: Cluster type: ocid apiserver_private_host: title: Cluster private endpoint IP type: copyableString dynamic_group_ids: title: Dynamic groups type: list policy_statements: title: Policy statements type: list worker_subnet_id: title: Subnet type: ocid worker_nsg_id: title: Network Security Group type: ocid worker_pool_ids: title: Pool type: list worker_image_id: title: Image type: ocid worker_shape: title: Shape type: string create_iam_autoscaler_policy: title: Create Cluster Autoscaler policy type: boolean create_iam_worker_policy: title: Create self-managed worker policy type: string autoscale: title: Manage with Cluster Autoscaler type: string ================================================ FILE: examples/rms/oke-workers-only/variables.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Identity # Automatically populated by Resource Manager variable "tenancy_ocid" { type = string } variable "current_user_ocid" { type = string } variable "compartment_ocid" { type = string } variable "region" { type = string } variable "use_defined_tags" { default = false description = "Add existing tags in the configured namespace to created resources when applicable." type = bool } variable "tag_namespace" { default = "oke" description = "Tag namespace containing standard tags for resources created by the module: [state_id, role, pool, cluster_autoscaler]." type = string } variable "create_iam_autoscaler_policy" { default = false } variable "create_iam_worker_policy" { default = false } variable "autoscale" { default = false } # Cluster variable "cluster_id" { default = null type = string } variable "cni_type" { default = "Flannel" } variable "kubernetes_version" { default = "v1.32.1" type = string } # Worker pools variable "worker_pool_mode" { default = "Instances" type = string validation { condition = contains(["Node Pool", "Instances", "Instance Pool", "Cluster Network"], var.worker_pool_mode) error_message = "Accepted values are Node Pool, Instances, Instance Pool, or Cluster Network" } } variable "worker_pool_size" { default = 1 type = number } # Workers: network variable "vcn_id" { default = null type = string } variable "assign_dns" { default = true } variable "pod_nsg_id" { default = "" } variable "pod_subnet_id" { default = "" } variable "worker_nsg_id" { default = "" } variable "worker_subnet_id" { type = string } variable "kubeproxy_mode" { type = string } # Workers: instance variable "worker_block_volume_type" { type = string } variable "worker_node_labels" { default = {} type = map(string) } variable "worker_image_type" { type = string } variable "worker_image_id" { default = null type = string } variable "worker_image_os" { default = "Oracle Linux" type = string } variable "worker_image_os_version" { default = "8" type = string } variable "worker_pool_name" { type = string } variable "worker_shape" { default = "VM.Standard.E4.Flex" } variable "worker_ocpus" { default = 2 } variable "worker_memory" { default = 16 } variable "worker_boot_volume_size" { default = 50 } variable "worker_pv_transit_encryption" { default = false } variable "worker_cloud_init_configure" { type = bool } variable "worker_cloud_init_oke" { default = <<-EOT #!/usr/bin/env bash curl --fail -H "Authorization: Bearer Oracle" -L0 http://169.254.169.254/opc/v2/instance/metadata/oke_init_script | base64 --decode >/var/run/oke-init.sh bash /etc/oke/oke-install.sh EOT type = string } variable "worker_cloud_init_byon" { default = <<-EOT #!/usr/bin/env bash #apiserver_host="10.0.0.1" #ca_base64="LS0tLS1...LS0tCg==" # kubectl config view --raw -o json | jq -rcM '.clusters[0].cluster["certificate-authority-data"]' bash /etc/oke/oke-install.sh --apiserver-endpoint "$\{apiserver_host}" --kubelet-ca-cert "$\{ca_base64}" EOT type = string } variable "worker_volume_kms_key_id" { default = null type = string } variable "worker_volume_kms_vault_id" { default = null type = string } variable "worker_image_platform_id" { default = null type = string } variable "worker_image_custom_id" { default = null type = string } variable "worker_tags" { default = {} type = map(any) } ================================================ FILE: examples/rms/oke-workers-only/versions.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { experiments = [module_variable_optional_attrs] required_version = ">= 1.2.0" required_providers { oci = { configuration_aliases = [oci.home] source = "oracle/oci" version = ">= 4.119.0" } } } ================================================ FILE: examples/utilities/README.md ================================================ # Utility Examples Example configurations for utility features: | File | Description | |------|-------------| | `vars-utilities-drain.auto.tfvars` | Worker pool draining configuration | | `vars-utilities-ocir.auto.tfvars` | Oracle Container Image Registry (OCIR) secret | | `vars-utilities-serviceaccount.auto.tfvars` | Kubernetes service account creation | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: examples/workers/README.md ================================================ # Worker Examples Example configurations for various worker pool modes and features: | File | Description | |------|-------------| | `vars-workers.auto.tfvars` | Basic worker pool defaults | | `vars-workers-basic.auto.tfvars` | Simple node pool | | `vars-workers-nodepool.auto.tfvars` | OKE-managed node pool | | `vars-workers-virtualnodepool.auto.tfvars` | OKE-managed virtual node pool | | `vars-workers-instance.auto.tfvars` | Self-managed compute instance | | `vars-workers-instancepool.auto.tfvars` | Self-managed instance pool | | `vars-workers-clusternetwork.auto.tfvars` | Cluster network (HPC/GPU with RDMA) | | `vars-workers-computecluster.auto.tfvars` | Shared compute cluster | | `vars-workers-autoscaling.auto.tfvars` | Autoscaled node pool | | `vars-workers-advanced.auto.tfvars` | Advanced configuration options | | `vars-workers-agent.auto.tfvars` | Management agent configuration | | `vars-workers-cloudinit-global.auto.tfvars` | Global cloud-init for all pools | | `vars-workers-cloudinit-pool.auto.tfvars` | Pool-specific cloud-init | | `vars-workers-drain.auto.tfvars` | Worker pool draining | | `vars-workers-network-nsgs.auto.tfvars` | Custom NSG configuration | | `vars-workers-network-subnets.auto.tfvars` | Custom subnet configuration | | `vars-workers-network-vnics.auto.tfvars` | Secondary VNIC configuration | | `vars-workers-node-cycling.auto.tfvars` | Node cycling for updates | ## Usage Copy the desired `.auto.tfvars` file(s) to your root module and adjust the values as needed. ================================================ FILE: migration.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Subnets moved { from = module.network.oci_core_subnet.cp to = module.network.oci_core_subnet.oke["cp"] } moved { from = module.bastion[0].oci_core_subnet.bastion to = module.network.oci_core_subnet.oke["bastion"] } moved { from = module.operator[0].oci_core_subnet.operator to = module.network.oci_core_subnet.oke["operator"] } moved { from = module.network.oci_core_subnet.int_lb[0] to = module.network.oci_core_subnet.oke["int_lb"] } moved { from = module.network.oci_core_subnet.pub_lb[0] to = module.network.oci_core_subnet.oke["pub_lb"] } moved { from = module.network.oci_core_subnet.workers to = module.network.oci_core_subnet.oke["workers"] } moved { from = module.network.oci_core_subnet.pods to = module.network.oci_core_subnet.oke["pods"] } moved { from = module.network.oci_core_subnet.fss to = module.network.oci_core_subnet.oke["fss"] } # Cluster moved { from = module.oke.oci_containerengine_cluster.k8s_cluster to = module.cluster[0].oci_containerengine_cluster.k8s_cluster } # Workers moved { from = module.oke.oci_containerengine_node_pool.nodepools to = module.workers[0].oci_containerengine_node_pool.workers } moved { from = module.workers[0].oci_containerengine_node_pool.workers to = module.workers[0].oci_containerengine_node_pool.tfscaled_workers } moved { from = module.workers[0].oci_core_instance_pool.workers to = module.workers[0].oci_core_instance_pool.tfscaled_workers } ================================================ FILE: module-bastion.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl // Used to retrieve available bastion images when enabled data "oci_core_images" "bastion" { count = var.create_bastion ? 1 : 0 compartment_id = local.compartment_id operating_system = var.bastion_image_os operating_system_version = var.bastion_image_os_version shape = lookup(var.bastion_shape, "shape", "VM.Standard.E4.Flex") state = "AVAILABLE" sort_by = "TIMECREATED" sort_order = "DESC" filter { name = "launch_mode" values = ["NATIVE"] } } locals { bastion_public_ip = (var.create_bastion && length(module.bastion) > 0 ? lookup(element(module.bastion, 0), "public_ip", var.bastion_public_ip) : var.bastion_public_ip ) bastion_images = try(data.oci_core_images.bastion[0].images, tolist([])) # Data source result or empty bastion_image_ids = local.bastion_images[*].id # Image OCIDs from data source bastion_image_id = (var.bastion_image_type == "custom" ? var.bastion_image_id : element(coalescelist(local.bastion_image_ids, ["none"]), 0) ) # Bastion SSH ProxyCommand argument used in e.g. ssh_to_operator command output if created/provided bastion_ssh_user_ip = join("@", compact([var.bastion_user, local.bastion_public_ip])) bastion_ssh_args = compact([local.ssh_key_arg, local.bastion_ssh_user_ip]) bastion_ssh_command = join(" ", concat(["ssh"], local.bastion_ssh_args)) bastion_proxy_command = try((local.bastion_public_ip == null ? [] : ["-o", format("ProxyCommand='ssh -W %%h:%%p %v'", join(" ", local.bastion_ssh_args))] ), []) } module "bastion" { count = var.create_bastion ? 1 : 0 source = "./modules/bastion" state_id = local.state_id compartment_id = local.compartment_id # Bastion await_cloudinit = var.bastion_await_cloudinit assign_dns = var.assign_dns availability_domain = coalesce(var.bastion_availability_domain, lookup(local.ad_numbers_to_names, local.ad_numbers[0])) bastion_image_os_version = var.bastion_image_os_version image_id = local.bastion_image_id nsg_ids = try(compact(flatten([var.bastion_nsg_ids, [try(module.network.bastion_nsg_id, null)]])), []) is_public = var.bastion_is_public shape = var.bastion_shape legacy_imds_endpoints_disabled = var.bastion_legacy_imds_endpoints_disabled ssh_private_key = sensitive(local.ssh_private_key) # to await cloud-init completion ssh_public_key = local.ssh_public_key subnet_id = try(module.network.bastion_subnet_id, "") # safe destroy; validated in submodule timezone = var.timezone upgrade = var.bastion_upgrade user = var.bastion_user volume_kms_key_id = var.bastion_volume_kms_key_id # Standard tags as defined if enabled for use, or freeform # User-provided tags are merged last and take precedence use_defined_tags = var.use_defined_tags tag_namespace = var.tag_namespace defined_tags = merge(var.use_defined_tags ? { "${var.tag_namespace}.state_id" = local.state_id, "${var.tag_namespace}.role" = "bastion", } : {}, local.bastion_defined_tags) freeform_tags = merge(var.use_defined_tags ? {} : { "state_id" = local.state_id, "role" = "bastion", }, local.bastion_freeform_tags) } output "bastion_id" { description = "ID of bastion instance" value = one(module.bastion[*].id) } output "bastion_public_ip" { description = "Public IP address of bastion host" value = local.bastion_public_ip } output "ssh_to_bastion" { description = "SSH command for bastion host" value = local.bastion_ssh_command } ================================================ FILE: module-cluster-addons.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl module "cluster-addons" { count = local.cluster_enabled && lower(var.cluster_type) == "enhanced" ? 1 : 0 source = "./modules/cluster-addons" operator_enabled = local.operator_enabled cluster_addons = var.cluster_addons cluster_addons_to_remove = var.cluster_addons_to_remove cluster_id = coalesce(var.cluster_id, one(module.cluster[*].cluster_id)) kubernetes_version = var.kubernetes_version # Bastion/operator connection ssh_private_key = sensitive(local.ssh_private_key) bastion_host = local.bastion_public_ip bastion_user = var.bastion_user operator_host = local.operator_private_ip operator_user = var.operator_user } # output "supported_addons" { # description = "Supported cluster addons" # value = var.output_detail ? try(one(module.cluster-addons[*].supported_addons), null) : null # } ================================================ FILE: module-cluster.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Used to retrieve cluster CA certificate or configure local kube context data "oci_containerengine_clusters" "existing_cluster" { count = var.cluster_id != null ? 1 : 0 compartment_id = local.compartment_id state = ["ACTIVE", "UPDATING"] filter { name = "id" values = [var.cluster_id] } } data "oci_containerengine_cluster_kube_config" "public" { count = local.cluster_enabled && local.public_endpoint_available ? 1 : 0 cluster_id = local.cluster_id endpoint = "PUBLIC_ENDPOINT" } data "oci_containerengine_cluster_kube_config" "private" { count = local.cluster_enabled && local.private_endpoint_available ? 1 : 0 cluster_id = local.cluster_id endpoint = "PRIVATE_ENDPOINT" } locals { cluster_enabled = var.create_cluster || coalesce(var.cluster_id, "none") != "none" cluster_id = var.create_cluster ? one(module.cluster[*].cluster_id) : var.cluster_id cluster_name = var.cluster_name cluster-context = try(format("context-%s", substr(local.cluster_id, -11, -1)), "") existing_cluster_endpoints = coalesce(one(flatten(data.oci_containerengine_clusters.existing_cluster[*].clusters[*].endpoints)), tomap({})) public_endpoint_available = var.cluster_id != null ? length(lookup(local.existing_cluster_endpoints, "public_endpoint", "")) > 0 : var.control_plane_is_public && var.assign_public_ip_to_control_plane private_endpoint_available = var.cluster_id != null ? length(lookup(local.existing_cluster_endpoints, "private_endpoint", "")) > 0 : true kubeconfig_public = var.control_plane_is_public ? try(yamldecode(replace(lookup(one(data.oci_containerengine_cluster_kube_config.public), "content", ""), local.cluster-context, var.cluster_name)), tomap({})) : null kubeconfig_private = try(yamldecode(replace(lookup(one(data.oci_containerengine_cluster_kube_config.private), "content", ""), local.cluster-context, var.cluster_name)), tomap({})) kubeconfig_clusters = try(lookup(local.kubeconfig_private, "clusters", []), []) apiserver_private_host = (var.create_cluster ? try(split(":", one(module.cluster[*].endpoints.private_endpoint))[0], "") : split(":", replace(try(lookup(lookup(local.kubeconfig_clusters[0], "cluster", {}), "server", ""), "none"), "https://", ""))[0]) kubeconfig_ca_cert = try(lookup(lookup(local.kubeconfig_clusters[0], "cluster", {}), "certificate-authority-data", ""), "none") cluster_ca_cert = coalesce(var.cluster_ca_cert, local.kubeconfig_ca_cert) } module "cluster" { count = var.create_cluster ? 1 : 0 source = "./modules/cluster" compartment_id = local.compartment_id state_id = local.state_id # Network vcn_id = local.vcn_id cni_type = var.cni_type control_plane_is_public = var.control_plane_is_public ip_families = length(var.oke_ip_families) > 0 ? var.oke_ip_families : var.enable_ipv6 ? ["IPv4", "IPv6"] : ["IPv4"] assign_public_ip_to_control_plane = var.assign_public_ip_to_control_plane control_plane_nsg_ids = compact(flatten([var.control_plane_nsg_ids, try(module.network.control_plane_nsg_id, null)])) control_plane_subnet_id = try(module.network.control_plane_subnet_id, "") # safe destroy; validated in submodule pods_cidr = var.pods_cidr services_cidr = var.services_cidr service_lb_subnet_id = (var.preferred_load_balancer == "public" ? try(module.network.pub_lb_subnet_id, "") # safe destroy; validated in submodule : try(module.network.int_lb_subnet_id, "") ) backend_nsg_ids = compact(flatten([ var.backend_nsg_ids, try(module.network.worker_nsg_id, null), var.cni_type == "npn" ? try(module.network.pod_nsg_id, null) : null ])) # Cluster cluster_kms_key_id = var.cluster_kms_key_id cluster_name = local.cluster_name cluster_type = lookup({ "basic" = "BASIC_CLUSTER", "enhanced" = "ENHANCED_CLUSTER" }, lower(var.cluster_type), "BASIC_CLUSTER") kubernetes_version = var.kubernetes_version # KMS use_signed_images = var.use_signed_images image_signing_keys = var.image_signing_keys # Tags use_defined_tags = var.use_defined_tags tag_namespace = var.tag_namespace # Standard tags as defined if enabled for use, or freeform # User-provided tags are merged last and take precedence cluster_defined_tags = merge( var.use_defined_tags ? { "${var.tag_namespace}.state_id" = local.state_id, "${var.tag_namespace}.role" = "cluster", } : {}, local.cluster_defined_tags, ) cluster_freeform_tags = merge( var.use_defined_tags ? {} : { "state_id" = local.state_id, "role" = "cluster", }, local.cluster_freeform_tags, ) persistent_volume_defined_tags = merge( var.use_defined_tags ? { "${var.tag_namespace}.state_id" = local.state_id, "${var.tag_namespace}.role" = "persistent_volume", } : {}, local.persistent_volume_defined_tags, ) persistent_volume_freeform_tags = merge( var.use_defined_tags ? {} : { "state_id" = local.state_id, "role" = "persistent_volume", }, local.persistent_volume_freeform_tags, ) service_lb_defined_tags = merge( var.use_defined_tags ? { "${var.tag_namespace}.state_id" = local.state_id, "${var.tag_namespace}.role" = "service_lb" } : {}, local.service_lb_defined_tags, ) service_lb_freeform_tags = merge( var.use_defined_tags ? {} : { "state_id" = local.state_id, "role" = "service_lb" }, local.service_lb_freeform_tags, ) oidc_discovery_enabled = var.oidc_discovery_enabled oidc_token_auth_enabled = var.oidc_token_auth_enabled oidc_token_authentication_config = var.oidc_token_authentication_config depends_on = [ module.iam_cluster_prerequisites, ] } output "cluster_id" { description = "ID of the OKE cluster" value = one(module.cluster[*].cluster_id) } output "cluster_endpoints" { description = "Endpoints for the OKE cluster" value = var.create_cluster ? one(module.cluster[*].endpoints) : local.existing_cluster_endpoints } output "cluster_oidc_discovery_endpoint" { description = "OIDC discovery endpoint for the OKE cluster" value = var.create_cluster && var.oidc_discovery_enabled ? one(module.cluster[*].oidc_discovery_endpoint) : null } output "cluster_kubeconfig" { description = "OKE kubeconfig" value = var.output_detail ? ( local.public_endpoint_available ? local.kubeconfig_public : local.kubeconfig_private ) : null } output "cluster_ca_cert" { description = "OKE cluster CA certificate" value = var.output_detail && length(local.cluster_ca_cert) > 0 ? local.cluster_ca_cert : null } output "apiserver_private_host" { description = "Private OKE cluster endpoint address" value = local.apiserver_private_host } ================================================ FILE: module-extensions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { cluster_private_endpoint = (var.create_cluster ? coalesce(split(":", lookup(one(module.cluster[*].endpoints), "private_endpoint", ""))...) : (length(local.existing_cluster_endpoints) > 0 ? coalesce(split(":", lookup(local.existing_cluster_endpoints, "private_endpoint", ""))...) : null ) ) } module "extensions" { source = "./modules/extensions" depends_on = [module.network] count = alltrue([var.create_cluster, local.operator_enabled]) ? 1 : 0 region = var.region state_id = local.state_id # Cluster kubernetes_version = var.kubernetes_version expected_node_count = local.worker_count_expected worker_pools = one(module.workers[*].worker_pools) cluster_private_endpoint = local.cluster_private_endpoint # Bastion/operator connection ssh_private_key = sensitive(local.ssh_private_key) bastion_host = local.bastion_public_ip bastion_user = var.bastion_user operator_host = local.operator_private_ip operator_user = var.operator_user # CNI vcn_cidrs = local.vcn_cidrs cni_type = var.cni_type pods_cidr = var.pods_cidr # CNI: Cilium cilium_install = var.cilium_install cilium_reapply = var.cilium_reapply cilium_namespace = var.cilium_namespace cilium_helm_version = var.cilium_helm_version cilium_helm_values = var.cilium_helm_values cilium_helm_values_files = var.cilium_helm_values_files # CNI: Multus multus_install = var.multus_install multus_namespace = var.multus_namespace multus_daemonset_url = var.multus_daemonset_url multus_version = var.multus_version # Metrics server metrics_server_install = var.metrics_server_install metrics_server_namespace = var.metrics_server_namespace metrics_server_helm_version = var.metrics_server_helm_version metrics_server_helm_values = var.metrics_server_helm_values metrics_server_helm_values_files = var.metrics_server_helm_values_files # Cluster autoscaler cluster_autoscaler_install = var.cluster_autoscaler_install cluster_autoscaler_namespace = var.cluster_autoscaler_namespace cluster_autoscaler_helm_version = var.cluster_autoscaler_helm_version cluster_autoscaler_helm_values = var.cluster_autoscaler_helm_values cluster_autoscaler_helm_values_files = var.cluster_autoscaler_helm_values_files expected_autoscale_worker_pools = coalesce(one(module.workers[*].worker_pool_autoscale_expected), 0) # Gatekeeper gatekeeper_install = var.gatekeeper_install gatekeeper_namespace = var.gatekeeper_namespace gatekeeper_helm_version = var.gatekeeper_helm_version gatekeeper_helm_values = var.gatekeeper_helm_values gatekeeper_helm_values_files = var.gatekeeper_helm_values_files # Prometheus prometheus_install = var.prometheus_install prometheus_reapply = var.prometheus_reapply prometheus_namespace = var.prometheus_namespace prometheus_helm_version = var.prometheus_helm_version prometheus_helm_values = var.prometheus_helm_values prometheus_helm_values_files = var.prometheus_helm_values_files # DCGM exporter dcgm_exporter_install = var.dcgm_exporter_install dcgm_exporter_reapply = var.dcgm_exporter_reapply dcgm_exporter_namespace = var.dcgm_exporter_namespace dcgm_exporter_helm_version = var.dcgm_exporter_helm_version dcgm_exporter_helm_values = var.dcgm_exporter_helm_values dcgm_exporter_helm_values_files = var.dcgm_exporter_helm_values_files # SR-IOV device plugin sriov_device_plugin_install = var.sriov_device_plugin_install sriov_device_plugin_namespace = var.sriov_device_plugin_namespace sriov_device_plugin_daemonset_url = var.sriov_device_plugin_daemonset_url sriov_device_plugin_version = var.sriov_device_plugin_version # SR-IOV CNI plugin sriov_cni_plugin_install = var.sriov_cni_plugin_install sriov_cni_plugin_namespace = var.sriov_cni_plugin_namespace sriov_cni_plugin_daemonset_url = var.sriov_cni_plugin_daemonset_url sriov_cni_plugin_version = var.sriov_cni_plugin_version # SR-IOV CNI plugin rdma_cni_plugin_install = var.rdma_cni_plugin_install rdma_cni_plugin_namespace = var.rdma_cni_plugin_namespace rdma_cni_plugin_daemonset_url = var.rdma_cni_plugin_daemonset_url rdma_cni_plugin_version = var.rdma_cni_plugin_version # Whereabouts IPAM plugin whereabouts_install = var.whereabouts_install whereabouts_namespace = var.whereabouts_namespace whereabouts_daemonset_url = var.whereabouts_daemonset_url whereabouts_version = var.whereabouts_version # MPI operator mpi_operator_install = var.mpi_operator_install mpi_operator_namespace = var.mpi_operator_namespace mpi_operator_deployment_url = var.mpi_operator_deployment_url mpi_operator_version = var.mpi_operator_version # Service Account create_service_account = var.create_service_account service_accounts = var.service_accounts # Argocd argocd_install = var.argocd_install argocd_namespace = var.argocd_namespace argocd_helm_version = var.argocd_helm_version argocd_helm_values = var.argocd_helm_values argocd_helm_values_files = var.argocd_helm_values_files } ================================================ FILE: module-iam.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_identity_availability_domains" "all" { compartment_id = local.tenancy_id != "unknown" ? local.tenancy_id : local.compartment_id } locals { // Tenancy-specific availability domains in region // Common reference for data source re-used throughout module ads = data.oci_identity_availability_domains.all.availability_domains // Map of parsed availability domain numbers to tenancy-specific names // Used by resources with AD placement for generic selection ad_numbers_to_names = local.ads != null ? { for ad in local.ads : parseint(substr(ad.name, -1, -1), 10) => ad.name } : { -1 : "" } # Fallback handles failure when unavailable but not required // List of availability domain numbers in region // Used to intersect desired AD lists against presence in region ad_numbers = local.ads != null ? sort(keys(local.ad_numbers_to_names)) : [] create_iam_worker_policy = anytrue([ var.create_iam_worker_policy == "always", var.create_iam_worker_policy == "auto" && var.create_cluster && anytrue([for k, v in var.worker_pools : tobool(lookup(v, "create", true)) && lookup(v, "mode", var.worker_pool_mode) != "node-pool" ]) ]) create_iam_autoscaler_policy = anytrue([ var.create_iam_autoscaler_policy == "always", var.create_iam_autoscaler_policy == "auto" && var.create_cluster && anytrue([for k, v in var.worker_pools : tobool(lookup(v, "create", true)) && tobool(lookup(v, "allow_autoscaler", false)) ]) ]) create_iam_operator_policy = anytrue([ var.create_iam_operator_policy == "always", var.create_iam_operator_policy == "auto" && local.operator_enabled ]) create_iam_kms_policy = anytrue([ var.create_iam_kms_policy == "always", var.create_iam_kms_policy == "auto" && anytrue([ # coalesce(var.worker_volume_kms_key_id, "none") != "none", ## Validated in group-workers.tf in the IAM module. coalesce(var.cluster_kms_key_id, "none") != "none", ]) ]) default_policy_name = format("oke-cluster-%v", local.state_id) prerequisites_policy_name = format("oke-cluster-prerequisites-%v", local.state_id) } # Default IAM sub-module implementation for OKE cluster module "iam_cluster_prerequisites" { source = "./modules/iam" compartment_id = local.compartment_id state_id = local.state_id tenancy_id = local.tenancy_id cluster_id = var.cluster_id create_iam_resources = var.create_iam_resources create_iam_autoscaler_policy = false create_iam_kms_policy = local.create_iam_kms_policy create_iam_operator_policy = false create_iam_worker_policy = false create_iam_cluster_policy = false policy_name = local.prerequisites_policy_name create_iam_tag_namespace = var.create_iam_tag_namespace create_iam_defined_tags = var.create_iam_defined_tags defined_tags = local.iam_defined_tags freeform_tags = local.iam_freeform_tags tag_namespace = var.tag_namespace use_defined_tags = var.use_defined_tags cluster_kms_key_id = var.cluster_kms_key_id operator_volume_kms_key_id = var.operator_volume_kms_key_id worker_volume_kms_key_id = var.worker_volume_kms_key_id autoscaler_compartments = [] worker_compartments = [] enable_ipv6 = false network_compartment_id = var.network_compartment_id providers = { oci.home = oci.home } } # Default IAM sub-module implementation for OKE cluster module "iam" { source = "./modules/iam" compartment_id = local.compartment_id state_id = local.state_id tenancy_id = local.tenancy_id cluster_id = local.cluster_id create_iam_resources = var.create_iam_resources create_iam_autoscaler_policy = local.create_iam_autoscaler_policy create_iam_kms_policy = false create_iam_operator_policy = local.create_iam_operator_policy create_iam_worker_policy = local.create_iam_worker_policy create_iam_cluster_policy = true policy_name = local.default_policy_name create_iam_tag_namespace = var.create_iam_tag_namespace create_iam_defined_tags = var.create_iam_defined_tags defined_tags = local.iam_defined_tags freeform_tags = local.iam_freeform_tags tag_namespace = var.tag_namespace use_defined_tags = var.use_defined_tags cluster_kms_key_id = var.cluster_kms_key_id operator_volume_kms_key_id = var.operator_volume_kms_key_id worker_volume_kms_key_id = var.worker_volume_kms_key_id autoscaler_compartments = local.autoscaler_compartments worker_compartments = local.worker_compartments enable_ipv6 = var.enable_ipv6 network_compartment_id = var.network_compartment_id providers = { oci.home = oci.home } } output "availability_domains" { description = "Availability domains for tenancy & region" value = local.ad_numbers_to_names } output "dynamic_group_ids" { description = "Cluster IAM dynamic group IDs" value = concat( coalesce(module.iam_cluster_prerequisites.dynamic_group_ids, []), coalesce(module.iam.dynamic_group_ids, []) ) } output "policy_statements" { description = "Cluster IAM policy statements" value = concat( coalesce(module.iam_cluster_prerequisites.policy_statements, []), coalesce(module.iam.policy_statements, []) ) } ================================================ FILE: module-network.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_core_vcn" "oke" { count = var.create_vcn ? 0 : 1 vcn_id = coalesce(var.vcn_id, "none") } locals { # Created VCN if enabled, else var.vcn_id vcn_id = var.create_vcn ? try(one(module.vcn[*].vcn_id), var.vcn_id) : var.vcn_id # Configured VCN CIDRs if creating, else from provided vcn_id vcn_lookup = coalesce(one(data.oci_core_vcn.oke[*].cidr_blocks), []) vcn_lookup_cidr_blocks = flatten(local.vcn_lookup) vcn_cidrs = var.create_vcn ? var.vcn_cidrs : local.vcn_lookup_cidr_blocks vcn_ipv6_cidrs = var.create_vcn ? concat(module.vcn[0].vcn_all_attributes["ipv6cidr_blocks"], module.vcn[0].vcn_all_attributes["byoipv6cidr_blocks"], module.vcn[0].vcn_all_attributes["ipv6private_cidr_blocks"]) : concat(data.oci_core_vcn.oke[0].ipv6cidr_blocks, data.oci_core_vcn.oke[0].byoipv6cidr_blocks, data.oci_core_vcn.oke[0].ipv6private_cidr_blocks) # Created route table if enabled, else var.ig_route_table_id ig_route_table_id = var.create_vcn ? try(one(module.vcn[*].ig_route_id), var.ig_route_table_id) : var.ig_route_table_id # Created route table if enabled, else var.nat_route_table_id nat_route_table_id = var.create_vcn ? try(one(module.vcn[*].nat_route_id), var.ig_route_table_id) : var.nat_route_table_id create_internet_gateway = alltrue([ var.vcn_create_internet_gateway != "never", # always disable anytrue([ # enable for configurations that generally utilize it var.vcn_create_internet_gateway == "always", # always enable var.create_bastion && var.bastion_is_public, # enable for public bastion var.control_plane_is_public, # enable for cluster w/ public endpoint var.load_balancers != "internal", # enable for cluster w/ public load balancers ]) ]) create_nat_gateway = alltrue([ var.vcn_create_nat_gateway != "never", # always disable anytrue([ # enable for configurations that generally utilize it var.vcn_create_nat_gateway == "always", # always enable !var.worker_is_public, # enable for private workers var.create_operator, # enable for operator !var.control_plane_is_public, # enable for cluster w/ private endpoint contains(["internal", "both"], var.load_balancers), # enable for cluster w/ private load balancers ]) ]) internet_gateway_id = var.create_vcn ? try(one(module.vcn[*].internet_gateway_id), var.internet_gateway_id) : var.internet_gateway_id nat_gateway_id = var.create_vcn ? try(one(module.vcn[*].nat_gateway_id), var.nat_gateway_id) : var.nat_gateway_id } module "vcn" { count = var.create_vcn ? 1 : 0 source = "oracle-terraform-modules/vcn/oci" version = "3.6.0" compartment_id = coalesce(var.network_compartment_id, local.compartment_id) # Standard tags as defined if enabled for use, or freeform # User-provided tags are merged last and take precedence defined_tags = merge(var.use_defined_tags ? { "${var.tag_namespace}.state_id" = local.state_id, "${var.tag_namespace}.role" = "network", } : {}, local.network_defined_tags, ) freeform_tags = merge(var.use_defined_tags ? {} : { "state_id" = local.state_id, "role" = "network", }, local.network_freeform_tags, ) attached_drg_id = var.drg_id != null ? var.drg_id : (tobool(var.create_drg) ? module.drg[0].drg_id : null) create_internet_gateway = local.create_internet_gateway create_nat_gateway = local.create_nat_gateway create_service_gateway = var.vcn_create_service_gateway != "never" internet_gateway_route_rules = var.internet_gateway_route_rules local_peering_gateways = var.local_peering_gateways lockdown_default_seclist = var.lockdown_default_seclist nat_gateway_public_ip_id = var.nat_gateway_public_ip_id nat_gateway_route_rules = var.nat_gateway_route_rules enable_ipv6 = var.enable_ipv6 vcn_cidrs = local.vcn_cidrs vcn_dns_label = var.assign_dns ? coalesce(var.vcn_dns_label, local.state_id) : null vcn_name = coalesce(var.vcn_name, "oke-${local.state_id}") } module "drg" { count = tobool(var.create_drg) || var.drg_id != null ? 1 : 0 source = "oracle-terraform-modules/drg/oci" version = "1.0.6" compartment_id = coalesce(var.network_compartment_id, local.compartment_id) drg_compartment_id = var.drg_compartment_id drg_id = one([var.drg_id]) # existing DRG ID or null drg_display_name = coalesce(var.drg_display_name, "oke-${local.state_id}") drg_vcn_attachments = tobool(var.create_drg) || var.drg_id != null ? { for k, v in module.vcn : k => { # gets the vcn_id values dynamically from the vcn module vcn_id : v.vcn_id vcn_transit_routing_rt_id : null drg_route_table_id : null } } : var.drg_attachments # rpc parameters remote_peering_connections = { for k, v in var.remote_peering_connections : k => { "rpc_acceptor_id" = try(v.rpc_acceptor_id, null), "rpc_acceptor_region" = try(v.rpc_acceptor_region, null) } } } module "network" { source = "./modules/network" state_id = local.state_id compartment_id = coalesce(var.network_compartment_id, local.compartment_id) defined_tags = local.network_defined_tags freeform_tags = local.network_freeform_tags tag_namespace = var.tag_namespace use_defined_tags = var.use_defined_tags allow_node_port_access = var.allow_node_port_access allow_pod_internet_access = var.allow_pod_internet_access allow_rules_cp = var.allow_rules_cp allow_rules_internal_lb = var.allow_rules_internal_lb allow_rules_pods = var.allow_rules_pods allow_rules_public_lb = var.allow_rules_public_lb allow_rules_workers = var.allow_rules_workers allow_worker_internet_access = var.allow_worker_internet_access allow_worker_ssh_access = var.allow_worker_ssh_access allow_bastion_cluster_access = var.allow_bastion_cluster_access assign_dns = var.assign_dns bastion_allowed_cidrs = var.bastion_allowed_cidrs bastion_is_public = var.bastion_is_public cni_type = var.cni_type control_plane_allowed_cidrs = var.control_plane_allowed_cidrs control_plane_is_public = var.control_plane_is_public create_cluster = var.create_cluster create_bastion = var.create_bastion create_internet_gateway = local.create_internet_gateway create_nat_gateway = local.create_nat_gateway enable_ipv6 = var.enable_ipv6 nsgs = var.nsgs create_operator = local.operator_enabled drg_attachments = var.drg_attachments enable_waf = var.enable_waf ig_route_table_id = local.ig_route_table_id igw_ngw_mixed_route_id = var.igw_ngw_mixed_route_id internet_gateway_id = local.internet_gateway_id load_balancers = var.load_balancers nat_gateway_id = local.nat_gateway_id nat_route_table_id = local.nat_route_table_id subnets = var.subnets use_stateless_rules = var.use_stateless_rules vcn_cidrs = local.vcn_cidrs vcn_ipv6_cidrs = local.vcn_ipv6_cidrs vcn_id = local.vcn_id worker_is_public = var.worker_is_public } # VCN output "vcn_id" { description = "VCN ID" value = try(local.vcn_id, null) } output "ig_route_table_id" { description = "Internet gateway route table ID" value = try(local.ig_route_table_id, null) } output "nat_route_table_id" { description = "NAT gateway route table ID" value = try(local.nat_route_table_id, null) } # Subnets output "bastion_subnet_id" { value = try(module.network.bastion_subnet_id, null) } output "bastion_subnet_cidr" { value = try(module.network.bastion_subnet_cidr, null) } output "operator_subnet_id" { value = try(module.network.operator_subnet_id, null) } output "operator_subnet_cidr" { value = try(module.network.operator_subnet_cidr, null) } output "control_plane_subnet_id" { value = try(module.network.control_plane_subnet_id, null) } output "control_plane_subnet_cidr" { value = try(module.network.control_plane_subnet_cidr, null) } output "worker_subnet_id" { value = try(module.network.worker_subnet_id, null) } output "worker_subnet_cidr" { value = try(module.network.worker_subnet_cidr, null) } output "pod_subnet_id" { value = try(module.network.pod_subnet_id, null) } output "pod_subnet_cidr" { value = try(module.network.pod_subnet_cidr, null) } output "int_lb_subnet_id" { value = try(module.network.int_lb_subnet_id, null) } output "int_lb_subnet_cidr" { value = try(module.network.int_lb_subnet_cidr, null) } output "pub_lb_subnet_id" { value = try(module.network.pub_lb_subnet_id, null) } output "pub_lb_subnet_cidr" { value = try(module.network.pub_lb_subnet_cidr, null) } output "fss_subnet_id" { value = try(module.network.fss_subnet_id, null) } output "fss_subnet_cidr" { value = try(module.network.fss_subnet_cidr, null) } # NSGs output "bastion_nsg_id" { description = "Network Security Group for bastion host(s)." value = try(module.network.bastion_nsg_id, null) } output "operator_nsg_id" { description = "Network Security Group for operator host(s)." value = try(module.network.operator_nsg_id, null) } output "control_plane_nsg_id" { description = "Network Security Group for Kubernetes control plane(s)." value = try(module.network.control_plane_nsg_id, null) } output "int_lb_nsg_id" { description = "Network Security Group for internal load balancers." value = try(module.network.int_lb_nsg_id, null) } output "pub_lb_nsg_id" { description = "Network Security Group for public load balancers." value = try(module.network.pub_lb_nsg_id, null) } output "worker_nsg_id" { description = "Network Security Group for worker nodes." value = try(module.network.worker_nsg_id, null) } output "pod_nsg_id" { description = "Network Security Group for pods." value = try(module.network.pod_nsg_id, null) } output "fss_nsg_id" { description = "Network Security Group for File Storage Service resources." value = try(module.network.fss_nsg_id, null) } output "network_security_rules" { value = var.output_detail ? try(module.network.network_security_rules, null) : null } # DRG output "drg_id" { description = "Dynamic routing gateway ID" value = try(one(module.drg[*].drg_id), null) } # LPG output "lpg_all_attributes" { description = "all attributes of created lpg" value = try(one(module.vcn[*].lpg_all_attributes), null) } ================================================ FILE: module-operator.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl // Used to retrieve available operator images when enabled data "oci_core_images" "operator" { count = var.create_operator ? 1 : 0 compartment_id = local.compartment_id operating_system = var.operator_image_os operating_system_version = var.operator_image_os_version shape = lookup(var.operator_shape, "shape", "VM.Standard.E4.Flex") state = "AVAILABLE" sort_by = "TIMECREATED" sort_order = "DESC" filter { name = "launch_mode" values = ["NATIVE"] } } locals { // The private IP of the operator instance, whether created in this TF state or existing provided by ID operator_private_ip = (local.cluster_enabled && var.create_operator && length(module.operator) > 0 ? lookup(element(module.operator, 0), "private_ip", var.operator_private_ip) : var.operator_private_ip ) // Whether the operator is enabled, i.e. created in this TF state or existing provided by ID operator_enabled = anytrue([ (local.cluster_enabled || coalesce(var.cluster_id, "none") != "none"), (var.create_operator || coalesce(var.operator_private_ip, "none") != "none"), ]) // The resolved image ID for the created operator instance operator_images = try(data.oci_core_images.operator[0].images, tolist([])) # Data source result or empty operator_image_ids = local.operator_images[*].id # Image OCIDs from data source operator_image_id = (var.operator_image_type == "custom" ? var.operator_image_id : element(coalescelist(local.operator_image_ids, ["none"]), 0) ) # Operator SSH command args for.ssh_to_operator output if created/provided operator_ssh_user_ip = join("@", compact([var.operator_user, local.operator_private_ip])) operator_ssh_args = compact([local.ssh_key_arg, local.operator_ssh_user_ip]) } module "operator" { count = var.create_operator ? 1 : 0 source = "./modules/operator" state_id = local.state_id compartment_id = local.compartment_id # Bastion (to await cloud-init completion) bastion_host = local.bastion_public_ip bastion_user = var.bastion_user # Operator await_cloudinit = var.operator_await_cloudinit assign_dns = var.assign_dns availability_domain = coalesce(var.operator_availability_domain, lookup(local.ad_numbers_to_names, local.ad_numbers[0])) cloud_init = var.operator_cloud_init image_id = local.operator_image_id install_cilium = var.cilium_install install_helm = var.operator_install_helm install_helm_from_repo = var.operator_install_helm_from_repo install_oci_cli_from_repo = var.operator_install_oci_cli_from_repo install_istioctl = var.operator_install_istioctl install_k8sgpt = var.operator_install_k8sgpt install_k9s = var.operator_install_k9s install_kubectx = var.operator_install_kubectx install_kubectl_from_repo = var.operator_install_kubectl_from_repo install_stern = var.operator_install_stern kubeconfig = yamlencode(local.kubeconfig_private) kubernetes_version = var.kubernetes_version nsg_ids = compact(flatten([var.operator_nsg_ids, try(module.network.operator_nsg_id, null)])) operator_image_os_version = var.operator_image_os_version pv_transit_encryption = var.operator_pv_transit_encryption shape = var.operator_shape legacy_imds_endpoints_disabled = var.operator_legacy_imds_endpoints_disabled ssh_private_key = sensitive(local.ssh_private_key) # to await cloud-init completion ssh_public_key = local.ssh_public_key subnet_id = try(module.network.operator_subnet_id, "") # safe destroy; validated in submodule timezone = var.timezone upgrade = var.operator_upgrade user = var.operator_user volume_kms_key_id = var.operator_volume_kms_key_id # Standard tags as defined if enabled for use, or freeform # User-provided tags are merged last and take precedence defined_tags = merge(var.use_defined_tags ? { "${var.tag_namespace}.state_id" = local.state_id, "${var.tag_namespace}.role" = "operator", } : {}, local.operator_defined_tags) freeform_tags = merge(var.use_defined_tags ? {} : { "state_id" = local.state_id, "role" = "operator", }, local.operator_freeform_tags) use_defined_tags = var.use_defined_tags tag_namespace = var.tag_namespace depends_on = [ module.iam, ] } output "operator_id" { description = "ID of operator instance" value = one(module.operator[*].id) } output "operator_private_ip" { description = "Private IP address of operator host" value = local.operator_private_ip } output "ssh_to_operator" { description = "SSH command for operator host" value = local.operator_enabled ? join(" ", concat(["ssh"], local.bastion_proxy_command, local.operator_ssh_args) ) : null } ================================================ FILE: module-utilities.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl module "utilities" { count = local.cluster_enabled && local.operator_enabled ? 1 : 0 source = "./modules/utilities" region = var.region # Cluster await_node_readiness = var.await_node_readiness expected_node_count = local.worker_count_expected worker_pools = one(module.workers[*].worker_pools) # Bastion/operator connection ssh_private_key = sensitive(local.ssh_private_key) bastion_host = local.bastion_public_ip bastion_user = var.bastion_user operator_host = local.operator_private_ip operator_user = var.operator_user # OCIR ocir_email_address = var.ocir_email_address ocir_secret_id = var.ocir_secret_id ocir_secret_name = var.ocir_secret_name ocir_secret_namespace = var.ocir_secret_namespace ocir_username = var.ocir_username # Worker pool draining expected_drain_count = local.worker_drain_expected worker_drain_delete_local_data = var.worker_drain_delete_local_data worker_drain_ignore_daemonsets = var.worker_drain_ignore_daemonsets worker_drain_timeout_seconds = var.worker_drain_timeout_seconds } ================================================ FILE: module-workers.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { worker_count_expected = coalesce(one(module.workers[*].worker_count_expected), 0) worker_drain_expected = coalesce(one(module.workers[*].worker_drain_expected), 0) # Distinct list of compartments for enabled worker pools worker_compartments = distinct(compact([ for k, v in var.worker_pools : lookup(v, "compartment_id", local.compartment_id) if tobool(lookup(v, "create", true)) ])) # Worker pools with cluster autoscaler management enabled autoscaler_compartments = distinct(compact([ for k, v in var.worker_pools : lookup(v, "compartment_id", local.compartment_id) if tobool(lookup(v, "create", true)) && tobool(lookup(v, "allow_autoscaler", false)) ])) } # Default workers sub-module implementation for OKE cluster module "workers" { count = local.cluster_enabled ? 1 : 0 source = "./modules/workers" # Common compartment_id = local.worker_compartment_id tenancy_id = local.tenancy_id state_id = local.state_id ad_numbers = local.ad_numbers ad_numbers_to_names = local.ad_numbers_to_names # Cluster apiserver_private_host = local.apiserver_private_host cluster_ca_cert = local.cluster_ca_cert cluster_dns = var.cluster_dns cluster_id = coalesce(var.cluster_id, one(module.cluster[*].cluster_id)) cluster_type = var.cluster_type kubernetes_version = var.kubernetes_version allow_short_container_image_names = var.allow_short_container_image_names # Compute clusters compute_clusters = var.worker_compute_clusters # GPU memory cluster scale config defaults gmc_scale_is_upsize_enabled = var.worker_gmc_scale_is_upsize_enabled gmc_scale_is_downsize_enabled = var.worker_gmc_scale_is_downsize_enabled gmc_scale_target_size = var.worker_gmc_scale_target_size # Worker pools worker_pool_mode = var.worker_pool_mode worker_pool_size = var.worker_pool_size worker_pools = var.worker_pools # Workers assign_dns = var.assign_dns assign_public_ip = var.worker_is_public block_volume_type = var.worker_block_volume_type capacity_reservation_id = var.worker_capacity_reservation_id cloud_init = var.worker_cloud_init disable_default_cloud_init = var.worker_disable_default_cloud_init cni_type = var.cni_type image_id = var.worker_image_id image_ids = local.image_ids image_os = var.worker_image_os image_os_version = var.worker_image_os_version image_type = var.worker_image_type indexed_images = local.indexed_images kubeproxy_mode = var.kubeproxy_mode legacy_imds_endpoints_disabled = var.worker_legacy_imds_endpoints_disabled max_pods_per_node = var.max_pods_per_node node_labels = alltrue([var.cluster_type == "basic", var.cilium_install == true]) ? merge(var.worker_node_labels, { "oci.oraclecloud.com/custom-k8s-networking" = true }) : var.worker_node_labels node_metadata = var.worker_node_metadata agent_config = var.agent_config platform_config = var.platform_config pod_nsg_ids = concat(var.pod_nsg_ids, var.cni_type == "npn" ? [try(module.network.pod_nsg_id, null)] : []) pod_subnet_id = try(module.network.pod_subnet_id, "") # safe destroy; validated in submodule pv_transit_encryption = var.worker_pv_transit_encryption shape = var.worker_shape ssh_public_key = local.ssh_public_key timezone = var.timezone volume_kms_key_id = var.worker_volume_kms_key_id worker_nsg_ids = concat(var.worker_nsg_ids, [try(module.network.worker_nsg_id, null)]) worker_subnet_id = try(module.network.worker_subnet_id, "") # safe destroy; validated in submodule preemptible_config = var.worker_preemptible_config enable_ipv6 = var.enable_ipv6 # Tagging tag_namespace = var.tag_namespace defined_tags = local.workers_defined_tags freeform_tags = local.workers_freeform_tags use_defined_tags = var.use_defined_tags depends_on = [ module.iam, ] } output "worker_pools" { description = "Created worker pools (mode != 'instance')" value = var.output_detail && local.worker_count_expected > 0 ? try(one(module.workers[*].worker_pools), null) : null } output "worker_instances" { description = "Created worker pools (mode == 'instance')" value = var.output_detail && local.worker_count_expected > 0 ? try(one(module.workers[*].worker_instances), null) : null } output "worker_pool_ids" { description = "Enabled worker pool IDs" value = local.worker_count_expected > 0 ? try(one(module.workers[*].worker_pool_ids), null) : null } output "worker_pool_ips" { description = "Created worker instance private IPs by pool for available modes ('node-pool', 'instance')." value = local.worker_count_expected > 0 ? try(one(module.workers[*].worker_pool_ips), null) : null } output "worker_gpu_memory_clusters" { description = "Created GPU Memory Clusters keyed by '###'." value = var.output_detail && local.worker_count_expected > 0 ? try(one(module.workers[*].worker_gpu_memory_clusters), null) : null } output "worker_gpu_memory_cluster_ids" { description = "OCIDs of created GPU Memory Clusters keyed by '###'." value = local.worker_count_expected > 0 ? try(one(module.workers[*].worker_gpu_memory_cluster_ids), null) : null } ================================================ FILE: modules/bastion/README.md ================================================ # Bastion This sub-module creates a bastion host in a public subnet for SSH access into the VCN. ## Usage Refer to the [Bastion section](../../docs/terraformoptions.md#bastion) of the module documentation. ================================================ FILE: modules/bastion/cloudinit.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html data "cloudinit_config" "bastion" { gzip = true base64_encode = true part { content_type = "text/cloud-config" # https://cloudinit.readthedocs.io/en/latest/reference/examples.html#run-commands-on-first-boot content = <<-EOT runcmd: - ${format("dnf config-manager --disable ol%v_addons --disable ol%v_appstream", var.bastion_image_os_version, var.bastion_image_os_version)} EOT } part { content_type = "text/cloud-config" # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#package-update-upgrade-install content = jsonencode({ package_upgrade = var.upgrade }) filename = "10-packages.yml" } part { content_type = "text/cloud-config" # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#timezone content = jsonencode({ timezone = var.timezone }) filename = "10-timezone.yml" } part { content_type = "text/cloud-config" # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#package-update-upgrade-install content = jsonencode({ users = ["default", var.user] }) filename = "10-user.yml" } } resource "null_resource" "await_cloudinit" { count = var.await_cloudinit ? 1 : 0 connection { host = oci_core_instance.bastion.public_ip user = var.user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } lifecycle { replace_triggered_by = [oci_core_instance.bastion] } provisioner "remote-exec" { inline = ["cloud-init status --wait &> /dev/null"] } } ================================================ FILE: modules/bastion/compute.tf ================================================ # Copyright (c) 2019, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { boot_volume_size = tonumber(lookup(var.shape, "boot_volume_size", 50)) memory = tonumber(lookup(var.shape, "memory", 4)) ocpus = max(1, tonumber(lookup(var.shape, "ocpus", 1))) shape = lookup(var.shape, "shape", "VM.Standard.E4.Flex") baseline_ocpu_utilization_map = { "12.5" = "BASELINE_1_8" "50" = "BASELINE_1_2" "100" = "BASELINE_1_1" } baseline_ocpu_utilization = lookup(local.baseline_ocpu_utilization_map, lookup(var.shape, "baseline_ocpu_utilization", "100")) } output "id" { value = oci_core_instance.bastion.id } output "public_ip" { value = oci_core_instance.bastion.public_ip } resource "oci_core_instance" "bastion" { availability_domain = var.availability_domain compartment_id = var.compartment_id display_name = "bastion-${var.state_id}" defined_tags = var.defined_tags freeform_tags = var.freeform_tags shape = lookup(var.shape, "shape") agent_config { are_all_plugins_disabled = false is_management_disabled = false is_monitoring_disabled = false plugins_config { desired_state = "DISABLED" name = "Bastion" } } create_vnic_details { assign_public_ip = var.is_public display_name = "bastion-${var.state_id}" hostname_label = var.assign_dns ? "b-${var.state_id}" : null nsg_ids = var.nsg_ids subnet_id = var.subnet_id } instance_options { are_legacy_imds_endpoints_disabled = var.legacy_imds_endpoints_disabled } metadata = { ssh_authorized_keys = var.ssh_public_key user_data = data.cloudinit_config.bastion.rendered } dynamic "shape_config" { for_each = length(regexall("Flex", local.shape)) > 0 ? [1] : [] content { baseline_ocpu_utilization = local.baseline_ocpu_utilization ocpus = local.ocpus memory_in_gbs = (local.memory / local.ocpus) > 64 ? (local.ocpus * 4) : local.memory } } source_details { boot_volume_size_in_gbs = local.boot_volume_size source_id = var.image_id source_type = "image" kms_key_id = var.volume_kms_key_id } lifecycle { ignore_changes = [ availability_domain, defined_tags, freeform_tags, display_name, create_vnic_details, metadata, source_details, ] precondition { condition = coalesce(var.image_id, "none") != "none" error_message = "Missing image_id for bastion. Check provided value for image_id if image_type is 'custom', or image_os/image_os_version if image_type is 'platform'." } } timeouts { create = "60m" } } ================================================ FILE: modules/bastion/variables.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "compartment_id" { type = string } variable "state_id" { type = string } # Bastion variable "await_cloudinit" { type = string } variable "assign_dns" { type = bool } variable "availability_domain" { type = string } variable "bastion_image_os_version" { type = string } variable "image_id" { type = string } variable "is_public" { type = bool } variable "nsg_ids" { type = list(string) } variable "shape" { type = map(any) } variable "legacy_imds_endpoints_disabled" { type = bool default = true } variable "ssh_private_key" { type = string sensitive = true } variable "ssh_public_key" { type = string } variable "subnet_id" { type = string } variable "timezone" { type = string } variable "upgrade" { type = bool } variable "user" { type = string } variable "volume_kms_key_id" { type = string } # Tags variable "defined_tags" { type = map(string) } variable "freeform_tags" { type = map(string) } variable "tag_namespace" { type = string } variable "use_defined_tags" { type = bool } ================================================ FILE: modules/bastion/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { cloudinit = { source = "hashicorp/cloudinit" version = ">= 2.2.0" } null = { source = "hashicorp/null" version = ">= 3.2.1" } oci = { source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/cluster/README.md ================================================ # Cluster This sub-module creates an OKE cluster with configurable CNI, Kubernetes version, and OIDC authentication. ## Usage Refer to the [Cluster section](../../docs/terraformoptions.md#cluster) of the module documentation. ================================================ FILE: modules/cluster/cluster.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl resource "oci_containerengine_cluster" "k8s_cluster" { compartment_id = var.compartment_id kms_key_id = coalesce(var.cluster_kms_key_id, "none") != "none" ? var.cluster_kms_key_id : null kubernetes_version = var.kubernetes_version name = var.cluster_name type = var.cluster_type defined_tags = var.cluster_defined_tags freeform_tags = var.cluster_freeform_tags vcn_id = var.vcn_id cluster_pod_network_options { cni_type = var.cni_type == "flannel" ? "FLANNEL_OVERLAY" : "OCI_VCN_IP_NATIVE" } endpoint_config { is_public_ip_enabled = var.control_plane_is_public && var.assign_public_ip_to_control_plane nsg_ids = var.control_plane_nsg_ids subnet_id = var.control_plane_subnet_id } dynamic "image_policy_config" { for_each = var.use_signed_images ? [1] : [] content { is_policy_enabled = true dynamic "key_details" { iterator = signing_keys_iterator for_each = var.image_signing_keys content { kms_key_id = signing_keys_iterator.value } } } } options { dynamic "open_id_connect_token_authentication_config" { for_each = var.oidc_token_auth_enabled ? [1] : [] content { is_open_id_connect_auth_enabled = var.oidc_token_auth_enabled issuer_url = lookup(var.oidc_token_authentication_config, "issuer_url", null) ca_certificate = lookup(var.oidc_token_authentication_config, "ca_certificate", null) client_id = lookup(var.oidc_token_authentication_config, "client_id", null) signing_algorithms = lookup(var.oidc_token_authentication_config, "signing_algorithms", null) groups_claim = lookup(var.oidc_token_authentication_config, "groups_claim", null) groups_prefix = lookup(var.oidc_token_authentication_config, "groups_prefix", null) username_claim = lookup(var.oidc_token_authentication_config, "username_claim", null) username_prefix = lookup(var.oidc_token_authentication_config, "username_prefix", null) dynamic "required_claims" { for_each = lookup(var.oidc_token_authentication_config, "required_claims", []) content { key = lookup(required_claims.value, "key") value = lookup(required_claims.value, "value") } } configuration_file = lookup(var.oidc_token_authentication_config, "configuration_file", null) } } dynamic "open_id_connect_discovery" { for_each = var.oidc_discovery_enabled ? [1] : [] content { is_open_id_connect_discovery_enabled = var.oidc_discovery_enabled } } kubernetes_network_config { pods_cidr = var.pods_cidr services_cidr = var.services_cidr } persistent_volume_config { defined_tags = var.persistent_volume_defined_tags freeform_tags = var.persistent_volume_freeform_tags } service_lb_config { backend_nsg_ids = var.backend_nsg_ids defined_tags = var.service_lb_defined_tags freeform_tags = var.service_lb_freeform_tags } service_lb_subnet_ids = compact([var.service_lb_subnet_id]) ip_families = var.ip_families } timeouts { update = "120m" } lifecycle { ignore_changes = [ defined_tags, options[0].kubernetes_network_config ] precondition { condition = !var.use_signed_images || length(var.image_signing_keys) > 0 error_message = "Must provide at least 1 image signing key when use_signed_images is enabled." } precondition { condition = var.service_lb_subnet_id != null error_message = <<-EOT Must have a service load balancer subnet ID for the preferred load balancer type. control_plane_is_public: ${coalesce(var.control_plane_is_public, "none")} service_lb_subnet_id: ${coalesce(var.service_lb_subnet_id, "none")} EOT } } } ================================================ FILE: modules/cluster/outputs.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl output "cluster_id" { value = oci_containerengine_cluster.k8s_cluster.id } output "endpoints" { value = one(oci_containerengine_cluster.k8s_cluster.endpoints) } output "oidc_discovery_endpoint" { value = oci_containerengine_cluster.k8s_cluster.open_id_connect_discovery_endpoint } ================================================ FILE: modules/cluster/variables.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "compartment_id" { type = string } variable "state_id" { type = string } # Cluster variable "cluster_kms_key_id" { type = string } variable "cluster_name" { type = string } variable "cluster_type" { type = string } variable "cni_type" { type = string } variable "ip_families" { type = list(string) } variable "control_plane_is_public" { type = bool } variable "control_plane_nsg_ids" { type = set(string) } variable "assign_public_ip_to_control_plane" { type = bool } variable "control_plane_subnet_id" { type = string } variable "image_signing_keys" { type = set(string) } variable "kubernetes_version" { type = string } variable "pods_cidr" { type = string } variable "service_lb_subnet_id" { type = string } variable "services_cidr" { type = string } variable "tag_namespace" { type = string } variable "use_defined_tags" { type = string } variable "use_signed_images" { type = bool } variable "vcn_id" { type = string } variable "backend_nsg_ids" { type = set(string) } # Tagging variable "cluster_defined_tags" { type = map(string) } variable "cluster_freeform_tags" { type = map(string) } variable "persistent_volume_defined_tags" { type = map(string) } variable "persistent_volume_freeform_tags" { type = map(string) } variable "service_lb_defined_tags" { type = map(string) } variable "service_lb_freeform_tags" { type = map(string) } # OIDC variable "oidc_discovery_enabled" { type = bool } variable "oidc_token_auth_enabled" { type = bool } variable "oidc_token_authentication_config" { type = any } ================================================ FILE: modules/cluster/versions.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { oci = { source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/cluster-addons/README.md ================================================ # Cluster Add-ons This sub-module manages OKE cluster add-ons and their configurations. ## Usage Refer to the [Cluster Add-ons section](../../docs/terraformoptions.md#cluster-add-ons) of the module documentation. ================================================ FILE: modules/cluster-addons/addons.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_containerengine_addon_options" "k8s_addon_options" { kubernetes_version = var.kubernetes_version } locals { supported_addons = [for entry in data.oci_containerengine_addon_options.k8s_addon_options.addon_options : entry.name] primary_addons = ["CertManager"] addons_defaults = { remove_addon_resources_on_delete = true configurations = [] version = null } addons_with_defaults = { for addon_name, addon_value in var.cluster_addons : addon_name => merge(local.addons_defaults, addon_value) } } resource "oci_containerengine_addon" "primary_addon" { for_each = { for k, v in local.addons_with_defaults : k => v if contains(local.primary_addons, k) } addon_name = each.key cluster_id = var.cluster_id remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true) dynamic "configurations" { for_each = lookup(each.value, "configurations", []) iterator = config content { key = tostring(lookup(config.value, "key")) value = tostring(lookup(config.value, "value")) } } override_existing = lookup(each.value, "override_existing", false) version = lookup(each.value, "version", null) timeouts { create = "30m" } lifecycle { precondition { condition = contains(local.supported_addons, each.key) error_message = <<-EOT The addon ${each.key} is not supported. The list of supported addons is: ${join(", ", local.supported_addons)}. EOT } } } resource "oci_containerengine_addon" "secondary_addon" { for_each = { for k, v in local.addons_with_defaults : k => v if !contains(local.primary_addons, k) } depends_on = [oci_containerengine_addon.primary_addon] addon_name = each.key cluster_id = var.cluster_id remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true) dynamic "configurations" { for_each = lookup(each.value, "configurations", []) iterator = config content { key = tostring(lookup(config.value, "key")) value = tostring(lookup(config.value, "value")) } } override_existing = lookup(each.value, "override_existing", false) version = lookup(each.value, "version", null) timeouts { create = "30m" } lifecycle { precondition { condition = contains(local.supported_addons, each.key) error_message = <<-EOT The addon ${each.key} is not supported. The list of supported addons is: ${join(", ", local.supported_addons)}. EOT } } } ================================================ FILE: modules/cluster-addons/delete_addons.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { remove_addon_command = "oci ce cluster disable-addon --addon-name %s --cluster-id %s --is-remove-existing-add-on %t --force" remove_addons_defaults = { custom_commands = [] remove_k8s_resources = true } remove_addons_with_defaults = { for addon_name, addon_value in var.cluster_addons_to_remove : addon_name => merge(local.remove_addons_defaults, addon_value) } } resource "null_resource" "remove_addons" { for_each = var.operator_enabled ? local.remove_addons_with_defaults : {} depends_on = [oci_containerengine_addon.primary_addon, oci_containerengine_addon.secondary_addon] connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = concat( [ "echo 'Removing ${each.key} addon'", format(local.remove_addon_command, each.key, var.cluster_id, lookup(each.value, "remove_k8s_resources")) ], lookup(each.value, "custom_commands") ) } lifecycle { precondition { condition = contains(local.supported_addons, each.key) error_message = <<-EOT The addon ${each.key} is not supported. The list of supported addons is: ${join(", ", local.supported_addons)}. EOT } } } ================================================ FILE: modules/cluster-addons/outputs.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl output "supported_addons" { value = data.oci_containerengine_addon_options.k8s_addon_options.addon_options } ================================================ FILE: modules/cluster-addons/variables.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # General variables variable "cluster_id" { type = string } variable "cluster_addons" { type = any } variable "cluster_addons_to_remove" { type = any } variable "kubernetes_version" { type = string } # Variables required to access the operator host variable "bastion_host" { type = string } variable "bastion_user" { type = string } variable "operator_enabled" { type = bool } variable "operator_host" { type = string } variable "operator_user" { type = string } variable "ssh_private_key" { type = string } ================================================ FILE: modules/cluster-addons/versions.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { oci = { source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/extensions/README.md ================================================ # Extensions This sub-module deploys Kubernetes extensions via Helm charts or YAML manifests, including Cilium, Multus, Prometheus, Gatekeeper, ArgoCD, and more. ## Usage Refer to the [Extensions section](../../docs/terraformoptions.md#extensions) of the module documentation. ================================================ FILE: modules/extensions/argocd.tf ================================================ # Copyright (c) 2021, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { argocd_enabled = var.argocd_install && var.expected_node_count > 0 argocd_manifest = sensitive(one(data.helm_template.argocd[*].manifest)) argocd_manifest_path = join("/", [local.yaml_manifest_path, "argocd.yaml"]) } data "helm_template" "argocd" { count = local.argocd_enabled ? 1 : 0 chart = "argo-cd" repository = "https://argoproj.github.io/argo-helm" version = var.argocd_helm_version kube_version = var.kubernetes_version name = "argocd" namespace = var.argocd_namespace create_namespace = true include_crds = true skip_tests = true values = length(var.argocd_helm_values_files) > 0 ? [ for path in var.argocd_helm_values_files : file(path) ] : null set = concat( [ for k, v in var.argocd_helm_values: { name = k, value = v } ] ) lifecycle { precondition { condition = alltrue([for path in var.argocd_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.argocd_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "argocd" { count = local.argocd_enabled ? 1 : 0 triggers = { manifest_md5 = try(md5(local.argocd_manifest), null) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.argocd_manifest destination = local.argocd_manifest_path } provisioner "remote-exec" { inline = [for c in compact([ (contains(["kube-system", "default"], var.argocd_namespace) ? null : format(local.kubectl_create_missing_ns, var.argocd_namespace)), format(local.kubectl_apply_server_file, local.argocd_manifest_path), ]) : format(local.output_log, c, "argocd") ] } } ================================================ FILE: modules/extensions/autoscaler.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler # https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengautoscalingclusters.htm locals { # Active worker pools that should be managed by the cluster autoscaler worker_pools_autoscaling = { for k, v in var.worker_pools : k => v if tobool(lookup(v, "autoscale", false)) } # Whether to enable cluster autoscaler deployment based on configuration, active nodes, and autoscaling pools cluster_autoscaler_enabled = alltrue([ var.cluster_autoscaler_install, var.expected_node_count > 0, var.expected_autoscale_worker_pools > 0, ]) # Templated Helm manifest values cluster_autoscaler_manifest = sensitive(one(data.helm_template.cluster_autoscaler[*].manifest)) cluster_autoscaler_manifest_path = join("/", [local.yaml_manifest_path, "cluster_autoscaler.yaml"]) cluster_autoscaler_defaults = { "cloudProvider" = "oci-oke", "extraArgs.logtostderr" = "true", "extraArgs.v" = "4", "extraArgs.stderrthreshold" = "info", "extraArgs.max-node-provision-time" = "25m", "extraArgs.scale-down-unneeded-time" = "2m", "extraArgs.unremovable-node-recheck-timeout" = "5m", "extraArgs.balance-similar-node-groups" = "true", "extraArgs.balancing-ignore-label" = "displayName", "extraArgs.balancing-ignore-label" = "hostname", "extraArgs.balancing-ignore-label" = "internal_addr", "extraArgs.balancing-ignore-label" = "oci.oraclecloud.com/fault-domain", "extraEnv.OCI_REGION" = var.region, "extraEnv.OCI_USE_INSTANCE_PRINCIPAL" = "true", "extraEnv.OKE_USE_INSTANCE_PRINCIPAL" = "true", "extraEnv.OCI_SDK_APPEND_USER_AGENT" = "oci-oke-cluster-autoscaler", "image.repository" = "iad.ocir.io/oracle/oci-cluster-autoscaler", "image.tag" = "1.26.2-7", } } data "helm_template" "cluster_autoscaler" { count = local.cluster_autoscaler_enabled ? 1 : 0 chart = "cluster-autoscaler" repository = "https://kubernetes.github.io/autoscaler" version = var.cluster_autoscaler_helm_version kube_version = var.kubernetes_version name = "cluster-autoscaler" namespace = var.cluster_autoscaler_namespace create_namespace = true include_crds = true skip_tests = true values = length(var.cluster_autoscaler_helm_values_files) > 0 ? [ for path in var.cluster_autoscaler_helm_values_files : file(path) ] : null set = concat( [ { name = "nodeSelector.oke\\.oraclecloud\\.com/cluster_autoscaler" value = "allowed" } ], [ for k, v in merge(local.cluster_autoscaler_defaults, var.cluster_autoscaler_helm_values) : { name = k, value = v } ], [ for k, v in local.worker_pools_autoscaling : { name = "autoscalingGroups[${index(keys(local.worker_pools_autoscaling), k)}].name", value = lookup(v, "id") } ], [ for k, v in local.worker_pools_autoscaling : { name = "autoscalingGroups[${index(keys(local.worker_pools_autoscaling), k)}].minSize", value = lookup(v, "min_size", lookup(v, "size")) } ], [ for k, v in local.worker_pools_autoscaling : { name = "autoscalingGroups[${index(keys(local.worker_pools_autoscaling), k)}].maxSize", value = lookup(v, "max_size", lookup(v, "size")) } ], ) lifecycle { precondition { condition = alltrue([for path in var.cluster_autoscaler_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.cluster_autoscaler_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "cluster_autoscaler" { count = local.cluster_autoscaler_enabled ? 1 : 0 triggers = { manifest_md5 = try(md5(local.cluster_autoscaler_manifest), null) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.cluster_autoscaler_manifest destination = local.cluster_autoscaler_manifest_path } provisioner "remote-exec" { inline = [ "kubectl apply -f ${local.cluster_autoscaler_manifest_path}" ] } } ================================================ FILE: modules/extensions/cilium.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { cilium_helm_crds_file = join("/", [local.yaml_manifest_path, "cilium.crds.yaml"]) cilium_helm_manifest_file = join("/", [local.yaml_manifest_path, "cilium.manifest.yaml"]) cilium_helm_values_file = join("/", [local.yaml_manifest_path, "cilium.values.yaml"]) cilium_helm_values_override_file = join("/", [local.yaml_manifest_path, "cilium.values-override.yaml"]) cilium_net_attach_def_file = join("/", [local.yaml_manifest_path, "cilium.net_attach_def.yaml"]) cilium_veth_config_map_file = join("/", [local.yaml_manifest_path, "cilium.cni_config_map.yaml"]) cilium_helm_crds = one(data.helm_template.cilium[*].crds) cilium_helm_values_override = one(data.helm_template.cilium[*].values) cilium_helm_repository = "https://helm.cilium.io" cilium_vxlan_cni = { install = true exclusive = true # !var.multus_install } cilium_helm_values = { annotateK8sNode = true cluster = { name = "oke-${var.state_id}" id = 1 } clustermesh = { useAPIServer = false apiserver = { kvstoremesh = { enabled = false } } } cni = local.cilium_vxlan_cni installNoConntrackIptablesRules = false ipam = { mode = "kubernetes" } kubeProxyReplacement = false k8sServiceHost = var.cluster_private_endpoint k8sServicePort = "6443" pmtuDiscovery = { enabled = true } rollOutCiliumPods = true tunnelProtocol = local.cilium_tunnel hubble = { metrics = { dashboards = { enabled = var.prometheus_install } # serviceMonitor = { enabled = var.prometheus_enabled } } relay = { enabled = true } ui = { enabled = true } } k8s = { requireIPv4PodCIDR = true # wait for Kubernetes to provide the PodCIDR (ipam kubernetes) } # Prometheus metrics operator = { prometheus = { enabled = var.prometheus_install # serviceMonitor = { enabled = var.prometheus_enabled } } } } # TODO Support Flannel w/ generic-veth & tunnel disabled cilium_tunnel = "vxlan" # var.cni_type == "flannel" ? "disabled" : "vxlan" cilium_flannel_cni = { install = true chainingMode = "generic-veth" configMap = "cni-configuration" customConf = var.cni_type == "flannel" exclusive = !var.multus_install } cilium_net_attach_def_conf = { cniVersion = "0.3.1" name = "cilium" plugins = [ { cniVersion = "0.3.1" name = "cilium" type = "cilium-cni" }, { name = "cilium-sbr" type = "sbr" } ], } cilium_net_attach_def = { apiVersion = "k8s.cni.cncf.io/v1" kind = "NetworkAttachmentDefinition" metadata = { name = "cilium" } spec = { config = jsonencode(local.cilium_net_attach_def_conf) } } cilium_veth_conf = { cniVersion = "0.3.1" name = "cbr0" "plugins" = [ { type = "flannel" delegate = { hairpinMode = true isDefaultGateway = true } }, { type = "portmap" capabilities = { portMappings = true } }, { type = "cilium-cni" }, ] } cilium_veth_config_map = { apiVersion = "v1" kind = "ConfigMap" metadata = { name = "cni-configuration" namespace = var.cilium_namespace } data = { "cni-config" = jsonencode(local.cilium_veth_conf) } } cilium_net_attach_def_yaml = yamlencode(local.cilium_net_attach_def) cilium_veth_config_map_yaml = yamlencode(local.cilium_veth_config_map) cilium_helm_values_yaml = yamlencode(merge(local.cilium_helm_values, var.cilium_helm_values)) cilium_helm_values_override_yaml = local.cilium_helm_values_override != null ? join("\n", local.cilium_helm_values_override) : "" } data "helm_template" "cilium" { count = var.cilium_install ? 1 : 0 chart = "cilium" repository = local.cilium_helm_repository version = var.cilium_helm_version kube_version = var.kubernetes_version name = "cilium" namespace = var.cilium_namespace create_namespace = true include_crds = true skip_tests = true values = concat( [local.cilium_helm_values_yaml], [for path in var.cilium_helm_values_files : file(path)], ) lifecycle { precondition { condition = alltrue([for path in var.cilium_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.cilium_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "cilium" { count = var.cilium_install ? 1 : 0 depends_on = [null_resource.prometheus] triggers = { helm_version = var.cilium_helm_version crds_md5 = try(md5(join("\n", local.cilium_helm_crds)), null) manifest_md5 = try(md5(local.cilium_helm_values_override_yaml), null) reapply = var.cilium_reapply ? uuid() : null } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = join("\n", local.cilium_helm_crds) destination = local.cilium_helm_crds_file } provisioner "file" { content = local.cilium_helm_values_override_yaml destination = local.cilium_helm_values_override_file } # provisioner "file" { # content = local.cilium_net_attach_def_yaml # destination = local.cilium_net_attach_def_file # } # provisioner "file" { # content = local.cilium_veth_config_map_yaml # destination = local.cilium_veth_config_map_file # } provisioner "remote-exec" { inline = [for c in compact([ # Create namespace if non-standard and missing (contains(["kube-system", "default"], var.cilium_namespace) ? null : format(local.kubectl_create_missing_ns, var.cilium_namespace)), # Install CRDs first format(local.kubectl_apply_server_ns_file, var.cilium_namespace, local.cilium_helm_crds_file), # Install full manifest format(local.helm_upgrade_install, "cilium", "cilium", local.cilium_helm_repository, var.cilium_helm_version, var.cilium_namespace, local.cilium_helm_values_override_file), # Install Network Attachment Definition when Multus is enabled # var.multus_install ? format(local.kubectl_apply_file, local.cilium_net_attach_def_file) : null, # Install CNI ConfigMap for Flannel # var.cni_type == "flannel" ? format(local.kubectl_apply_file, local.cilium_veth_config_map_file) : null, ]) : format(local.output_log, c, "cilium") ] } lifecycle { precondition { condition = var.cni_type == "flannel" error_message = "Incompatible cni_type for installation - must be 'flannel'." } } } ================================================ FILE: modules/extensions/dcgm_exporter.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { dcgm_exporter_helm_crds_file = join("/", [local.yaml_manifest_path, "dcgm_exporter.crds.yaml"]) dcgm_exporter_helm_manifest_file = join("/", [local.yaml_manifest_path, "dcgm_exporter.manifest.yaml"]) dcgm_exporter_helm_crds = sensitive(one(data.helm_template.dcgm_exporter[*].crds)) dcgm_exporter_helm_manifest = sensitive(one(data.helm_template.dcgm_exporter[*].manifest)) dcgm_exporter_scrape_config = [ { job_name = "gpu-metrics" scrape_interval = "10s" metrics_path = "/metrics" scheme = "http" kubernetes_sd_configs = [ { role = "endpoints" namespaces = { names = ["metrics"] } selectors = [{ label = "app.kubernetes.io/component=dcgm-exporter" }] } ] relabel_configs = [ { source_labels = ["__meta_kubernetes_pod_node_name"] action = "replace" target_label = "kubernetes_node" } ] } ] } data "helm_template" "dcgm_exporter" { count = var.dcgm_exporter_install ? 1 : 0 chart = "dcgm-exporter" repository = "https://nvidia.github.io/dcgm-exporter/helm-charts" version = var.dcgm_exporter_helm_version kube_version = var.kubernetes_version name = "dcgm-exporter" namespace = var.dcgm_exporter_namespace create_namespace = true include_crds = true skip_tests = true values = length(var.dcgm_exporter_helm_values_files) > 0 ? [ for path in var.dcgm_exporter_helm_values_files : file(path) ] : null set = concat( [ for k, v in var.dcgm_exporter_helm_values: { name = k, value = v } ] ) lifecycle { precondition { condition = alltrue([for path in var.dcgm_exporter_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.dcgm_exporter_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "dcgm_exporter" { count = var.dcgm_exporter_install ? 1 : 0 triggers = { helm_version = var.dcgm_exporter_helm_version crds_md5 = try(md5(join("\n", local.dcgm_exporter_helm_crds)), null) manifest_md5 = try(md5(local.dcgm_exporter_helm_manifest), null) reapply = var.dcgm_exporter_reapply ? uuid() : null } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = join("\n", local.dcgm_exporter_helm_crds) destination = local.dcgm_exporter_helm_crds_file } provisioner "file" { content = local.dcgm_exporter_helm_manifest destination = local.dcgm_exporter_helm_manifest_file } provisioner "remote-exec" { inline = [for c in compact([ (contains(["kube-system", "default"], var.dcgm_exporter_namespace) ? null : format(local.kubectl_create_missing_ns, var.dcgm_exporter_namespace)), format(local.kubectl_apply_server_file, local.dcgm_exporter_helm_crds_file), format(local.kubectl_apply_server_file, local.dcgm_exporter_helm_manifest_file), ]) : format(local.output_log, c, "dcgm_exporter") ] } } ================================================ FILE: modules/extensions/gatekeeper.tf ================================================ # Copyright (c) 2021, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { gatekeeper_enabled = var.gatekeeper_install && var.expected_node_count > 0 gatekeeper_manifest = sensitive(one(data.helm_template.gatekeeper[*].manifest)) gatekeeper_manifest_path = join("/", [local.yaml_manifest_path, "gatekeeper.yaml"]) } data "helm_template" "gatekeeper" { count = local.gatekeeper_enabled ? 1 : 0 chart = "gatekeeper" repository = "https://open-policy-agent.github.io/gatekeeper/charts" version = var.gatekeeper_helm_version kube_version = var.kubernetes_version name = "gatekeeper" namespace = var.gatekeeper_namespace create_namespace = true include_crds = true skip_tests = true values = length(var.gatekeeper_helm_values_files) > 0 ? [ for path in var.gatekeeper_helm_values_files : file(path) ] : null set = concat( [ for k, v in var.gatekeeper_helm_values: { name = k, value = v } ] ) lifecycle { precondition { condition = alltrue([for path in var.gatekeeper_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.gatekeeper_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "gatekeeper" { count = local.gatekeeper_enabled ? 1 : 0 triggers = { manifest_md5 = try(md5(local.gatekeeper_manifest), null) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.gatekeeper_manifest destination = local.gatekeeper_manifest_path } provisioner "remote-exec" { inline = ["kubectl apply --force-conflicts=true --server-side -f ${local.gatekeeper_manifest_path}"] } } ================================================ FILE: modules/extensions/locals.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { yaml_manifest_path = "/home/${var.operator_user}/yaml" kubectl = "set -o pipefail; kubectl" helm = "set -o pipefail; helm" kubectl_apply_ns_file = "${local.kubectl} apply -n %s -f %s" kubectl_apply_file = "${local.kubectl} apply -f %s" kubectl_apply_server_file = "${local.kubectl} apply --force-conflicts=true --server-side -f %s" kubectl_apply_server_ns_file = "${local.kubectl} apply -n %s --force-conflicts=true --server-side -f %s" kubectl_create_missing_ns = "${local.kubectl} create ns %s --dry-run=client -o yaml | kubectl apply -f -" selector_linux = { "kubernetes.io/os" = "linux" } output_log = "bash -c \"%s | tee >(systemd-cat -t %s -p info)\"" helm_upgrade_install = "${local.helm} upgrade --install %s %s --repo %s --version %s --namespace %s --create-namespace --skip-crds -f %s" } ================================================ FILE: modules/extensions/metricserver.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { metrics_server_enabled = var.metrics_server_install && var.expected_node_count > 0 metrics_server_manifest = sensitive(one(data.helm_template.metrics_server[*].manifest)) metrics_server_manifest_path = join("/", [local.yaml_manifest_path, "metrics_server.manifest.yaml"]) } data "helm_template" "metrics_server" { count = local.metrics_server_enabled ? 1 : 0 chart = "metrics-server" repository = "https://kubernetes-sigs.github.io/metrics-server" version = var.metrics_server_helm_version kube_version = var.kubernetes_version name = "metrics-server" namespace = var.metrics_server_namespace create_namespace = true include_crds = true skip_tests = true values = length(var.metrics_server_helm_values_files) > 0 ? [ for path in var.metrics_server_helm_values_files : file(path) ] : null set = concat( [ for k, v in var.metrics_server_helm_values: { name = k, value = v } ] ) lifecycle { precondition { condition = alltrue([for path in var.metrics_server_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.metrics_server_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "metrics_server" { count = local.metrics_server_enabled ? 1 : 0 triggers = { manifest_md5 = try(md5(local.metrics_server_manifest), null) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.metrics_server_manifest destination = local.metrics_server_manifest_path } provisioner "remote-exec" { inline = compact([ (contains(["kube-system", "default"], var.metrics_server_namespace) ? null : format(local.kubectl_create_missing_ns, var.metrics_server_namespace)), format(local.kubectl_apply_server_file, local.metrics_server_manifest_path), ]) } } ================================================ FILE: modules/extensions/mpi_operator.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { mpi_operator_url = "https://raw.githubusercontent.com/kubeflow/mpi-operator/v${var.mpi_operator_version}/deploy/v2beta1" mpi_operator_deployment_url = coalesce( var.mpi_operator_deployment_url, "${local.mpi_operator_url}/mpi-operator.yaml" ) mpi_operator_manifest_path = join("/", [local.yaml_manifest_path, "mpi-operator.manifest.yaml"]) mpi_operator_manifest_status_code = one(data.http.mpi_operator[*].status_code) mpi_operator_manifest_content = sensitive(one(data.http.mpi_operator[*].response_body)) } data "http" "mpi_operator" { count = var.mpi_operator_install ? 1 : 0 url = local.mpi_operator_deployment_url } resource "null_resource" "mpi_operator" { count = var.mpi_operator_install ? 1 : 0 triggers = { mpi_operator_deployment_url = local.mpi_operator_deployment_url mpi_operator_deployment_md5 = md5(local.mpi_operator_manifest_content) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.mpi_operator_manifest_content destination = local.mpi_operator_manifest_path } provisioner "remote-exec" { inline = [ format(local.kubectl_apply_file, local.mpi_operator_manifest_path), ] } lifecycle { precondition { condition = local.mpi_operator_manifest_status_code == 200 error_message = <<-EOT Error retrieving MPI Operator manifest URL: ${local.mpi_operator_deployment_url} Status code: ${local.mpi_operator_manifest_status_code} Response: ${local.mpi_operator_manifest_content} EOT } } } ================================================ FILE: modules/extensions/multus.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { multus_url = "https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni" multus_daemonset_url = coalesce( var.multus_daemonset_url, "${local.multus_url}/v${var.multus_version}/deployments/multus-daemonset.yml" ) multus_manifest_path = join("/", [local.yaml_manifest_path, "multus.manifest.yaml"]) multus_manifest_status_code = one(data.http.multus[*].status_code) multus_manifest_content = sensitive(one(data.http.multus[*].response_body)) } data "http" "multus" { count = var.multus_install ? 1 : 0 url = local.multus_daemonset_url } resource "null_resource" "multus" { count = var.multus_install ? 1 : 0 triggers = { multus_daemonset_url = local.multus_daemonset_url multus_daemonset_md5 = md5(local.multus_manifest_content) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.multus_manifest_content destination = local.multus_manifest_path } provisioner "remote-exec" { inline = [ format(local.kubectl_apply_file, local.multus_manifest_path), ] } lifecycle { precondition { condition = local.multus_manifest_status_code == 200 error_message = <<-EOT Error retrieving Multus Daemonset manifest Status code: ${local.multus_manifest_status_code} Response: ${local.multus_manifest_content} EOT } } } ================================================ FILE: modules/extensions/prometheus.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { prometheus_helm_crds_file = join("/", [local.yaml_manifest_path, "prometheus.crds.yaml"]) prometheus_helm_manifest_file = join("/", [local.yaml_manifest_path, "prometheus.manifest.yaml"]) prometheus_helm_values_file = join("/", [local.yaml_manifest_path, "prometheus.values.yaml"]) prometheus_helm_crds = sensitive(one(data.helm_template.prometheus[*].crds)) prometheus_helm_manifest = sensitive(one(data.helm_template.prometheus[*].manifest)) prometheus_helm_values = { additionalScrapeConfigs = local.dcgm_exporter_scrape_config } prometheus_helm_values_yaml = jsonencode(local.prometheus_helm_values) } data "helm_template" "prometheus" { count = var.prometheus_install ? 1 : 0 chart = "kube-prometheus-stack" repository = "https://prometheus-community.github.io/helm-charts" version = var.prometheus_helm_version kube_version = var.kubernetes_version name = "prometheus" namespace = var.prometheus_namespace create_namespace = true include_crds = true skip_tests = true values = concat( [local.prometheus_helm_values_yaml], [for path in var.prometheus_helm_values_files : file(path)], ) set = concat( [ { name = "podSecurityPolicy.enabled" value = "false" }, ], [ for k, v in var.prometheus_helm_values: { name = k, value = v } ] ) lifecycle { precondition { condition = alltrue([for path in var.prometheus_helm_values_files : fileexists(path)]) error_message = format("Missing Helm values files in configuration: %s", jsonencode([for path in var.prometheus_helm_values_files : path if !fileexists(path)]) ) } } } resource "null_resource" "prometheus" { count = var.prometheus_install ? 1 : 0 triggers = { helm_version = var.prometheus_helm_version crds_md5 = try(md5(join("\n", local.prometheus_helm_crds)), null) manifest_md5 = try(md5(local.prometheus_helm_manifest), null) reapply = var.prometheus_reapply ? uuid() : null } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = join("\n", local.prometheus_helm_crds) destination = local.prometheus_helm_crds_file } provisioner "file" { content = local.prometheus_helm_manifest destination = local.prometheus_helm_manifest_file } provisioner "file" { content = local.prometheus_helm_values_yaml destination = local.prometheus_helm_values_file } provisioner "remote-exec" { inline = [for c in compact([ (contains(["kube-system", "default"], var.prometheus_namespace) ? null : format(local.kubectl_create_missing_ns, var.prometheus_namespace)), format(local.kubectl_apply_server_file, local.prometheus_helm_crds_file), format(local.kubectl_apply_server_file, local.prometheus_helm_manifest_file), ]) : format(local.output_log, c, "prometheus") ] } } ================================================ FILE: modules/extensions/rdma_cni_plugin.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { rdma_cni_plugin_url = "https://raw.githubusercontent.com/k8snetworkplumbingwg/rdma-cni" rdma_cni_plugin_daemonset_url = coalesce( var.rdma_cni_plugin_daemonset_url, "${local.rdma_cni_plugin_url}/${var.rdma_cni_plugin_version}/deployment/rdma-cni-daemonset.yaml" ) rdma_cni_plugin_manifest_path = join("/", [local.yaml_manifest_path, "rdma-cni-daemonset.yaml"]) rdma_cni_plugin_manifest_status_code = one(data.http.rdma_cni_plugin[*].status_code) rdma_cni_plugin_manifest_content = one(data.http.rdma_cni_plugin[*].response_body) } data "http" "rdma_cni_plugin" { count = var.rdma_cni_plugin_install ? 1 : 0 url = local.rdma_cni_plugin_daemonset_url } resource "null_resource" "rdma_cni_plugin" { count = var.rdma_cni_plugin_install ? 1 : 0 triggers = { rdma_cni_plugin_daemonset_url = local.rdma_cni_plugin_daemonset_url rdma_cni_plugin_daemonset_md5 = md5(local.rdma_cni_plugin_manifest_content) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.rdma_cni_plugin_manifest_content destination = local.rdma_cni_plugin_manifest_path } provisioner "remote-exec" { inline = [ format(local.kubectl_apply_file, local.rdma_cni_plugin_manifest_path), ] } lifecycle { precondition { condition = local.rdma_cni_plugin_manifest_status_code == 200 error_message = <<-EOT Error retrieving RDMA CNI Daemonset manifest URL: ${local.rdma_cni_plugin_daemonset_url} Status code: ${local.rdma_cni_plugin_manifest_status_code} Response: ${local.rdma_cni_plugin_manifest_content} EOT } } } ================================================ FILE: modules/extensions/service_account.tf ================================================ # Copyright (c) 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { sa_with_cluster_role_bindings = { for k, v in var.service_accounts : k => v if lookup(v, "sa_cluster_role_binding", null) != null } sa_with_role_bindings = { for k, v in var.service_accounts : k => v if lookup(v, "sa_role_binding", null) != null } } resource "null_resource" "service_account_crb" { for_each = var.create_service_account ? local.sa_with_cluster_role_bindings : {} triggers = { service_account_name = each.value.sa_name service_account_namespace = each.value.sa_namespace service_account_cluster_role = each.value.sa_cluster_role service_account_cluster_role_binding = each.value.sa_cluster_role_binding # Parameters ignored as triggers in the life_cycle block. Required to establish connections. bastion_host = var.bastion_host bastion_user = var.bastion_user ssh_private_key = var.ssh_private_key operator_host = var.operator_host operator_user = var.operator_user } connection { bastion_host = self.triggers.bastion_host bastion_user = self.triggers.bastion_user bastion_private_key = self.triggers.ssh_private_key host = self.triggers.operator_host user = self.triggers.operator_user private_key = self.triggers.ssh_private_key timeout = "10m" type = "ssh" } provisioner "remote-exec" { inline = [ "kubectl get ns ${self.triggers.service_account_namespace} || kubectl create ns ${self.triggers.service_account_namespace}", "kubectl create sa -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_name}", "kubectl create clusterrolebinding ${self.triggers.service_account_cluster_role_binding} --clusterrole=${self.triggers.service_account_cluster_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" ] } provisioner "remote-exec" { when = destroy on_failure = continue inline = [ "kubectl delete clusterrolebinding ${self.triggers.service_account_cluster_role_binding}", "kubectl delete sa -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_name}" ] } lifecycle { ignore_changes = [ triggers["bastion_host"], triggers["bastion_user"], triggers["ssh_private_key"], triggers["operator_host"], triggers["operator_user"] ] } } resource "null_resource" "service_account_rb" { for_each = var.create_service_account ? local.sa_with_role_bindings : {} triggers = { service_account_name = each.value.sa_name service_account_namespace = each.value.sa_namespace service_account_cluster_role = each.value.sa_cluster_role service_account_role = lookup(each.value, "sa_role", "") service_account_role_binding = each.value.sa_role_binding # Parameters ignored as triggers in the life_cycle block. Required to establish connections. bastion_host = var.bastion_host bastion_user = var.bastion_user ssh_private_key = var.ssh_private_key operator_host = var.operator_host operator_user = var.operator_user } connection { bastion_host = self.triggers.bastion_host bastion_user = self.triggers.bastion_user bastion_private_key = self.triggers.ssh_private_key host = self.triggers.operator_host user = self.triggers.operator_user private_key = self.triggers.ssh_private_key timeout = "10m" type = "ssh" } provisioner "remote-exec" { inline = [ "kubectl get ns ${self.triggers.service_account_namespace} || kubectl create ns ${self.triggers.service_account_namespace}", "kubectl create sa -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_name}", self.triggers.service_account_role != "" ? "kubectl create rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding} --role=${self.triggers.service_account_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" : "kubectl create rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding} --clusterrole=${self.triggers.service_account_cluster_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" ] } provisioner "remote-exec" { when = destroy on_failure = continue inline = [ "kubectl delete rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding}", "kubectl delete sa -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_name}" ] } lifecycle { ignore_changes = [ triggers["bastion_host"], triggers["bastion_user"], triggers["ssh_private_key"], triggers["operator_host"], triggers["operator_user"] ] } } ================================================ FILE: modules/extensions/sriov_cni_plugin.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { sriov_cni_plugin_url = "https://raw.githubusercontent.com/openshift/sriov-cni" sriov_cni_plugin_daemonset_url = coalesce( var.sriov_cni_plugin_daemonset_url, "${local.sriov_cni_plugin_url}/${var.sriov_cni_plugin_version}/images/k8s-v1.16/sriov-cni-daemonset.yaml" ) sriov_cni_plugin_manifest_path = join("/", [local.yaml_manifest_path, "sriov_cni_plugin-manifest.yaml"]) sriov_cni_plugin_manifest_status_code = one(data.http.sriov_cni_plugin[*].status_code) sriov_cni_plugin_manifest_content = sensitive(one(data.http.sriov_cni_plugin[*].response_body)) } data "http" "sriov_cni_plugin" { count = var.sriov_cni_plugin_install ? 1 : 0 url = local.sriov_cni_plugin_daemonset_url } resource "null_resource" "sriov_cni_plugin" { count = var.sriov_cni_plugin_install ? 1 : 0 triggers = { sriov_cni_plugin_daemonset_url = local.sriov_cni_plugin_daemonset_url sriov_cni_plugin_daemonset_md5 = md5(local.sriov_cni_plugin_manifest_content) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.sriov_cni_plugin_manifest_content destination = local.sriov_cni_plugin_manifest_path } provisioner "remote-exec" { inline = [ format(local.kubectl_apply_file, local.sriov_cni_plugin_manifest_path), ] } lifecycle { precondition { condition = local.sriov_cni_plugin_manifest_status_code == 200 error_message = <<-EOT Error retrieving SR-IOV CNI Daemonset manifest URL: ${local.sriov_cni_plugin_daemonset_url} Status code: ${local.sriov_cni_plugin_manifest_status_code} Response: ${local.sriov_cni_plugin_manifest_content} EOT } } } ================================================ FILE: modules/extensions/sriov_device_plugin.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { sriov_device_plugin_url = "https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin" sriov_device_plugin_daemonset_url = coalesce( var.sriov_device_plugin_daemonset_url, "${local.sriov_device_plugin_url}/${var.sriov_device_plugin_version}/deployments/sriovdp-daemonset.yaml" ) sriov_device_plugin_manifest_path = join("/", [local.yaml_manifest_path, "sriov_device_plugin-manifest.yaml"]) sriov_device_plugin_manifest_status_code = one(data.http.sriov_device_plugin[*].status_code) sriov_device_plugin_manifest_content = sensitive(one(data.http.sriov_device_plugin[*].response_body)) } data "http" "sriov_device_plugin" { count = var.sriov_device_plugin_install ? 1 : 0 url = local.sriov_device_plugin_daemonset_url } resource "null_resource" "sriov_device_plugin" { count = var.sriov_device_plugin_install ? 1 : 0 triggers = { sriov_device_plugin_daemonset_url = local.sriov_device_plugin_daemonset_url sriov_device_plugin_daemonset_md5 = md5(local.sriov_device_plugin_manifest_content) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.sriov_device_plugin_manifest_content destination = local.sriov_device_plugin_manifest_path } provisioner "remote-exec" { inline = [ format(local.kubectl_apply_file, local.sriov_device_plugin_manifest_path), ] } lifecycle { precondition { condition = local.sriov_device_plugin_manifest_status_code == 200 error_message = <<-EOT Error retrieving SR-IOV Daemonset manifest URL: ${local.sriov_device_plugin_daemonset_url} Status code: ${local.sriov_device_plugin_manifest_status_code} Response: ${local.sriov_device_plugin_manifest_content} EOT } } } ================================================ FILE: modules/extensions/variables.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "region" { type = string } variable "state_id" { type = string } variable "worker_pools" { type = any } variable "kubernetes_version" { type = string } variable "expected_node_count" { type = number } variable "cluster_private_endpoint" { type = string } # Connection variable "bastion_host" { type = string } variable "bastion_user" { type = string } variable "operator_host" { type = string } variable "operator_user" { type = string } variable "ssh_private_key" { type = string } # CNI variable "vcn_cidrs" { type = list(string) } variable "cni_type" { type = string } variable "pods_cidr" { type = string } # CNI: Cilium variable "cilium_install" { type = bool } variable "cilium_reapply" { type = bool } variable "cilium_namespace" { type = string } variable "cilium_helm_version" { type = string } variable "cilium_helm_values" { type = any } variable "cilium_helm_values_files" { type = list(string) } # CNI: Multus variable "multus_install" { type = bool } variable "multus_namespace" { type = string } variable "multus_daemonset_url" { type = string } variable "multus_version" { type = string } # SR-IOV Device Plugin variable "sriov_device_plugin_install" { type = bool } variable "sriov_device_plugin_namespace" { type = string } variable "sriov_device_plugin_daemonset_url" { type = string } variable "sriov_device_plugin_version" { type = string } # SR-IOV CNI Plugin variable "sriov_cni_plugin_install" { type = bool } variable "sriov_cni_plugin_namespace" { type = string } variable "sriov_cni_plugin_daemonset_url" { type = string } variable "sriov_cni_plugin_version" { type = string } # RDMA CNI Plugin variable "rdma_cni_plugin_install" { type = bool } variable "rdma_cni_plugin_namespace" { type = string } variable "rdma_cni_plugin_daemonset_url" { type = string } variable "rdma_cni_plugin_version" { type = string } # Whereabouts variable "whereabouts_install" { type = bool } variable "whereabouts_namespace" { type = string } variable "whereabouts_daemonset_url" { type = string } variable "whereabouts_version" { type = string } # Metrics server variable "metrics_server_install" { type = bool } variable "metrics_server_namespace" { type = string } variable "metrics_server_helm_version" { type = string } variable "metrics_server_helm_values" { type = map(string) } variable "metrics_server_helm_values_files" { type = list(string) } # Cluster autoscaler variable "cluster_autoscaler_install" { type = bool } variable "cluster_autoscaler_namespace" { type = string } variable "cluster_autoscaler_helm_version" { type = string } variable "cluster_autoscaler_helm_values" { type = map(string) } variable "cluster_autoscaler_helm_values_files" { type = list(string) } variable "expected_autoscale_worker_pools" { type = number } # Prometheus variable "prometheus_install" { type = bool } variable "prometheus_reapply" { type = bool } variable "prometheus_namespace" { type = string } variable "prometheus_helm_version" { type = string } variable "prometheus_helm_values" { type = map(string) } variable "prometheus_helm_values_files" { type = list(string) } # DCGM exporter variable "dcgm_exporter_install" { type = bool } variable "dcgm_exporter_reapply" { type = bool } variable "dcgm_exporter_namespace" { type = string } variable "dcgm_exporter_helm_version" { type = string } variable "dcgm_exporter_helm_values" { type = map(string) } variable "dcgm_exporter_helm_values_files" { type = list(string) } # MPI Operator variable "mpi_operator_install" { type = bool } variable "mpi_operator_namespace" { type = string } variable "mpi_operator_deployment_url" { type = string } variable "mpi_operator_version" { type = string } # Gatekeeper variable "gatekeeper_install" { type = bool } variable "gatekeeper_namespace" { type = string } variable "gatekeeper_helm_version" { type = string } variable "gatekeeper_helm_values" { type = map(string) } variable "gatekeeper_helm_values_files" { type = list(string) } # Service Account variable "create_service_account" { type = bool } variable "service_accounts" { type = map(any) } # Argocd variable "argocd_install" { type = bool } variable "argocd_namespace" { type = string } variable "argocd_helm_version" { type = string } variable "argocd_helm_values" { type = map(string) } variable "argocd_helm_values_files" { type = list(string) } ================================================ FILE: modules/extensions/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { helm = { source = "hashicorp/helm" version = ">= 3.0.1" } http = { source = "hashicorp/http" version = ">= 3.2.1" } null = { source = "hashicorp/null" version = ">= 3.2.1" } } } ================================================ FILE: modules/extensions/whereabouts.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { whereabouts_url = "https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/${var.whereabouts_version}/doc/crds" whereabouts_ippools_url = "${local.whereabouts_url}/whereabouts.cni.cncf.io_ippools.yaml" whereabouts_overlap_res_url = "${local.whereabouts_url}/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml" whereabouts_daemonset_url = coalesce(var.whereabouts_daemonset_url, "${local.whereabouts_url}/daemonset-install.yaml") whereabouts_ippools_path = join("/", [local.yaml_manifest_path, "whereabouts.ippools.crd.yaml"]) whereabouts_ippools_status_code = one(data.http.whereabouts_ippools[*].status_code) whereabouts_ippools_content = sensitive(one(data.http.whereabouts_ippools[*].response_body)) whereabouts_overlap_res_path = join("/", [local.yaml_manifest_path, "whereabouts.overlap_res.crd.yaml"]) whereabouts_overlap_res_status_code = one(data.http.whereabouts_overlap_res[*].status_code) whereabouts_overlap_res_content = sensitive(one(data.http.whereabouts_overlap_res[*].response_body)) whereabouts_manifest_path = join("/", [local.yaml_manifest_path, "whereabouts.manifest.yaml"]) whereabouts_manifest_status_code = one(data.http.whereabouts_daemonset[*].status_code) whereabouts_manifest_content = sensitive(one(data.http.whereabouts_daemonset[*].response_body)) } data "http" "whereabouts_ippools" { count = var.whereabouts_install ? 1 : 0 url = local.whereabouts_ippools_url } data "http" "whereabouts_overlap_res" { count = var.whereabouts_install ? 1 : 0 url = local.whereabouts_overlap_res_url } data "http" "whereabouts_daemonset" { count = var.whereabouts_install ? 1 : 0 url = local.whereabouts_daemonset_url } resource "null_resource" "whereabouts" { count = var.whereabouts_install ? 1 : 0 triggers = { whereabouts_ippools_url = local.whereabouts_ippools_url whereabouts_overlap_res_url = local.whereabouts_overlap_res_url whereabouts_daemonset_url = local.whereabouts_daemonset_url whereabouts_ippools_md5 = md5(local.whereabouts_ippools_content) whereabouts_overlap_res_md5 = md5(local.whereabouts_overlap_res_content) whereabouts_daemonset_md5 = md5(local.whereabouts_manifest_content) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = ["mkdir -p ${local.yaml_manifest_path}"] } provisioner "file" { content = local.whereabouts_ippools_content destination = local.whereabouts_ippools_path } provisioner "file" { content = local.whereabouts_overlap_res_content destination = local.whereabouts_overlap_res_path } provisioner "file" { content = local.whereabouts_manifest_content destination = local.whereabouts_manifest_path } provisioner "remote-exec" { inline = [ format("${local.kubectl} apply -f %s -f %s -f %s", local.whereabouts_ippools_path, local.whereabouts_overlap_res_path, local.whereabouts_manifest_path ) ] } lifecycle { precondition { condition = local.whereabouts_ippools_status_code == 200 error_message = <<-EOT Error retrieving Whereabouts CRD URL: ${local.whereabouts_ippools_url} Status code: ${local.whereabouts_ippools_status_code} Response: ${local.whereabouts_ippools_content} EOT } precondition { condition = local.whereabouts_overlap_res_status_code == 200 error_message = <<-EOT Error retrieving Whereabouts CRD URL: ${local.whereabouts_overlap_res_url} Status code: ${local.whereabouts_overlap_res_status_code} Response: ${local.whereabouts_overlap_res_content} EOT } precondition { condition = local.whereabouts_manifest_status_code == 200 error_message = <<-EOT Error retrieving Whereabouts manifest URL: ${local.whereabouts_daemonset_url} Status code: ${local.whereabouts_manifest_status_code} Response: ${local.whereabouts_manifest_content} EOT } } } ================================================ FILE: modules/iam/README.md ================================================ # IAM This sub-module creates IAM dynamic groups, policies, and optional tag namespaces for OKE resources. ## Usage Refer to the [Identity & Access Management section](../../docs/terraformoptions.md#identity--access-management) of the module documentation. ================================================ FILE: modules/iam/await.tf ================================================ resource "time_sleep" "await_iam_resources" { count = anytrue([ local.has_policy_statements, local.create_iam_tag_namespace, ]) ? 1 : 0 create_duration = "30s" destroy_duration = "0s" } ================================================ FILE: modules/iam/group-autoscaling.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { autoscaler_group_name = format("oke-autoscaler-%v", var.state_id) autoscaler_compartments = coalescelist(var.autoscaler_compartments, [var.compartment_id]) autoscaler_compartment_matches = formatlist("instance.compartment.id = '%v'", local.autoscaler_compartments) autoscaler_compartment_rule = format("ANY {%v}", join(", ", local.autoscaler_compartment_matches)) autoscaler_group_rules = var.use_defined_tags ? format("ALL {%v}", join(", ", [ format("tag.%v.role.value='worker'", var.tag_namespace), format("tag.%v.cluster_autoscaler.value='allowed'", var.tag_namespace), local.autoscaler_compartment_rule, # "tag.${var.tag_namespace}.state_id.value='${var.state_id}'", # TODO optional use w/ config ])) : local.autoscaler_compartment_rule autoscaler_templates = [ "Allow dynamic-group %v to manage cluster-node-pools in compartment id %v", "Allow dynamic-group %v to manage compute-management-family in compartment id %v", "Allow dynamic-group %v to manage instance-family in compartment id %v", "Allow dynamic-group %v to manage volume-family in compartment id %v", "Allow dynamic-group %v to use subnets in compartment id %v", "Allow dynamic-group %v to read virtual-network-family in compartment id %v", "Allow dynamic-group %v to use vnics in compartment id %v", "Allow dynamic-group %v to inspect compartments in compartment id %v", ] autoscaler_policy_statements = var.create_iam_autoscaler_policy ? tolist([ for statement in local.autoscaler_templates : formatlist(statement, local.autoscaler_group_name, local.worker_compartments, ) ]) : [] } resource "oci_identity_dynamic_group" "autoscaling" { provider = oci.home count = var.create_iam_resources && var.create_iam_autoscaler_policy ? 1 : 0 compartment_id = var.tenancy_id # dynamic groups exist in root compartment (tenancy) description = format("Dynamic group of cluster autoscaler-capable worker nodes for OKE Terraform state %v", var.state_id) matching_rule = local.autoscaler_group_rules name = local.autoscaler_group_name defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } ================================================ FILE: modules/iam/group-cluster.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { cluster_group_name = format("oke-cluster-%v", var.state_id) cluster_rule = format("ALL {%v}", join(", ", compact([ "resource.type = 'cluster'", format("resource.compartment.id = '%v'", var.compartment_id), var.use_defined_tags ? format("tag.%v.state_id.value='%v'", var.tag_namespace, var.state_id) : null, ]))) # Cluster secrets encryption using OCI Key Management System (KMS) cluster_policy_statements = coalesce(var.cluster_kms_key_id, "none") != "none" ? tolist([format( "Allow dynamic-group %v to use keys in compartment id %v where target.key.id = '%v'", local.cluster_group_name, var.compartment_id, var.cluster_kms_key_id, ), format("Allow dynamic-group %v to read instance-images in compartment id %v", local.cluster_group_name, var.compartment_id) ]) : [] } resource "oci_identity_dynamic_group" "cluster" { provider = oci.home count = var.create_iam_resources && var.create_iam_kms_policy ? 1 : 0 compartment_id = var.tenancy_id # dynamic groups exist in root compartment (tenancy) description = format("Dynamic group with cluster for OKE Terraform state %v", var.state_id) matching_rule = local.cluster_rule name = local.cluster_group_name defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } ================================================ FILE: modules/iam/group-operator.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { operator_group_name = format("oke-operator-%v", var.state_id) operator_group_rules = var.use_defined_tags ? format("ALL {%v}", join(", ", [ format("tag.%v.role.value='operator'", var.tag_namespace), format("tag.%v.state_id.value='%v'", var.tag_namespace, var.state_id), ])) : "ALL {instance.compartment.id = '${var.compartment_id}'}" cluster_manage_statement = format( "Allow dynamic-group %v to MANAGE clusters in compartment id %v", local.operator_group_name, var.compartment_id, ) # TODO support keys defined at worker group level operator_kms_volume_templates = [ "Allow service blockstorage to USE keys in compartment id %v where target.key.id = '%v'", "Allow dynamic-group ${local.operator_group_name} to USE key-delegates in compartment id %v where target.key.id = '%v'" ] # Block volume encryption using OCI Key Management System (KMS) operator_kms_volume_statements = coalesce(var.operator_volume_kms_key_id, "none") != "none" ? tolist([ for statement in local.operator_kms_volume_templates : format(statement, var.compartment_id, var.operator_volume_kms_key_id) ]) : [] operator_policy_statements = var.create_iam_operator_policy ? concat( [local.cluster_manage_statement], local.operator_kms_volume_statements, ) : [] } resource "oci_identity_dynamic_group" "operator" { provider = oci.home count = var.create_iam_resources && var.create_iam_operator_policy ? 1 : 0 compartment_id = var.tenancy_id # dynamic groups exist in root compartment (tenancy) description = format("Dynamic group of operator instance(s) for OKE Terraform state %v", var.state_id) matching_rule = local.operator_group_rules name = local.operator_group_name defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } ================================================ FILE: modules/iam/group-workers.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { worker_group_name = format("oke-workers-%v", var.state_id) worker_compartments = coalescelist(var.worker_compartments, [var.compartment_id]) worker_compartment_matches = formatlist("instance.compartment.id = '%v'", local.worker_compartments) worker_compartment_rule = format("ANY {%v}", join(", ", local.worker_compartment_matches)) worker_group_rules = var.use_defined_tags ? format("ALL {%v}", join(", ", [ format("tag.%v.role.value='worker'", var.tag_namespace), format("tag.%v.state_id.value='%v'", var.tag_namespace, var.state_id), ])) : local.worker_compartment_rule cluster_join_where_clause = format("ALL {%v}", join(", ", compact([ var.create_iam_worker_policy && var.cluster_id != null ? format("target.cluster.id = %v", var.cluster_id) : null ]))) cluster_join_statements = formatlist( "Allow dynamic-group %v to {CLUSTER_JOIN} in compartment id %v where %v", local.worker_group_name, local.worker_compartments, local.cluster_join_where_clause ) # TODO support keys defined at worker group level worker_kms_volume_templates = tolist([ "Allow service oke to USE key-delegates in compartment id %v where target.key.id = '%v'", "Allow service blockstorage to USE keys in compartment id %v where target.key.id = '%v'", "Allow dynamic-group ${local.worker_group_name} to USE key-delegates in compartment id %v where target.key.id = '%v'" ]) # Block volume encryption using OCI Key Management System (KMS) worker_kms_volume_statements = coalesce(var.worker_volume_kms_key_id, "none") != "none" ? flatten(tolist([ for statement in local.worker_kms_volume_templates : formatlist(statement, local.worker_compartments, var.worker_volume_kms_key_id) ])) : [] worker_policy_statements = var.create_iam_worker_policy ? tolist(concat( local.cluster_join_statements, local.worker_kms_volume_statements, )) : [] } resource "oci_identity_dynamic_group" "workers" { provider = oci.home count = var.create_iam_resources && var.create_iam_worker_policy ? 1 : 0 compartment_id = var.tenancy_id # dynamic groups exist in root compartment (tenancy) description = format("Dynamic group of self-managed worker nodes for OKE Terraform state %v", var.state_id) matching_rule = local.worker_group_rules name = local.worker_group_name defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } ================================================ FILE: modules/iam/outputs.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl output "dynamic_group_ids" { description = "Cluster IAM dynamic group IDs" value = local.has_policy_statements ? compact([ one(oci_identity_dynamic_group.cluster[*].id), one(oci_identity_dynamic_group.workers[*].id), one(oci_identity_dynamic_group.autoscaling[*].id), one(oci_identity_dynamic_group.operator[*].id), ]) : null } output "policy_statements" { description = "Cluster IAM policy statements" value = local.has_policy_statements ? local.policy_statements : null } ================================================ FILE: modules/iam/policy.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { policy_statements = distinct(compact(flatten([ local.cluster_policy_statements, local.worker_policy_statements, local.operator_policy_statements, local.autoscaler_policy_statements, ]))) has_policy_statements = var.create_iam_resources && anytrue([ var.create_iam_autoscaler_policy, var.create_iam_kms_policy, var.create_iam_operator_policy, var.create_iam_worker_policy, ]) } resource "oci_identity_policy" "cluster" { provider = oci.home count = local.has_policy_statements ? 1 : 0 compartment_id = var.compartment_id description = format("Policies for OKE Terraform state %v", var.state_id) name = var.policy_name statements = local.policy_statements defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } resource "oci_identity_policy" "networking_policies" { provider = oci.home count = alltrue([var.create_iam_resources, var.create_iam_cluster_policy]) ? 1 : 0 compartment_id = var.network_compartment_id != null ? var.network_compartment_id : var.compartment_id description = format("Policies for OKE Terraform state %v", var.state_id) name = format("%v-network", var.policy_name) statements = compact([ var.enable_ipv6 ? format("Allow any-user to use ipv6s in compartment id %v where all { request.principal.type = 'cluster' }", coalesce(var.network_compartment_id, var.compartment_id)) : null, format("Allow any-user to manage network-security-groups in compartment id %v where request.principal.type = 'cluster'", coalesce(var.network_compartment_id, var.compartment_id)), ]) defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } ================================================ FILE: modules/iam/tagging.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_identity_tag_namespaces" "oke" { count = var.create_iam_resources ? 1 : 0 provider = oci.home compartment_id = var.compartment_id filter { name = "name" values = [var.tag_namespace] } state = "ACTIVE" // TODO Support reactivation of retired namespace w/ update } locals { # Filtered value from data source (only 1 by name, or null) # Identified tag namespace ID when not created and used tag_namespace_id_found = one( one(data.oci_identity_tag_namespaces.oke[*].tag_namespaces)[*].id ) create_iam_tag_namespace = alltrue([ var.create_iam_resources, var.create_iam_tag_namespace, local.tag_namespace_id_found == null, ]) # Map of standard tags & descriptions to be created if enabled tags = var.create_iam_resources && var.create_iam_defined_tags ? { "role" = "Functional role of a resource" "state_id" = "Terraform state ID associated with a resource" "cluster_autoscaler" = "Granted permissions for Kubernetes cluster autoscaler" "pool" = "Named group of resources with shared configuration" } : {} # Standard tags as freeform if defined tags are disabled freeform_tags = merge(var.freeform_tags, !var.use_defined_tags ? { "state_id" = var.state_id, "role" = "iam", } : {}, ) # Standard tags as defined if enabled for use defined_tags = merge(var.defined_tags, var.use_defined_tags ? { "${var.tag_namespace}.state_id" = var.state_id, "${var.tag_namespace}.role" = "iam", } : {}, ) } resource "oci_identity_tag_namespace" "oke" { provider = oci.home count = local.create_iam_tag_namespace ? 1 : 0 compartment_id = var.compartment_id description = "Tag namespace for OKE resources" name = var.tag_namespace # defined_tags = local.defined_tags freeform_tags = local.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags] } } resource "oci_identity_tag" "oke" { provider = oci.home for_each = local.create_iam_tag_namespace ? local.tags : {} #{ for k, v in oci_identity_tag_namespace.oke : k => local.tags } # local.create_iam_tag_namespace ? local.tags : {} description = each.value name = each.key # defined_tags = local.defined_tags freeform_tags = local.freeform_tags tag_namespace_id = one(oci_identity_tag_namespace.oke[*].id) lifecycle { ignore_changes = [defined_tags, freeform_tags] } } ================================================ FILE: modules/iam/variables.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "cluster_id" { type = string } variable "compartment_id" { type = string } variable "network_compartment_id" { type = string } variable "state_id" { type = string } variable "tenancy_id" { type = string } variable "worker_compartments" { type = list(string) } variable "enable_ipv6" { type = bool } # Tags variable "create_iam_defined_tags" { type = bool } variable "create_iam_tag_namespace" { type = bool } variable "defined_tags" { type = map(string) } variable "freeform_tags" { type = map(string) } variable "tag_namespace" { type = string } variable "use_defined_tags" { type = bool } # Policy variable "autoscaler_compartments" { type = list(string) } variable "create_iam_resources" { type = bool } variable "create_iam_cluster_policy" { type = bool } variable "create_iam_autoscaler_policy" { type = bool } variable "create_iam_kms_policy" { type = bool } variable "create_iam_operator_policy" { type = bool } variable "create_iam_worker_policy" { type = bool } variable "policy_name" { type = string } # KMS variable "cluster_kms_key_id" { type = string } variable "operator_volume_kms_key_id" { type = string } variable "worker_volume_kms_key_id" { type = string } ================================================ FILE: modules/iam/versions.tf ================================================ // Copyright (c) 2024 Oracle and/or its affiliates # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { oci = { configuration_aliases = [oci.home] source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/network/README.md ================================================ # Network This sub-module creates the VCN, subnets, network security groups, gateways, routing, DRG, and LPG configurations. ## Usage Refer to the [Network section](../../docs/terraformoptions.md#network) of the module documentation. ================================================ FILE: modules/network/datasources.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_core_services" "all_oci_services" { filter { name = "name" values = ["All .* Services In Oracle Services Network"] regex = true } } data "oci_waas_edge_subnets" "waf_cidr_blocks" { count = var.enable_waf ? 1 : 0 } ================================================ FILE: modules/network/drgs.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { drg_attachments = (var.drg_attachments == null ? {} : { for k, v in var.drg_attachments : k => v if tobool(lookup(v, "create", true)) } ) } // Create a DRG if any attachments are defined resource "oci_core_drg" "oke" { count = length(local.drg_attachments) > 0 ? 1 : 0 compartment_id = var.compartment_id display_name = "oke-${var.state_id}" defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [freeform_tags, defined_tags] } } // Attach the current VCN to the DRG resource "oci_core_drg_attachment" "oke" { count = length(local.drg_attachments) > 0 && length(oci_core_drg.oke[*]) > 0 ? 1 : 0 drg_id = one(oci_core_drg.oke[*].id) display_name = "drg-oke-${var.state_id}" defined_tags = var.defined_tags freeform_tags = var.freeform_tags network_details { id = var.vcn_id type = "VCN" } lifecycle { ignore_changes = [freeform_tags, defined_tags] } } // Attach configured VCNs to the DRG resource "oci_core_drg_attachment" "extra" { for_each = local.drg_attachments drg_id = one(oci_core_drg.oke[*].id) display_name = format("%v-%v", each.key, var.state_id) defined_tags = var.defined_tags freeform_tags = var.freeform_tags network_details { id = lookup(each.value, "vcn_id") type = "VCN" } lifecycle { precondition { condition = alltrue([for k, v in local.drg_attachments : contains(keys(v), "vcn_id")]) error_message = "DRG attachments must specify a 'vcn_id'." } ignore_changes = [freeform_tags, defined_tags] } } ================================================ FILE: modules/network/locals.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { # Port numbers all_ports = -1 apiserver_port = 6443 fss_nfs_portmapper_port = 111 fss_nfs_port_min = 2048 fss_nfs_port_max = 2050 health_check_port = 10256 kubelet_api_port = 10250 oke_port = 12250 node_port_min = 30000 node_port_max = 32767 ssh_port = 22 # Protocols # See https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml all_protocols = "all" icmp_protocol = 1 icmpv6_protocol = 58 tcp_protocol = 6 udp_protocol = 17 anywhere = "0.0.0.0/0" anywhere_ipv6 = "::/0" rule_type_nsg = "NETWORK_SECURITY_GROUP" rule_type_cidr = "CIDR_BLOCK" rule_type_service = "SERVICE_CIDR_BLOCK" # Oracle Services Network (OSN) osn = one(data.oci_core_services.all_oci_services.services[*].cidr_block) } ================================================ FILE: modules/network/nsg-bastion.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { bastion_nsg_config = try(var.nsgs.bastion, { create = "never" }) bastion_nsg_create = coalesce(lookup(local.bastion_nsg_config, "create", null), "auto") bastion_nsg_enabled = anytrue([ local.bastion_nsg_create == "always", alltrue([ local.bastion_nsg_create == "auto", coalesce(lookup(local.bastion_nsg_config, "id", null), "none") == "none", var.create_bastion, ]), ]) # Return provided NSG when configured with an existing ID or created resource ID bastion_nsg_id = one(compact([try(var.nsgs.bastion.id, null), one(oci_core_network_security_group.bastion[*].id)])) bastion_rules = local.bastion_nsg_enabled ? ( var.use_stateless_rules ? local.bastion_stateless_rules: local.bastion_stateful_rules ) : {} bastion_stateful_rules = merge( { for cidr in var.bastion_allowed_cidrs : "Allow SSH ingress to bastion from ${cidr}" => { protocol = local.tcp_protocol, port = local.ssh_port, source = cidr, source_type = local.rule_type_cidr, } }, { "Allow TCP egress from bastion to OCI services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, }, }, local.operator_nsg_enabled ? { "Allow SSH egress from bastion to operator" = { protocol = local.tcp_protocol, port = local.ssh_port, destination = local.operator_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.allow_worker_ssh_access && local.worker_nsg_enabled ? { "Allow SSH egress from bastion to workers" = { protocol = local.tcp_protocol, port = local.ssh_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, (var.allow_bastion_cluster_access && local.control_plane_nsg_enabled) ? { "Allow TCP egress from bastion to cluster endpoint" = { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, ) bastion_stateless_rules = merge( { for cidr in var.bastion_allowed_cidrs : "Allow SSH ingress to bastion from ${cidr}" => { protocol = local.tcp_protocol, port = local.ssh_port, source = cidr, source_type = local.rule_type_cidr, } }, { "Allow TCP egress from bastion to OCI services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, }, }, local.operator_nsg_enabled ? { "Allow SSH egress from bastion to operator" = { protocol = local.tcp_protocol, port = local.ssh_port, destination = local.operator_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.allow_worker_ssh_access && local.worker_nsg_enabled ? { "Allow SSH egress from bastion to workers" = { protocol = local.tcp_protocol, port = local.ssh_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, (var.allow_bastion_cluster_access && local.control_plane_nsg_enabled) ? { "Allow TCP egress from bastion to cluster endpoint" = { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, ) } resource "oci_core_network_security_group" "bastion" { count = local.bastion_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "bastion-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "bastion_nsg_id" { value = local.bastion_nsg_id } ================================================ FILE: modules/network/nsg-controlplane.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { control_plane_nsg_config = try(var.nsgs.cp, { create = "never" }) control_plane_nsg_create = coalesce(lookup(local.control_plane_nsg_config, "create", null), "auto") control_plane_nsg_enabled = anytrue([ local.control_plane_nsg_create == "always", alltrue([ local.control_plane_nsg_create == "auto", coalesce(lookup(local.control_plane_nsg_config, "id", null), "none") == "none", var.create_cluster, ]), ]) # Return provided NSG when configured with an existing ID or created resource ID control_plane_nsg_id = one(compact([try(var.nsgs.cp.id, null), one(oci_core_network_security_group.cp[*].id)])) control_plane_rules = local.control_plane_nsg_enabled ? ( var.use_stateless_rules ? local.control_plane_stateless_rules: local.control_plane_stateful_rules ) : {} control_plane_stateful_rules= merge( { "Allow TCP egress from OKE control plane to OCI services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, }, "Allow TCP egress from OKE control plane to Kubelet on worker nodes" : { protocol = local.tcp_protocol, port = local.kubelet_api_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP ingress to OKE control plane from worker nodes" : { protocol = local.tcp_protocol, port = local.oke_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP egress from OKE control plane to worker nodes" : { protocol = local.tcp_protocol, port = local.oke_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress for Kubernetes control plane inter-communication" : { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP ingress for Kubernetes control plane inter-communication" : { protocol = local.tcp_protocol, port = local.apiserver_port, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress to kube-apiserver from worker nodes" : { protocol = local.tcp_protocol, port = local.apiserver_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, "Allow ICMP egress for path discovery to worker nodes" : { protocol = local.icmp_protocol, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ICMP ingress for path discovery from worker nodes" : { protocol = local.icmp_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, }, var.enable_ipv6 ? { "Allow ICMPv6 egress for path discovery to worker nodes" : { protocol = local.icmpv6_protocol, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ICMPv6 ingress for path discovery from worker nodes" : { protocol = local.icmpv6_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, } : {}, local.operator_nsg_enabled ? { "Allow TCP ingress to kube-apiserver from operator instance" : { protocol = local.tcp_protocol, port = local.apiserver_port, source = local.operator_nsg_id, source_type = local.rule_type_nsg, }, } : {}, local.pod_nsg_enabled ? { "Allow TCP ingress to kube-apiserver from pods" : { protocol = local.tcp_protocol, port = local.apiserver_port, source = local.pod_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress to OKE control plane from pods" : { protocol = local.tcp_protocol, port = local.oke_port, source = local.pod_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP egress from OKE control plane to pods" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, } } : {}, (var.allow_bastion_cluster_access && local.bastion_nsg_enabled) ? { "Allow TCP ingress to kube-apiserver from bastion host" = { protocol = local.tcp_protocol, port = local.apiserver_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, }, } : {}, { for allowed_cidr in var.control_plane_allowed_cidrs : "Allow TCP ingress to kube-apiserver from ${allowed_cidr}" => { protocol = local.tcp_protocol, port = local.apiserver_port, source = allowed_cidr, source_type = local.rule_type_cidr } }, var.allow_rules_cp ) control_plane_stateless_rules= merge( { "Allow TCP egress from OKE control plane to OCI services" : { protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true }, "Allow TCP ingress to OKE control plane from OCI services" : { protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true }, "Allow ingress to OKE control plane from worker nodes" : { protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow egress from OKE control plane to worker nodes" : { protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress for Kubernetes control plane inter-communication" : { protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress for Kubernetes control plane inter-communication" : { protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, }, var.enable_ipv6 ? { "Allow ICMPv6 egress for path discovery to worker nodes" : { protocol = local.icmpv6_protocol, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ICMPv6 ingress for path discovery from worker nodes" : { protocol = local.icmpv6_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, } : {}, local.operator_nsg_enabled ? { "Allow TCP ingress to kube-apiserver from operator instance" : { protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = local.operator_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from kube-apiserver to operator instance" : { protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, destination = local.operator_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, } : {}, local.pod_nsg_enabled ? { "Allow TCP ingress to OKE control plane from pods" : { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true } "Allow TCP egress from OKE control plane to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true } } : {}, (var.allow_bastion_cluster_access && local.bastion_nsg_enabled) ? { "Allow TCP ingress to kube-apiserver from bastion host" = { protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg }, } : {}, { for allowed_cidr in var.control_plane_allowed_cidrs : "Allow TCP ingress to kube-apiserver from ${allowed_cidr}" => { protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = allowed_cidr, source_type = local.rule_type_cidr } }, var.allow_rules_cp ) } resource "oci_core_network_security_group" "cp" { count = local.control_plane_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "cp-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "control_plane_nsg_id" { value = local.control_plane_nsg_id } ================================================ FILE: modules/network/nsg-fss.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { fss_nsg_config = try(var.nsgs.fss, { create = "never" }) fss_nsg_create = coalesce(lookup(local.fss_nsg_config, "create", null), "auto") fss_nsg_enabled = anytrue([ local.fss_nsg_create == "always", alltrue([ local.fss_nsg_create == "auto", coalesce(lookup(local.fss_nsg_config, "id", null), "none") == "none", var.create_cluster, ]), ]) # Return provided NSG when configured with an existing ID or created resource ID fss_nsg_id = one(compact([try(var.nsgs.fss.id, null), one(oci_core_network_security_group.fss[*].id)])) fss_rules = local.fss_nsg_enabled ? ( var.use_stateless_rules ? local.fss_stateless_rules: local.fss_stateful_rules ) : {} fss_stateful_rules = { # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm # Ingress "Allow UDP ingress for NFS portmapper from workers" : { protocol = local.udp_protocol, port = local.fss_nfs_portmapper_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress for NFS portmapper from workers" : { protocol = local.tcp_protocol, port = local.fss_nfs_portmapper_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, "Allow UDP ingress for NFS from workers" : { protocol = local.udp_protocol, port = local.fss_nfs_port_min, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress for NFS from workers" : { protocol = local.tcp_protocol, port_min = local.fss_nfs_port_min, port_max = local.fss_nfs_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, # Egress "Allow UDP egress for NFS portmapper to the workers" : { protocol = local.udp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress for NFS portmapper to the workers" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress for NFS to the workers" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } fss_stateless_rules = { # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm # Ingress "Allow UDP ingress for NFS portmapper from workers" : { protocol = local.udp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress for NFS portmapper to workers" : { protocol = local.udp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress for NFS portmapper from workers" : { protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress for NFS portmapper to workers" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow UDP ingress for NFS from workers" : { protocol = local.udp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_min, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress for NFS to workers" : { protocol = local.udp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_min, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress for NFS from workers" : { protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress for NFS to workers" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, } } resource "oci_core_network_security_group" "fss" { count = local.fss_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "fss-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "fss_nsg_id" { value = local.fss_nsg_id } ================================================ FILE: modules/network/nsg-loadbalancers-int.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { int_lb_nsg_config = try(var.nsgs.int_lb, { create = "never" }) int_lb_nsg_create = coalesce(lookup(local.int_lb_nsg_config, "create", null), "auto") int_lb_nsg_enabled = anytrue([ local.int_lb_nsg_create == "always", alltrue([ local.int_lb_nsg_create == "auto", coalesce(lookup(local.int_lb_nsg_config, "id", null), "none") == "none", var.create_cluster, var.load_balancers == "internal" || var.load_balancers == "both", ]), ]) # Return provided NSG when configured with an existing ID or created resource ID int_lb_nsg_id = one(compact([try(var.nsgs.int_lb.id, null), one(oci_core_network_security_group.int_lb[*].id)])) int_lb_rules = local.int_lb_nsg_enabled ? ( var.use_stateless_rules ? local.int_lb_stateless_rules: local.int_lb_stateful_rules ) : {} int_lb_stateful_rules = merge( { "Allow TCP egress from internal load balancers to workers for Node Ports" : { protocol = local.tcp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow UDP egress from internal load balancers to workers for Node Ports" : { protocol = local.udp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ICMP egress from internal load balancers to worker nodes for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress from internal load balancers to workers for health checks" : { protocol = local.tcp_protocol, port = local.health_check_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, }, local.pod_nsg_enabled ? { "Allow all egress from internal load balancers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.enable_ipv6 ? { "Allow ICMPv6 egress from internal load balancers to worker nodes for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_internal_lb, ) int_lb_stateless_rules = merge( { "Allow TCP egress from internal load balancers to workers for Node Ports" : { protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to internal load balancers from workers for Node Ports" : { protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress from internal load balancers to workers for Node Ports" : { protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow UDP ingress to internal load balancers from workers for Node Ports" : { protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from internal load balancers to pods for health checks" : { protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress to internal load balancers from pods for health checks" : { protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow ICMP egress from internal load balancers to worker nodes for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, }, local.pod_nsg_enabled ? { "Allow all egress from internal load balancers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow all ingress from pods to internal load balancers" : { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true }, } : {}, var.enable_ipv6 ? { "Allow ICMPv6 egress from internal load balancers to worker nodes for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_internal_lb, ) } resource "oci_core_network_security_group" "int_lb" { count = local.int_lb_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "int_lb-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "int_lb_nsg_id" { value = local.int_lb_nsg_id } ================================================ FILE: modules/network/nsg-loadbalancers-pub.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { pub_lb_nsg_config = try(var.nsgs.pub_lb, { create = "never" }) pub_lb_nsg_create = coalesce(lookup(local.pub_lb_nsg_config, "create", null), "auto") pub_lb_nsg_enabled = anytrue([ local.pub_lb_nsg_create == "always", alltrue([ local.pub_lb_nsg_create == "auto", coalesce(lookup(local.pub_lb_nsg_config, "id", null), "none") == "none", var.create_cluster, var.load_balancers == "public" || var.load_balancers == "both", ]), ]) # Return provided NSG when configured with an existing ID or created resource ID pub_lb_nsg_id = one(compact([try(var.nsgs.pub_lb.id, null), one(oci_core_network_security_group.pub_lb[*].id)])) pub_lb_rules = local.pub_lb_nsg_enabled ? ( var.use_stateless_rules ? local.pub_lb_stateless_rules: local.pub_lb_stateful_rules ) : {} pub_lb_stateful_rules = merge( { "Allow TCP egress from public load balancers to workers nodes for NodePort traffic" : { protocol = local.tcp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow UDP egress from public load balancers to workers nodes for NodePort traffic" : { protocol = local.udp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress from public load balancers to worker nodes for health checks" : { protocol = local.tcp_protocol, port = local.health_check_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ICMP egress from public load balancers to worker nodes for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, }, local.pod_nsg_enabled ? { "Allow all egress from public load balancers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.enable_ipv6 ? { "Allow ICMPv6 egress from public load balancers to worker nodes for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_public_lb, ) pub_lb_stateless_rules = merge( { "Allow TCP egress from public load balancers to workers nodes for NodePort traffic" : { protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to public load balancers from workers nodes for NodePort traffic" : { protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress from public load balancers to workers nodes for NodePort traffic" : { protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow UDP ingress to public load balancers from workers nodes for NodePort traffic" : { protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from public load balancers to worker nodes for health checks" : { protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to public load balancers from worker nodes for health checks" : { protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow ICMP egress from public load balancers to worker nodes for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, }, local.pod_nsg_enabled ? { "Allow all egress from public load balancers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow all ingress from pods to public load balancers" : { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true }, } : {}, var.enable_ipv6 ? { "Allow ICMPv6 egress from public load balancers to worker nodes for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_public_lb, ) } resource "oci_core_network_security_group" "pub_lb" { count = local.pub_lb_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "pub_lb-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "pub_lb_nsg_id" { value = local.pub_lb_nsg_id } ================================================ FILE: modules/network/nsg-operator.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { operator_nsg_config = try(var.nsgs.operator, { create = "never" }) operator_nsg_create = coalesce(lookup(local.operator_nsg_config, "create", null), "auto") operator_nsg_enabled = anytrue([ local.operator_nsg_create == "always", alltrue([ local.operator_nsg_create == "auto", coalesce(lookup(local.operator_nsg_config, "id", null), "none") == "none", var.create_cluster, var.create_operator, ]), ]) # Return provided NSG when configured with an existing ID or created resource ID operator_nsg_id = one(compact([try(var.nsgs.operator.id, null), one(oci_core_network_security_group.operator[*].id)])) operator_rules = local.operator_nsg_enabled ? ( var.use_stateless_rules ? local.operator_stateless_rules: local.operator_stateful_rules ) : {} operator_stateful_rules = merge( { "Allow TCP egress from operator to OCI services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, }, "Allow TCP egress from operator to Kubernetes API server" : { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ALL egress from operator to internet" : { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, }, }, local.bastion_nsg_enabled ? merge( var.enable_ipv6 ? { "Allow ICMPv6 ingress to operator from bastion for path discovery" : { protocol = local.icmpv6_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } } : {}, { "Allow ICMP ingress to operator from bastion for path discovery" : { protocol = local.icmp_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } "Allow SSH ingress to operator from bastion" : { protocol = local.tcp_protocol, port = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } }) : {}, ) operator_stateless_rules = merge( { "Allow TCP egress from operator to OCI services" : { protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service }, "Allow TCP egress from operator to Kubernetes API server" : { protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to operator from Kubernetes API server" : { protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow ALL egress from operator to all" : { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr } }, local.bastion_nsg_enabled ? merge( var.enable_ipv6 ? { "Allow ICMPv6 ingress to operator from bastion for path discovery" : { protocol = local.icmpv6_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } } : {}, { "Allow ICMP ingress to operator from bastion for path discovery" : { protocol = local.icmp_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } "Allow ingress to operator SSH from bastion" : { protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg }, }) : {}, ) } resource "oci_core_network_security_group" "operator" { count = local.operator_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "operator-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "operator_nsg_id" { value = local.operator_nsg_id } ================================================ FILE: modules/network/nsg-pods.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { pod_nsg_config = try(var.nsgs.pods, { create = "never" }) pod_nsg_create = coalesce(lookup(local.pod_nsg_config, "create", null), "auto") pod_nsg_enabled = anytrue([ local.pod_nsg_create == "always", alltrue([ local.pod_nsg_create == "auto", coalesce(lookup(local.pod_nsg_config, "id", null), "none") == "none", var.create_cluster, var.cni_type == "npn", ]), ]) # Return provided NSG when configured with an existing ID or created resource ID pod_nsg_id = one(compact([try(var.nsgs.pods.id, null), one(oci_core_network_security_group.pods[*].id)])) pods_rules = local.pod_nsg_enabled ? ( var.use_stateless_rules ? local.pod_stateless_rules: local.pod_stateful_rules ) : {} pod_stateful_rules = merge( { "Allow TCP egress from pods to OCI Services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, }, "Allow ALL egress from pods to other pods" = { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, } "Allow ALL ingress to pods from other pods" = { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, } "Allow TCP egress from pods to Kubernetes API server" = { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, } "Allow ALL ingress to pods from Kubernetes control plane for webhooks served by pods" = { protocol = local.all_protocols, port = local.all_ports, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, } "Allow ALL egress from pods for cross-node pod communication when using NodePorts or hostNetwork: true" = { protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, } "Allow ALL ingress to pods for cross-node pod communication when using NodePorts or hostNetwork: true" = { protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, } "Allow ICMP egress from pods for path discovery" = { protocol = local.icmp_protocol, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, } "Allow ICMP ingress to pods for path discovery" = { protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, }, }, local.int_lb_nsg_enabled ? { "Allow ALL egress from pods to internal_lb" = { protocol = local.all_protocols, port = local.all_ports, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, } "Allow ALL ingress from internal_lb to pods" = { protocol = local.all_protocols, port = local.all_ports, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, } } : {}, local.pub_lb_nsg_enabled ? { "Allow ALL egress from pods to pub_lb" = { protocol = local.all_protocols, port = local.all_ports, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, } "Allow ALL ingress from pub_lb to pods" = { protocol = local.all_protocols, port = local.all_ports, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, } }: {}, var.enable_ipv6 ? { "Allow ICMPv6 ingress to pods for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, }, "Allow ICMPv6 egress from pods for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, }, } : {}, var.allow_pod_internet_access ? merge( var.enable_ipv6 ? { "Allow ALL IPv6 egress from pods to internet" = { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, } } : {}, { "Allow ALL egress from pods to internet" = { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, } }) : {}, var.allow_rules_pods ) pod_stateless_rules = merge( { "Allow TCP egress from pods to OCI Services" : { protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true }, "Allow TCP egress to pods from OCI Services" : { protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true }, "Allow ALL egress from pods to other pods" = { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true } "Allow ALL ingress to pods from other pods" = { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true } "Allow TCP egress from pods to Kubernetes control_plane" = { protocol = local.tcp_protocol, port = local.all_ports, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true } "Allow TCP ingress to pods from Kubernetes control_plane" = { protocol = local.tcp_protocol, port = local.all_ports, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true } "Allow ALL egress from pods for cross-node pod communication when using NodePorts or hostNetwork: true" = { protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true } "Allow ALL ingress to pods for cross-node pod communication when using NodePorts or hostNetwork: true" = { protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true } "Allow ICMP egress from pods for path discovery" = { protocol = local.icmp_protocol, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, } "Allow ICMP ingress to pods for path discovery" = { protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, } }, local.int_lb_nsg_enabled ? { "Allow ALL egress from pods to internal_lb" = { protocol = local.all_protocols, port = local.all_ports, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true } "Allow ALL egress from pods to internal_lb" = { protocol = local.all_protocols, port = local.all_ports, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true } } : {}, local.pub_lb_nsg_enabled ? { "Allow ALL egress from pods to pub_lb" = { protocol = local.all_protocols, port = local.all_ports, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true } "Allow ALL egress from pods to pub_lb" = { protocol = local.all_protocols, port = local.all_ports, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true } }: {}, var.enable_ipv6 ? { "Allow ICMPv6 ingress to pods for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, }, "Allow ICMPv6 egress from pods for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, }, } : {}, var.allow_pod_internet_access ? merge( var.enable_ipv6 ? { "Allow ALL IPv6 egress from pods to internet (not using stateless rules because of security concern with IPv6 GUA and routing over IGW)" = { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, } } : {}, { "Allow ALL egress from pods to anywhere" = { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, stateless = true } "Allow ALL ingress to pods from anywhere" = { protocol = local.all_protocols, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, stateless = true } }) : {}, var.allow_rules_pods ) } resource "oci_core_network_security_group" "pods" { count = local.pod_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "pods-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "pod_nsg_id" { value = local.pod_nsg_id } ================================================ FILE: modules/network/nsg-workers.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { worker_nsg_config = try(var.nsgs.workers, { create = "never" }) worker_nsg_create = coalesce(lookup(local.worker_nsg_config, "create", null), "auto") worker_nsg_enabled = anytrue([ local.worker_nsg_create == "always", alltrue([ local.worker_nsg_create == "auto", coalesce(lookup(local.worker_nsg_config, "id", null), "none") == "none", var.create_cluster, ]), ]) # Return provided NSG when configured with an existing ID or created resource ID worker_nsg_id = one(compact([try(var.nsgs.workers.id, null), one(oci_core_network_security_group.workers[*].id)])) workers_rules = local.worker_nsg_enabled ? ( var.use_stateless_rules ? local.workers_stateless_rules: local.workers_stateful_rules ) : {} workers_stateful_rules = merge( { "Allow TCP egress from workers to OCI Services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, }, "Allow ALL egress from workers to other workers" : { protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ALL ingress to workers from other workers" : { protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP egress from workers to Kubernetes API server" : { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress from workers to OKE control plane" : { protocol = local.tcp_protocol, port = local.oke_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress to OKE control plane from workers for health check" : { protocol = local.tcp_protocol, port = local.kubelet_api_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ALL ingress to workers from Kubernetes control plane for webhooks served by workers" : { protocol = local.all_protocols, port = local.all_ports, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, }, "Allow ICMP egress from workers for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, }, "Allow ICMP ingress to workers for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, }, }, var.enable_ipv6 ? { "Allow ICMPv6 ingress to workers for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, }, "Allow ICMPv6 egress from workers for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, }, } : {}, local.pod_nsg_enabled ? { "Allow ALL egress from workers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, }, "Allow ALL ingress to workers from pods" : { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, }, } : {}, var.allow_worker_internet_access ? merge( var.enable_ipv6 ? { "Allow ALL IPv6 egress from workers to internet" = { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, } } : {}, { "Allow ALL egress from workers to internet" : { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, }, }) : {}, local.int_lb_nsg_enabled ? { "Allow TCP ingress to workers from internal load balancers" : { protocol = local.tcp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, }, "Allow UDP ingress to workers from internal load balancers" : { protocol = local.udp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress to workers for health check from internal load balancers" : { protocol = local.tcp_protocol, port = local.health_check_port, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, }, } : {}, local.pub_lb_nsg_enabled ? { "Allow TCP ingress to workers from public load balancers" : { protocol = local.tcp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, }, "Allow UDP ingress to workers from public load balancers" : { protocol = local.udp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress to workers for health check from public load balancers" : { protocol = local.tcp_protocol, port = local.health_check_port, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, }, } : {}, local.bastion_nsg_enabled && var.allow_worker_ssh_access ? { "Allow SSH ingress to workers from bastion" : { protocol = local.tcp_protocol, port = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } } : {}, local.fss_nsg_enabled ? { # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm # Ingress "Allow TCP ingress to workers for NFS portmapper from FSS mounts" : { protocol = local.tcp_protocol, port = local.fss_nfs_portmapper_port, source = local.fss_nsg_id, source_type = local.rule_type_nsg, }, "Allow UDP ingress to workers for NFS portmapper from FSS mounts" : { protocol = local.udp_protocol, port = local.fss_nfs_portmapper_port, source = local.fss_nsg_id, source_type = local.rule_type_nsg, }, "Allow TCP ingress to workers for NFS from FSS mounts" : { protocol = local.tcp_protocol, port_min = local.fss_nfs_port_min, port_max = local.fss_nfs_port_max, source = local.fss_nsg_id, source_type = local.rule_type_nsg, }, # Egress "Allow TCP egress from workers for NFS portmapper to FSS mounts" : { protocol = local.tcp_protocol, port = local.fss_nfs_portmapper_port, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, }, "Allow UDP egress from workers for NFS portmapper to FSS mounts" : { protocol = local.udp_protocol, port = local.fss_nfs_portmapper_port, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, }, "Allow TCP egress from workers for NFS to FSS mounts" : { protocol = local.tcp_protocol, port_min = local.fss_nfs_port_min, port_max = local.fss_nfs_port_max, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, }, "Allow UDP egress from workers for NFS to FSS mounts" : { protocol = local.udp_protocol, port = local.fss_nfs_port_min, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, var.allow_rules_workers ) workers_stateless_rules = merge( { "Allow TCP egress from workers to OCI Services" : { protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true }, "Allow TCP ingress to workers from OCI Services" : { protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true }, "Allow ALL egress from workers to other workers" : { protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow ALL ingress to workers from other workers" : { protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow ALL egress from workers to Kubernetes control plane" : { protocol = local.all_protocols, port = local.all_ports, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow ALL ingress to workers from Kubernetes control plane" : { protocol = local.all_protocols, port = local.all_ports, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow ICMP egress from workers for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, }, "Allow ICMP ingress to workers for path discovery" : { protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, }, }, var.enable_ipv6 ? { "Allow ICMPv6 ingress to workers for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, }, "Allow ICMPv6 egress from workers for path discovery" : { protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, }, } : {}, local.pod_nsg_enabled ? { "Allow ALL egress from workers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow ALL ingress to workers from pods" : { protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true }, } : {}, var.allow_worker_internet_access ? merge( var.enable_ipv6 ? { "Allow ALL IPv6 egress from workers to internet (not using stateless rules because of security concern with IPv6 GUA and routing over IGW)" = { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, } } : {}, { "Allow ALL egress from workers to internet (not using stateless rules because of security concern with Public IPs and routing over IGW)" : { protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, }, }) : {}, local.int_lb_nsg_enabled ? { "Allow TCP ingress to workers from internal load balancers" : { protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from workers to internal load balancers" : { protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow UDP ingress to workers from internal load balancers" : { protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress from workers to internal load balancers" : { protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to workers for health check from internal load balancers" : { protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from workers for health check to internal load balancers" : { protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, } : {}, local.pub_lb_nsg_enabled ? { "Allow TCP ingress to workers from public load balancers" : { protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from workers to public load balancers" : { protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow UDP ingress to workers from public load balancers" : { protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress from workers to public load balancers" : { protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to workers for health check from public load balancers" : { protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from workers for health check to public load balancers" : { protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, } : {}, local.bastion_nsg_enabled && var.allow_worker_ssh_access ? { "Allow ingress to workers SSH from bastion" : { protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg }, } : {}, local.fss_nsg_enabled ? { # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm # Ingress "Allow UDP ingress to workers for NFS portmapper from FSS mounts" : { protocol = local.udp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress from workers for NFS portmapper to FSS mounts" : { protocol = local.udp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to workers for NFS portmapper from FSS mounts" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from workers for NFS portmapper to FSS mounts" : { protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow TCP ingress to workers for NFS from FSS mounts" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_max, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow TCP egress from workers for NFS to FSS mounts" : { protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_max, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true }, "Allow UDP ingress to workers for NFS from FSS mounts" : { protocol = local.udp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_min, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true }, "Allow UDP egress from workers for NFS to FSS mounts" : { protocol = local.udp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_min, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true } } : {}, var.allow_rules_workers ) } resource "oci_core_network_security_group" "workers" { count = local.worker_nsg_enabled ? 1 : 0 compartment_id = var.compartment_id display_name = "workers-${var.state_id}" vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [defined_tags, freeform_tags, display_name, vcn_id] } } output "worker_nsg_id" { value = local.worker_nsg_id } ================================================ FILE: modules/network/rules.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { waf_rules = var.enable_waf ? { # Used in load balancer NSGs if enabled for waf_subnet in data.oci_waas_edge_subnets.waf_cidr_blocks[0].edge_subnets : "Allow SSL ingress from WAF ${waf_subnet.cidr}" => { protocol = local.tcp_protocol, port = 443, source = waf_subnet.cidr, source_type = local.rule_type_cidr, } } : {} # Dynamic map of all NSG rules for enabled NSGs all_rules = { for x, y in merge( { for k, v in local.bastion_rules : k => merge(v, { "nsg_id" = local.bastion_nsg_id }) }, { for k, v in local.control_plane_rules : k => merge(v, { "nsg_id" = local.control_plane_nsg_id }) }, { for k, v in local.int_lb_rules : k => merge(v, { "nsg_id" = local.int_lb_nsg_id }) }, { for k, v in local.pub_lb_rules : k => merge(v, { "nsg_id" = local.pub_lb_nsg_id }) }, { for k, v in local.workers_rules : k => merge(v, { "nsg_id" = local.worker_nsg_id }) }, { for k, v in local.pods_rules : k => merge(v, { "nsg_id" = local.pod_nsg_id }) }, { for k, v in local.operator_rules : k => merge(v, { "nsg_id" = local.operator_nsg_id }) }, { for k, v in local.fss_rules : k => merge(v, { "nsg_id" = local.fss_nsg_id }) }, ) : x => merge(y, { description = x stateless = lookup(y, "stateless", false) network_security_group_id = lookup(y, "nsg_id") direction = contains(keys(y), "source") ? "INGRESS" : "EGRESS" protocol = lookup(y, "protocol") source = ( alltrue([ upper(lookup(y, "source_type", "")) == local.rule_type_nsg, length(regexall("ocid\\d+\\.networksecuritygroup", lower(lookup(y, "source", "")))) == 0]) ? lookup(local.all_nsg_ids, lower(lookup(y, "source", "")), null) : lookup(y, "source", null) ) source_type = lookup(y, "source_type", null) destination = ( alltrue([ upper(lookup(y, "destination_type", "")) == local.rule_type_nsg, length(regexall("ocid\\d+\\.networksecuritygroup", lower(lookup(y, "destination", "")))) == 0]) ? lookup(local.all_nsg_ids, lower(lookup(y, "destination", "")), null) : lookup(y, "destination", null) ) destination_type = lookup(y, "destination_type", null) }) } # Dynamic map of all NSG IDs for enabled NSGs all_nsg_ids = { for x, y in merge( local.bastion_nsg_enabled ? { "bastion" = local.bastion_nsg_id } : {}, local.control_plane_nsg_enabled ? { "cp" = local.control_plane_nsg_id } : {}, local.int_lb_nsg_enabled ? { "int_lb" = local.int_lb_nsg_id } : {}, local.pub_lb_nsg_enabled ? { "pub_lb" = local.pub_lb_nsg_id } : {}, local.worker_nsg_enabled ? { "workers" = local.worker_nsg_id } : {}, local.pod_nsg_enabled ? { "pods" = local.pod_nsg_id } : {}, local.operator_nsg_enabled ? { "operator" = local.operator_nsg_id } : {}, local.fss_nsg_enabled ? { "fss" = local.fss_nsg_id } : {}, ) : x => y } } resource "oci_core_network_security_group_security_rule" "oke" { for_each = local.all_rules stateless = each.value.stateless description = each.value.description destination = each.value.destination destination_type = each.value.destination_type direction = each.value.direction network_security_group_id = each.value.network_security_group_id protocol = each.value.protocol source = each.value.source source_type = each.value.source_type dynamic "tcp_options" { for_each = (tostring(each.value.protocol) == tostring(local.tcp_protocol) && tonumber(lookup(each.value, "port", 0)) != local.all_ports ? [each.value] : [] ) content { dynamic "destination_port_range" { for_each = ( (contains(keys(tcp_options.value), "destination_port_min") && contains(keys(tcp_options.value), "destination_port_max")) || (contains(keys(tcp_options.value), "source_port_min") && contains(keys(tcp_options.value), "source_port_max")) ) ? [] : [tcp_options.value] content { min = tonumber(lookup(destination_port_range.value, "port_min", lookup(destination_port_range.value, "port", 0))) max = tonumber(lookup(destination_port_range.value, "port_max", lookup(destination_port_range.value, "port", 0))) } } dynamic "destination_port_range" { for_each = (contains(keys(tcp_options.value), "destination_port_min") && contains(keys(tcp_options.value), "destination_port_max")) ? [tcp_options.value] : [] content { min = tonumber(lookup(destination_port_range.value, "destination_port_min", 0)) max = tonumber(lookup(destination_port_range.value, "destination_port_max", 0)) } } dynamic "source_port_range" { for_each = (contains(keys(tcp_options.value), "source_port_min") && contains(keys(tcp_options.value), "source_port_max")) ? [tcp_options.value] : [] content { min = tonumber(lookup(source_port_range.value, "source_port_min", 0)) max = tonumber(lookup(source_port_range.value, "source_port_max", 0)) } } } } dynamic "udp_options" { for_each = (tostring(each.value.protocol) == tostring(local.udp_protocol) && tonumber(lookup(each.value, "port", 0)) != local.all_ports ? [each.value] : [] ) content { dynamic "destination_port_range" { for_each = ( (contains(keys(udp_options.value), "destination_port_min") && contains(keys(udp_options.value), "destination_port_max")) || (contains(keys(udp_options.value), "source_port_min") && contains(keys(udp_options.value), "source_port_max")) ) ? [] : [udp_options.value] content { min = tonumber(lookup(destination_port_range.value, "port_min", lookup(destination_port_range.value, "port", 0))) max = tonumber(lookup(destination_port_range.value, "port_max", lookup(destination_port_range.value, "port", 0))) } } dynamic "destination_port_range" { for_each = (contains(keys(udp_options.value), "destination_port_min") && contains(keys(udp_options.value), "destination_port_max")) ? [udp_options.value] : [] content { min = tonumber(lookup(destination_port_range.value, "destination_port_min", 0)) max = tonumber(lookup(destination_port_range.value, "destination_port_max", 0)) } } dynamic "source_port_range" { for_each = (contains(keys(udp_options.value), "source_port_min") && contains(keys(udp_options.value), "source_port_max")) ? [udp_options.value] : [] content { min = tonumber(lookup(source_port_range.value, "source_port_min", 0)) max = tonumber(lookup(source_port_range.value, "source_port_max", 0)) } } } } dynamic "icmp_options" { for_each = tostring(each.value.protocol) == tostring(local.icmp_protocol) ? [1] : [] content { type = 3 code = 4 } } dynamic "icmp_options" { for_each = tostring(each.value.protocol) == tostring(local.icmpv6_protocol) ? [1] : [] content { type = 2 code = 0 } } lifecycle { precondition { condition = contains([tostring(local.icmp_protocol), tostring(local.icmpv6_protocol)], tostring(each.value.protocol)) || contains(keys(each.value), "port") || ( contains(keys(each.value), "port_min") && contains(keys(each.value), "port_max")) || ( contains(keys(each.value), "source_port_min") && contains(keys(each.value), "source_port_max") || ( contains(keys(each.value), "destination_port_min") && contains(keys(each.value), "destination_port_max") ) ) error_message = "TCP/UDP rule must contain a port or port range: '${each.key}'" } precondition { condition = ( contains([tostring(local.icmp_protocol), tostring(local.icmpv6_protocol)], tostring(each.value.protocol)) || can(tonumber(each.value.port)) || (can(tonumber(each.value.port_min)) && can(tonumber(each.value.port_max))) || (can(tonumber(each.value.source_port_min)) && can(tonumber(each.value.source_port_max))) || (can(tonumber(each.value.destination_port_min)) && can(tonumber(each.value.destination_port_max))) ) error_message = "TCP/UDP ports must be numeric: '${each.key}'" } precondition { condition = each.value.direction == "EGRESS" || coalesce(each.value.source, "none") != "none" error_message = "Ingress rule must have a source: '${each.key}'" } precondition { condition = each.value.direction == "INGRESS" || coalesce(each.value.destination, "none") != "none" error_message = "Egress rule must have a destination: '${each.key}'" } # Extra precaution against unexpected allow-all ingress rules created by the module # Generated rules will produce errors unless any of the follow conditions are true precondition { condition = var.use_stateless_rules || anytrue([ tostring(each.value.protocol) == tostring(local.icmp_protocol), # Traffic is ICMP each.value.direction == "EGRESS", # Traffic is outbound each.value.source != local.anywhere, # Rule does not allow all traffic # SSH ingress to bastion from anywhere has been configured explicitly alltrue([ tonumber(lookup(each.value, "port", 0)) == local.ssh_port, contains(var.bastion_allowed_cidrs, local.anywhere), ]), # TCP ingress to Kubernetes endpoint from anywhere has been configured explicitly alltrue([ tonumber(lookup(each.value, "port", 0)) == local.apiserver_port, contains(var.control_plane_allowed_cidrs, local.anywhere), ]), # TCP ingress to internal load balancer from anywhere has been configured explicitly contains(keys(var.allow_rules_internal_lb), each.key), # TCP ingress to public load balancer from anywhere has been configured explicitly contains(keys(var.allow_rules_public_lb), each.key), ]) error_message = "Unexpected open ingress rule: ${each.key}" } } } output "network_security_rules" { value = local.all_rules } output "nsg_ids" { value = length(local.all_nsg_ids) > 0 ? local.all_nsg_ids : null } ================================================ FILE: modules/network/subnets.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { # VCN subnet configuration # See https://docs.oracle.com/en-us/iaas/Content/ContEng/Concepts/contengnetworkconfig.htm#vcnconfig # May be undefined when VCN is neither created nor required, e.g. when creating only workers for # an existing cluster. Fallback value is unused. vcn_cidr = length(var.vcn_cidrs) > 0 ? element(var.vcn_cidrs, 0) : "0.0.0.0/16" # Filter configured subnets eligible for resource creation subnet_cidrs_new = { for k, v in var.subnets : k => merge(v, { "type" = (lookup(v, "netnum", null) == null && lookup(v, "newbits", null) != null ? "newbits" : (lookup(v, "netnum", null) != null && lookup(v, "newbits", null) != null ? "netnum" : (lookup(v, "cidr", null) != null ? "cidr" : (lookup(v, "id", null) != null ? "id" : "invalid")))) }) if lookup(v, "create", "auto") != "never" } # Handle subnets configured with provided CIDRs subnet_cidrs_cidr_input = { for k, v in local.subnet_cidrs_new : k => lookup(v, "cidr") if v.type == "cidr" } # Handle subnets configured with only newbits for sizing subnet_cidrs_newbits_input = { for k, v in local.subnet_cidrs_new : k => lookup(v, "newbits") if v.type == "newbits" } # Generate CIDR ranges for subnets to be created subnet_cidrs_newbits_ranges = cidrsubnets(local.vcn_cidr, values(local.subnet_cidrs_newbits_input)...) subnet_cidrs_newbits_resolved = length(local.vcn_cidr) > 0 ? { for k, v in local.subnet_cidrs_newbits_input : k => element(local.subnet_cidrs_newbits_ranges, index(keys(local.subnet_cidrs_newbits_input), k)) } : {} # Handle subnets configured with netnum + newbits for sizing subnet_cidrs_netnum_newbits_ranges = { for k, v in local.subnet_cidrs_new : k => cidrsubnet(local.vcn_cidr, lookup(v, "newbits"), lookup(v, "netnum")) if v.type == "netnum" } // Combine provided and calculated subnet CIDRs subnet_cidrs_all = merge( local.subnet_cidrs_cidr_input, local.subnet_cidrs_newbits_resolved, local.subnet_cidrs_netnum_newbits_ranges, ) # IPv6 Default CIDRs default_ipv6_cidrs = { bastion = { ipv6_cidr = "8, 0" } operator = { ipv6_cidr = "8, 1" } cp = { ipv6_cidr = "8, 2" } int_lb = { ipv6_cidr = "8, 3" } pub_lb = { ipv6_cidr = "8, 4" } workers = { ipv6_cidr = "8, 5" } pods = { ipv6_cidr = "8, 6" } fss = { ipv6_cidr = "8, 7" } } # Add default ipv6 cidrs to var.subnets if missing subnets_with_ipv6_cidr_defaults = { for k, v in var.subnets : k => merge(v, lookup(v, "ipv6_cidr", null) == null ? lookup(local.default_ipv6_cidrs, k, { "ipv6_cidr" : "::/0" }) : {}) } # Generate IPv6 CIDRs subnets_ipv6_cidr = var.enable_ipv6 == true ? { for k, v in local.subnets_with_ipv6_cidr_defaults : k => merge(v, { "ipv6_cidr" = length(regexall("^\\d+,[ ]?\\d+$", lookup(v, "ipv6_cidr"))) > 0 ? cidrsubnet(var.vcn_ipv6_cidrs[0], tonumber(split(",", lookup(v, "ipv6_cidr"))[0]), tonumber(trim(split(",", lookup(v, "ipv6_cidr"))[1], " "))) : lookup(v, "ipv6_cidr") }) if try(v.create, "auto") != "never" } : { for k, v in var.subnets : k => merge(v, { "ipv6_cidr" : null }) if try(v.create, "auto") != "never" } # Map of subnets for standard components with additional configuration derived # TODO enumerate worker pools for public/private overrides, conditional subnets for both subnet_info = { bastion = { create = var.create_bastion, is_public = var.bastion_is_public } cp = { create = var.create_cluster, is_public = var.enable_ipv6 == true ? true : var.control_plane_is_public } workers = { create = var.create_cluster, is_public = var.enable_ipv6 == true ? true : var.worker_is_public } pods = { create = var.create_cluster && var.cni_type == "npn", is_public = var.enable_ipv6 == true ? true : false } operator = { create = var.create_operator } fss = { create = contains(keys(var.subnets), "fss") } int_lb = { create = var.create_cluster && contains(["both", "internal"], var.load_balancers), create_seclist = true, dns_label = "ilb", } pub_lb = { create = var.create_cluster && contains(["both", "public"], var.load_balancers), create_seclist = true, is_public = true, dns_label = "plb", } } # Map of configured subnets to specified/generated dns_label when enabled # If `assign_dns = true`, use dns_label for subnet if specified or first 2 characters of subnet key subnet_dns_labels = { for k, v in var.subnets : k => coalesce(lookup(v, "dns_label", null), substr(k, 0, 2)) if var.assign_dns } # Create subnets if when all are true: # - Associated component is enabled OR configured with create == 'always' # - Subnet is configured with newbits and/or netnum/cidr # - Not configured with create == 'never' # - Not configured with an existing 'id' subnets_to_create = merge( { for k, v in local.subnet_info : k => # Override `create = true` if configured with "always" merge(v, lookup(try(lookup(var.subnets, k), { create = "never" }), "create", "auto") == "always" ? { "create" = true } : {}) if alltrue([ # Filter disabled subnets from output contains(keys(local.subnet_cidrs_all), k), # has a calculated CIDR range (not id input) lookup(try(lookup(var.subnets, k), { create = "never" }), "create", "auto") != "never", # not disabled anytrue([ tobool(lookup(v, "create", true)), # automatically enabled lookup(try(lookup(var.subnets, k), { create = "never" }), "create", "auto") == "always" # force enabled ]), ]) } ) subnet_output = { for k, v in var.subnets : k => lookup(v, "id", null) != null ? v.id : lookup(lookup(oci_core_subnet.oke, k, {}), "id", null) } create_mixed_igw_ngw_route_table = alltrue([ var.enable_ipv6 == true, var.create_internet_gateway == true, var.create_nat_gateway == true, var.igw_ngw_mixed_route_id == null ]) } resource "null_resource" "validate_subnets" { count = anytrue([for k, v in local.subnet_cidrs_new : contains(["netnum", "newbits", "cidr"], v.type) if lookup(v, "create", "auto") != "never" ]) ? 1 : 0 lifecycle { precondition { condition = !contains([for k, v in local.subnet_cidrs_new : v.type], "invalid") error_message = format("Invalid subnet specification: %s", jsonencode({ for k, v in local.subnet_cidrs_new : k => v if v.type == "invalid" })) } precondition { condition = !(contains([for k, v in local.subnet_cidrs_new : v.type], "netnum") && contains([for k, v in local.subnet_cidrs_new : v.type], "newbits")) error_message = format( "Must omit or include `netnum` for all subnet defintions uniformely: %s", jsonencode({ for k, v in local.subnet_cidrs_new : k => v if contains(["netnum", "newbits"], v.type) }) ) } } } resource "oci_core_route_table" "igw_ngw_mixed_route_id" { count = local.create_mixed_igw_ngw_route_table ? 1 : 0 compartment_id = var.compartment_id display_name = format("%v-%v", "igw-ngw-mixed-rt", var.state_id) vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags route_rules { #Required network_entity_id = var.nat_gateway_id description = "Default route for IPv4." destination = local.anywhere destination_type = local.rule_type_cidr } route_rules { #Required network_entity_id = var.internet_gateway_id description = "Default route for IPv6." destination = local.anywhere_ipv6 destination_type = local.rule_type_cidr } lifecycle { ignore_changes = [ freeform_tags, defined_tags, ] } } resource "oci_core_subnet" "oke" { for_each = local.subnets_to_create compartment_id = var.compartment_id vcn_id = var.vcn_id cidr_block = lookup(local.subnet_cidrs_all, each.key) display_name = (lookup(var.subnets, each.key, null) != null ? (lookup(var.subnets[each.key], "display_name", null) != null ? var.subnets[each.key]["display_name"] : format("%v-%v", each.key, var.state_id) ) : format("%v-%v", each.key, var.state_id) ) dns_label = lookup(local.subnet_dns_labels, each.key, null) prohibit_public_ip_on_vnic = !tobool(lookup(each.value, "is_public", false)) route_table_id = var.enable_ipv6 && var.cni_type == "npn" && each.key == "pods" ? coalesce(one(oci_core_route_table.igw_ngw_mixed_route_id[*].id), var.igw_ngw_mixed_route_id) : !tobool(lookup(each.value, "is_public", false)) ? var.nat_route_table_id : var.ig_route_table_id security_list_ids = compact([lookup(lookup(oci_core_security_list.oke, each.key, {}), "id", null)]) defined_tags = var.defined_tags freeform_tags = var.freeform_tags ipv6cidr_block = var.enable_ipv6 ? lookup(local.subnets_ipv6_cidr[each.key], "ipv6_cidr", null) : null lifecycle { ignore_changes = [ freeform_tags, defined_tags, cidr_block, dns_label, security_list_ids, vcn_id, ] } } # Create an associated security list for subnets when enabled # e.g. for load balancers to prevent CCM management of default security list resource "oci_core_security_list" "oke" { for_each = { for k, v in local.subnets_to_create : k => v if tobool(lookup(v, "create_seclist", false)) } compartment_id = var.compartment_id display_name = format("%v-%v", each.key, var.state_id) vcn_id = var.vcn_id defined_tags = var.defined_tags freeform_tags = var.freeform_tags lifecycle { ignore_changes = [ freeform_tags, defined_tags, display_name, vcn_id, ingress_security_rules, egress_security_rules, # ignore for CCM-management ] } } # Return configured/created subnet IDs and CIDRs when applicable output "bastion_subnet_id" { value = lookup(local.subnet_output, "bastion", null) } output "bastion_subnet_cidr" { value = contains(keys(local.subnet_output), "bastion") ? lookup(local.subnet_cidrs_all, "bastion", null) : null } output "operator_subnet_id" { value = lookup(local.subnet_output, "operator", null) } output "operator_subnet_cidr" { value = contains(keys(local.subnet_output), "operator") ? lookup(local.subnet_cidrs_all, "operator", null) : null } output "control_plane_subnet_id" { value = lookup(local.subnet_output, "cp", null) } output "control_plane_subnet_cidr" { value = contains(keys(local.subnet_output), "cp") ? lookup(local.subnet_cidrs_all, "cp", null) : null } output "int_lb_subnet_id" { value = lookup(local.subnet_output, "int_lb", null) } output "int_lb_subnet_cidr" { value = contains(keys(local.subnet_output), "int_lb") ? lookup(local.subnet_cidrs_all, "int_lb", null) : null } output "pub_lb_subnet_id" { value = lookup(local.subnet_output, "pub_lb", null) } output "pub_lb_subnet_cidr" { value = contains(keys(local.subnet_output), "pub_lb") ? lookup(local.subnet_cidrs_all, "pub_lb", null) : null } output "worker_subnet_id" { value = lookup(local.subnet_output, "workers", null) } output "worker_subnet_cidr" { value = contains(keys(local.subnet_output), "workers") ? lookup(local.subnet_cidrs_all, "workers", null) : null } output "pod_subnet_id" { value = lookup(local.subnet_output, "pods", null) } output "pod_subnet_cidr" { value = contains(keys(local.subnet_output), "pods") ? lookup(local.subnet_cidrs_all, "pods", null) : null } output "fss_subnet_id" { value = lookup(local.subnet_output, "fss", null) } output "fss_subnet_cidr" { value = contains(keys(local.subnet_output), "fss") ? lookup(local.subnet_cidrs_all, "fss", null) : null } ================================================ FILE: modules/network/variables.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "compartment_id" { type = string } variable "state_id" { type = string } # Tags variable "defined_tags" { type = map(string) } variable "freeform_tags" { type = map(string) } variable "tag_namespace" { type = string } variable "use_defined_tags" { type = bool } # Network variable "allow_node_port_access" { type = bool } variable "allow_pod_internet_access" { type = bool } variable "allow_rules_cp" { type = any } variable "allow_rules_internal_lb" { type = any } variable "allow_rules_pods" { type = any } variable "allow_rules_public_lb" { type = any } variable "allow_rules_workers" { type = any } variable "allow_worker_internet_access" { type = bool } variable "allow_worker_ssh_access" { type = bool } variable "allow_bastion_cluster_access" { type = bool } variable "assign_dns" { type = bool } variable "bastion_allowed_cidrs" { type = set(string) } variable "bastion_is_public" { type = bool } variable "cni_type" { type = string } variable "control_plane_allowed_cidrs" { type = set(string) } variable "control_plane_is_public" { type = bool } variable "create_cluster" { type = bool } variable "create_bastion" { type = bool } variable "create_internet_gateway" { type = bool } variable "create_nat_gateway" { type = bool } variable "create_operator" { type = bool } variable "drg_attachments" { type = any } variable "enable_ipv6" { type = bool } variable "enable_waf" { type = bool } variable "ig_route_table_id" { type = string } variable "igw_ngw_mixed_route_id" { type = string } variable "internet_gateway_id" { type = string } variable "load_balancers" { type = string } variable "nat_gateway_id" { type = string } variable "nat_route_table_id" { type = string } variable "vcn_cidrs" { type = list(string) } variable "vcn_ipv6_cidrs" { type = list(string) } variable "vcn_id" { type = string } variable "worker_is_public" { type = bool } variable "subnets" { type = map(object({ create = optional(string) id = optional(string) newbits = optional(string) netnum = optional(string) cidr = optional(string) display_name = optional(string) dns_label = optional(string) ipv6_cidr = optional(string) })) } variable "nsgs" { type = map(object({ create = optional(string) id = optional(string) })) } variable "use_stateless_rules" { type = bool } ================================================ FILE: modules/network/versions.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { oci = { source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/operator/README.md ================================================ # Operator This sub-module creates an operator host in a private subnet pre-installed with kubectl, Helm, and optional tools. ## Usage Refer to the [Operator section](../../docs/terraformoptions.md#operator) of the module documentation. ================================================ FILE: modules/operator/cloudinit.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { # https://cloudinit.readthedocs.io/en/latest/explanation/format.html#mime-multi-part-archive default_cloud_init_content_type = "text/x-shellscript" # https://canonical-cloud-init.readthedocs-hosted.com/en/latest/reference/merging.html default_cloud_init_merge_type = "list(append)+dict(no_replace,recurse_list)+str(append)" baserepo = "ol${var.operator_image_os_version}" developer_EPEL = "${local.baserepo}_developer_EPEL" olcne19 = "${local.baserepo}_olcne19" developer_olcne = "${local.baserepo}_developer_olcne" arch_amd = "amd64" arch_arm = "aarch64" } # https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html data "cloudinit_config" "operator" { gzip = true base64_encode = true # Repository/package installation part { content_type = "text/cloud-config" content = jsonencode({ # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#package-update-upgrade-install package_update = true package_upgrade = var.upgrade packages = compact([ "git", "jq", "python3-oci-cli", "golang", var.install_helm ? "helm" : null, var.install_istioctl ? "istio-istioctl" : null, var.install_kubectl_from_repo ? "kubectl" : null, ]) yum_repos = { "${local.developer_EPEL}" = { name = "Oracle Linux $releasever EPEL Packages for Development ($basearch)" baseurl = "https://yum$ociregion.$ocidomain/repo/OracleLinux/OL${var.operator_image_os_version}/developer/EPEL/$basearch/" gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle" gpgcheck = true enabled = true } "${local.olcne19}" = { name = "Oracle Linux Cloud Native Environment 1.8 ($basearch)" baseurl = "https://yum$ociregion.$ocidomain/repo/OracleLinux/OL${var.operator_image_os_version}/olcne19/$basearch/" gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle" gpgcheck = true enabled = true } "${local.developer_olcne}" = { name = "Developer Preview for Oracle Linux Cloud Native Environment ($basearch)" baseurl = "https://yum$ociregion.$ocidomain/repo/OracleLinux/OL${var.operator_image_os_version}/developer/olcne/$basearch/" gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle" gpgcheck = true enabled = false } } }) filename = "10-packages.yml" merge_type = local.default_cloud_init_merge_type } # Set timezone part { # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#timezone content_type = "text/cloud-config" content = jsonencode({ timezone = var.timezone }) filename = "10-timezone.yml" } # Create configured user part { content_type = "text/cloud-config" # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#users-and-groups content = jsonencode({ users = ["default", var.user] }) filename = "10-user.yml" } # Expand root filesystem to fill available space on volume part { content_type = "text/cloud-config" content = jsonencode({ # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#growpart growpart = { mode = "auto" devices = ["/"] ignore_growroot_disabled = false } # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#resizefs resize_rootfs = true # Resize logical LVM root volume when utility is present bootcmd = ["if [[ -f /usr/libexec/oci-growfs ]]; then /usr/libexec/oci-growfs -y; fi"] }) filename = "10-growpart.yml" merge_type = local.default_cloud_init_merge_type } # OCI CLI installation from repo dynamic "part" { for_each = var.install_oci_cli_from_repo ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "curl -LO https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh", "su -c 'bash /install.sh --accept-all-defaults' - ${var.user}", ] }) filename = "20-oci_cli_from_repo.yml" merge_type = local.default_cloud_init_merge_type } } # kubectl installation dynamic "part" { for_each = var.install_kubectl_from_repo ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "CLI_ARCH='${local.arch_amd}'", "if [ \"$(uname -m)\" = ${local.arch_arm} ]; then CLI_ARCH='arm64'; fi", "curl -LO https://dl.k8s.io/release/${var.kubernetes_version}/bin/linux/$CLI_ARCH/kubectl", "install -o root -g root -m 0755 kubectl /usr/bin/kubectl", ] }) filename = "20-kubectl.yml" merge_type = local.default_cloud_init_merge_type } } # k8sgpt installation dynamic "part" { for_each = var.install_k8sgpt ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "CLI_ARCH='${local.arch_amd}'", "if [ \"$(uname -m)\" = ${local.arch_arm} ]; then CLI_ARCH='arm64'; fi", "if [ -f /etc/os-release ]; then os_id=$(grep '^ID=' /etc/os-release | awk -F= '{print $2}' | tr -d '\"'); fi", "if [ \"$os_id\" == \"ubuntu\" ]; then curl -LO https://github.com/k8sgpt-ai/k8sgpt/releases/latest/download/k8sgpt_$CLI_ARCH.deb; dpkg -i k8sgpt_$CLI_ARCH.deb; rm k8sgpt_$CLI_ARCH.deb; fi", "if [ \"$os_id\" == \"ol\" ]; then while fuser /var/lib/rpm/.rpm.lock >/dev/null 2>&1; do sleep 5; done; rpm -ivh https://github.com/k8sgpt-ai/k8sgpt/releases/latest/download/k8sgpt_$CLI_ARCH.rpm; fi" ] }) filename = "20-k8sgpt.yml" merge_type = local.default_cloud_init_merge_type } } # kubectx/kubens installation dynamic "part" { for_each = var.install_kubectx ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "git clone https://github.com/ahmetb/kubectx /opt/kubectx", "ln -s /opt/kubectx/kubectx /usr/bin/kubectx", "ln -s /opt/kubectx/kubens /usr/bin/kubens", ] }) filename = "20-kubectx.yml" merge_type = local.default_cloud_init_merge_type } } # Helm installation from repo dynamic "part" { for_each = var.install_helm_from_repo ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3", "chmod 700 get_helm.sh", "./get_helm.sh", ] }) filename = "20-helm_from_repo.yml" merge_type = local.default_cloud_init_merge_type } } # Optional Helm installation bashrc dynamic "part" { for_each = var.install_helm ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#write-files write_files = [ { content = <<-EOT source <(helm completion bash) alias h='helm' EOT path = "/tmp/helm.bashrc" # see 30-bashrc.yml for final move }, ] }) filename = "20-helm.bashrc.yml" merge_type = local.default_cloud_init_merge_type } } # Optional k9s installation dynamic "part" { for_each = var.install_k9s ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "curl -LO https://github.com/derailed/k9s/releases/latest/download/k9s_Linux_amd64.tar.gz", "tar -xvzf k9s_Linux_amd64.tar.gz && mv ./k9s /usr/bin/k9s", "echo 'export K9S_FEATURE_GATE_NODE_SHELL=true' | tee -a /home/${var.user}/.bashrc", "mkdir -p /home/${var.user}/.config/k9s", <<-EOT cat << 'EOF' | tee /home/${var.user}/.config/k9s/views.yaml views: v1/nodes: columns: - NAME - HOSTNAME:.metadata.labels.hostname - SHAPE:.metadata.labels.node\.kubernetes\.io/instance-type - SERIAL:.metadata.labels.oci\.oraclecloud\.com/host\.serial_number - ROLE:|H EOF EOT ] }) filename = "20-k9s.yml" merge_type = local.default_cloud_init_merge_type } } # Optional cilium cli installation dynamic "part" { for_each = var.install_cilium ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)", "CLI_ARCH='${local.arch_amd}'", "if [ \"$(uname -m)\" = ${local.arch_arm} ]; then CLI_ARCH='arm64'; fi", "curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/$CILIUM_CLI_VERSION/cilium-linux-$CLI_ARCH.tar.gz", "tar xzvfC cilium-linux-$CLI_ARCH.tar.gz /usr/local/bin" ] }) filename = "20-cilium.yml" merge_type = local.default_cloud_init_merge_type } } # stern installation dynamic "part" { for_each = var.install_stern ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "go install github.com/stern/stern@v1.30", "mv $HOME/go/bin/stern /usr/local/bin/", "ln -s /usr/local/bin/stern /usr/bin/stern" ] }) filename = "20-stern.yml" merge_type = local.default_cloud_init_merge_type } } # Write user bashrc to filesystem part { content_type = "text/cloud-config" content = jsonencode({ # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#write-files write_files = [ { content = <<-EOT export OCI_CLI_AUTH=instance_principal export TERM=xterm-256color export OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=True source <(kubectl completion bash) alias k='kubectl' alias ktx='kubectx' alias kns='kubens' EOT path = "/tmp/user.bashrc" # see 30-home.yml for final move }, ] }) filename = "20-bashrc.yml" merge_type = local.default_cloud_init_merge_type } # Write user kubeconfig to filesystem part { content_type = "text/cloud-config" content = jsonencode({ # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#write-files write_files = [ { content = var.kubeconfig path = "/tmp/kubeconfig" # see 30-home.yml for final move }, ] }) filename = "20-kubeconfig.yml" merge_type = local.default_cloud_init_merge_type } # Bug w/ write_files defer: parent directory created as root if not present. # https://github.com/canonical/cloud-init/pull/916#issuecomment-1254732400 # Or: defer not supported on older versions of cloud-init. # Created in tmp first and moved into user's home directory using runcmd. part { content_type = "text/cloud-config" content = jsonencode({ runcmd = [ "cat /tmp/*.bashrc >> /home/${var.user}/.bashrc && rm /tmp/*.bashrc", "chmod 600 /home/${var.user}/.bashrc", "mkdir -p /home/${var.user}/.kube", "mv /tmp/kubeconfig /home/${var.user}/.kube/config", "chmod 700 /home/${var.user}/.kube", "chmod 600 /home/${var.user}/.kube/config", "chown -R ${var.user}:${var.user} /home/${var.user}", ] }) filename = "30-home.yml" merge_type = local.default_cloud_init_merge_type } # Include custom cloud init MIME parts dynamic "part" { for_each = var.cloud_init iterator = part content { # Load content from file if local path, attempt base64 decode, or use raw value content = contains(keys(part.value), "content") ? ( try(fileexists(lookup(part.value, "content")), false) ? file(lookup(part.value, "content")) : try(base64decode(lookup(part.value, "content")), lookup(part.value, "content")) ) : "" content_type = lookup(part.value, "content_type", local.default_cloud_init_content_type) filename = lookup(part.value, "filename", null) merge_type = lookup(part.value, "merge_type", local.default_cloud_init_merge_type) } } lifecycle { precondition { condition = alltrue([for c in var.cloud_init : trimspace(lookup(c, "content", "")) != ""]) error_message = <<-EOT Each operator cloud_init map entry must include a non-empty 'content' field. See https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html. var.cloud_init: ${try(jsonencode(var.cloud_init), "invalid")} EOT } precondition { condition = alltrue([for c in var.cloud_init : length(regexall("^text/[a-z-]*$", trimspace(lookup(c, "content_type", local.default_cloud_init_content_type)))) > 0 ]) error_message = <<-EOT Each operator cloud_init map entry must include a 'content_type' field prefixed with 'text/'. See https://cloudinit.readthedocs.io/en/latest/explanation/format.html#mime-multi-part-archive. var.cloud_init: ${try(jsonencode(var.cloud_init), "invalid")} EOT } } } resource "null_resource" "await_cloudinit" { count = var.await_cloudinit ? 1 : 0 connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = oci_core_instance.operator.private_ip user = var.user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } lifecycle { replace_triggered_by = [oci_core_instance.operator] } provisioner "remote-exec" { inline = ["cloud-init status --wait &> /dev/null"] } } ================================================ FILE: modules/operator/compute.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { boot_volume_size = lookup(var.shape, "boot_volume_size", 50) memory = lookup(var.shape, "memory", 4) ocpus = max(1, lookup(var.shape, "ocpus", 1)) shape = lookup(var.shape, "shape", "VM.Standard.E4.Flex") baseline_ocpu_utilization_map = { "12.5" = "BASELINE_1_8" "50" = "BASELINE_1_2" "100" = "BASELINE_1_1" } baseline_ocpu_utilization = lookup(local.baseline_ocpu_utilization_map, lookup(var.shape, "baseline_ocpu_utilization", "100")) } output "id" { value = oci_core_instance.operator.id } output "private_ip" { value = oci_core_instance.operator.private_ip } resource "oci_core_instance" "operator" { availability_domain = var.availability_domain compartment_id = var.compartment_id display_name = "operator-${var.state_id}" defined_tags = var.defined_tags freeform_tags = var.freeform_tags is_pv_encryption_in_transit_enabled = var.pv_transit_encryption shape = local.shape agent_config { are_all_plugins_disabled = false is_management_disabled = false is_monitoring_disabled = false plugins_config { desired_state = "ENABLED" name = "Bastion" } plugins_config { desired_state = "DISABLED" name = "OS Management Service Agent" } } create_vnic_details { assign_public_ip = false display_name = "operator-${var.state_id}" hostname_label = var.assign_dns ? "o-${var.state_id}" : null nsg_ids = var.nsg_ids subnet_id = var.subnet_id } launch_options { boot_volume_type = "PARAVIRTUALIZED" network_type = "PARAVIRTUALIZED" } instance_options { are_legacy_imds_endpoints_disabled = var.legacy_imds_endpoints_disabled } metadata = { ssh_authorized_keys = var.ssh_public_key user_data = data.cloudinit_config.operator.rendered } dynamic "shape_config" { for_each = length(regexall("Flex", local.shape)) > 0 ? [1] : [] content { baseline_ocpu_utilization = local.baseline_ocpu_utilization ocpus = local.ocpus memory_in_gbs = (local.memory / local.ocpus) > 64 ? (local.ocpus * 4) : local.memory } } source_details { boot_volume_size_in_gbs = local.boot_volume_size source_type = "image" source_id = var.image_id kms_key_id = var.volume_kms_key_id } lifecycle { ignore_changes = [ availability_domain, defined_tags, freeform_tags, display_name, create_vnic_details, metadata, source_details, ] replace_triggered_by = [null_resource.operator_changed] precondition { condition = coalesce(var.image_id, "none") != "none" error_message = "Missing image_id for operator. Check provided value for image_id if image_type is 'custom', or image_os/image_os_version if image_type is 'platform'." } } timeouts { create = "60m" } } resource "null_resource" "operator_changed" { triggers = { cloud_init = jsonencode(var.cloud_init) image_id = var.image_id install_helm = var.install_helm install_k9s = var.install_k9s install_kubectx = var.install_kubectx ssh_public_key = var.ssh_public_key } } ================================================ FILE: modules/operator/variables.tf ================================================ # Copyright (c) 2019, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "compartment_id" { type = string } variable "state_id" { type = string } # Bastion (to await cloud-init completion) variable "bastion_host" { type = string } variable "bastion_user" { type = string } # Operator variable "await_cloudinit" { type = string } variable "assign_dns" { type = bool } variable "availability_domain" { type = string } variable "cloud_init" { type = list(map(string)) } variable "image_id" { type = string } variable "install_cilium" { type = bool } variable "install_oci_cli_from_repo" { type = bool } variable "install_helm" { type = bool } variable "install_helm_from_repo" { type = bool } variable "install_istioctl" { type = bool } variable "install_k8sgpt" { type = bool } variable "install_k9s" { type = bool } variable "install_kubectl_from_repo" { type = bool default = true } variable "install_kubectx" { type = bool } variable "install_stern" { type = bool } variable "kubeconfig" { type = string } variable "kubernetes_version" { type = string } variable "nsg_ids" { type = list(string) } variable "operator_image_os_version" { type = string } variable "pv_transit_encryption" { type = bool } variable "shape" { type = map(any) } variable "legacy_imds_endpoints_disabled" { type = bool default = true } variable "ssh_private_key" { type = string sensitive = true } variable "ssh_public_key" { type = string } variable "subnet_id" { type = string } variable "timezone" { type = string } variable "upgrade" { type = bool } variable "user" { type = string } variable "volume_kms_key_id" { type = string } # Tags variable "defined_tags" { type = map(string) } variable "freeform_tags" { type = map(string) } variable "tag_namespace" { type = string } variable "use_defined_tags" { type = bool } ================================================ FILE: modules/operator/versions.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { cloudinit = { source = "hashicorp/cloudinit" version = ">= 2.2.0" } null = { source = "hashicorp/null" version = ">= 3.2.1" } oci = { source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/utilities/README.md ================================================ # Utilities This sub-module provides helper resources including node readiness checks, OCIR secret creation, and worker pool draining. ## Usage Refer to the [Utilities section](../../docs/terraformoptions.md#utilities) of the module documentation. ================================================ FILE: modules/utilities/drain.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { drain_enabled = var.expected_drain_count > 0 drain_pools = (local.drain_enabled ? tolist([for k, v in var.worker_pools : k if tobool(lookup(v, "drain", false))]) : [] ) drain_commands = formatlist( format( "kubectl drain %v %v %v %v", format("--timeout=%vs", var.worker_drain_timeout_seconds), format("--ignore-daemonsets=%v", var.worker_drain_ignore_daemonsets), format("--delete-emptydir-data=%v", var.worker_drain_delete_local_data), "-l oke.oraclecloud.com/pool.name=%v" # interpolation deferred to formatlist ), local.drain_pools ) } resource "null_resource" "drain_workers" { count = local.drain_enabled ? 1 : 0 triggers = { drain_pools = jsonencode(sort(local.drain_pools)) drain_commands = jsonencode(local.drain_commands) } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = local.drain_commands } } ================================================ FILE: modules/utilities/nodeready.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { node_ready_script = "/home/${var.operator_user}/await_node_ready.sh" node_ready_template = templatefile("${path.module}/resources/await_node_readiness.tpl.sh", { await_node_readiness = var.await_node_readiness expected_node_count = var.expected_node_count } ) } resource "null_resource" "await_node_readiness" { count = var.await_node_readiness != "none" && var.expected_node_count > 0 ? 1 : 0 triggers = { expected_node_count = var.expected_node_count } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "file" { content = local.node_ready_template destination = local.node_ready_script } provisioner "remote-exec" { inline = ["bash ${local.node_ready_script}"] } } ================================================ FILE: modules/utilities/ocir.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Retrieve object storage namespace for secret data "oci_objectstorage_namespace" "object_storage_namespace" { count = coalesce(var.ocir_secret_id, "none") != "none" ? 1 : 0 } locals { oci_secret_query = "'data.\"secret-bundle-content\".content'" oci_secret_get = format( "oci secrets secret-bundle get --raw-output --secret-id %s --query %s | base64 -d", coalesce(var.ocir_secret_id, "none"), local.oci_secret_query, ) region_registry = "${var.region}.ocir.io" tenancy_namespace = one(data.oci_objectstorage_namespace.object_storage_namespace[*].namespace) } resource "null_resource" "ocir_secret" { count = coalesce(var.ocir_secret_id, "none") != "none" ? 1 : 0 triggers = { always_run = timestamp() } connection { bastion_host = var.bastion_host bastion_user = var.bastion_user bastion_private_key = var.ssh_private_key host = var.operator_host user = var.operator_user private_key = var.ssh_private_key timeout = "40m" type = "ssh" } provisioner "remote-exec" { inline = formatlist("%s --dry-run=client -o yaml | kubectl apply -f -", [ format("kubectl create ns %s", var.ocir_secret_namespace), format("kubectl create secret docker-registry %s", join(" ", [ var.ocir_secret_name, format("-n %s", var.ocir_secret_namespace), format("--docker-server=%s", local.region_registry), format("--docker-ocir_username=%s/%s", join("/", compact([local.tenancy_namespace, var.ocir_username]))), format("--docker-email=%s", var.ocir_email_address), format("--docker-password=%s", local.oci_secret_get) ]) ), ]) } lifecycle { precondition { condition = alltrue([ local.region_registry != null, local.tenancy_namespace != null, var.ocir_email_address != null, var.ocir_secret_name != null, var.ocir_secret_namespace != null, var.ocir_username != null, ]) error_message = "Missing required configuration for OCIR; check variables." } } } ================================================ FILE: modules/utilities/resources/await_node_readiness.tpl.sh ================================================ #!/usr/bin/env bash # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # shellcheck disable=SC1083,SC2309,SC2154,SC2157,SC2034 # Ignore templated/escaped/unused file variables export ALL_FILE=~/all_node.active ONE_FILE=~/one_node.active function clean_node_active() { rm -f "$${ALL_FILE}" "$${ONE_FILE}" } function get_actual_node_count() ( (kubectl get --no-headers nodes | grep -v NotReady | awk '{print $1}' | wc -l) 2>/dev/null || echo '0' ) function wait_for_active() { clean_node_active while true; do local actual_node_count actual_node_count=$(get_actual_node_count) if [[ $${actual_node_count} -ge ${expected_node_count} ]]; then touch all_node.active; fi if [[ $${actual_node_count} -ge 1 ]]; then touch one_node.active; fi if [[ -f "$${ONE_FILE}" ]] && [[ "${await_node_readiness}" == 'one' ]]; then echo "$(date): Ready with $${actual_node_count} node(s)" >&2 break fi if [[ -f "$${ALL_FILE}" ]] && [[ "${await_node_readiness}" == 'all' ]]; then echo "$(date): Ready with $${actual_node_count} node(s)" >&2 break fi echo "$(date): Waiting for ${await_node_readiness} of ${expected_node_count} node(s) to become ready ($${actual_node_count} found)" >&2 sleep 30 done } if [[ ${expected_node_count} -ge 1 ]]; then time wait_for_active; fi ================================================ FILE: modules/utilities/variables.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "region" { type = string } variable "worker_pools" { type = any } # Connection variable "bastion_host" { type = string } variable "bastion_user" { type = string } variable "operator_host" { type = string } variable "operator_user" { type = string } variable "ssh_private_key" { type = string sensitive = true } # OCIR variable "ocir_email_address" { type = string } variable "ocir_secret_id" { type = string } variable "ocir_secret_name" { type = string } variable "ocir_secret_namespace" { type = string } variable "ocir_username" { type = string } # Node readiness check, drain variable "await_node_readiness" { type = string } variable "expected_drain_count" { type = number } variable "expected_node_count" { type = number } variable "worker_drain_ignore_daemonsets" { type = bool } variable "worker_drain_delete_local_data" { type = bool } variable "worker_drain_timeout_seconds" { type = number } ================================================ FILE: modules/utilities/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { null = { source = "hashicorp/null" version = ">= 3.2.1" } oci = { source = "oracle/oci" version = ">= 7.30.0" } } } ================================================ FILE: modules/workers/README.md ================================================ # Worker pools This sub-module supports different modes of OKE worker node management with advanced configuration. ## Usage Refer to the [Workers section](../../docs/terraformoptions.md#workers) of the module documentation. ================================================ FILE: modules/workers/cloudinit-oke.sh ================================================ #!/usr/bin/env bash # Copyright (c) 2022, 2025 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # shellcheck disable=SC1091 # Ignore unresolved file path present on base images set -o pipefail function get_imds_base_url() { imds_base_url=$(cat /tmp/imds_base_url || echo "") if [[ -z $imds_base_url ]]; then for url in "http://169.254.169.254/" "http://[fd00:c1::a9fe:a9fe]/"; do if curl -sSf -m 5 --retry 5 --retry-delay 1 -H 'Authorization: Bearer Oracle' -L0 "${url}opc/v2/instance/state" > /dev/null; then imds_base_url="$url" echo "$imds_base_url" > /tmp/imds_base_url break fi done fi if [ -z "${imds_base_url}" ]; then echo "Unable to determine imds base url" >&2 exit 1 fi echo "${imds_base_url}" } function curl_instance_metadata() { local imds_base="$(get_imds_base_url)" local url="${imds_base}$1" local retries=10 local output while (( retries-- > 0 )); do if output=$(curl -sSf -m 5 -H 'Authorization: Bearer Oracle' -L0 "$url"); then echo "$output" return 0 fi sleep 1 done echo "Failed to fetch metadata from $url" >&2 return 1 } function get_imds_instance() { find "${INSTANCE_FILE}" -mmin -1 -not -empty > /dev/null 2>&1 || (curl_instance_metadata 'opc/v2/instance' | jq -rcM '.' > "${INSTANCE_FILE}") INSTANCE="$(cat "${INSTANCE_FILE}" || echo -n '')" export INSTANCE echo "${INSTANCE}" } function get_imds_metadata() { get_imds_instance | jq -rcM '.metadata // {}' } function run_oke_init() { # Initialize OKE worker node if [[ -f /etc/systemd/system/oke-init.service ]]; then systemctl --no-block enable --now oke-init.service return fi if [[ -f /etc/oke/oke-install.sh ]]; then local apiserver_host cluster_ca if [[ -f "/etc/oke/oke-apiserver" ]]; then apiserver_host=$(< /etc/oke/oke-apiserver) else apiserver_host=$(get_imds_metadata | jq -rcM '.apiserver_host') fi if [[ -f "/etc/kubernetes/ca.crt" ]]; then cluster_ca=$(base64 -w0 /etc/kubernetes/ca.crt) else cluster_ca=$(get_imds_metadata | jq -rcM '.cluster_ca_cert') fi bash /etc/oke/oke-install.sh \ --apiserver-endpoint "${apiserver_host}" \ --kubelet-ca-cert "${cluster_ca}" return fi local retries=5 local delay=2 local oke_init_relative_path="opc/v2/instance/metadata/oke_init_script" local script_path="/var/run/oke-init.sh" for (( i=0; i "${script_path}"; then bash "${script_path}" exit 0 fi done echo "Retry $((i+1)) failed, retrying in $delay seconds..." done } INSTANCE_FILE="/etc/oke/imds_instance.json" time run_oke_init || { echo "Error in OKE startup" >&2; exit 1; } ================================================ FILE: modules/workers/cloudinit-ubuntu.sh.tftpl ================================================ #!/bin/bash set -x source /etc/os-release oke_package_name="oci-oke-node-all-${oke_minor_version}" # Add OKE Ubuntu package repo tee /etc/apt/sources.list.d/oke-node-client.sources </dev/null 2>&1; do echo "Waiting for dpkg/apt lock" sleep 1 done apt-get -y update && apt-get -y install $oke_package_name # OKE bootstrap oke bootstrap ================================================ FILE: modules/workers/cloudinit.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { # https://cloudinit.readthedocs.io/en/latest/explanation/format.html#mime-multi-part-archive default_cloud_init_content_type = "text/x-shellscript" # https://canonical-cloud-init.readthedocs-hosted.com/en/latest/reference/merging.html default_cloud_init_merge_type = "list(append)+dict(no_replace,recurse_list)+str(append)" } data "oci_core_image" "workers" { for_each = { # Skip generation for mode = virtual-node-pool for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", var.worker_pool_mode) != "virtual-node-pool" } image_id = each.value.image_id } # https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html data "cloudinit_config" "workers" { for_each = { # Skip generation for mode = virtual-node-pool for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", var.worker_pool_mode) != "virtual-node-pool" } gzip = true base64_encode = true # Include global and pool-specific custom cloud init MIME parts dynamic "part" { for_each = each.value.cloud_init iterator = part content { content = lookup(part.value, "content", "") content_type = lookup(part.value, "content_type", local.default_cloud_init_content_type) filename = lookup(part.value, "filename", null) merge_type = lookup(part.value, "merge_type", local.default_cloud_init_merge_type) } } # Set timezone dynamic "part" { for_each = each.value.disable_default_cloud_init ? [] : [1] content { content_type = "text/cloud-config" # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#timezone content = jsonencode({ timezone = var.timezone }) filename = "10-timezone.yml" } } # Expand root filesystem to fill available space on volume dynamic "part" { for_each = each.value.disable_default_cloud_init ? [] : [1] content { content_type = "text/cloud-config" content = jsonencode({ # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#growpart growpart = { mode = "auto" devices = ["/"] ignore_growroot_disabled = false } # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#resizefs resize_rootfs = true # Resize logical LVM root volume when utility is present bootcmd = ["if [[ -f /usr/libexec/oci-growfs ]]; then /usr/libexec/oci-growfs -y; fi"] }) filename = "10-growpart.yml" merge_type = local.default_cloud_init_merge_type } } # Write extra OKE configuration to filesystem dynamic "part" { for_each = each.value.disable_default_cloud_init ? [] : [1] content { content_type = "text/cloud-config" content = jsonencode({ write_files = [ { content = var.apiserver_private_host path = "/etc/oke/oke-apiserver" }, { content = var.cluster_ca_cert encoding = "base64" path = "/etc/kubernetes/ca.crt" }, ] }) filename = "50-oke-config.yml" merge_type = local.default_cloud_init_merge_type } } # Disable CRI-O enforce shortnames mode (for versions greater than 1.34) dynamic "part" { for_each = tonumber(split(".", each.value.kubernetes_version)[1]) >= 34 && var.allow_short_container_image_names ? [1] : [] content { content_type = "text/cloud-config" content = jsonencode({ write_files = [ { content = <<-EOT [crio.image] short_name_mode = "disabled" EOT path = "/etc/crio/crio.conf.d/11-default.conf" } ] }) filename = "50-crio-config.yml" merge_type = local.default_cloud_init_merge_type } } # OKE setup and initialization for Ubuntu images dynamic "part" { for_each = !each.value.disable_default_cloud_init && lookup(local.ubuntu_worker_pools, each.key, null) != null ? [1] : [] content { content_type = "text/x-shellscript" content = templatefile( "${path.module}/cloudinit-ubuntu.sh.tftpl", { version_codename = lookup(local.ubuntu_supported_versions, lookup(lookup(local.ubuntu_worker_pools, each.key, {}), "ubuntu_release", lookup(each.value, "os_version")), "unsupported_ubuntu_version"), oke_major_version = lookup(lookup(local.ubuntu_worker_pools, each.key, {}), "kubernetes_major_version", "") oke_minor_version = lookup(lookup(local.ubuntu_worker_pools, each.key, {}), "kubernetes_minor_version", "") } ) filename = "50-oke-ubuntu.sh" merge_type = local.default_cloud_init_merge_type } } # OKE startup initialization dynamic "part" { for_each = !each.value.disable_default_cloud_init && lookup(local.ubuntu_worker_pools, each.key, null) == null ? [1] : [] content { content_type = "text/x-shellscript" content = file("${path.module}/cloudinit-oke.sh") filename = "50-oke.sh" merge_type = local.default_cloud_init_merge_type } } lifecycle { precondition { condition = alltrue([for c in var.cloud_init : trimspace(lookup(c, "content", "")) != "" ]) error_message = <<-EOT Each global cloud_init map entry must include a non-empty 'content' field. See https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html. var.cloud_init (${each.key}): ${try(jsonencode(var.cloud_init), "invalid")} EOT } precondition { condition = alltrue([for c in var.cloud_init : length(regexall("^text/[a-z-]*$", trimspace(lookup(c, "content_type", local.default_cloud_init_content_type)))) > 0 ]) error_message = <<-EOT Each global cloud_init map entry must include a 'content_type' field prefixed with 'text/'. See https://cloudinit.readthedocs.io/en/latest/explanation/format.html#mime-multi-part-archive. var.cloud_init (${each.key}): ${try(jsonencode(var.cloud_init), "invalid")} EOT } precondition { condition = alltrue([for c in each.value.cloud_init : trimspace(lookup(c, "content", "")) != "" ]) error_message = <<-EOT Each pool-specific cloud_init map entry must include a non-empty 'content' field. See https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html. ${each.key}["cloud_init"]: ${try(jsonencode(each.value.cloud_init), "invalid")} EOT } precondition { condition = alltrue([for c in each.value.cloud_init : length(regexall("^text/[a-z-]+$", trimspace(lookup(c, "content_type", local.default_cloud_init_content_type)))) > 0 ]) error_message = <<-EOT Each pool-specific cloud_init map entry must include a 'content_type' field prefixed with 'text/'. See https://cloudinit.readthedocs.io/en/latest/explanation/format.html#mime-multi-part-archive. ${each.key}["cloud_init"]: ${try(jsonencode(each.value.cloud_init), "invalid")} EOT } precondition { condition = lookup(local.ubuntu_worker_pools, each.key, null) == null || ( lookup(local.ubuntu_worker_pools, each.key, null) != null && contains(keys(local.ubuntu_supported_versions), lookup(lookup(local.ubuntu_worker_pools, each.key, {}), "ubuntu_release", "")) ) error_message = <<-EOT Supported Ubuntu versions are "22.04" and "24.04". See https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengcreatingubuntubasedworkernodes.htm#contengcreatingubuntubasedworkernodes_availabilitycompatibility. ${each.key}: ${jsonencode(lookup(local.ubuntu_worker_pools, each.key, {}))} EOT } } } ================================================ FILE: modules/workers/clusternetworks.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Dynamic resource block for Cluster Network groups defined in worker_pools resource "oci_core_cluster_network" "workers" { # Create an OCI Cluster Network resource for each enabled entry of the worker_pools map with that mode. for_each = local.enabled_cluster_networks compartment_id = each.value.compartment_id display_name = each.key defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags instance_pools { instance_configuration_id = oci_core_instance_configuration.workers[each.key].id display_name = each.key size = each.value.size defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags } placement_configuration { availability_domain = element(each.value.availability_domains, 1) primary_subnet_id = each.value.subnet_id dynamic "secondary_vnic_subnets" { for_each = lookup(each.value, "secondary_vnics", {}) iterator = vnic content { display_name = vnic.key subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id) } } } lifecycle { ignore_changes = [ display_name, defined_tags, freeform_tags, instance_pools[0].defined_tags, instance_pools[0].freeform_tags, ] precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = "Missing image_id for pool ${each.key}. Check provided value for image_id if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'." } precondition { condition = each.value.autoscale == false error_message = "Cluster Networks do not support cluster autoscaler management." } } # First-boot hardware config for bare metal instances takes extra time timeouts { create = "2h" } } ================================================ FILE: modules/workers/computecluster.tf ================================================ # Copyright (c) 2022, 2025 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Create the shared compute clusters defined in workers_compute_clusters resource "oci_core_compute_cluster" "shared" { # Create an OCI Compute Cluster resource for each enabled entry of the worker_pools map with that mode. for_each = var.compute_clusters compartment_id = lookup(each.value, "compartment_id", var.compartment_id) display_name = each.key defined_tags = merge( var.defined_tags, lookup(each.value, "defined_tags", {}) ) freeform_tags = merge( var.freeform_tags, lookup(each.value, "freeform_tags", {}) ) availability_domain = lookup(var.ad_numbers_to_names, lookup(each.value, "placement_ad", 1)) lifecycle { ignore_changes = [ display_name, defined_tags, freeform_tags, ] } } # Dynamic resource block for Compute Cluster groups defined in worker_pools resource "oci_core_compute_cluster" "workers" { # Create an OCI Compute Cluster resource for each enabled entry of the worker_pools map with that mode. for_each = { for k, v in local.enabled_compute_clusters : k => v if length(lookup(v, "instance_ids", [])) > 0 && lookup(v, "compute_cluster", null) == null } compartment_id = each.value.compartment_id display_name = each.key defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags availability_domain = lookup(each.value, "placement_ad", null) != null ? lookup(var.ad_numbers_to_names, lookup(each.value, "placement_ad")) : element(each.value.availability_domains, 0) lifecycle { ignore_changes = [ display_name, defined_tags, freeform_tags, ] } } resource "oci_core_instance" "compute_cluster_workers" { for_each = local.compute_cluster_instance_map availability_domain = (lookup(oci_core_compute_cluster.shared, lookup(each.value, "compute_cluster", ""), null) != null ? oci_core_compute_cluster.shared[lookup(each.value, "compute_cluster", "")].availability_domain : lookup(each.value, "placement_ad", null) != null ? lookup(var.ad_numbers_to_names, lookup(each.value, "placement_ad")) : element(each.value.availability_domains, 0) ) fault_domain = try(each.value.placement_fds[0], null) compartment_id = each.value.compartment_id display_name = format("%s-%s", element(split("###", each.key), 0), element(split("###", each.key), 1)) preserve_boot_volume = false shape = each.value.shape defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags extended_metadata = each.value.extended_metadata capacity_reservation_id = each.value.capacity_reservation_id compute_cluster_id = (lookup(oci_core_compute_cluster.shared, lookup(each.value, "compute_cluster", ""), null) != null ? oci_core_compute_cluster.shared[lookup(each.value, "compute_cluster", "")].id : (lookup(oci_core_compute_cluster.workers, element(split("###", each.key), 0), null) != null ? oci_core_compute_cluster.workers[element(split("###", each.key), 0)].id : lookup(each.value, "compute_cluster", "") ) ) dynamic "platform_config" { for_each = each.value.platform_config != null ? [1] : [] content { type = lookup( # Attempt lookup against data source for the associated 'type' of configured worker shape lookup(local.platform_config_by_shape, each.value.shape, {}), "type", # Fall back to 'type' on pool with custom platform_config, or INTEL_VM default lookup(each.value.platform_config, "type", "INTEL_VM") ) # Remaining parameters as configured, validated by instance/instance config resource are_virtual_instructions_enabled = lookup(each.value.platform_config, "are_virtual_instructions_enabled", null) is_access_control_service_enabled = lookup(each.value.platform_config, "is_access_control_service_enabled", null) is_input_output_memory_management_unit_enabled = lookup(each.value.platform_config, "is_input_output_memory_management_unit_enabled", null) is_measured_boot_enabled = lookup(each.value.platform_config, "is_measured_boot_enabled", null) is_memory_encryption_enabled = lookup(each.value.platform_config, "is_memory_encryption_enabled", null) is_secure_boot_enabled = lookup(each.value.platform_config, "is_secure_boot_enabled", null) is_symmetric_multi_threading_enabled = lookup(each.value.platform_config, "is_symmetric_multi_threading_enabled", null) is_trusted_platform_module_enabled = lookup(each.value.platform_config, "is_trusted_platform_module_enabled", null) numa_nodes_per_socket = lookup(each.value.platform_config, "numa_nodes_per_socket", null) percentage_of_cores_enabled = lookup(each.value.platform_config, "percentage_of_cores_enabled", null) } } agent_config { are_all_plugins_disabled = each.value.agent_config.are_all_plugins_disabled is_management_disabled = each.value.agent_config.is_management_disabled is_monitoring_disabled = each.value.agent_config.is_monitoring_disabled dynamic "plugins_config" { for_each = merge( { "Compute HPC RDMA Authentication" : "ENABLED", "Compute HPC RDMA Auto-Configuration" : "ENABLED" }, each.value.agent_config.plugins_config ) content { name = plugins_config.key desired_state = plugins_config.value } } } create_vnic_details { assign_private_dns_record = var.assign_dns assign_public_ip = each.value.assign_public_ip nsg_ids = each.value.nsg_ids subnet_id = each.value.subnet_id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags } instance_options { are_legacy_imds_endpoints_disabled = each.value.legacy_imds_endpoints_disabled } metadata = merge( { apiserver_host = var.apiserver_private_host cluster_ca_cert = var.cluster_ca_cert oke-k8version = var.kubernetes_version oke-kubeproxy-proxy-mode = var.kubeproxy_mode oke-tenancy-id = var.tenancy_id oke-initial-node-labels = join(",", [for k, v in each.value.node_labels : format("%v=%v", k, v)]) secondary_vnics = jsonencode(lookup(each.value, "secondary_vnics", {})) ssh_authorized_keys = var.ssh_public_key user_data = lookup(lookup(data.cloudinit_config.workers, element(split("###", each.key), 0), {}), "rendered", "") }, # Add labels required for NPN CNI. var.cni_type == "npn" ? { oke-native-pod-networking = true oke-max-pods = each.value.max_pods_per_node pod-subnets = each.value.pod_subnet_id pod-nsgids = join(",", each.value.pod_nsg_ids) } : {}, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, # Extra user-defined fields merged last var.node_metadata, # global lookup(each.value, "node_metadata", {}), # pool-specific ) source_details { boot_volume_size_in_gbs = each.value.boot_volume_size boot_volume_vpus_per_gb = each.value.boot_volume_vpus_per_gb source_id = each.value.image_id source_type = "image" } lifecycle { precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = <<-EOT Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. pool: ${element(split("###", each.key), 0)} image_type: ${coalesce(each.value.image_type, "none")} image_id: ${coalesce(each.value.image_id, "none")} EOT } ignore_changes = [ agent_config, # TODO Not updateable; remove when supported defined_tags, freeform_tags, display_name, metadata["cluster_ca_cert"], metadata["user_data"], create_vnic_details[0].defined_tags, create_vnic_details[0].freeform_tags, ] } } ================================================ FILE: modules/workers/data-faultdomains.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_identity_fault_domains" "all" { for_each = var.ad_numbers_to_names availability_domain = each.value compartment_id = var.compartment_id } ================================================ FILE: modules/workers/data-shapes.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl data "oci_core_shapes" "oke" { compartment_id = var.compartment_id } locals { shapes_by_name = { # Group by shape name, yielding a list of objects for each for shape in data.oci_core_shapes.oke.shapes : lookup(shape, "name") => shape... if contains(keys(shape), "name") } platform_config_by_shape = { # Merge objects for each shape; we only need the consistent 'type' for k, v in local.shapes_by_name : k => merge(lookup(merge(v...), "platform_config_options", [])...) } } ================================================ FILE: modules/workers/gpumemorycluster.tf ================================================ # Copyright (c) 2026 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # One shared compute cluster per gpu-memory-cluster pool. All GMCs in a pool bind to this compute cluster. # compute_cluster_id is not updatable on the GMC, so any replacement here cascades into a destroy+create of every GMC bound to it. resource "oci_core_compute_cluster" "gmc" { for_each = local.enabled_gmc_pools compartment_id = each.value.compartment_id display_name = each.key defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags availability_domain = element(each.value.availability_domains, 1) lifecycle { ignore_changes = [ display_name, defined_tags, freeform_tags, ] } } # One GPU Memory Cluster per (pool, GMF). Keyed by "###" so list edits don't shift other GMCs. # Size is omitted: when unset, the OCI control plane sizes the GMC from the fabric's available_host_count. # lifecycle.ignore_changes keeps ongoing size management with the OCI scaler via gpu_memory_cluster_scale_config. resource "oci_core_compute_gpu_memory_cluster" "workers" { for_each = local.enabled_gmc_fabric_map availability_domain = element(each.value.availability_domains, 1) compartment_id = each.value.compartment_id compute_cluster_id = oci_core_compute_cluster.gmc[each.value.pool_name].id instance_configuration_id = oci_core_instance_configuration.workers[each.value.pool_name].id gpu_memory_fabric_id = each.value.gpu_memory_fabric_id display_name = format("%s-%s", each.value.pool_name, substr(each.value.gpu_memory_fabric_id, -11, 11)) defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags gpu_memory_cluster_scale_config { is_upsize_enabled = lookup(each.value.gpu_memory_cluster_scale_config, "is_upsize_enabled", var.gmc_scale_is_upsize_enabled) is_downsize_enabled = lookup(each.value.gpu_memory_cluster_scale_config, "is_downsize_enabled", var.gmc_scale_is_downsize_enabled) target_size = lookup(each.value.gpu_memory_cluster_scale_config, "target_size", var.gmc_scale_target_size) } lifecycle { precondition { condition = length(each.value.gpu_memory_fabric_ids) == length(toset(each.value.gpu_memory_fabric_ids)) error_message = "Duplicate GMF OCIDs detected in pool ${each.value.pool_name}'s gpu_memory_fabric_ids list. Each GMF must appear at most once." } precondition { condition = each.value.autoscale == false error_message = "GPU Memory Clusters do not support cluster autoscaler management; the OCI control plane manages scaling via gpu_memory_cluster_scale_config." } ignore_changes = [ size, gpu_memory_cluster_scale_config, defined_tags, freeform_tags, display_name, ] } } ================================================ FILE: modules/workers/instance.tf ================================================ resource "oci_core_instance" "workers" { for_each = local.enabled_instances availability_domain = element(each.value.availability_domains, 1) fault_domain = try(each.value.placement_fds[0], null) compartment_id = each.value.compartment_id display_name = each.key preserve_boot_volume = false shape = each.value.shape defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags extended_metadata = each.value.extended_metadata capacity_reservation_id = each.value.capacity_reservation_id dynamic "shape_config" { for_each = length(regexall("Flex", each.value.shape)) > 0 ? [1] : [] content { baseline_ocpu_utilization = lookup(each.value, "burst", "BASELINE_1_1") ocpus = each.value.ocpus memory_in_gbs = ( # If > 64GB memory/core, correct input to exactly 64GB memory/core (each.value.memory / each.value.ocpus) > 64 ? each.value.ocpus * 64 : each.value.memory ) } } dynamic "platform_config" { for_each = each.value.platform_config != null ? [1] : [] content { type = lookup( # Attempt lookup against data source for the associated 'type' of configured worker shape lookup(local.platform_config_by_shape, each.value.shape, {}), "type", # Fall back to 'type' on pool with custom platform_config, or INTEL_VM default lookup(each.value.platform_config, "type", "INTEL_VM") ) # Remaining parameters as configured, validated by instance/instance config resource are_virtual_instructions_enabled = lookup(each.value.platform_config, "are_virtual_instructions_enabled", null) is_access_control_service_enabled = lookup(each.value.platform_config, "is_access_control_service_enabled", null) is_input_output_memory_management_unit_enabled = lookup(each.value.platform_config, "is_input_output_memory_management_unit_enabled", null) is_measured_boot_enabled = lookup(each.value.platform_config, "is_measured_boot_enabled", null) is_memory_encryption_enabled = lookup(each.value.platform_config, "is_memory_encryption_enabled", null) is_secure_boot_enabled = lookup(each.value.platform_config, "is_secure_boot_enabled", null) is_symmetric_multi_threading_enabled = lookup(each.value.platform_config, "is_symmetric_multi_threading_enabled", null) is_trusted_platform_module_enabled = lookup(each.value.platform_config, "is_trusted_platform_module_enabled", null) numa_nodes_per_socket = lookup(each.value.platform_config, "numa_nodes_per_socket", null) percentage_of_cores_enabled = lookup(each.value.platform_config, "percentage_of_cores_enabled", null) } } agent_config { are_all_plugins_disabled = each.value.agent_config.are_all_plugins_disabled is_management_disabled = each.value.agent_config.is_management_disabled is_monitoring_disabled = each.value.agent_config.is_monitoring_disabled dynamic "plugins_config" { for_each = each.value.agent_config.plugins_config content { name = plugins_config.key desired_state = plugins_config.value } } } create_vnic_details { assign_private_dns_record = var.assign_dns assign_ipv6ip = each.value.assign_ipv6ip assign_public_ip = each.value.assign_public_ip nsg_ids = each.value.nsg_ids subnet_id = each.value.subnet_id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags } instance_options { are_legacy_imds_endpoints_disabled = each.value.legacy_imds_endpoints_disabled } metadata = merge( { apiserver_host = var.apiserver_private_host cluster_ca_cert = var.cluster_ca_cert oke-k8version = var.kubernetes_version oke-kubeproxy-proxy-mode = var.kubeproxy_mode oke-tenancy-id = var.tenancy_id oke-initial-node-labels = join(",", [for k, v in each.value.node_labels : format("%v=%v", k, v)]) secondary_vnics = jsonencode(lookup(each.value, "secondary_vnics", {})) ssh_authorized_keys = var.ssh_public_key user_data = lookup(lookup(data.cloudinit_config.workers, lookup(each.value, "key", ""), {}), "rendered", "") }, # Add labels required for NPN CNI. var.cni_type == "npn" ? merge( { oke-native-pod-networking = true oke-max-pods = each.value.max_pods_per_node pod-subnets = each.value.pod_subnet_id pod-nsgids = join(",", each.value.pod_nsg_ids) }, var.enable_ipv6 ? { ip-families = "IPv4,IPv6" }: {} ) : {}, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, # Extra user-defined fields merged last var.node_metadata, # global lookup(each.value, "node_metadata", {}), # pool-specific ) source_details { boot_volume_size_in_gbs = each.value.boot_volume_size boot_volume_vpus_per_gb = each.value.boot_volume_vpus_per_gb source_id = each.value.image_id source_type = "image" } lifecycle { precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = <<-EOT Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. pool: ${each.key} image_type: ${coalesce(each.value.image_type, "none")} image_id: ${coalesce(each.value.image_id, "none")} EOT } ignore_changes = [ agent_config, # TODO Not updateable; remove when supported defined_tags, freeform_tags, display_name, metadata["cluster_ca_cert"], metadata["user_data"], create_vnic_details[0].defined_tags, create_vnic_details[0].freeform_tags, ] } } ================================================ FILE: modules/workers/instanceconfig.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl resource "oci_core_instance_configuration" "workers" { # Create an OCI Instance Configuration resource for each enabled entry of the worker_pools map with a mode that uses one. for_each = local.enabled_instance_configs compartment_id = each.value.compartment_id display_name = each.key defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags instance_details { instance_type = "compute" launch_details { agent_config { are_all_plugins_disabled = each.value.agent_config.are_all_plugins_disabled is_management_disabled = each.value.agent_config.is_management_disabled is_monitoring_disabled = each.value.agent_config.is_monitoring_disabled dynamic "plugins_config" { for_each = each.value.agent_config.plugins_config content { name = plugins_config.key desired_state = plugins_config.value } } } availability_domain = element(each.value.availability_domains, 1) # First value specified on pool, or null to select automatically fault_domain = try(each.value.placement_fds[0], null) compartment_id = each.value.compartment_id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags extended_metadata = each.value.extended_metadata capacity_reservation_id = each.value.capacity_reservation_id instance_options { are_legacy_imds_endpoints_disabled = each.value.legacy_imds_endpoints_disabled } create_vnic_details { assign_private_dns_record = var.assign_dns assign_ipv6ip = each.value.assign_ipv6ip assign_public_ip = each.value.assign_public_ip nsg_ids = each.value.nsg_ids subnet_id = each.value.subnet_id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags } metadata = merge( { apiserver_host = var.apiserver_private_host cluster_ca_cert = var.cluster_ca_cert oke-k8version = var.kubernetes_version oke-kubeproxy-proxy-mode = var.kubeproxy_mode oke-tenancy-id = var.tenancy_id oke-initial-node-labels = join(",", [for k, v in each.value.node_labels : format("%v=%v", k, v)]) secondary_vnics = jsonencode(lookup(each.value, "secondary_vnics", {})) ssh_authorized_keys = var.ssh_public_key user_data = lookup(lookup(data.cloudinit_config.workers, each.key, {}), "rendered", "") }, # Add labels required for NPN CNI. var.cni_type == "npn" ? merge( { oke-native-pod-networking = true oke-max-pods = each.value.max_pods_per_node pod-subnets = each.value.pod_subnet_id pod-nsgids = join(",", each.value.pod_nsg_ids) }, var.enable_ipv6 ? { ip-families = "IPv4,IPv6" }: {} ) : {}, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, # Extra user-defined fields merged last var.node_metadata, # global lookup(each.value, "node_metadata", {}), # pool-specific ) shape = each.value.shape dynamic "shape_config" { for_each = length(regexall("Flex", each.value.shape)) > 0 ? [1] : [] content { baseline_ocpu_utilization = lookup(each.value, "burst", "BASELINE_1_1") ocpus = each.value.ocpus memory_in_gbs = ( # If > 64GB memory/core, correct input to exactly 64GB memory/core (each.value.memory / each.value.ocpus) > 64 ? each.value.ocpus * 64 : each.value.memory ) } } dynamic "platform_config" { for_each = each.value.platform_config != null ? [1] : [] content { type = lookup( # Attempt lookup against data source for the associated 'type' of configured worker shape lookup(local.platform_config_by_shape, each.value.shape, {}), "type", # Fall back to 'type' on pool with custom platform_config, or INTEL_VM default lookup(each.value.platform_config, "type", "INTEL_VM") ) # Remaining parameters as configured, validated by instance/instance config resource are_virtual_instructions_enabled = lookup(each.value.platform_config, "are_virtual_instructions_enabled", null) is_access_control_service_enabled = lookup(each.value.platform_config, "is_access_control_service_enabled", null) is_input_output_memory_management_unit_enabled = lookup(each.value.platform_config, "is_input_output_memory_management_unit_enabled", null) is_measured_boot_enabled = lookup(each.value.platform_config, "is_measured_boot_enabled", null) is_memory_encryption_enabled = lookup(each.value.platform_config, "is_memory_encryption_enabled", null) is_secure_boot_enabled = lookup(each.value.platform_config, "is_secure_boot_enabled", null) is_symmetric_multi_threading_enabled = lookup(each.value.platform_config, "is_symmetric_multi_threading_enabled", null) is_trusted_platform_module_enabled = lookup(each.value.platform_config, "is_trusted_platform_module_enabled", null) numa_nodes_per_socket = lookup(each.value.platform_config, "numa_nodes_per_socket", null) percentage_of_cores_enabled = lookup(each.value.platform_config, "percentage_of_cores_enabled", null) } } source_details { boot_volume_size_in_gbs = each.value.boot_volume_size boot_volume_vpus_per_gb = each.value.boot_volume_vpus_per_gb image_id = each.value.image_id source_type = "image" } is_pv_encryption_in_transit_enabled = each.value.pv_transit_encryption } dynamic "block_volumes" { for_each = (lookup(each.value, "disable_block_volume", false) != true) ? [1] : [] content { attach_details { type = each.value.block_volume_type is_pv_encryption_in_transit_enabled = each.value.pv_transit_encryption } create_details { // Limit to first candidate placement AD for cluster-network and gpu-memory-cluster; undefined for all otherwise availability_domain = contains(["cluster-network", "gpu-memory-cluster"], each.value.mode) ? element(each.value.availability_domains, 1) : null compartment_id = each.value.compartment_id display_name = each.key kms_key_id = each.value.volume_kms_key_id size_in_gbs = max(50, lookup(each.value, "block_volume_size_in_gbs", 50)) } } } dynamic "secondary_vnics" { for_each = lookup(each.value, "secondary_vnics", {}) iterator = vnic content { display_name = vnic.key nic_index = lookup(vnic.value, "nic_index", null) create_vnic_details { assign_private_dns_record = lookup(vnic.value, "assign_private_dns_record", null) assign_ipv6ip = lookup(vnic.value, "assign_ipv6ip", null) assign_public_ip = lookup(vnic.value, "assign_public_ip", null) display_name = vnic.key defined_tags = lookup(vnic.value, "defined_tags", null) freeform_tags = lookup(vnic.value, "freeform_tags", null) hostname_label = lookup(vnic.value, "hostname_label", null) nsg_ids = lookup(vnic.value, "nsg_ids", null) private_ip = lookup(vnic.value, "private_ip", null) skip_source_dest_check = lookup(vnic.value, "skip_source_dest_check", null) subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id) } } } } lifecycle { # TODO Instance Configuration replacement without delete when supported: # https://github.com/hashicorp/terraform/issues/15485 create_before_destroy = true ignore_changes = [ defined_tags, freeform_tags, display_name, instance_details[0].launch_details[0].defined_tags, instance_details[0].launch_details[0].freeform_tags, instance_details[0].launch_details[0].create_vnic_details[0].defined_tags, instance_details[0].launch_details[0].create_vnic_details[0].freeform_tags, instance_details[0].secondary_vnics, ] } } ================================================ FILE: modules/workers/instancepools.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Dynamic resource block for Instance Pool groups defined in worker_pools resource "oci_core_instance_pool" "tfscaled_workers" { # Create an OCI Instance Pool resource for each enabled entry of the worker_pools map with that mode. for_each = { for key, value in local.enabled_instance_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == false } compartment_id = each.value.compartment_id display_name = each.key size = each.value.size instance_configuration_id = oci_core_instance_configuration.workers[each.key].id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags dynamic "placement_configurations" { for_each = each.value.availability_domains iterator = ad content { availability_domain = ad.value primary_subnet_id = each.value.subnet_id # Value(s) specified on pool, or null to select automatically fault_domains = try(each.value.placement_fds, null) dynamic "secondary_vnic_subnets" { for_each = lookup(each.value, "secondary_vnics", {}) iterator = vnic content { display_name = vnic.key subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id) } } } } lifecycle { ignore_changes = [ display_name, defined_tags, freeform_tags, placement_configurations, ] precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = <<-EOT Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. pool: ${each.key} image_type: ${coalesce(each.value.image_type, "none")} image_id: ${coalesce(each.value.image_id, "none")} EOT } precondition { condition = each.value.autoscale == false error_message = "Instance Pools do not support cluster autoscaler management." } } } resource "oci_core_instance_pool" "autoscaled_workers" { # Create an OCI Instance Pool resource for each enabled entry of the worker_pools map with that mode. for_each = { for key, value in local.enabled_instance_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == true } compartment_id = each.value.compartment_id display_name = each.key size = each.value.size instance_configuration_id = oci_core_instance_configuration.workers[each.key].id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags dynamic "placement_configurations" { for_each = each.value.availability_domains iterator = ad content { availability_domain = ad.value primary_subnet_id = each.value.subnet_id # Value(s) specified on pool, or null to select automatically fault_domains = try(each.value.placement_fds, null) dynamic "secondary_vnic_subnets" { for_each = lookup(each.value, "secondary_vnics", {}) iterator = vnic content { display_name = vnic.key subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id) } } } } lifecycle { ignore_changes = [ display_name, defined_tags, freeform_tags, placement_configurations, size ] precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = <<-EOT Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. pool: ${each.key} image_type: ${coalesce(each.value.image_type, "none")} image_id: ${coalesce(each.value.image_id, "none")} EOT } precondition { condition = each.value.autoscale == false error_message = "Instance Pools do not support cluster autoscaler management." } } } ================================================ FILE: modules/workers/locals.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { boot_volume_size = lookup(var.shape, "boot_volume_size", 50) boot_volume_vpus_per_gb = lookup(var.shape, "boot_volume_vpus_per_gb", 10) memory = lookup(var.shape, "memory", 4) ocpus = max(1, lookup(var.shape, "ocpus", 1)) shape = lookup(var.shape, "shape", "VM.Standard.E4.Flex") # Used for default values of required input for virtual node pools fault_domains_all = formatlist("FD-%v", [1, 2, 3]) fault_domains_available = { for ad, fd in data.oci_identity_fault_domains.all : ad => fd } worker_pool_defaults = { agent_config = { are_all_plugins_disabled = false is_management_disabled = false is_monitoring_disabled = false plugins_config = {} } allow_autoscaler = false legacy_imds_endpoints_disabled = var.legacy_imds_endpoints_disabled assign_public_ip = var.assign_public_ip assign_ipv6ip = var.enable_ipv6 ? true : false autoscale = false block_volume_type = var.block_volume_type boot_volume_size = local.boot_volume_size boot_volume_vpus_per_gb = local.boot_volume_vpus_per_gb capacity_reservation_id = var.capacity_reservation_id cloud_init = [] # empty pool-specific default compartment_id = var.compartment_id create = true disable_default_cloud_init = var.disable_default_cloud_init drain = false eviction_grace_duration = 300 force_node_action = true force_node_delete = true extended_metadata = {} # empty pool-specific default gpu_memory_cluster_scale_config = { is_upsize_enabled = var.gmc_scale_is_upsize_enabled is_downsize_enabled = var.gmc_scale_is_downsize_enabled target_size = var.gmc_scale_target_size } gpu_memory_fabric_ids = [] # empty pool-specific default; required for mode = "gpu-memory-cluster" ignore_initial_pool_size = false image_id = var.image_id image_type = var.image_type kubernetes_version = var.kubernetes_version max_pods_per_node = min(max(var.max_pods_per_node, 1), 110) memory = local.memory mode = var.worker_pool_mode node_cycling_enabled = false node_cycling_max_surge = 1 node_cycling_max_unavailable = 0 node_cycling_mode = ["instance"] node_labels = var.node_labels nsg_ids = [] # empty pool-specific default ocpus = local.ocpus os = var.image_os os_version = var.image_os_version placement_ads = var.ad_numbers platform_config = var.platform_config pod_nsg_ids = var.pod_nsg_ids pod_subnet_id = coalesce(var.pod_subnet_id, var.worker_subnet_id, "none") preemptible_config = var.preemptible_config pv_transit_encryption = var.pv_transit_encryption shape = local.shape size = var.worker_pool_size subnet_id = var.worker_subnet_id taints = [] # empty pool-specific default volume_kms_key_id = var.volume_kms_key_id } # Merge desired pool configuration onto default values worker_pools_with_defaults = { for pool_name, pool in var.worker_pools : pool_name => merge(local.worker_pool_defaults, pool) } # Filter worker_pools map for enabled entries and add derived configuration enabled_worker_pools = { for pool_name, pool in local.worker_pools_with_defaults : pool_name => merge(pool, { preemptible_config = lookup(pool, "preemptible_config", pool.preemptible_config) # Bare metal instances must use iSCSI block volume attachments, not paravirtualized block_volume_type = length(regexall("^BM", pool.shape)) > 0 ? "iscsi" : var.block_volume_type pv_transit_encryption = alltrue([ var.pv_transit_encryption, pool.block_volume_type == "paravirtualized", length(regexall("^VM", pool.shape)) > 0 ]) # Combine global and pool-specific cloud init parts cloud_init = [for part in concat(var.cloud_init, pool.cloud_init) : { # Load content from file if local path, attempt base64 decode, or use raw value content = contains(keys(part), "content") ? ( try(fileexists(lookup(part, "content")), false) ? file(lookup(part, "content")) : try(base64decode(lookup(part, "content")), lookup(part, "content")) ) : "" content_type = lookup(part, "content_type", local.default_cloud_init_content_type) filename = lookup(part, "filename", null) merge_type = lookup(part, "merge_type", local.default_cloud_init_merge_type) } ] agent_config = coalesce(var.agent_config, pool.agent_config, local.worker_pool_defaults.agent_config) # Translate configured + available AD numbers e.g. 2 into tenancy/compartment-specific names availability_domains = compact([for ad_number in tolist(setintersection(pool.placement_ads, var.ad_numbers)) : lookup(var.ad_numbers_to_names, ad_number, null) ]) # Use provided image_id for 'custom' type, or first match for all shape + OS criteria image_id = ( pool.image_type == "custom" ? pool.image_id : element(split("###", element(reverse(sort([for entry in tolist(setintersection([ pool.image_type == "oke" ? setintersection( lookup(var.image_ids, "oke", null), lookup(var.image_ids, trimprefix(lower(pool.kubernetes_version), "v"), null) ) : lookup(var.image_ids, "platform", null), lookup(var.image_ids, pool.image_type, null), length(regexall("GPU", pool.shape)) > 0 ? var.image_ids.gpu : var.image_ids.nongpu, length(regexall("A[12]\\.", pool.shape)) > 0 ? var.image_ids.aarch64 : var.image_ids.x86_64, lookup(var.image_ids, format("%v %v", pool.os, split(".", pool.os_version)[0]), null), ]...)) : "${var.indexed_images[entry].sort_key}###${entry}"])), 0)), 1) ) # Standard tags as defined if enabled for use # User-provided freeform tags are merged and take precedence defined_tags = merge( var.use_defined_tags ? merge( { "${var.tag_namespace}.state_id" = var.state_id, "${var.tag_namespace}.role" = "worker", "${var.tag_namespace}.pool" = pool_name, "${var.tag_namespace}.cluster_autoscaler" = pool.allow_autoscaler ? "allowed" : "disabled", }, pool.autoscale ? { "${var.tag_namespace}.cluster_autoscaler" = "managed" } : {}, ) : {}, var.defined_tags, lookup(pool, "defined_tags", {}) ) # Standard tags as freeform if defined tags are disabled # User-provided freeform tags are merged and take precedence freeform_tags = merge( var.use_defined_tags ? {} : merge( { "state_id" = var.state_id, "role" = "worker", "pool" = pool_name, "cluster_autoscaler" = pool.allow_autoscaler ? "allowed" : "disabled", }, pool.autoscale ? { "cluster_autoscaler" = "managed" } : {}, ), var.freeform_tags, lookup(pool, "freeform_tags", {}) ) # Combine global and pool-specific NSGs nsg_ids = compact(concat(var.worker_nsg_ids, pool.nsg_ids)) pods_nsg_ids = compact(concat(var.pod_nsg_ids, pool.pod_nsg_ids)) # Add a node label for cluster autoscaler where scheduling is supported node_labels = merge( { "oke.oraclecloud.com/tf.module" = "terraform-oci-oke" "oke.oraclecloud.com/tf.state_id" = var.state_id "oke.oraclecloud.com/pool.name" = pool_name "oke.oraclecloud.com/pool.mode" = pool.mode "oke.oraclecloud.com/cluster_autoscaler" = pool.allow_autoscaler ? "allowed" : "disabled" "oci.oraclecloud.com/vcn-native-ip-cni" = var.cni_type == "npn" ? true : false }, pool.autoscale ? { "oke.oraclecloud.com/cluster_autoscaler" = "managed" } : {}, pool.node_labels, ) # Override Node-cycling mode node_cycling_mode = pool.node_cycling_mode != null ? [for entry in pool.node_cycling_mode : lookup(local.supported_node_cycling_mode, lower(entry))] : null }) if tobool(pool.create) } supported_node_cycling_mode = { instance = "INSTANCE_REPLACE" boot_volume = "BOOT_VOLUME_REPLACE" } enabled_modes = distinct([for w in values(local.enabled_worker_pools) : w.mode]) # Number of nodes expected from enabled worker pools expected_node_count = length(local.enabled_worker_pools) == 0 ? 0 : sum([ for k, v in local.enabled_worker_pools : lookup(v, "size", var.worker_pool_size) ]) # Number of nodes expected to be draining in worker pools expected_drain_count = length(local.enabled_worker_pools) == 0 ? 0 : sum([ for k, v in local.enabled_worker_pools : tobool(v.drain) ? lookup(v, "size", var.worker_pool_size) : 0 ]) # Number of work pools in the worker pools with autoscale enabled expected_autoscale_worker_pools = length(local.enabled_worker_pools) == 0 ? 0 : sum([ for k, v in local.enabled_worker_pools : tobool(v.autoscale) ? 1 : 0 ]) # Enabled worker_pool map entries for node pools enabled_node_pools = { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "node-pool" } # Enabled worker_pool map entries for virtual node pools enabled_virtual_node_pools = { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "virtual-node-pool" } # Enabled worker_pool map entries for instance pools enabled_instance_configs = { for k, v in local.enabled_worker_pools : k => v if contains(["cluster-network", "instance-pool", "gpu-memory-cluster"], lookup(v, "mode", "")) } # Enabled worker_pool map entries for instance pools enabled_instance_pools = { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "instance-pool" } # Enabled worker_pool map entries for individual instances enabled_instances = { for e in concat([], [ for k, v in local.enabled_worker_pools : [ for i in range(0, lookup(v, "size", 0)) : merge(v, { "key" = k, "index" = i }) ] if lookup(v, "mode", "") == "instance" ]...) : format("%v-%v", lookup(e, "key"), lookup(e, "index")) => e } # Enabled worker_pool map entries for cluster networks enabled_cluster_networks = { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "cluster-network" } # Enabled worker_pool map entries for compute clusters enabled_compute_clusters = { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "compute-cluster" } # Enabled worker_pool map entries for GPU memory clusters enabled_gmc_pools = { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "gpu-memory-cluster" } # Map keyed by "###" of merged pool config + pool_name + gpu_memory_fabric_id, used to drive the GMC resource for_each. # Keying off OCID (not list index) keeps adds/removes stable. enabled_gmc_fabric_map = { for entry in flatten([ for k, v in local.enabled_gmc_pools : [ for gmf_id in lookup(v, "gpu_memory_fabric_ids", []) : merge(v, { pool_name = k gpu_memory_fabric_id = gmf_id }) ] ]) : format("%s###%s", entry.pool_name, entry.gpu_memory_fabric_id) => entry } # Prepare a map workers node enabled for compute_clusters { "pool_id###worker_id" => pool_values } compute_cluster_instance_ids_map = { for k, v in local.enabled_compute_clusters : k => toset(lookup(v, "instance_ids", [])) } compute_cluster_instance_ids = toset(concat(flatten([for k, v in local.compute_cluster_instance_ids_map : [for id in v : format("%s###%s", k, id)]]))) compute_cluster_instance_map = { for id in local.compute_cluster_instance_ids : id => lookup(local.enabled_compute_clusters, element(split("###", id), 0), {}) } # Sanitized worker_pools output; some conditionally-used defaults would be misleading worker_pools_final = { for pool_name, pool in local.enabled_worker_pools : pool_name => { for a, b in pool : a => b if a != "create" # implied && b != null && try(length(b), -1) != 0 && try(!!tobool(b), true) # exclude empty/disabled values && !(contains(["os", "os_version"], a) && pool.image_type == "custom") # unused defaults for custom && !(contains(["pod_nsg_ids", "pod_subnet_id"], a) && var.cni_type != "npn") # unused defaults for NPN && !(contains(["ocpus", "memory"], a) && length(regexall("Flex", pool.shape)) == 0) # unused defaults for non-Flex shapes } } # Maps of worker pool OCI resources by pool name enriched with desired/custom parameters for various modes worker_node_pools = { for k, v in merge(oci_containerengine_node_pool.tfscaled_workers, oci_containerengine_node_pool.autoscaled_workers) : k => merge(lookup(local.worker_pools_final, k, {}), v) } worker_virtual_node_pools = { for k, v in oci_containerengine_virtual_node_pool.workers : k => merge(lookup(local.worker_pools_final, k, {}), v) } worker_instance_pools = { for k, v in merge(oci_core_instance_pool.tfscaled_workers, oci_core_instance_pool.autoscaled_workers) : k => merge(lookup(local.worker_pools_final, k, {}), v) } worker_cluster_networks = { for k, v in oci_core_cluster_network.workers : k => merge(lookup(local.worker_pools_final, k, {}), v) } worker_instances = { for k, v in oci_core_instance.workers : k => merge(lookup(local.worker_pools_final, k, {}), v) } # Combined map of outputs by pool name for all modes excluding 'instance' (output separately) worker_pools_output = merge( local.worker_node_pools, local.worker_virtual_node_pools, local.worker_instance_pools, local.worker_cluster_networks, ) # OCIDs of pool resources by pool name for modes: 'node-pool', 'virtual-node-pool', 'instance-pool', 'cluster-network' worker_pool_ids = { for k, v in local.worker_pools_output : k => v.id } # Map of pool name to list of instance IP addresses for modes: 'instance' worker_instance_ips = { for x, y in { for k, v in local.worker_instances : replace(k, "/-[^-]*$/", "") => # remove index suffix { lookup(v, "id", "") = lookup(v, "private_ip", null) }... # instances grouped by "pool" } : x => merge(y...) } # Map of pool name to list of instance IP addresses for modes: 'node-pool' worker_nodepool_ips = { for k, v in local.worker_node_pools : k => { for n in lookup(v, "nodes", []) : lookup(n, "id", "") => lookup(n, "private_ip", null) } } # Yields { = { = }} for modes: 'node-pool', 'instance' worker_pool_ips = merge(local.worker_instance_ips, local.worker_nodepool_ips) # Map of nodepools using Ubuntu images. ubuntu_supported_versions = { "22.04" = "jammy" "24.04" = "noble" "22.04 Minimal" = "jammy" "22.04 Minimal aarch64" = "jammy" "24.04 Minimal" = "noble" "24.04 Minimal aarch64" = "noble" } ubuntu_worker_pools = { for k, v in local.enabled_worker_pools : k => { kubernetes_major_version = substr(lookup(v, "kubernetes_version", ""), 1, 4) kubernetes_minor_version = substr(lookup(v, "kubernetes_version", ""), 1, -1) ubuntu_release = lookup(lookup(data.oci_core_image.workers, k, {}), "operating_system_version", null) != null ? lookup(lookup(data.oci_core_image.workers, k, {}), "operating_system_version") : lookup(v, "os_version", null) } if lookup(v, "mode", var.worker_pool_mode) != "virtual-node-pool" && contains(coalescelist(split(" ", lookup(lookup(data.oci_core_image.workers, k, {}), "operating_system", "")), [lookup(v, "os", "")]), "Ubuntu") } } ================================================ FILE: modules/workers/nodepools.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Dynamic resource block for Node Pool groups defined in worker_pools resource "oci_containerengine_node_pool" "tfscaled_workers" { # Create an OKE node pool resource for each enabled entry of the worker_pools map with that mode. for_each = { for key, value in local.enabled_node_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == false } cluster_id = var.cluster_id compartment_id = each.value.compartment_id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags kubernetes_version = each.value.kubernetes_version name = each.key node_shape = each.value.shape ssh_public_key = var.ssh_public_key node_config_details { size = each.value.size is_pv_encryption_in_transit_enabled = each.value.pv_transit_encryption kms_key_id = each.value.volume_kms_key_id nsg_ids = each.value.nsg_ids defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags dynamic "placement_configs" { for_each = each.value.availability_domains iterator = ad content { availability_domain = ad.value capacity_reservation_id = each.value.capacity_reservation_id subnet_id = each.value.subnet_id # Value(s) specified on pool, or null to select automatically fault_domains = try(each.value.placement_fds, null) dynamic "preemptible_node_config" { for_each = each.value.preemptible_config.enable ? [1] : [] content { preemption_action { type = "TERMINATE" is_preserve_boot_volume = each.value.preemptible_config.is_preserve_boot_volume } } } } } dynamic "node_pool_pod_network_option_details" { for_each = var.cni_type == "flannel" ? [1] : [] content { # Flannel requires cni type only cni_type = "FLANNEL_OVERLAY" } } dynamic "node_pool_pod_network_option_details" { for_each = var.cni_type == "npn" ? [1] : [] content { # VCN-Native requires max pods/node, nsg ids, subnet ids cni_type = "OCI_VCN_IP_NATIVE" max_pods_per_node = each.value.max_pods_per_node pod_nsg_ids = compact(tolist(each.value.pod_nsg_ids)) pod_subnet_ids = compact(tolist([each.value.pod_subnet_id])) } } } node_metadata = merge( { apiserver_host = var.apiserver_private_host oke-kubeproxy-proxy-mode = var.kubeproxy_mode user_data = lookup(lookup(data.cloudinit_config.workers, each.key, {}), "rendered", "") }, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, # Extra user-defined fields merged last var.node_metadata, # global lookup(each.value, "node_metadata", {}), # pool-specific ) node_eviction_node_pool_settings { eviction_grace_duration = (floor(tonumber(each.value.eviction_grace_duration) / 60) > 0 ? (each.value.eviction_grace_duration > 3600 ? format("PT%dM", 60) : (each.value.eviction_grace_duration % 60 == 0 ? format("PT%dM", floor(each.value.eviction_grace_duration / 60)) : format("PT%dM%dS", floor(each.value.eviction_grace_duration / 60), each.value.eviction_grace_duration % 60) ) ) : format("PT%dS", each.value.eviction_grace_duration) ) is_force_delete_after_grace_duration = tobool(each.value.force_node_delete) is_force_action_after_grace_duration = tobool(each.value.force_node_action) } dynamic "node_shape_config" { for_each = length(regexall("Flex", each.value.shape)) > 0 ? [1] : [] content { ocpus = each.value.ocpus memory_in_gbs = ( # If > 64GB memory/core, correct input to exactly 64GB memory/core (each.value.memory / each.value.ocpus) > 64 ? each.value.ocpus * 64 : each.value.memory ) } } node_pool_cycling_details { is_node_cycling_enabled = each.value.node_cycling_enabled maximum_surge = each.value.node_cycling_max_surge maximum_unavailable = each.value.node_cycling_max_unavailable cycle_modes = each.value.node_cycling_mode } node_source_details { boot_volume_size_in_gbs = each.value.boot_volume_size image_id = each.value.image_id source_type = "image" } lifecycle { # prevent resources changes for changed fields ignore_changes = [ # kubernetes_version, # e.g. if changed as part of an upgrade name, defined_tags, freeform_tags, node_config_details[0].placement_configs, # dynamic placement configs # node_source_details[0], # dynamic image lookup ] precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = <<-EOT Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. pool: ${each.key} image_type: ${coalesce(each.value.image_type, "none")} image_id: ${coalesce(each.value.image_id, "none")} EOT } precondition { condition = anytrue([ contains(["instance-pool", "cluster-network"], each.value.mode), # supported modes length(lookup(each.value, "secondary_vnics", {})) == 0, # unrestricted when empty/unset ]) error_message = "Unsupported option for mode=${each.value.mode}: secondary_vnics" } precondition { condition = coalesce(each.value.capacity_reservation_id, "none") == "none" || length(each.value.availability_domains) == 1 error_message = "A single availability domain must be specified when using a capacity reservation with mode=${each.value.mode}" } } dynamic "initial_node_labels" { for_each = each.value.node_labels content { key = initial_node_labels.key value = initial_node_labels.value } } } resource "oci_containerengine_node_pool" "autoscaled_workers" { # Create an OKE node pool resource for each enabled entry of the worker_pools map with that mode. for_each = { for key, value in local.enabled_node_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == true } cluster_id = var.cluster_id compartment_id = each.value.compartment_id defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags kubernetes_version = each.value.kubernetes_version name = each.key node_shape = each.value.shape ssh_public_key = var.ssh_public_key node_config_details { size = each.value.size is_pv_encryption_in_transit_enabled = each.value.pv_transit_encryption kms_key_id = each.value.volume_kms_key_id nsg_ids = each.value.nsg_ids defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags dynamic "placement_configs" { for_each = each.value.availability_domains iterator = ad content { availability_domain = ad.value capacity_reservation_id = each.value.capacity_reservation_id subnet_id = each.value.subnet_id # Value(s) specified on pool, or null to select automatically fault_domains = try(each.value.placement_fds, null) dynamic "preemptible_node_config" { for_each = each.value.preemptible_config.enable ? [1] : [] content { preemption_action { type = "TERMINATE" is_preserve_boot_volume = each.value.preemptible_config.is_preserve_boot_volume } } } } } dynamic "node_pool_pod_network_option_details" { for_each = var.cni_type == "flannel" ? [1] : [] content { # Flannel requires cni type only cni_type = "FLANNEL_OVERLAY" } } dynamic "node_pool_pod_network_option_details" { for_each = var.cni_type == "npn" ? [1] : [] content { # VCN-Native requires max pods/node, nsg ids, subnet ids cni_type = "OCI_VCN_IP_NATIVE" max_pods_per_node = each.value.max_pods_per_node pod_nsg_ids = compact(tolist(each.value.pod_nsg_ids)) pod_subnet_ids = compact(tolist([each.value.pod_subnet_id])) } } } node_metadata = merge( { apiserver_host = var.apiserver_private_host oke-kubeproxy-proxy-mode = var.kubeproxy_mode user_data = lookup(lookup(data.cloudinit_config.workers, each.key, {}), "rendered", "") }, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, # Extra user-defined fields merged last var.node_metadata, # global lookup(each.value, "node_metadata", {}), # pool-specific ) node_eviction_node_pool_settings { eviction_grace_duration = (floor(tonumber(each.value.eviction_grace_duration) / 60) > 0 ? (each.value.eviction_grace_duration > 3600 ? format("PT%dM", 60) : (each.value.eviction_grace_duration % 60 == 0 ? format("PT%dM", floor(each.value.eviction_grace_duration / 60)) : format("PT%dM%dS", floor(each.value.eviction_grace_duration / 60), each.value.eviction_grace_duration % 60) ) ) : format("PT%dS", each.value.eviction_grace_duration) ) is_force_delete_after_grace_duration = tobool(each.value.force_node_delete) is_force_action_after_grace_duration = tobool(each.value.force_node_action) } dynamic "node_shape_config" { for_each = length(regexall("Flex", each.value.shape)) > 0 ? [1] : [] content { ocpus = each.value.ocpus memory_in_gbs = ( # If > 64GB memory/core, correct input to exactly 64GB memory/core (each.value.memory / each.value.ocpus) > 64 ? each.value.ocpus * 64 : each.value.memory ) } } node_pool_cycling_details { is_node_cycling_enabled = each.value.node_cycling_enabled maximum_surge = each.value.node_cycling_max_surge maximum_unavailable = each.value.node_cycling_max_unavailable cycle_modes = each.value.node_cycling_mode } node_source_details { boot_volume_size_in_gbs = each.value.boot_volume_size image_id = each.value.image_id source_type = "image" } lifecycle { # prevent resources changes for changed fields ignore_changes = [ # kubernetes_version, # e.g. if changed as part of an upgrade name, defined_tags, freeform_tags, node_config_details[0].placement_configs, # dynamic placement configs node_config_details[0].size # size ] precondition { condition = coalesce(each.value.image_id, "none") != "none" error_message = <<-EOT Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. pool: ${each.key} image_type: ${coalesce(each.value.image_type, "none")} image_id: ${coalesce(each.value.image_id, "none")} EOT } precondition { condition = anytrue([ contains(["instance-pool", "cluster-network"], each.value.mode), # supported modes length(lookup(each.value, "secondary_vnics", {})) == 0, # unrestricted when empty/unset ]) error_message = "Unsupported option for mode=${each.value.mode}: secondary_vnics" } precondition { condition = coalesce(each.value.capacity_reservation_id, "none") == "none" || length(each.value.availability_domains) == 1 error_message = "A single availability domain must be specified when using a capacity reservation with mode=${each.value.mode}" } } dynamic "initial_node_labels" { for_each = each.value.node_labels content { key = initial_node_labels.key value = initial_node_labels.value } } } ================================================ FILE: modules/workers/outputs.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl output "worker_pools" { description = "Created worker pools (mode != 'instance')" value = local.worker_pools_output } output "worker_instances" { description = "Created worker pools (mode == 'instance')" value = local.worker_instances } output "worker_pool_ids" { description = "Created worker pool IDs" value = local.worker_pool_ids } output "worker_pool_ips" { description = "Created worker instance private IPs by pool for available modes ('node-pool', 'instance')." value = local.worker_pool_ips } output "worker_count_expected" { description = "# of nodes expected from created worker pools" value = local.expected_node_count } output "worker_drain_expected" { description = "# of nodes expected to be draining in worker pools" value = local.expected_drain_count } output "worker_pool_autoscale_expected" { description = "# of worker pools expected with autoscale enabled from created worker pools" value = local.expected_autoscale_worker_pools } output "worker_gpu_memory_clusters" { description = "Created GPU Memory Clusters keyed by '###'." value = oci_core_compute_gpu_memory_cluster.workers } output "worker_gpu_memory_cluster_ids" { description = "OCIDs of created GPU Memory Clusters keyed by '###'." value = { for k, v in oci_core_compute_gpu_memory_cluster.workers : k => v.id } } ================================================ FILE: modules/workers/variables.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Common variable "state_id" { default = null description = "Optional Terraform state_id from an existing deployment of the module to re-use with created resources." type = string } variable "compartment_id" { default = null description = "The compartment id where resources will be created." type = string } variable "tenancy_id" { default = null description = "The tenancy id of the OCI Cloud Account in which to create the resources." type = string } # Tags variable "freeform_tags" { default = {} description = "Freeform tags to be applied to created resources." type = map(string) } variable "defined_tags" { default = {} description = "Defined tags to be applied to created resources. Must already exist in the tenancy." type = map(string) } variable "use_defined_tags" { default = false description = "Whether to apply defined tags to created resources for IAM policy and tracking." type = bool } variable "tag_namespace" { default = "oke" description = "The tag namespace for standard OKE defined tags." type = string } # Cluster variable "apiserver_private_host" { type = string } variable "cluster_id" { default = null description = "An existing OKE cluster OCID when `create_cluster = false`." type = string } variable "cluster_type" { default = "basic" description = "The cluster type. See Working with Enhanced Clusters and Basic Clusters for more information." type = string validation { condition = contains(["basic", "enhanced"], lower(var.cluster_type)) error_message = "Accepted values are 'basic' or 'enhanced'." } } variable "cluster_ca_cert" { default = null description = "Base64+PEM-encoded cluster CA certificate for unmanaged instance pools. Determined automatically when 'create_cluster' = true or 'cluster_id' is provided." type = string } variable "cluster_dns" { default = null description = "Cluster DNS resolver IP address. Determined automatically when not set (recommended)." type = string } variable "kubernetes_version" { default = "v1.26.2" description = "The version of Kubernetes used for worker nodes." type = string } # Network variable "assign_dns" { type = bool } variable "assign_public_ip" { type = bool } variable "cni_type" { default = "flannel" description = "The CNI for the cluster: 'flannel' or 'npn'. See Pod Networking." type = string validation { condition = contains(["flannel", "npn"], var.cni_type) error_message = "Accepted values are flannel or npn" } } variable "enable_ipv6" { default = false description = "Whether to create a dual-stack (IPv4/IPv6) cluster." type = bool } variable "pod_subnet_id" { type = string } variable "worker_subnet_id" { type = string } # Worker pools variable "worker_pools" { default = {} description = "Tuple of OKE worker pools where each key maps to the OCID of an OCI resource, and value contains its definition." type = any } variable "worker_pool_mode" { default = "node-pool" description = "Default management mode for workers when unspecified on a pool. Only 'node-pool' is currently supported." type = string validation { condition = contains(["node-pool", "instance", "instance-pool", "cluster-network", "gpu-memory-cluster"], var.worker_pool_mode) error_message = "Accepted values are node-pool, instance-pool, cluster-network, or gpu-memory-cluster" } } variable "worker_pool_size" { default = 0 description = "Default size for worker pools when unspecified on a pool." type = number } # GPU memory cluster scale config defaults variable "gmc_scale_is_upsize_enabled" { default = true description = "Default gpu_memory_cluster_scale_config.is_upsize_enabled when unspecified on a 'gpu-memory-cluster' pool." type = bool } variable "gmc_scale_is_downsize_enabled" { default = true description = "Default gpu_memory_cluster_scale_config.is_downsize_enabled when unspecified on a 'gpu-memory-cluster' pool." type = bool } variable "gmc_scale_target_size" { default = 18 description = "Default gpu_memory_cluster_scale_config.target_size when unspecified on a 'gpu-memory-cluster' pool." type = number } # Workers: instance variable "ad_numbers_to_names" { type = map(string) } variable "ad_numbers" { type = list(number) } variable "image_ids" { default = {} description = "Map of images for filtering with image_os and image_os_version." type = any } variable "indexed_images" { default = {} description = "Map of images." type = any } variable "ssh_public_key" { default = null description = "The contents of the SSH public key file. Used to allow login for workers/bastion/operator with corresponding private key." type = string } variable "timezone" { type = string } variable "worker_nsg_ids" { default = [] description = "An additional list of network security group (NSG) IDs for node security. Combined with 'nsg_ids' specified on each pool." type = list(string) } variable "pod_nsg_ids" { default = [] description = "An additional list of network security group (NSG) IDs for pod security. Combined with 'pod_nsg_ids' specified on each pool." type = list(string) } variable "kubeproxy_mode" { default = "iptables" description = "The mode in which to run kube-proxy when unspecified on a pool." type = string validation { condition = contains(["iptables", "ipvs"], var.kubeproxy_mode) error_message = "Accepted values are iptables or ipvs." } } # # Workers: instance # variable "block_volume_type" { default = "paravirtualized" description = "Default block volume attachment type for Instance Configurations when unspecified on a pool." type = string validation { condition = contains(["iscsi", "paravirtualized"], var.block_volume_type) error_message = "Accepted values are 'iscsi' or 'paravirtualized'." } } variable "node_labels" { default = {} description = "Default worker node labels. Merged with labels defined on each pool." type = map(string) } variable "node_metadata" { default = {} description = "Map of additional worker node instance metadata. Merged with metadata defined on each pool." type = map(string) } variable "image_id" { default = null description = "Default image for worker pools when unspecified on a pool." type = string } variable "image_type" { default = "oke" description = "Whether to use a platform, OKE, or custom image for worker nodes by default when unspecified on a pool. When custom is set, the worker_image_id must be specified." type = string validation { condition = contains(["custom", "oke", "platform"], var.image_type) error_message = "Accepted values are custom, oke, platform" } } variable "image_os" { default = "Oracle Linux" description = "Default worker image operating system name when worker_image_type = 'oke' or 'platform' and unspecified on a pool." type = string } variable "image_os_version" { default = "8" description = "Default worker image operating system version when worker_image_type = 'oke' or 'platform' and unspecified on a pool." type = string } variable "shape" { default = { shape = "VM.Standard.E4.Flex", ocpus = 2, memory = 16, boot_volume_size = 50 } description = "Default shape of the created worker instance when unspecified on a pool." type = map(any) } variable "capacity_reservation_id" { default = null description = "The ID of the Compute capacity reservation the worker node will be launched under. See Capacity Reservations for more information." type = string } variable "preemptible_config" { default = { enable = false, is_preserve_boot_volume = false } description = "Default preemptible Compute configuration when unspecified on a pool. See Preemptible Worker Nodes for more information." type = map(any) } variable "cloud_init" { default = [] description = "List of maps containing cloud init MIME part configuration for worker nodes. Merged with pool-specific definitions. See https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html#part for expected schema of each element." type = list(map(string)) } variable "disable_default_cloud_init" { default = false description = "Whether to disable the default OKE cloud init and only use the cloud init explicitly passed to the worker pool in 'worker_cloud_init'." type = bool } variable "volume_kms_key_id" { default = null description = "The ID of the OCI KMS key to be used as the master encryption key for Boot Volume and Block Volume encryption by default when unspecified on a pool." type = string } variable "pv_transit_encryption" { default = false description = "Whether to enable in-transit encryption for the data volume's paravirtualized attachment by default when unspecified on a pool." type = bool } variable "max_pods_per_node" { default = 31 description = "The default maximum number of pods to deploy per node when unspecified on a pool. Absolute maximum is 110. Ignored when when cni_type != 'npn'." type = number validation { condition = var.max_pods_per_node > 0 && var.max_pods_per_node <= 110 error_message = "Must be between 1 and 110." } } variable "legacy_imds_endpoints_disabled" { default = false description = "Whether to disable requests to the IMDSv1 endpoint and only allow requests to the IMDSv2 endpoint. See Instance Metadata for more information." type = bool } variable "allow_short_container_image_names" { default = false description = "Whether to allow short container image names for K8s version >= 1.34.0. See CRI-O pull request for more information." type = bool } variable "platform_config" { default = null description = "Default platform_config for self-managed worker pools created with mode: 'instance', 'instance-pool', or 'cluster-network'. See PlatformConfig for more information." type = object({ type = optional(string), are_virtual_instructions_enabled = optional(bool), is_access_control_service_enabled = optional(bool), is_input_output_memory_management_unit_enabled = optional(bool), is_measured_boot_enabled = optional(bool), is_memory_encryption_enabled = optional(bool), is_secure_boot_enabled = optional(bool), is_symmetric_multi_threading_enabled = optional(bool), is_trusted_platform_module_enabled = optional(bool), numa_nodes_per_socket = optional(number), percentage_of_cores_enabled = optional(bool), }) } variable "agent_config" { description = "Default agent_config for self-managed worker pools created with mode: 'instance', 'instance-pool', or 'cluster-network'. See InstanceConfig for more information." type = object({ are_all_plugins_disabled = bool, is_management_disabled = bool, is_monitoring_disabled = bool, plugins_config = map(string), }) } # # Workers: compute-cluster # variable "compute_clusters" { default = {} description = "Whether to create compute clusters shared by nodes across multiple worker pools enabled for 'compute-cluster'." type = map(any) } ================================================ FILE: modules/workers/versions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl terraform { required_version = ">= 1.2.0" required_providers { cloudinit = { source = "hashicorp/cloudinit" version = ">= 2.2.0" } oci = { source = "oracle/oci" version = ">= 8.2.0" } } } ================================================ FILE: modules/workers/virtualnodepools.tf ================================================ # Copyright (c) 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # Dynamic resource block for Virtual Node Pool groups defined in worker_pools resource "oci_containerengine_virtual_node_pool" "workers" { # Create an OKE Virtual Node Pool resource for each enabled entry of the worker_pools map with that mode. for_each = local.enabled_virtual_node_pools cluster_id = var.cluster_id compartment_id = each.value.compartment_id display_name = each.key size = each.value.size defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags nsg_ids = each.value.nsg_ids dynamic "placement_configurations" { for_each = each.value.availability_domains iterator = ad content { availability_domain = ad.value subnet_id = each.value.subnet_id # Intersect the list of available and configured FDs for this AD fault_domain = tolist(setintersection( try(each.value.placement_fds, local.fault_domains_all), lookup(local.fault_domains_available, ad.value, local.fault_domains_all) )) } } dynamic "initial_virtual_node_labels" { for_each = each.value.node_labels content { key = initial_virtual_node_labels.key value = initial_virtual_node_labels.value } } pod_configuration { shape = each.value.shape subnet_id = coalesce(each.value.pod_subnet_id, each.value.subnet_id) nsg_ids = toset(compact(coalescelist(each.value.pod_nsg_ids, each.value.nsg_ids, []))) } dynamic "taints" { for_each = each.value.taints content { effect = lookup(taints.value, "effect", "NoSchedule") key = taints.key value = lookup(taints.value, "value", null) } } virtual_node_tags { defined_tags = each.value.defined_tags freeform_tags = each.value.freeform_tags } lifecycle { ignore_changes = [ display_name, virtual_node_tags, placement_configurations, defined_tags, freeform_tags, ] precondition { condition = var.cni_type == "npn" error_message = "Virtual Node Pools require a cluster with `cni_type = npn`." } precondition { condition = each.value.autoscale == false error_message = "Virtual Node Pools do not support cluster autoscaler management." } precondition { condition = var.cluster_type == "enhanced" error_message = "Virtual Node Pools require `cluster_type = enhanced`." } precondition { condition = contains(["Pod.Standard.E3.Flex", "Pod.Standard.E4.Flex", "Pod.Standard.A1.Flex"], each.value.shape) error_message = "Virtual Node Pools must be 'Pod.Standard.E3.Flex', 'Pod.Standard.E4.Flex' or 'Pod.Standard.A1.Flex'." } } } ================================================ FILE: variables-bastion.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_bastion" { default = true description = "Whether to create a bastion host." type = bool } variable "bastion_public_ip" { default = null description = "The IP address of an existing bastion host, if create_bastion = false." type = string } variable "bastion_allowed_cidrs" { default = [] description = "A list of CIDR blocks to allow SSH access to the bastion host. NOTE: Default is empty i.e. no access permitted. Allow access from anywhere with '0.0.0.0/0'." type = list(string) } variable "bastion_availability_domain" { default = null description = "The availability domain for bastion placement. Defaults to first available." type = string } variable "bastion_nsg_ids" { description = "An additional list of network security group (NSG) IDs for bastion security." default = [] type = list(string) } variable "bastion_user" { default = "opc" description = "User for SSH access through bastion host." type = string } variable "bastion_image_id" { default = null description = "Image ID for created bastion instance." type = string } variable "bastion_image_type" { default = "platform" description = "Whether to use a platform or custom image for the created bastion instance. When custom is set, the bastion_image_id must be specified." type = string validation { condition = contains(["custom", "platform"], var.bastion_image_type) error_message = "Accepted values are custom or platform" } } variable "bastion_image_os" { default = "Oracle Autonomous Linux" description = "Bastion image operating system name when bastion_image_type = 'platform'." type = string } variable "bastion_image_os_version" { default = "8" description = "Bastion image operating system version when bastion_image_type = 'platform'." type = string } variable "bastion_shape" { default = { shape = "VM.Standard.E4.Flex", ocpus = 1, memory = 4, boot_volume_size = 50, baseline_ocpu_utilization = 100 } description = "The shape of bastion instance. Baseline OCPU utilization can be used to provision burstable shapes." type = map(any) } variable "bastion_is_public" { default = true description = "Whether to create allocate a public IP and subnet for the created bastion host." type = bool } variable "bastion_upgrade" { default = false description = "Whether to upgrade bastion packages after provisioning." type = bool } variable "bastion_await_cloudinit" { default = true description = "Whether to block until successful connection to bastion and completion of cloud-init." type = bool } variable "bastion_volume_kms_key_id" { default = null description = "The OCID of the OCI KMS key to assign as the master encryption key for the bastion host boot volume." type = string } variable "bastion_legacy_imds_endpoints_disabled" { default = true description = "Whether to disable requests to the IMDSv1 endpoint and only allow requests to the IMDSv2 endpoint for the bastion instance." type = bool } ================================================ FILE: variables-cluster-addons.tf ================================================ # Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "cluster_addons" { description = "Map with cluster addons that should be enabled. See ClusterAddon documentation for the supported configuration of each addon." type = any default = {} } variable "cluster_addons_to_remove" { description = "Map with cluster addons not created by Terraform that should be removed. This operation is performed using oci-cli and requires the operator host to be deployed." type = any default = {} } ================================================ FILE: variables-cluster.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_cluster" { default = true description = "Whether to create the OKE cluster and dependent resources." type = bool } variable "cluster_name" { default = "oke" description = "The name of oke cluster." type = string } variable "cluster_type" { default = "basic" description = "The cluster type. See Working with Enhanced Clusters and Basic Clusters for more information." type = string validation { condition = contains(["basic", "enhanced"], lower(var.cluster_type)) error_message = "Accepted values are 'basic' or 'enhanced'." } } variable "control_plane_is_public" { default = false description = "Whether the Kubernetes control plane endpoint should be allocated a public IP address to enable access over public internet." type = bool } variable "assign_public_ip_to_control_plane" { default = false description = "Whether to assign a public IP address to the API endpoint for public access. Requires the control plane subnet to be public to assign a public IP address." type = bool } variable "control_plane_nsg_ids" { default = [] description = "An additional list of network security groups (NSG) ids for the cluster endpoint." type = set(string) } variable "backend_nsg_ids" { default = [] description = "An additional list of network security groups (NSG) ids for the LB backends. Used when the service rule management mode is set to NSG via annotations. See Security Rule Management Options for Load Balancers and Network Load Balancers for more information." type = set(string) } variable "cni_type" { default = "flannel" description = "The CNI for the cluster: 'flannel' or 'npn'. See Pod Networking." type = string validation { condition = contains(["flannel", "npn"], var.cni_type) error_message = "Accepted values are flannel or npn" } } variable "enable_ipv6" { default = false description = "Whether to create a dual-stack (IPv4/IPv6) cluster." type = bool } variable "oke_ip_families" { default = [] type = list(string) description = "Override the ip_families attribute for the OKE cluster. Supported values: ['IPv4'] or ['IPV4', 'IPv6']" } variable "pods_cidr" { default = "10.244.0.0/16" description = "The CIDR range used for IP addresses by the pods. A /16 CIDR is generally sufficient. This CIDR should not overlap with any subnet range in the VCN (it can also be outside the VCN CIDR range). Ignored when cni_type = 'npn'." type = string } variable "services_cidr" { default = "10.96.0.0/16" description = "The CIDR range used within the cluster by Kubernetes services (ClusterIPs). This CIDR should not overlap with the VCN CIDR range." type = string } variable "kubernetes_version" { default = "v1.34.2" description = "The version of kubernetes to use when provisioning OKE or to upgrade an existing OKE cluster to." type = string } variable "cluster_kms_key_id" { default = "" description = "The id of the OCI KMS key to be used as the master encryption key for Kubernetes secrets encryption." type = string } variable "use_signed_images" { default = false description = "Whether to enforce the use of signed images. If set to true, at least 1 RSA key must be provided through image_signing_keys." type = bool } variable "image_signing_keys" { default = [] description = "A list of KMS key ids used by the worker nodes to verify signed images. The keys must use RSA algorithm." type = set(string) } variable "load_balancers" { default = "both" description = "The type of subnets to create for load balancers." type = string validation { condition = contains(["public", "internal", "both"], var.load_balancers) error_message = "Accepted values are public, internal or both." } } variable "preferred_load_balancer" { default = "public" description = "The preferred load balancer subnets that OKE will automatically choose when creating a load balancer. Valid values are 'public' or 'internal'. If 'public' is chosen, the value for load_balancers must be either 'public' or 'both'. If 'private' is chosen, the value for load_balancers must be either 'internal' or 'both'. NOTE: Service annotations for internal load balancers must still be specified regardless of this setting. See Load Balancer Annotations for more information." type = string validation { condition = contains(["public", "internal"], var.preferred_load_balancer) error_message = "Accepted values are public or internal." } } variable "oidc_discovery_enabled" { default = false description = "Whether the cluster has OIDC Discovery enabled. See OIDC Discovery configuration documentation." type = bool } variable "oidc_token_auth_enabled" { default = false description = "Whether the cluster has OIDC Auth Config enabled." type = bool } variable "oidc_token_authentication_config" { default = {} description = "The properties that configure OIDC token authentication in kube-apiserver. See OIDC Token Authentication configuration documentation." type = any } ================================================ FILE: variables-common.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { # SSH key precedence: base64-encoded PEM > raw PEM > file PEM > null ssh_key_arg = (coalesce(var.ssh_private_key_path, "none") != "none" ? join(" ", ["-i", var.ssh_private_key_path]) : null) ssh_private_key = sensitive( coalesce(var.ssh_private_key, "none") != "none" ? try(base64decode(var.ssh_private_key), var.ssh_private_key) : coalesce(var.ssh_private_key_path, "none") != "none" ? file(var.ssh_private_key_path) : null ) ssh_public_key = ( coalesce(var.ssh_public_key, "none") != "none" ? try(base64decode(var.ssh_public_key), var.ssh_public_key) : coalesce(var.ssh_public_key_path, "none") != "none" ? file(var.ssh_public_key_path) : null ) } variable "state_id" { default = null description = "Optional Terraform state_id from an existing deployment of the module to re-use with created resources." type = string } variable "output_detail" { default = false description = "Whether to include detailed output in state." type = bool } variable "timezone" { default = "Etc/UTC" description = "The preferred timezone for workers, operator, and bastion instances." type = string } variable "ssh_private_key" { default = null description = "The contents of the SSH private key file, optionally base64-encoded. May be provided via SSH agent when unset." sensitive = true type = string } variable "ssh_private_key_path" { default = null description = "A path on the local filesystem to the SSH private key. May be provided via SSH agent when unset." type = string } variable "ssh_public_key" { default = null description = "The contents of the SSH public key file, optionally base64-encoded. Used to allow login for workers/bastion/operator with corresponding private key." type = string } variable "ssh_public_key_path" { default = null description = "A path on the local filesystem to the SSH public key. Used to allow login for workers/bastion/operator with corresponding private key." type = string } ================================================ FILE: variables-extensions.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # CNI: Cilium variable "cilium_install" { default = false description = "Whether to deploy the Cilium Helm chart. May only be enabled when cni_type = 'flannel'. See https://docs.cilium.io. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "cilium_reapply" { default = false description = "Whether to force reapply of the chart when no changes are detected, e.g. with state modified externally." type = bool } variable "cilium_namespace" { default = "kube-system" description = "Kubernetes namespace for deployed resources." type = string } variable "cilium_helm_version" { default = "1.16.3" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "cilium_helm_values" { default = {} description = "Map of individual Helm chart values. See https://registry.terraform.io/providers/hashicorp/helm/latest/docs/data-sources/template." type = any } variable "cilium_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } # CNI: Multus variable "multus_install" { default = false description = "Whether to deploy Multus. See k8snetworkplumbingwg/multus-cni. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "multus_namespace" { default = "network" description = "Kubernetes namespace for deployed resources." type = string } variable "multus_daemonset_url" { default = null description = "The URL path to the Multus manifest. Leave unset for tags of k8snetworkplumbingwg/multus-cni using multus_version." type = string } variable "multus_version" { default = "3.9.3" description = "Version of Multus to install. Ignored when an explicit value for multus_daemonset_url is provided." type = string } # SR-IOV Device Plugin variable "sriov_device_plugin_install" { default = false description = "Whether to deploy the SR-IOV Network Device Plugin. See k8snetworkplumbingwg/sriov-network-device-plugin. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "sriov_device_plugin_namespace" { default = "network" description = "Kubernetes namespace for deployed resources." type = string } variable "sriov_device_plugin_daemonset_url" { default = null description = "The URL path to the manifest. Leave unset for tags of k8snetworkplumbingwg/sriov-network-device-plugin using sriov_device_plugin_version." type = string } variable "sriov_device_plugin_version" { default = "master" description = "Version to install. Ignored when an explicit value for sriov_device_plugin_daemonset_url is provided." type = string } # SR-IOV CNI Plugin variable "sriov_cni_plugin_install" { default = false description = "Whether to deploy the SR-IOV CNI Plugin. See . NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "sriov_cni_plugin_namespace" { default = "network" description = "Kubernetes namespace for deployed resources." type = string } variable "sriov_cni_plugin_daemonset_url" { default = null description = "The URL path to the manifest. Leave unset for tags of using sriov_cni_plugin_version." type = string } variable "sriov_cni_plugin_version" { default = "master" description = "Version to install. Ignored when an explicit value for sriov_cni_plugin_daemonset_url is provided." type = string } # RDMA CNI Plugin variable "rdma_cni_plugin_install" { default = false description = "Whether to deploy the RDMA CNI Plugin. See Mellanox/rdma-cni. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "rdma_cni_plugin_namespace" { default = "network" description = "Kubernetes namespace for deployed resources." type = string } variable "rdma_cni_plugin_daemonset_url" { default = null description = "The URL path to the manifest. Leave unset for tags of Mellanox/rdma-cni using rdma_cni_plugin_version." type = string } variable "rdma_cni_plugin_version" { default = "master" description = "Version to install. Ignored when an explicit value for rdma_cni_plugin_daemonset_url is provided." type = string } # Metrics server variable "metrics_server_install" { default = false description = "Whether to deploy the Kubernetes Metrics Server Helm chart. See kubernetes-sigs/metrics-server. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "metrics_server_namespace" { default = "metrics" description = "Kubernetes namespace for deployed resources." type = string } variable "metrics_server_helm_version" { default = "3.8.3" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "metrics_server_helm_values" { default = {} description = "Map of individual Helm chart values. See data.helm_template." type = map(string) } variable "metrics_server_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } # Cluster autoscaler variable "cluster_autoscaler_install" { default = false description = "Whether to deploy the Kubernetes Cluster Autoscaler Helm chart. See kubernetes/autoscaler. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "cluster_autoscaler_namespace" { default = "kube-system" description = "Kubernetes namespace for deployed resources." type = string } variable "cluster_autoscaler_helm_version" { default = "9.24.0" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "cluster_autoscaler_helm_values" { default = {} description = "Map of individual Helm chart values. See data.helm_template." type = map(string) } variable "cluster_autoscaler_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } # Prometheus variable "prometheus_install" { default = false description = "Whether to deploy the Prometheus Helm chart. See https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "prometheus_reapply" { default = false description = "Whether to force reapply of the Prometheus Helm chart when no changes are detected, e.g. with state modified externally." type = bool } variable "prometheus_namespace" { default = "metrics" description = "Kubernetes namespace for deployed resources." type = string } variable "prometheus_helm_version" { default = "45.2.0" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "prometheus_helm_values" { default = {} description = "Map of individual Helm chart values. See data.helm_template." type = map(string) } variable "prometheus_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } # DCGM exporter variable "dcgm_exporter_install" { default = false description = "Whether to deploy the DCGM exporter Helm chart. See DCGM-Exporter. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "dcgm_exporter_reapply" { default = false description = "Whether to force reapply of the Helm chart when no changes are detected, e.g. with state modified externally." type = bool } variable "dcgm_exporter_namespace" { default = "metrics" description = "Kubernetes namespace for deployed resources." type = string } variable "dcgm_exporter_helm_version" { default = "3.1.5" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "dcgm_exporter_helm_values" { default = {} description = "Map of individual Helm chart values. See data.helm_template." type = map(string) } variable "dcgm_exporter_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } # MPI Operator variable "mpi_operator_install" { default = false description = "Whether to deploy the MPI Operator. See kubeflow/mpi-operator. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "mpi_operator_namespace" { default = "default" description = "Kubernetes namespace for deployed resources." type = string } variable "mpi_operator_deployment_url" { default = null description = "The URL path to the manifest. Leave unset for tags of kubeflow/mpi-operator using mpi_operator_version." type = string } variable "mpi_operator_version" { default = "0.4.0" description = "Version to install. Ignored when an explicit value for mpi_operator_deployment_url is provided." type = string } # Whereabouts variable "whereabouts_install" { default = false description = "Whether to deploy Whereabouts IPAM. See k8snetworkplumbingwg/whereabouts. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "whereabouts_namespace" { default = "default" description = "Kubernetes namespace for deployed resources." type = string } variable "whereabouts_daemonset_url" { default = null description = "The URL path to the manifest. Leave unset for tags of k8snetworkplumbingwg/whereabouts using whereabouts_version." type = string } variable "whereabouts_version" { default = "master" description = "Version to install. Ignored when an explicit value for whereabouts_daemonset_url is provided." type = string } # Gatekeeper variable "gatekeeper_install" { default = false description = "Whether to deploy the Gatekeeper Helm chart. See https://github.com/open-policy-agent/gatekeeper. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "gatekeeper_namespace" { default = "kube-system" description = "Kubernetes namespace for deployed resources." type = string } variable "gatekeeper_helm_version" { default = "3.11.0" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "gatekeeper_helm_values" { default = {} description = "Map of individual Helm chart values. See data.helm_template." type = map(string) } variable "gatekeeper_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } # Service Account variable "create_service_account" { default = false description = "Whether to create a service account or not." type = bool } variable "service_accounts" { default = { kubeconfigsa = { sa_name = "kubeconfigsa" sa_namespace = "kube-system" sa_cluster_role = "cluster-admin" sa_cluster_role_binding = "kubeconfigsa-crb" } } description = "Map of service accounts and associated parameters." type = map(any) } # Argocd variable "argocd_install" { default = false description = "Whether to deploy the Argocd Helm chart. See https://github.com/argoproj/argo-cd. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "argocd_namespace" { default = "argocd" description = "Kubernetes namespace for deployed resources." type = string } variable "argocd_helm_version" { default = "8.1.2" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } variable "argocd_helm_values" { default = {} description = "Map of individual Helm chart values. See data.helm_template." type = map(string) } variable "argocd_helm_values_files" { default = [] description = "Paths to a local YAML files with Helm chart values (as with `helm install -f` which supports multiple). Generate with defaults using `helm show values [CHART] [flags]`." type = list(string) } ================================================ FILE: variables-iam.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { tenancy_id = coalesce(var.tenancy_id, var.tenancy_ocid, "unknown") compartment_id = coalesce(var.compartment_id, var.compartment_ocid, var.tenancy_id) worker_compartment_id = coalesce(var.worker_compartment_id, var.compartment_id) user_id = var.user_id != "" ? var.user_id : var.current_user_ocid home_region = coalesce(var.home_region, var.region) api_private_key = sensitive( var.api_private_key != "" ? try(base64decode(var.api_private_key), var.api_private_key) : var.api_private_key_path != "" ? file(var.api_private_key_path) : null ) # Merge freeform tags from map & individual inputs better suited to Resource Manager bastion_freeform_tags = merge(lookup(var.freeform_tags, "bastion", {}), var.bastion_freeform_tags) cluster_freeform_tags = merge(lookup(var.freeform_tags, "cluster", {}), var.cluster_freeform_tags) iam_freeform_tags = merge(lookup(var.freeform_tags, "iam", {}), var.iam_freeform_tags) network_freeform_tags = merge(lookup(var.freeform_tags, "network", {}), var.network_freeform_tags) operator_freeform_tags = merge(lookup(var.freeform_tags, "operator", {}), var.operator_freeform_tags) persistent_volume_freeform_tags = merge(lookup(var.freeform_tags, "persistent_volume", {}), var.persistent_volume_freeform_tags) service_lb_freeform_tags = merge(lookup(var.freeform_tags, "service_lb", {}), var.service_lb_freeform_tags) workers_freeform_tags = merge(lookup(var.freeform_tags, "workers", {}), var.workers_freeform_tags) # Merge defined tags from map & individual inputs better suited to Resource Manager bastion_defined_tags = merge(lookup(var.defined_tags, "bastion", {}), var.bastion_defined_tags) cluster_defined_tags = merge(lookup(var.defined_tags, "cluster", {}), var.cluster_defined_tags) iam_defined_tags = merge(lookup(var.defined_tags, "iam", {}), var.iam_defined_tags) network_defined_tags = merge(lookup(var.defined_tags, "network", {}), var.network_defined_tags) operator_defined_tags = merge(lookup(var.defined_tags, "operator", {}), var.operator_defined_tags) persistent_volume_defined_tags = merge(lookup(var.defined_tags, "persistent_volume", {}), var.persistent_volume_defined_tags) service_lb_defined_tags = merge(lookup(var.defined_tags, "service_lb", {}), var.service_lb_defined_tags) workers_defined_tags = merge(lookup(var.defined_tags, "workers", {}), var.workers_defined_tags) } # Overrides Resource Manager variable "tenancy_id" { default = null description = "The tenancy id of the OCI Cloud Account in which to create the resources." type = string } variable "tenancy_ocid" { default = null description = "A tenancy OCID automatically populated by Resource Manager." type = string } # Overrides Resource Manager variable "user_id" { default = null description = "The id of the user that terraform will use to create the resources." type = string } # Automatically populated by Resource Manager variable "current_user_ocid" { default = null description = "A user OCID automatically populated by Resource Manager." type = string } # Overrides Resource Manager variable "compartment_id" { default = null description = "The compartment id where resources will be created." type = string } # Automatically populated by Resource Manager variable "compartment_ocid" { default = null description = "A compartment OCID automatically populated by Resource Manager." type = string } # Overrides compartment_[oc]id variable "worker_compartment_id" { default = null description = "The compartment id where worker group resources will be created." type = string } variable "network_compartment_id" { default = null description = "The compartment id where network resources will be created." type = string } # Automatically populated by Resource Manager # List of regions: https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm#ServiceAvailabilityAcrossRegions variable "region" { default = "us-ashburn-1" description = "The OCI region where OKE resources will be created." type = string } # List of regions: https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm#ServiceAvailabilityAcrossRegions variable "home_region" { default = null description = "The tenancy's home region. Required to perform identity operations." type = string } variable "api_fingerprint" { default = null description = "Fingerprint of the API private key to use with OCI API." type = string } variable "api_private_key" { default = null description = "The contents of the private key file to use with OCI API, optionally base64-encoded. This takes precedence over private_key_path if both are specified in the provider." sensitive = true type = string } variable "api_private_key_password" { default = null description = "The corresponding private key password to use with the api private key if it is encrypted." sensitive = true type = string } variable "api_private_key_path" { default = null description = "The path to the OCI API private key." type = string } variable "config_file_profile" { default = "DEFAULT" description = "The profile within the OCI config file to use." type = string } variable "create_iam_resources" { default = false description = "Whether to create IAM dynamic groups, policies, and tags. Resources for components may be controlled individually with 'create_iam_*' variables when enabled. Ignored when 'create_iam_resources' is false." type = bool } variable "create_iam_autoscaler_policy" { default = "auto" description = "Whether to create an IAM dynamic group and policy rules for Cluster Autoscaler management. Depends on configuration of associated component when set to 'auto'. Ignored when 'create_iam_resources' is false." type = string validation { condition = contains(["never", "auto", "always"], var.create_iam_autoscaler_policy) error_message = "Accepted values are never, auto, or always" } } variable "create_iam_kms_policy" { default = "auto" description = "Whether to create an IAM dynamic group and policy rules for KMS encryption. Depends on configuration of associated components when set to 'auto'. Ignored when 'create_iam_resources' is false." type = string validation { condition = contains(["never", "auto", "always"], var.create_iam_kms_policy) error_message = "Accepted values are never, auto, or always" } } variable "create_iam_operator_policy" { default = "auto" description = "Whether to create an IAM dynamic group and policy rules for operator access to the OKE control plane. Depends on configuration of associated components when set to 'auto'. Ignored when 'create_iam_resources' is false." type = string validation { condition = contains(["never", "auto", "always"], var.create_iam_operator_policy) error_message = "Accepted values are never, auto, or always" } } variable "create_iam_worker_policy" { default = "auto" description = "Whether to create an IAM dynamic group and policy rules for self-managed worker nodes. Depends on configuration of associated components when set to 'auto'. Ignored when 'create_iam_resources' is false." type = string validation { condition = contains(["never", "auto", "always"], var.create_iam_worker_policy) error_message = "Accepted values are never, auto, or always" } } # Tagging variable "create_iam_tag_namespace" { default = false description = "Whether to create a namespace for defined tags used for IAM policy and tracking. Ignored when 'create_iam_resources' is false." type = bool } variable "create_iam_defined_tags" { default = false description = "Whether to create defined tags used for IAM policy and tracking. Ignored when 'create_iam_resources' is false." type = bool } variable "use_defined_tags" { default = false description = "Whether to apply defined tags to created resources for IAM policy and tracking." type = bool } variable "tag_namespace" { default = "oke" description = "The tag namespace for standard OKE defined tags." type = string } variable "freeform_tags" { default = { bastion = {} cluster = {} iam = {} network = {} operator = {} persistent_volume = {} service_lb = {} workers = {} } description = "Freeform tags to be applied to created resources." type = any } variable "defined_tags" { default = { bastion = {} cluster = {} iam = {} network = {} operator = {} persistent_volume = {} service_lb = {} workers = {} } description = "Defined tags to be applied to created resources. Must already exist in the tenancy." type = any } # Individual inputs better suited to Resource Manager are merged in locals variable "bastion_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "bastion_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "cluster_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "cluster_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "iam_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "iam_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "network_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "network_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "operator_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "operator_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "persistent_volume_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "persistent_volume_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "service_lb_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "service_lb_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } variable "workers_defined_tags" { type = map(string) description = "Defined tags applied to created resources." default = {} } variable "workers_freeform_tags" { type = map(string) description = "Freeform tags applied to created resources." default = {} } ================================================ FILE: variables-network.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_vcn" { default = true description = "Whether to create a Virtual Cloud Network." type = bool } variable "vcn_name" { default = null description = "Display name for the created VCN. Defaults to 'oke' suffixed with the generated Terraform 'state_id' value." type = string } variable "vcn_id" { default = null description = "Optional ID of existing VCN. Takes priority over vcn_name filter. Ignored when `create_vcn = true`." type = string } variable "vcn_create_nat_gateway" { default = "auto" description = "Whether to create a NAT gateway with the VCN. Defaults to automatic creation when private network resources are expected to utilize it." type = string validation { condition = contains(["never", "auto", "always"], var.vcn_create_nat_gateway) error_message = "Accepted values are never, auto, or always" } } variable "vcn_create_internet_gateway" { default = "auto" description = "Whether to create an internet gateway with the VCN. Defaults to automatic creation when public network resources are expected to utilize it." type = string validation { condition = contains(["never", "auto", "always"], var.vcn_create_internet_gateway) error_message = "Accepted values are never, auto, or always" } } variable "vcn_create_service_gateway" { default = "always" description = "Whether to create a service gateway with the VCN. Defaults to always created." type = string validation { condition = contains(["never", "auto", "always"], var.vcn_create_service_gateway) error_message = "Accepted values are never, auto, or always" } } variable "vcn_enable_ipv6_gua" { default = true description = "Whether to enable IPv6 GUA when IPv6 is enabled." type = bool } variable "vcn_ipv6_ula_cidrs" { default = [] description = "IPv6 ULA CIDR blocks to be used for the VCN." type = list(string) } variable "internet_gateway_id" { default = null description = "Optional ID of existing Internet gateway in VCN." type = string } variable "ig_route_table_id" { default = null description = "Optional ID of existing internet gateway route table in VCN." type = string } variable "nat_gateway_id" { default = null description = "Optional ID of existing NAT gateway in VCN." type = string } variable "nat_route_table_id" { default = null description = "Optional ID of existing NAT gateway route table in VCN." type = string } variable "igw_ngw_mixed_route_id" { default = null description = "Optional ID of the existing route table in VCN using IGW for IPv6 and NGW for IPv4." type = string } variable "create_drg" { default = false description = "Whether to create a Dynamic Routing Gateway and attach it to the VCN." type = bool } variable "drg_display_name" { default = null description = "(Updatable) Name of the created Dynamic Routing Gateway. Does not have to be unique. Defaults to 'oke' suffixed with the generated Terraform 'state_id' value." type = string } variable "drg_id" { default = null description = "ID of an external created Dynamic Routing Gateway to be attached to the VCN." type = string } variable "drg_compartment_id" { default = null description = "Compartment for the DRG resource. Can be used to override network_compartment_id." type = string } variable "drg_attachments" { description = "DRG attachment configurations." type = any default = {} } variable "remote_peering_connections" { description = "Map of parameters to add and optionally to peer to remote peering connections. Key-only items represent local acceptors and no peering attempted; items containing key and values represent local requestor and must have the OCID and region of the remote acceptor to peer to" type = map(any) default = {} } variable "internet_gateway_route_rules" { default = null description = "(Updatable) List of routing rules to add to Internet Gateway Route Table." type = list(map(string)) } variable "local_peering_gateways" { default = null description = "Map of Local Peering Gateways to attach to the VCN." type = map(any) } variable "lockdown_default_seclist" { default = true description = "Whether to remove all default security rules from the VCN Default Security List." type = bool } variable "nat_gateway_route_rules" { default = null description = "(Updatable) List of routing rules to add to NAT Gateway Route Table." type = list(map(string)) } variable "nat_gateway_public_ip_id" { default = null description = "OCID of reserved IP address for NAT gateway. The reserved public IP address needs to be manually created." type = string } variable "subnets" { default = { bastion = { newbits = 13, ipv6_cidr = "8, 0" } operator = { newbits = 13, ipv6_cidr = "8, 1" } cp = { newbits = 13, ipv6_cidr = "8, 2" } int_lb = { newbits = 11, ipv6_cidr = "8, 3" } pub_lb = { newbits = 11, ipv6_cidr = "8, 4" } workers = { newbits = 4, ipv6_cidr = "8, 5" } pods = { newbits = 2, ipv6_cidr = "8, 6" } } description = "Configuration for standard subnets. The 'create' parameter of each entry defaults to 'auto', creating subnets when other enabled components are expected to utilize them, and may be configured with 'never' or 'always' to force disabled/enabled." type = map(object({ create = optional(string) id = optional(string) newbits = optional(string) netnum = optional(string) cidr = optional(string) display_name = optional(string) dns_label = optional(string) ipv6_cidr = optional(string) })) validation { condition = alltrue([ for k, v in var.subnets : contains(["never", "auto", "always"], coalesce(v.create, "auto")) ]) error_message = "Accepted values for 'create' are 'never', 'auto', or 'always'." } validation { condition = alltrue([ for v in flatten([for k, v in var.subnets : keys(v)]) : contains(["create", "id", "cidr", "netnum", "newbits", "display_name", "dns_label", "ipv6_cidr"], v) ]) error_message = format("Invalid subnet configuration keys: %s", jsonencode(distinct([ for v in flatten([for k, v in var.subnets : keys(v)]) : v if !contains(["create", "id", "cidr", "netnum", "newbits", "display_name", "dns_label", "ipv6_cidr"], v) ]))) } } variable "nsgs" { default = { bastion = {} operator = {} cp = {} int_lb = {} pub_lb = {} workers = {} pods = {} } description = "Configuration for standard network security groups (NSGs). The 'create' parameter of each entry defaults to 'auto', creating NSGs when other enabled components are expected to utilize them, and may be configured with 'never' or 'always' to force disabled/enabled." type = map(object({ create = optional(string) id = optional(string) })) validation { condition = alltrue([ for k, v in values(var.nsgs) : contains(["never", "auto", "always"], coalesce(v.create, "auto")) ]) error_message = "Accepted values for 'create' are 'never', 'auto', or 'always'." } validation { condition = alltrue([ for v in flatten([for k, v in var.nsgs : keys(v)]) : contains(["create", "id"], v) ]) error_message = format("Invalid NSG configuration keys: %s", jsonencode(distinct([ for v in flatten([for k, v in var.nsgs : keys(v)]) : v if !contains(["create", "id"], v) ]))) } validation { condition = alltrue([ for k, v in var.nsgs : contains(["bastion", "operator", "cp", "int_lb", "pub_lb", "workers", "pods", "fss"], k) ]) error_message = format("Invalid NSG keys: %s", jsonencode([for k, v in var.nsgs : k if !contains(["bastion", "operator", "cp", "int_lb", "pub_lb", "workers", "pods", "fss"], k) ])) } } variable "vcn_cidrs" { default = ["10.0.0.0/16"] description = "The list of IPv4 CIDR blocks the VCN will use." type = list(string) } variable "vcn_dns_label" { default = null description = "A DNS label for the VCN, used in conjunction with the VNIC's hostname and subnet's DNS label to form a fully qualified domain name (FQDN) for each VNIC within this subnet. Defaults to the generated Terraform 'state_id' value." type = string } variable "assign_dns" { default = true description = "Whether to assign DNS records to created instances or disable DNS resolution of hostnames in the VCN." type = bool } variable "allow_node_port_access" { default = false description = "Whether to allow access from worker NodePort range to load balancers." type = bool } variable "allow_worker_internet_access" { default = true description = "Allow worker nodes to egress to internet. Required if container images are in a registry other than OCIR." type = bool } variable "allow_pod_internet_access" { default = true description = "Allow pods to egress to internet. Ignored when cni_type != 'npn'." type = bool } variable "allow_worker_ssh_access" { default = false description = "Whether to allow SSH access to worker nodes." type = bool } variable "allow_bastion_cluster_access" { default = false description = "Whether to allow access to the Kubernetes cluster endpoint from the bastion host." type = bool } variable "allow_rules_cp" { default = {} description = "A map of additional rules to allow traffic for the OKE control plane." type = any } variable "allow_rules_internal_lb" { default = {} description = "A map of additional rules to allow incoming traffic for internal load balancers." type = any } variable "allow_rules_pods" { default = {} description = "A map of additional rules to allow traffic for the pods." type = any } variable "allow_rules_public_lb" { default = {} description = "A map of additional rules to allow incoming traffic for public load balancers." type = any } variable "allow_rules_workers" { default = {} description = "A map of additional rules to allow traffic for the workers." type = any } variable "control_plane_allowed_cidrs" { default = [] description = "The list of CIDR blocks from which the control plane can be accessed." type = list(string) } variable "enable_waf" { description = "Whether to enable WAF monitoring of load balancers." type = bool default = false } variable "use_stateless_rules" { description = "(experimental) Create NSGs with stateless rules instead of the default stateful rules." type = bool default = false } ================================================ FILE: variables-operator.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "create_operator" { default = true description = "Whether to create an operator server in a private subnet." type = bool } variable "operator_availability_domain" { default = null description = "The availability domain for operator placement. Defaults to first available." type = string } variable "operator_cloud_init" { default = [] description = "List of maps containing cloud init MIME part configuration for operator host. See https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html#part for expected schema of each element." type = list(map(string)) } variable "operator_nsg_ids" { description = "An optional and updatable list of network security groups that the operator will be part of." default = [] type = list(string) } variable "operator_user" { default = "opc" description = "User for SSH access to operator host." type = string } variable "operator_image_id" { default = null description = "Image ID for created operator instance." type = string } variable "operator_image_os" { default = "Oracle Linux" description = "Operator image operating system name when operator_image_type = 'platform'." type = string } variable "operator_image_os_version" { default = "8" description = "Operator image operating system version when operator_image_type = 'platform'." type = string } variable "operator_image_type" { default = "platform" description = "Whether to use a platform or custom image for the created operator instance. When custom is set, the operator_image_id must be specified." type = string validation { condition = contains(["custom", "platform"], var.operator_image_type) error_message = "Accepted values are custom or platform" } } variable "operator_install_helm" { default = true description = "Whether to install Helm on the created operator host." type = bool } variable "operator_install_helm_from_repo" { default = false description = "Whether to install Helm from the repo on the created operator host." type = bool } variable "operator_install_oci_cli_from_repo" { default = false description = "Whether to install OCI from repo on the created operator host." type = bool } variable "operator_install_istioctl" { default = false description = "Whether to install istioctl on the created operator host." type = bool } variable "operator_install_k8sgpt" { default = false description = "Whether to install k8sgpt on the created operator host. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "operator_install_k9s" { default = false description = "Whether to install k9s on the created operator host. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "operator_install_kubectl_from_repo" { default = true description = "Whether to install kubectl from the repo on the created operator host." type = bool } variable "operator_install_kubectx" { default = true description = "Whether to install kubectx/kubens on the created operator host. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "operator_install_stern" { default = false description = "Whether to install stern on the created operator host. NOTE: Provided only as a convenience and not supported by or sourced from Oracle - use at your own risk." type = bool } variable "operator_shape" { default = { shape = "VM.Standard.E4.Flex", ocpus = 1, memory = 4, boot_volume_size = 50, baseline_ocpu_utilization = 100 } description = "Shape of the created operator instance. Baseline OCPU utilization can be used to provision burstable shapes." type = map(any) } variable "operator_volume_kms_key_id" { default = null description = "The OCID of the OCI KMS key to assign as the master encryption key for the operator host boot volume." type = string } variable "operator_pv_transit_encryption" { default = false description = "Whether to enable in-transit encryption for the data volume's paravirtualized attachment." type = bool } variable "operator_upgrade" { default = false description = "Whether to upgrade operator packages after provisioning." type = bool } variable "operator_private_ip" { default = null description = "The IP address of an existing operator host. Ignored when create_operator = true." type = string } variable "operator_await_cloudinit" { default = true description = "Whether to block until successful connection to operator and completion of cloud-init." type = bool } variable "operator_legacy_imds_endpoints_disabled" { default = true description = "Whether to disable requests to the IMDSv1 endpoint and only allow requests to the IMDSv2 endpoint for the operator instance." type = bool } ================================================ FILE: variables-utilities.tf ================================================ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl variable "await_node_readiness" { default = "none" description = "Optionally block completion of Terraform apply until one/all worker nodes become ready." type = string validation { condition = contains(["none", "one", "all"], var.await_node_readiness) error_message = "Accepted values are 'none', 'one' or 'all'." } } # Oracle Container Image Registry (OCIR) variable "ocir_email_address" { default = null description = "The email address used for the Oracle Container Image Registry (OCIR)." type = string } variable "ocir_secret_id" { default = null description = "The OCI Vault secret ID for the OCIR authentication token." type = string } variable "ocir_secret_name" { default = "ocirsecret" description = "The name of the Kubernetes secret to be created with the OCIR authentication token." type = string } variable "ocir_secret_namespace" { default = "default" description = "The Kubernetes namespace in which to create the OCIR secret." type = string } variable "ocir_username" { default = null description = "A username with access to the OCI Vault secret for OCIR access. Required when 'ocir_secret_id' is provided." type = string } # Worker pool draining variable "worker_drain_ignore_daemonsets" { default = true description = "Whether to ignore DaemonSet-managed Pods when draining worker pools. See kubectl drain for more information." type = bool } variable "worker_drain_delete_local_data" { default = true description = "Whether to accept removal of data stored locally on draining worker pools. See kubectl drain for more information." type = bool } variable "worker_drain_timeout_seconds" { default = 900 description = "The length of time to wait before giving up on draining nodes in a pool. See kubectl drain for more information." type = number } ================================================ FILE: variables-workers.tf ================================================ # Copyright (c) 2022, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl # # Cluster # variable "cluster_id" { default = null description = "An existing OKE cluster OCID when `create_cluster = false`." type = string } variable "cluster_ca_cert" { default = null description = "Base64+PEM-encoded cluster CA certificate for unmanaged instance pools. Determined automatically when 'create_cluster' = true or 'cluster_id' is provided." type = string } variable "cluster_dns" { default = null description = "Cluster DNS resolver IP address. Determined automatically when not set (recommended)." type = string } # # Worker pools # variable "worker_pools" { default = {} description = "Tuple of OKE worker pools where each key maps to the OCID of an OCI resource, and value contains its definition." type = any } variable "worker_pool_mode" { default = "node-pool" description = "Default management mode for workers when unspecified on a pool." type = string validation { condition = contains([ "node-pool", "virtual-node-pool", "instance", "instance-pool", "cluster-network", "compute-cluster", "gpu-memory-cluster" ], var.worker_pool_mode) error_message = "Accepted values are node-pool, virtual-node-pool, instance, instance-pool, cluster-network, compute-cluster, or gpu-memory-cluster." } } variable "worker_pool_size" { default = 0 description = "Default size for worker pools when unspecified on a pool." type = number } # # Workers: Compute clusters # variable "worker_compute_clusters" { default = {} description = "Whether to create compute clusters shared by nodes across multiple worker pools enabled for 'compute-cluster'." type = map(any) } # # Workers: GPU memory clusters # variable "worker_gmc_scale_is_upsize_enabled" { default = true description = "Default for gpu_memory_cluster_scale_config.is_upsize_enabled when unspecified on a 'gpu-memory-cluster' pool. When true, the OCI control plane may grow the GPU Memory Cluster toward target_size." type = bool } variable "worker_gmc_scale_is_downsize_enabled" { default = true description = "Default for gpu_memory_cluster_scale_config.is_downsize_enabled when unspecified on a 'gpu-memory-cluster' pool. When true, the OCI control plane may shrink the GPU Memory Cluster toward target_size." type = bool } variable "worker_gmc_scale_target_size" { default = 18 description = "Default for gpu_memory_cluster_scale_config.target_size when unspecified on a 'gpu-memory-cluster' pool. The desired GPU Memory Cluster size that the OCI control plane converges toward." type = number } # # Workers: network # variable "worker_is_public" { default = false description = "Whether to provision workers with public IPs allocated by default when unspecified on a pool. It should be true when creating dual-stack clusters." type = bool } variable "worker_nsg_ids" { default = [] description = "An additional list of network security group (NSG) IDs for node security. Combined with 'nsg_ids' specified on each pool." type = list(string) } variable "pod_nsg_ids" { default = [] description = "An additional list of network security group (NSG) IDs for pod security. Combined with 'pod_nsg_ids' specified on each pool." type = list(string) } variable "kubeproxy_mode" { default = "iptables" description = "The mode in which to run kube-proxy when unspecified on a pool." type = string validation { condition = contains(["iptables", "ipvs"], var.kubeproxy_mode) error_message = "Accepted values are iptables or ipvs." } } # # Workers: instance # variable "worker_block_volume_type" { default = "paravirtualized" description = "Default block volume attachment type for Instance Configurations when unspecified on a pool." type = string validation { condition = contains(["iscsi", "paravirtualized"], var.worker_block_volume_type) error_message = "Accepted values are 'iscsi' or 'paravirtualized'." } } variable "worker_node_labels" { default = {} description = "Default worker node labels. Merged with labels defined on each pool." type = map(string) } variable "worker_node_metadata" { default = {} description = "Map of additional worker node instance metadata. Merged with metadata defined on each pool." type = map(string) } variable "worker_image_id" { default = null description = "Default image for worker pools when unspecified on a pool." type = string } variable "worker_image_type" { default = "oke" description = "Whether to use a platform, OKE, or custom image for worker nodes by default when unspecified on a pool. When custom is set, the worker_image_id must be specified." type = string validation { condition = contains(["custom", "oke", "platform"], var.worker_image_type) error_message = "Accepted values are custom, oke, platform" } } variable "worker_image_os" { default = "Oracle Linux" description = "Default worker image operating system name when worker_image_type = 'oke' or 'platform' and unspecified on a pool." type = string } variable "worker_image_os_version" { default = "8" description = "Default worker image operating system version when worker_image_type = 'oke' or 'platform' and unspecified on a pool." type = string } variable "worker_shape" { default = { shape = "VM.Standard.E4.Flex" ocpus = 2 memory = 16 boot_volume_size = 50 # https://docs.oracle.com/en-us/iaas/Content/Block/Concepts/blockvolumeperformance.htm # Supported for mode = "cluster-network" | "instance-pool" | "instance" (self-managed) only boot_volume_vpus_per_gb = 10 # 10: Balanced, 20: High, 30-120: Ultra High (requires multipath) } description = "Default shape of the created worker instance when unspecified on a pool." type = map(any) } variable "worker_capacity_reservation_id" { default = null description = "The ID of the Compute capacity reservation the worker node will be launched under. See Capacity Reservations for more information." type = string } variable "worker_preemptible_config" { default = { enable = false, is_preserve_boot_volume = false } description = "Default preemptible Compute configuration when unspecified on a pool. See Preemptible Worker Nodes for more information." type = map(any) } variable "worker_cloud_init" { default = [] description = "List of maps containing cloud init MIME part configuration for worker nodes. Merged with pool-specific definitions. See https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/cloudinit_config.html#part for expected schema of each element." type = list(map(string)) } variable "worker_disable_default_cloud_init" { default = false description = "Whether to disable the default OKE cloud init and only use the cloud init explicitly passed to the worker pool in 'worker_cloud_init'." type = bool } variable "worker_volume_kms_key_id" { default = null description = "The ID of the OCI KMS key to be used as the master encryption key for Boot Volume and Block Volume encryption by default when unspecified on a pool." type = string } variable "worker_pv_transit_encryption" { default = false description = "Whether to enable in-transit encryption for the data volume's paravirtualized attachment by default when unspecified on a pool." type = bool } variable "worker_legacy_imds_endpoints_disabled" { default = false description = "Whether to disable requests to the IMDSv1 endpoint and only allow requests to the IMDSv2 endpoint. See Instance Metadata for more information." type = bool } variable "max_pods_per_node" { default = 31 description = "The default maximum number of pods to deploy per node when unspecified on a pool. Absolute maximum is 110. Ignored when when cni_type != 'npn'." type = number validation { condition = var.max_pods_per_node > 0 && var.max_pods_per_node <= 110 error_message = "Must be between 1 and 110." } } variable "platform_config" { default = null description = "Default platform_config for self-managed worker pools created with mode: 'instance', 'instance-pool', or 'cluster-network'. See PlatformConfig for more information." type = object({ type = optional(string), are_virtual_instructions_enabled = optional(bool), is_access_control_service_enabled = optional(bool), is_input_output_memory_management_unit_enabled = optional(bool), is_measured_boot_enabled = optional(bool), is_memory_encryption_enabled = optional(bool), is_secure_boot_enabled = optional(bool), is_symmetric_multi_threading_enabled = optional(bool), is_trusted_platform_module_enabled = optional(bool), numa_nodes_per_socket = optional(number), percentage_of_cores_enabled = optional(bool), }) } variable "agent_config" { default = null description = "Default agent_config for self-managed worker pools created with mode: 'instance', 'instance-pool', or 'cluster-network'. See