Full Code of mesosphere/dcos-kubernetes-quickstart for AI

Repository: mesosphere/dcos-kubernetes-quickstart
Branch: master
Commit: cc5d592c2a9d
Files: 26
Total size: 53.5 KB

Directory structure:
gitextract_2769bp27/

├── .gitignore
├── LICENSE
├── Makefile
├── README.md
├── docs/
│   ├── aws.md
│   ├── cncf_conformance.md
│   ├── existing_cluster.md
│   ├── exposing_kubernetes_api.md
│   └── gcp.md
├── examples/
│   ├── README.md
│   └── os-detector/
│       ├── cassandra-cql.json
│       ├── cassandra.json
│       ├── os-detector.md
│       └── os-detector.yaml
├── resources/
│   ├── desired_cluster_profile.aws.tfvars
│   ├── desired_cluster_profile.gcp.tfvars
│   ├── kubeapi-proxy.json
│   ├── main.aws.tf
│   ├── main.gcp.tf
│   ├── options-ha.json
│   ├── options.json
│   ├── outputs.tf
│   ├── variables.aws.tf
│   └── variables.gcp.tf
└── scripts/
    ├── get_cli
    └── poll_api.sh

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
.DS_Store
.id_key
.master_ip
.master_lb_ip

# cli
dcos
kubectl

# terraform
.deploy
.terraform
/*.tf
/*.tf.disabled
*.tfstate
*.tfstate.backup
terraform.tfvars
.terraform.tfstate.lock.info
modules/
desired_cluster_profile.tfvars.example
desired_cluster_profile



================================================
FILE: LICENSE
================================================

                              Apache License
                        Version 2.0, January 2004
                     http://www.apache.org/licenses/

TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

1. Definitions.

   "License" shall mean the terms and conditions for use, reproduction,
   and distribution as defined by Sections 1 through 9 of this document.

   "Licensor" shall mean the copyright owner or entity authorized by
   the copyright owner that is granting the License.

   "Legal Entity" shall mean the union of the acting entity and all
   other entities that control, are controlled by, or are under common
   control with that entity. For the purposes of this definition,
   "control" means (i) the power, direct or indirect, to cause the
   direction or management of such entity, whether by contract or
   otherwise, or (ii) ownership of fifty percent (50%) or more of the
   outstanding shares, or (iii) beneficial ownership of such entity.

   "You" (or "Your") shall mean an individual or Legal Entity
   exercising permissions granted by this License.

   "Source" form shall mean the preferred form for making modifications,
   including but not limited to software source code, documentation
   source, and configuration files.

   "Object" form shall mean any form resulting from mechanical
   transformation or translation of a Source form, including but
   not limited to compiled object code, generated documentation,
   and conversions to other media types.

   "Work" shall mean the work of authorship, whether in Source or
   Object form, made available under the License, as indicated by a
   copyright notice that is included in or attached to the work
   (an example is provided in the Appendix below).

   "Derivative Works" shall mean any work, whether in Source or Object
   form, that is based on (or derived from) the Work and for which the
   editorial revisions, annotations, elaborations, or other modifications
   represent, as a whole, an original work of authorship. For the purposes
   of this License, Derivative Works shall not include works that remain
   separable from, or merely link (or bind by name) to the interfaces of,
   the Work and Derivative Works thereof.

   "Contribution" shall mean any work of authorship, including
   the original version of the Work and any modifications or additions
   to that Work or Derivative Works thereof, that is intentionally
   submitted to Licensor for inclusion in the Work by the copyright owner
   or by an individual or Legal Entity authorized to submit on behalf of
   the copyright owner. For the purposes of this definition, "submitted"
   means any form of electronic, verbal, or written communication sent
   to the Licensor or its representatives, including but not limited to
   communication on electronic mailing lists, source code control systems,
   and issue tracking systems that are managed by, or on behalf of, the
   Licensor for the purpose of discussing and improving the Work, but
   excluding communication that is conspicuously marked or otherwise
   designated in writing by the copyright owner as "Not a Contribution."

   "Contributor" shall mean Licensor and any individual or Legal Entity
   on behalf of whom a Contribution has been received by Licensor and
   subsequently incorporated within the Work.

2. Grant of Copyright License. Subject to the terms and conditions of
   this License, each Contributor hereby grants to You a perpetual,
   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
   copyright license to reproduce, prepare Derivative Works of,
   publicly display, publicly perform, sublicense, and distribute the
   Work and such Derivative Works in Source or Object form.

3. Grant of Patent License. Subject to the terms and conditions of
   this License, each Contributor hereby grants to You a perpetual,
   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
   (except as stated in this section) patent license to make, have made,
   use, offer to sell, sell, import, and otherwise transfer the Work,
   where such license applies only to those patent claims licensable
   by such Contributor that are necessarily infringed by their
   Contribution(s) alone or by combination of their Contribution(s)
   with the Work to which such Contribution(s) was submitted. If You
   institute patent litigation against any entity (including a
   cross-claim or counterclaim in a lawsuit) alleging that the Work
   or a Contribution incorporated within the Work constitutes direct
   or contributory patent infringement, then any patent licenses
   granted to You under this License for that Work shall terminate
   as of the date such litigation is filed.

4. Redistribution. You may reproduce and distribute copies of the
   Work or Derivative Works thereof in any medium, with or without
   modifications, and in Source or Object form, provided that You
   meet the following conditions:

   (a) You must give any other recipients of the Work or
       Derivative Works a copy of this License; and

   (b) You must cause any modified files to carry prominent notices
       stating that You changed the files; and

   (c) You must retain, in the Source form of any Derivative Works
       that You distribute, all copyright, patent, trademark, and
       attribution notices from the Source form of the Work,
       excluding those notices that do not pertain to any part of
       the Derivative Works; and

   (d) If the Work includes a "NOTICE" text file as part of its
       distribution, then any Derivative Works that You distribute must
       include a readable copy of the attribution notices contained
       within such NOTICE file, excluding those notices that do not
       pertain to any part of the Derivative Works, in at least one
       of the following places: within a NOTICE text file distributed
       as part of the Derivative Works; within the Source form or
       documentation, if provided along with the Derivative Works; or,
       within a display generated by the Derivative Works, if and
       wherever such third-party notices normally appear. The contents
       of the NOTICE file are for informational purposes only and
       do not modify the License. You may add Your own attribution
       notices within Derivative Works that You distribute, alongside
       or as an addendum to the NOTICE text from the Work, provided
       that such additional attribution notices cannot be construed
       as modifying the License.

   You may add Your own copyright statement to Your modifications and
   may provide additional or different license terms and conditions
   for use, reproduction, or distribution of Your modifications, or
   for any such Derivative Works as a whole, provided Your use,
   reproduction, and distribution of the Work otherwise complies with
   the conditions stated in this License.

5. Submission of Contributions. Unless You explicitly state otherwise,
   any Contribution intentionally submitted for inclusion in the Work
   by You to the Licensor shall be under the terms and conditions of
   this License, without any additional terms or conditions.
   Notwithstanding the above, nothing herein shall supersede or modify
   the terms of any separate license agreement you may have executed
   with Licensor regarding such Contributions.

6. Trademarks. This License does not grant permission to use the trade
   names, trademarks, service marks, or product names of the Licensor,
   except as required for reasonable and customary use in describing the
   origin of the Work and reproducing the content of the NOTICE file.

7. Disclaimer of Warranty. Unless required by applicable law or
   agreed to in writing, Licensor provides the Work (and each
   Contributor provides its Contributions) on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
   implied, including, without limitation, any warranties or conditions
   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
   PARTICULAR PURPOSE. You are solely responsible for determining the
   appropriateness of using or redistributing the Work and assume any
   risks associated with Your exercise of permissions under this License.

8. Limitation of Liability. In no event and under no legal theory,
   whether in tort (including negligence), contract, or otherwise,
   unless required by applicable law (such as deliberate and grossly
   negligent acts) or agreed to in writing, shall any Contributor be
   liable to You for damages, including any direct, indirect, special,
   incidental, or consequential damages of any character arising as a
   result of this License or out of the use or inability to use the
   Work (including but not limited to damages for loss of goodwill,
   work stoppage, computer failure or malfunction, or any and all
   other commercial damages or losses), even if such Contributor
   has been advised of the possibility of such damages.

9. Accepting Warranty or Additional Liability. While redistributing
   the Work or Derivative Works thereof, You may choose to offer,
   and charge a fee for, acceptance of support, warranty, indemnity,
   or other liability obligations and/or rights consistent with this
   License. However, in accepting such obligations, You may act only
   on Your own behalf and on Your sole responsibility, not on behalf
   of any other Contributor, and only if You agree to indemnify,
   defend, and hold each Contributor harmless for any liability
   incurred by, or claims asserted against, such Contributor by reason
   of your accepting any such warranty or additional liability.

END OF TERMS AND CONDITIONS

APPENDIX: How to apply the Apache License to your work.

   To apply the Apache License to your work, attach the following
   boilerplate notice, with the fields enclosed by brackets "[]"
   replaced with your own identifying information. (Don't include
   the brackets!)  The text should be enclosed in the appropriate
   comment syntax for the file format. We also recommend that a
   file or class name and description of purpose be included on the
   same "printed page" as the copyright notice for easier
   identification within third-party archives.

Copyright 2016 Mesosphere

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.


================================================
FILE: Makefile
================================================

RM := rm -f
SSH_USER := core
TERRAFORM_INSTALLER_URL := github.com/dcos/terraform-dcos
DCOS_CLI_VERSION := 1.12
CUSTOM_DCOS_DOWNLOAD_PATH := https://downloads.dcos.io/dcos/stable/1.12.3/dcos_generate_config.sh
KUBERNETES_VERSION ?= 1.16.9
KUBERNETES_FRAMEWORK_VERSION ?= 2.5.0-1.16.9
KUBERNETES_STUB_URL ?=
KUBERNETES_CLUSTER_STUB_URL ?=
# PATH_TO_PACKAGE_OPTIONS holds the path to the package options file to be used
# when installing DC/OS Kubernetes.
PATH_TO_PACKAGE_OPTIONS ?= "$(PWD)/.deploy/options.json"

# Set PATH (locally) to include local dir for locally downloaded binaries.
FAKEPATH := "$(PWD):$(PATH)"

# Get the path to relevant binaries.
DCOS_CMD := $(shell PATH=$(FAKEPATH) command -v dcos 2> /dev/null)
KUBECTL_CMD := $(shell PATH=$(FAKEPATH) command -v kubectl 2> /dev/null)
TERRAFORM_CMD := $(shell PATH=$(FAKEPATH) command -v terraform 2> /dev/null)
TERRAFORM_APPLY_ARGS ?=
TERRAFORM_DESTROY_ARGS ?=

UNAME := $(shell uname -s)
ifeq ($(UNAME),Linux)
OPEN := xdg-open
else
OPEN := open
endif

# Define a new line character to use in error strings.
define n


endef

.PHONY: get-cli
get-cli:
	$(eval export DCOS_CLI_VERSION)
	$(eval export KUBERNETES_VERSION)
	scripts/get_cli

.PHONY: check-cli
check-cli: check-terraform check-dcos check-kubectl

.PHONY: check-terraform
check-terraform:
ifndef TERRAFORM_CMD
	$(error "$n$nNo terraform command in $(FAKEPATH).$n$nPlease install via 'brew install terraform' on MacOS, or download from https://www.terraform.io/downloads.html.$n$n")
endif

.PHONY: check-dcos
check-dcos:
ifndef DCOS_CMD
	$(error "$n$nNo dcos command in $(FAKEPATH).$n$nPlease run 'make get-cli' to download required binaries.$n$n")
endif

.PHONY: check-kubectl
check-kubectl:
ifndef KUBECTL_CMD
	$(error "$n$nNo kubectl command in $(FAKEPATH).$n$nPlease run 'make get-cli' to download required binaries.$n$n")
endif

.PHONY: gcp aws
gcp aws: clean check-terraform
	mkdir -p .deploy && \
	cd .deploy && \
	cp ../resources/main.$@.tf main.tf && \
	cp ../resources/variables.$@.tf variables.tf && \
	$(TERRAFORM_CMD) init && \
	cp ../resources/desired_cluster_profile.$@.tfvars terraform.tfvars && \
	cp ../resources/options.json . && \
	cp ../resources/outputs.tf . && \
	cp ../resources/kubeapi-proxy.json .

.PHONY: get-master-lb-ip
get-master-lb-ip: check-terraform
	$(call get_master_lb_ip)
	@echo $(MASTER_LB_IP)

define get_master_lb_ip
$(eval MASTER_LB_IP := $(shell $(TERRAFORM_CMD) output -state=.deploy/terraform.tfstate "cluster-address"))
endef

.PHONY: get-public-agent-ip
get-public-agent-ip: check-terraform
	$(call get_public_agent_ip)
	@echo $(PUBLIC_AGENT_IP)

define get_public_agent_ip
$(eval PUBLIC_AGENT_IP := $(shell $(TERRAFORM_CMD) output -state=.deploy/terraform.tfstate  "public-agents-loadbalancer"))
endef

.PHONY: plan-dcos
plan-dcos: check-terraform
	@cd .deploy; \
	$(TERRAFORM_CMD) plan

.PHONY: launch-dcos
launch-dcos: check-terraform
	@cd .deploy; \
	$(TERRAFORM_CMD) apply $(TERRAFORM_APPLY_ARGS)

.PHONY: plan
plan: plan-dcos

.PHONY: deploy
deploy: check-cli launch-dcos setup-cli install

.PHONY: setup-cli
setup-cli: check-dcos
	$(call get_master_lb_ip)
	for i in {1..20}; do $(DCOS_CMD) cluster setup https://$(MASTER_LB_IP) --insecure && break || (sleep 3) ; done
	@scripts/poll_api.sh "DC/OS Master" $(MASTER_LB_IP) 443

.PHONY: ui
ui:
	$(call get_master_lb_ip)
	$(OPEN) https://$(MASTER_LB_IP)

.PHONY: install
install: check-dcos add-stubs
	@echo "Installing Mesosphere Kubernetes Engine..."
	$(DCOS_CMD) package install --yes kubernetes --package-version="$(KUBERNETES_FRAMEWORK_VERSION)"
	@echo "Waiting for Mesosphere Kubernetes Engine to be up..."
	@while [[ ! $$($(DCOS_CMD) kubernetes manager plan show deploy 2> /dev/null | head -n1 | grep COMPLETE ) ]]; do \
		sleep 1; \
	done
	@echo "Creating a Kubernetes cluster..."
	$(DCOS_CMD) kubernetes cluster create --yes --options="$(PATH_TO_PACKAGE_OPTIONS)" --package-version="$(KUBERNETES_FRAMEWORK_VERSION)"

.PHONY: add-stubs
add-stubs:
ifdef KUBERNETES_STUB_URL
	@echo "Adding 'kubernetes' stub"
	$(DCOS_CMD) package repo add --index=0 kubernetes-aws "$(KUBERNETES_STUB_URL)"
endif
ifdef KUBERNETES_CLUSTER_STUB_URL
	@echo "Adding 'kubernetes-cluster' stub"
	$(DCOS_CMD) package repo add --index=0 kubernetes-cluster-aws "$(KUBERNETES_CLUSTER_STUB_URL)"
endif

.PHONY: marathon-lb
marathon-lb:
	$(DCOS_CMD) package install --yes marathon-lb
	@sleep 30
	$(DCOS_CMD) marathon app add "$(PWD)/.deploy/kubeapi-proxy.json"

.PHONY: watch-kubernetes-cluster
watch-kubernetes-cluster:
	watch dcos kubernetes cluster debug --cluster-name=dev/kubernetes01 plan show deploy

.PHONY: watch-kubernetes
watch-kubernetes:
	watch dcos kubernetes manager plan show deploy

.PHONY: kubeconfig
kubeconfig:
	$(call get_public_agent_ip)
	$(DCOS_CMD) kubernetes cluster kubeconfig --cluster-name dev/kubernetes01 --apiserver-url https://$(PUBLIC_AGENT_IP):6443 --context-name devkubernetes01 --insecure-skip-tls-verify
	@scripts/poll_api.sh "Kubernetes API" $(PUBLIC_AGENT_IP) 6443

.PHONY: upgrade-infra
upgrade-infra: launch-dcos

.PHONY: uninstall
uninstall: check-dcos
	$(DCOS_CMD) marathon app remove kubeapi-proxy
	$(DCOS_CMD) package uninstall marathon-lb --yes
	$(DCOS_CMD) kubernetes cluster delete --cluster-name dev/kubernetes01 --yes
	for i in {1..8}; do ! $(DCOS_CMD) marathon app list --json | jq '.[].id' | grep '/dev/kubernetes01' >/dev/null && break || (echo "Kubernetes Cluster is still uninstalling. Retrying in 15 seconds..." && sleep 15) ; done
	$(DCOS_CMD) package uninstall kubernetes --yes
	for i in {1..8}; do ! $(DCOS_CMD) marathon app list --json | jq '.[].id' | grep '/kubernetes' >/dev/null && break || (echo "Mesosphere Kubernetes Engine is still uninstalling. Retrying in 15 seconds..." && sleep 15) ; done
ifdef KUBERNETES_STUB_URL
	@echo "Removing 'kubernetes' stub"
	$(DCOS_CMD) package repo remove kubernetes-aws
endif
ifdef KUBERNETES_CLUSTER_STUB_URL
	@echo "Removing 'kubernetes-cluster' stub"
	$(DCOS_CMD) package repo remove kubernetes-cluster-aws
endif

.PHONY: destroy
destroy: check-terraform
	cd .deploy; \
	$(TERRAFORM_CMD) destroy $(TERRAFORM_DESTROY_ARGS)

.PHONY: clean
clean:
	$(RM) -r .deploy dcos kubectl

.PHONY: kubectl-tunnel
kubectl-tunnel:
	$(KUBECTL_CMD) config set-cluster dcos-k8s --server=http://localhost:9000
	$(KUBECTL_CMD) config set-context dcos-k8s --cluster=dcos-k8s --namespace=default
	$(KUBECTL_CMD) config use-context dcos-k8s
	$(call get_public_agent_ip)
	ssh -4 -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -o "ServerAliveInterval=120" \
		-N -L 9000:apiserver-insecure.devkubernetes01.l4lb.thisdcos.directory:9000 \
		$(SSH_USER)@$(PUBLIC_AGENT_IP)


================================================
FILE: README.md
================================================
# Kubernetes on DC/OS

Kubernetes is now available as a DC/OS package to quickly, and reliably run Kubernetes clusters on Mesosphere DC/OS.

![](docs/assets/ui-install.gif)

**NOTE:** The latest `dcos-kubernetes-quickstart` doesn't support any Kubernetes framework version before `2.0.0-1.12.1`. The reason is that now creating Kubernetes clusters requires the installation of the [Mesosphere Kubernetes Engine](https://docs.mesosphere.com/services/kubernetes/2.5.0-1.16.9/overview/#cluster-manager).

## Known limitations

Before proceeding, please check the [current package limitations](https://docs.mesosphere.com/service-docs/kubernetes/2.5.0-1.16.9/limitations/).

## Pre-Requisites

Check the requirements for running this quickstart:

* Linux or MacOS
* [Terraform 0.11.x](https://www.terraform.io/downloads.html). On MacOS, you can install with [brew](https://brew.sh/):
  ```bash
  $ brew install terraform
  ```
* [Google Cloud](docs/gcp.md) or [AWS](docs/aws.md) account with enough permissions to provide the
  needed infrastructure

## Quickstart

Once the pre-requisites are met, clone this repo:

```bash
$ git clone git@github.com:mesosphere/dcos-kubernetes-quickstart.git && cd dcos-kubernetes-quickstart
```

### Prepare infrastructure configuration

**This quickstart defaults to Google Cloud**

First, make sure you have have followed the [Google Cloud setup instructions](docs/gcp.md).

Then, start by generating the default infrastructure configuration:

```bash
$ make gcp
```

This will output sane defaults to `.deploy/terraform.tfvars`.
Now, edit said file and set your `gcp_project` and the `ssh_public_key_file`
(the SSH public key you will use to log-in into your new VMs later).

**WARNING:** Please, do not set a smaller instance (VM) type on the risk of failing to
install Kubernetes.

```
cluster_name = "dcos-kubernetes"
cluster_name_random_string = true

dcos_version = "1.12.3"

num_of_masters = "1"
num_of_private_agents = "4"
num_of_public_agents = "1"

bootstrap_instance_type = "n1-standard-1"
master_instance_type = "n1-standard-8"
private_agent_instance_type = "n1-standard-8"
public_agent_instance_type = "n1-standard-8"

# admin_ips = "0.0.0.0/0" # uncomment to access master from any IP

gcp_project = "YOUR_GCP_PROJECT"
gcp_region = "us-central1"
ssh_public_key_file = "/PATH/YOUR_GCP_SSH_PUBLIC_KEY.pub"
#
# If you want to use GCP service account key instead of GCP SDK
# uncomment the line below and update it with the path to the key file
# gcp_credentials = "/PATH/YOUR_GCP_SERVICE_ACCOUNT_KEY.json"
#
```

**NOTE:** The current release of the DC/OS GCP Terraform module also requires the `GOOGLE_PROJECT`
and `GOOGLE_REGION` environment variables to be set. Please set them with appropriates values for
your deployment:

```
$ export GOOGLE_PROJECT="YOUR_GCP_PROJECT"
$ export GOOGLE_REGION="us-central1"
```

### Kubernetes configuration

#### RBAC

**NOTE:** This `quickstart` will provision a Kubernetes cluster with `RBAC` support.

To deploy a cluster with RBAC disabled [RBAC](https://docs.mesosphere.com/services/kubernetes/2.5.0-1.16.9/operations/authn-and-authz/#rbac) update `.deploy/options.json`:

```
{
  "service": {
    "name": "dev/kubernetes01"
  },
  "kubernetes": {
    "authorization_mode": "AlwaysAllow"
  }
}
```

If you want to give users access to the Kubernetes API check [documentation](https://docs.mesosphere.com/services/kubernetes/2.5.0-1.16.9/operations/authn-and-authz/#giving-users-access-to-the-kubernetes-api).

**NOTE:** The authorization mode for a cluster must be chosen when installing the package. Changing the authorization mode after installing the package is not supported.

#### HA Cluster

**NOTE:** By default, it will provision a Kubernetes cluster with one (1) worker node, and
a single instance of every control plane component.

To deploy a **highly-available** cluster with three (3) private Kubernetes nodes update `.deploy/options.json`:

```
{
  "service": {
    "name": "dev/kubernetes01"
  },
  "kubernetes": {
    "high_availability": true,
    "private_node_count": 3
  }
}
```

### Download command-line tools

If you haven't already, please download DC/OS client, `dcos` and Kubernetes
client, `kubectl`:

```bash
$ make get-cli
```

The `dcos` and `kubectl` binaries will be downloaded to the current workdir.
It's up to you to decided whether or not to copy or move them to another path,
e.g. a path included in `PATH`.

### Install

You are now ready to provision the DC/OS cluster and install the Kubernetes package:

```bash
$ make deploy
```

Terraform will now try and provision the infrastructure on your chosen cloud
provider, and then proceed to install DC/OS.

When DC/OS is up and running, the Kubernetes package installation will take place.

Wait until all tasks are running before trying to access the Kubernetes API.

You can watch the progress what was deployed so far with:

```bash
$ make watch-kubernetes-cluster
```

Below is an example of how it looks like when the install ran successfully:

```
Using Kubernetes cluster: dev/kubernetes01
deploy (serial strategy) (COMPLETE)
   etcd (serial strategy) (COMPLETE)
      etcd-0:[peer] (COMPLETE)
   control-plane (dependency strategy) (COMPLETE)
      kube-control-plane-0:[instance] (COMPLETE)
   mandatory-addons (serial strategy) (COMPLETE)
      mandatory-addons-0:[instance] (COMPLETE)
   node (dependency strategy) (COMPLETE)
      kube-node-0:[kubelet] (COMPLETE)
   public-node (dependency strategy) (COMPLETE)
```

You can access DC/OS Dashboard and check Kubernetes package tasks under Services:

```bash
$ make ui
```

### Exposing the Kubernetes API

Check the [exposing Kubernetes API doc](docs/exposing_kubernetes_api.md) to understand how
the Kubernetes API gets exposed.
To actually expose the Kubernetes API for the new Kubernetes cluster using Marathon-LB, run:

```bash
$ make marathon-lb
```

**NOTE:** If you have changed in `.deploy/terraform.tfvars` file the number of
`num_of_public_agents` to more than `1`, please scale `marathon-lb` service to the same number,
so you can access Kubernetes API from any DC/OS public agent.

### Accessing the Kubernetes API

In order to access the Kubernetes API from outside the DC/OS cluster, one needs
to configure `kubectl`, the Kubernetes CLI tool:

```bash
$ make kubeconfig
```

Let's test accessing the Kubernetes API and list the Kubernetes cluster nodes:

```bash
$ ./kubectl --context devkubernetes01 get nodes
NAME                                                  STATUS   ROLES    AGE     VERSION
kube-control-plane-0-instance.devkubernetes01.mesos   Ready    master   5m18s   v1.16.9
kube-node-0-kubelet.devkubernetes01.mesos             Ready    <none>   2m58s   v1.16.9
```

And now, let's check how the system Kubernetes pods are doing:

```bash
$ ./kubectl --context devkubernetes01 -n kube-system get pods
NAME                                                                          READY   STATUS    RESTARTS   AGE
calico-node-s9828                                                             2/2     Running   0          3m21s
calico-node-zc8qw                                                             2/2     Running   0          3m38s
coredns-6c7669957f-rvz85                                                      1/1     Running   0          3m38s
kube-apiserver-kube-control-plane-0-instance.devkubernetes01.mesos            1/1     Running   0          4m43s
kube-controller-manager-kube-control-plane-0-instance.devkubernetes01.mesos   1/1     Running   0          4m42s
kube-proxy-kube-control-plane-0-instance.devkubernetes01.mesos                1/1     Running   0          4m48s
kube-proxy-kube-node-0-kubelet.devkubernetes01.mesos                          1/1     Running   0          3m21s
kube-scheduler-kube-control-plane-0-instance.devkubernetes01.mesos            1/1     Running   0          4m26s
kubernetes-dashboard-5cbf45898-nkjsm                                          1/1     Running   0          3m37s
local-dns-dispatcher-kube-node-0-kubelet.devkubernetes01.mesos                1/1     Running   0          3m21s
metrics-server-594576c7d8-cb4pj                                               1/1     Running   0          3m35s
```

### Accessing the Kubernetes Dashboard

You will be able to access the Kubernetes Dashboard by running:

```bash
$ kubectl --context devkubernetes01 proxy
```

Then pointing your browser at:

```
http://127.0.0.1:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/
```

Please note that you will have to sign-in into the [Kubernetes Dashboard](https://docs.mesosphere.com/services/kubernetes/2.5.0-1.16.9/operations/kubernetes-dashboard/#login-view-and-authorization) before being able to perform any action.

## Uninstall Kubernetes

To uninstall the DC/OS Kubernetes package while leaving your DC/OS cluster up,
run:

```bash
$ make uninstall
```

**NOTE:** This will only uninstall Kubernetes. Make sure you destroy your DC/OS
cluster using the instructions below when you finish testing, or otherwise you
will need to delete all cloud resources manually!

## Destroy cluster

To destroy the whole deployment:

```bash
$ make destroy
```

Last, clean generated resources:
```bash
$ make clean
```

## Documentation

For more details, please see the [docs folder](docs) and as well check the official [service docs](https://docs.mesosphere.com/service-docs/kubernetes/2.5.0-1.16.9)

## Community
Get help and connect with other users on the [mailing list](https://groups.google.com/a/dcos.io/forum/#!forum/kubernetes) or on DC/OS community [Slack](http://chat.dcos.io/) in the #kubernetes channel.


================================================
FILE: docs/aws.md
================================================
# AWS

**WARNING:** When running this quickstart, you might experience some issues
with cloud resource limits. Please, verify your [quotas](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-resource-limits.html)
before proceeding.

## Install AWS CLI

Make sure to have previously installed [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/installing.html).

## Setup access

First, you will need to [retrieve your AWS credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html).
The default location is `$HOME/.aws/credentials` on Linux and OS X, or `"%USERPROFILE%\.aws\credentials"` for Windows users.

Before proceeding, we recommend you create a file with your AWS credentials,
exposed as (the commonly) recognized environment variables, so you can [`source`](http://tldp.org/HOWTO/Bash-Prompt-HOWTO/x237.html)
it later, in between shell sessions:

```bash
$ cat << EOF > ~/.aws/my_credentials
export AWS_ACCESS_KEY_ID=<YOUR ACCESS KEY>
export AWS_SECRET_ACCESS_KEY=<YOUR SECRET KEY>
EOF
```

Last, set-up SSH keys as detailed in [the official documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html#how-to-generate-your-own-key-and-import-it-to-aws).

Don't forget to add your new SSH private key to your session:

```bash
$ ssh-add ~/.ssh/path_to_your_new_key.pem
```

## Prepare infrastructure configuration

Make sure Terraform knows where to find your AWS credentials:

```bash
$ source ~/.aws/my_credentials
```

Now, let's generate the default infrastructure configuration:

```bash
$ make aws
```

This will output sane defaults to `.deploy/terraform.tfvars`.
Now, edit said file and set `ssh_public_key_file`, the public SSH key you will use to
log-in into your new VMs later.

**WARNING:** Please, do not set a smaller instance (VM) type on the risk of
failing to install Kubernetes.

```
cluster_name = "dcos-kubernetes"
cluster_name_random_string = true

dcos_version = "1.12.3"
dcos_security = "strict" # valid values are strict, permissive, disabled

num_of_masters = "1"
num_of_private_agents = "4"
num_of_public_agents = "1"

instance_os = "centos_7.5"
bootstrap_instance_type = "m5.large"
master_instance_type = "m5.2xlarge"
private_agent_instance_type = "m5.2xlarge"
public_agent_instance_type = "m5.2xlarge"

aws_region = "us-west-2"
# ssh_public_key_file = ""
# aws_key_name = "default" # uncomment to use an already defined AWS key
# admin_ips = "0.0.0.0/0" # uncomment to access master from any IP

```

### Kubernetes configuration

#### Highly Available cluster

**NOTE:** By default, it will provision a Kubernetes cluster with one (1) worker node, and
a single instance of every control plane component.

To deploy a **highly-available** cluster with three (3) private Kubernetes nodes update `.deploy/options.json`:

```
{
  "service": {
    "name": "dev/kubernetes01"
  },
  "kubernetes": {
    "high_availability": true,
    "private_node_count": 3
  }
}
```

Let's continue with [Kubernetes cluster configuration](../README.md#kubernetes-configuration).


================================================
FILE: docs/cncf_conformance.md
================================================
# CNCF Conformance

## Prerequisites

The following prerequisites apply to follow these instructions. You will need:

* A Linux or MacOS machine with
  [Terraform 0.11.x](https://www.terraform.io/downloads.html) installed.
* A [Google Cloud](gcp.md), or [AWS](aws.md) account with enough permissions to provide the needed
  infrastructure

## Preparation

**NOTE:** These instructions are targeted at a
[Google Cloud Platform](gcp.md) deployment. To deploy in [AWS](aws.md),
please run `make aws` instead of
`make gcp` in the step below, and edit the resulting file accordingly.

**NOTE:** To install `dcos-kubernetes` in an existing cluster, please follow
[these instructions](existing_cluster.md).

First, clone this repository:

```shell
$ git clone git@github.com:mesosphere/dcos-kubernetes-quickstart.git
$ cd dcos-kubernetes-quickstart
```

Then generate the default infrastructure configuration:

```shell
$ make gcp
```

This will output sane defaults to `.deploy/terraform.tfvars`. Now, edit
said file and set the `gcp_project` and the `ssh_public_key_file` variables.
Please, do not set a smaller instance (VM) type on the risk of failing to
install Kubernetes. In the end, the `.deploy/terraform.tfvars` file
should look something like this:

```
cluster_name = "dcos-kubernetes"
cluster_name_random_string = true

dcos_version = "1.12.3"

num_of_masters = "1"
num_of_private_agents = "4"
num_of_public_agents = "1"

bootstrap_instance_type = "n1-standard-1"
master_instance_type = "n1-standard-8"
private_agent_instance_type = "n1-standard-8"
public_agent_instance_type = "n1-standard-8"

# admin_ips = "0.0.0.0/0" # uncomment to access master from any IP

gcp_project = "YOUR_GCP_PROJECT"
gcp_region = "us-central1"
ssh_public_key_file = "/PATH/YOUR_GCP_SSH_PUBLIC_KEY.pub"
#
# If you want to use GCP service account key instead of GCP SDK
# uncomment the line below and update it with the path to the key file
# gcp_credentials = "/PATH/YOUR_GCP_SERVICE_ACCOUNT_KEY.json"
#
```

Now, launch the DC/OS cluster by running:

```shell
$ KUBERNETES_VERSION=1.16.9 make get-cli launch-dcos setup-cli
```

This command will:

1. Download the `dcos` CLI and `kubectl` to your machine.
1. Provision the necessary infrastructure in GCP and install DC/OS.
1. Setup the `dcos` CLI to access the newly created DC/OS cluster.

As part of the last step, your browser will open and ask you to login with
a Google, GitHub or Microsoft account. Choose an option and copy the resulting
OpenID token to the shell where you ran the above mentioned command.

## Installing Mesosphere Kubernetes Engine

To install Mesosphere Kuberentes Engine and create a Kubernetes cluster in the newly created DC/OS cluster run:

```shell
$ KUBERNETES_FRAMEWORK_VERSION=2.5.0-1.16.9 \
  PATH_TO_PACKAGE_OPTIONS=./resources/options-ha.json make install
```

Wait until all tasks are running before proceeding.
You can track installation progress as follows:

```shell
$ make watch-kubernetes-cluster
```

When installation is successful you will see the following output:

```
Using Kubernetes cluster: dev/kubernetes01
deploy (serial strategy) (COMPLETE)
   etcd (serial strategy) (COMPLETE)
      etcd-0:[peer] (COMPLETE)
      etcd-1:[peer] (COMPLETE)
      etcd-2:[peer] (COMPLETE)
   control-plane (dependency strategy) (COMPLETE)
      kube-control-plane-0:[instance] (COMPLETE)
      kube-control-plane-1:[instance] (COMPLETE)
      kube-control-plane-2:[instance] (COMPLETE)
   mandatory-addons (serial strategy) (COMPLETE)
      mandatory-addons-0:[instance] (COMPLETE)
   node (dependency strategy) (COMPLETE)
      kube-node-0:[kubelet] (COMPLETE)
      kube-node-1:[kubelet] (COMPLETE)
      kube-node-2:[kubelet] (COMPLETE)
   public-node (dependency strategy) (COMPLETE)
```

When all tasks are in state `COMPLETE`, press `Ctrl-C` to terminate the `watch`
process and proceed to access your Kubernetes cluster.

## Accessing the Kubernetes API

In order to access the Kubernetes API from outside the DC/OS cluster, we must
first be able to access it. This can be achieved by running the following
command:

```shell
$ make marathon-lb kubeconfig
```

This command will expose the Kubernetes API for our newly created Kubernetes cluster, and configure `kubectl` to access said Kubernetes cluster.
Let's try and list this cluster's nodes:

```shell
$ ./kubectl --context devkubernetes01 get nodes
NAME                                                  STATUS   ROLES    AGE     VERSION
kube-control-plane-0-instance.devkubernetes01.mesos   Ready    master   5m18s   v1.16.9
kube-control-plane-1-instance.devkubernetes01.mesos   Ready    master   5m12s   v1.16.9
kube-control-plane-2-instance.devkubernetes01.mesos   Ready    master   5m11s   v1.16.9
kube-node-0-kubelet.devkubernetes01.mesos             Ready    <none>   2m58s   v1.16.9
kube-node-1-kubelet.devkubernetes01.mesos             Ready    <none>   2m42s   v1.16.9
kube-node-2-kubelet.devkubernetes01.mesos             Ready    <none>   2m39s   v1.16.9
```

If the output is similar to what is shown above, you're good to go and run the
conformance test suite.

## Running the test suite

To run the test suite and grab the results, follow the
[official instructions](https://github.com/cncf/k8s-conformance/blob/master/instructions.md).

## Destroy the infrastructure

In order to delete the DC/OS cluster created above, run:

```shell
$ make destroy
```


================================================
FILE: docs/existing_cluster.md
================================================
# Existing Cluster

If you already have a DC/OS 1.11+ cluster, Kubernetes is publicly available in the Catalog.

Before proceeding, make sure your cluster fulfils the [Kubernetes package default requirements](https://docs.mesosphere.com/services/kubernetes/2.5.0-1.16.9/getting-started/install-basic/#prerequisites).

Then, install is as easy as:

```shell
$ dcos package install kubernetes
```

## Kubernetes configuration

**NOTE:** By default, it will provision a Kubernetes cluster with one (1) private worker node, and
a single instance of every control plane component.

To deploy a **highly-available** cluster with three (3) private and one (1) public workers node update, run:

```shell
$ dcos package install --options=./resources/options-ha.json kubernetes
```


================================================
FILE: docs/exposing_kubernetes_api.md
================================================
# Exposing the Kubernetes API

DC/OS Kubernetes doesn’t automatically expose the Kubernetes API outside of the DC/OS cluster.
It can be achieved using Marathon-LB and dummy marathon application.

## Using Marathon-LB instance

Marathon-LB instance and dummy `kubeapi-proxy` marathon application get installed as part of Kubernetes
framework install. This allows to expose Kubernetes API via DC/OS public agent IP.

The dummy Marathon application `kubeapi-proxy` definition:

```json
{
  "id": "/kubeapi-proxy",
  "instances": 1,
  "cpus": 0.001,
  "mem": 16,
  "cmd": "tail -F /dev/null",
  "container": {
    "type": "MESOS"
  },
  "portDefinitions": [
    {
      "protocol": "tcp",
      "port": 0
    }
  ],
  "labels": {
    "HAPROXY_GROUP": "external",
    "HAPROXY_0_MODE": "http",
    "HAPROXY_0_PORT": "6443",
    "HAPROXY_0_SSL_CERT": "/etc/ssl/cert.pem",
    "HAPROXY_0_BACKEND_SERVER_OPTIONS": "  timeout connect 10s\n  timeout client 86400s\n  timeout server 86400s\n  timeout tunnel 86400s\n  server kube-apiserver apiserver.devkubernetes01.l4lb.thisdcos.directory:6443 ssl verify none\n"   
  }
}
```

Here is how this works:
1. Marathon-LB identifies that the application `kubeapi-proxy` has the `HAPROXY_GROUP` label set to `external` (change this if you're using a different `HAPROXY_GROUP` for your Marathon-LB configuration).
1. The `instances`, `cpus`, `mem`, `cmd`, and `container` fields basically create a dummy container that takes up minimal space and performs no operation.
1. The single port indicates that this application has one "port" (this information is used by Marathon-LB)
1. `"HAPROXY_0_MODE": "http"` indicates to Marathon-LB that the frontend and backend configuration for this particular service should be configured with `http`.
1. `"HAPROXY_0_PORT": "6443"` tells Marathon-LB to expose the service on port 6443 (rather than the randomly-generated service port, which is ignored)
1. `"HAPROXY_0_SSL_CERT": "/etc/ssl/cert.pem"` tells Marathon-LB to expose the service with the self-signed Marathon-LB certificate (which has **no CN**)
1. The last label `HAPROXY_0_BACKEND_SERVER_OPTIONS` indicates that Marathon-LB should forward traffic to the endpoint `apiserver.kubernetes.l4lb.thisdcos.directory:6443` rather than to the dummy application, and that the connection should be made using TLS without verification.

For more options of exposing Kubernetes API, please check the [documentation](https://docs.mesosphere.com/services/kubernetes/2.5.0-1.16.9/operations/exposing-the-kubernetes-api/).


================================================
FILE: docs/gcp.md
================================================
# Google Cloud

**WARNING**: When running this quickstart, you might experience some issues
with cloud resource limits. Please, verify your [quotas](https://cloud.google.com/compute/quotas)
before proceeding.

## Install Google Cloud SDK

Make sure to have previously installed [Google Cloud SDK](https://cloud.google.com/sdk/downloads).

### Setup access

First, you need to retrieve the credentials needed for Terraform to manage your
Google Cloud resources:

```bash
$ gcloud auth login
$ gcloud auth application-default login
```

## Google Cloud Service Account

If you want to use GCP Service Account key instead of GCP SDK, uncomment the line as shown below in `desired_cluster_profile` and update it with the path to the ssh key file:

```
...
gcp_credentials_key_file = "/PATH/YOUR_GCP_SERVICE_ACCOUNT_KEY.json"
...

```

## Setup SSH key

Next, you need to setup SSH as per [official GCP documentation](https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys) if you setup Google Cloud SDK or Google Cloud Service Account.

Add the SSH private key:

```bash
$ ssh-add ~/.ssh/google_compute_engine
```

Later, you will be asked to add the SSH public key to the Terraform cluster profile.

## Infrastructure configuration

Let's  move on to [infrastructure configuration](../README.md#prepare-infrastructure-configuration).


================================================
FILE: examples/README.md
================================================
# Overview

A collection of examples to run Kubernetes workloads on DC/OS.

* [OS Detector](os-detector/os-detector.md)

================================================
FILE: examples/os-detector/cassandra-cql.json
================================================
{
  "id": "/cassandra-cql",
  "instances": 1,
  "portDefinitions": [],
  "container": {
    "type": "MESOS",
    "volumes": [],
    "docker": {
      "image": "cassandra:3.0.13"
    }
  },
  "cpus": 0.1,
  "mem": 256,
  "requirePorts": false,
  "networks": [],
  "healthChecks": [],
  "fetch": [],
  "constraints": [],
  "cmd": "while true; do sleep 1000000; done"
}


================================================
FILE: examples/os-detector/cassandra.json
================================================
{
    "nodes": {
        "count": 3,
        "seeds": 2
    }
}

================================================
FILE: examples/os-detector/os-detector.md
================================================
# Kubernetes + Cassandra

This sample application will walk through

* Deploying a Cassandra cluster on DC/OS
* Deploying a web application on Kubernetes, that reads/writes data in Cassandra, over the DC/OS network

## Pre-Requisites

* A DC/OS 1.11+ cluster, with at least 4 private nodes
* Kubernetes running on DC/OS
* [Configured kubectl](https://github.com/mesosphere/dcos-kubernetes-quickstart#installing-kubectl)

## OS Detector

We are going to deploy a web application that counts the number of operating systems that visit the site.  First, deploy a 3 node Cassandra cluster onto DC/OS.

```
# TODO: CHANGE THE NAME ONCE THE NEW SDK PACKAGE GOES LIVE
dcos package install cassandra
```

We need to set up the Cassandra keyspace and table so let's start the `csql` terminal.
Deploy the following marathon app definition (e.g., using `dcos marathon app add https://raw.githubusercontent.com/mesosphere/dcos-kubernetes-quickstart/master/examples/os-detector/cassandra-cql.json`).

```json
{
  "id": "/cassandra-cql",
  "instances": 1,
  "portDefinitions": [],
  "container": {
    "type": "MESOS",
    "volumes": [],
    "docker": {
      "image": "cassandra:3.0.13"
    }
  },
  "cpus": 0.1,
  "mem": 256,
  "requirePorts": false,
  "networks": [],
  "healthChecks": [],
  "fetch": [],
  "constraints": [],
  "cmd": "while true; do sleep 1000000; done"
}
```

Next, let us connect to that container using the DC/OS CLI and connect to Cassandra:

```bash
dcos task exec -it cassandra-cql bash

cqlsh node-0-server.cassandra.autoip.dcos.thisdcos.directory
```

Once connected to Cassandra, create the keyspace and table

```
CREATE KEYSPACE browsers WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };

CREATE TABLE browsers.browser_counts (
  counter counter,
  os varchar,
  PRIMARY KEY (os)
);
```

Now we are ready to deploy our application via Kubernetes.  The example manifest will create a deployment, as well as a `NodePort` service so we can access the application from outside the cluster.

```
kubectl apply -f https://raw.githubusercontent.com/mesosphere/dcos-kubernetes-quickstart/master/examples/os-detector/os-detector.yaml
```

From here, the application is deployed, and available at port `31000` on the nodes which Kubernetes is running.  Ensure your firewall is allowing traffic on that port, and you can navigate to your browser to interact with the app.

![](../../docs/assets/os-detector.gif)


================================================
FILE: examples/os-detector/os-detector.yaml
================================================
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: osdetect
spec:
  replicas: 10
  template:
    metadata:
      name: osdetector
      labels:
        app: osdetector
    spec:
      containers:
      - name: osdetector
        image: smugcloud/osdetector:blue
        imagePullPolicy: Always
        args: ["--cassandra-host", "node.cassandra.l4lb.thisdcos.directory:9042"]
        ports:
        - containerPort: 8080
---

kind: Service
apiVersion: v1
metadata:
  name: osdetect
spec:
  selector:
    app: osdetector
  ports:
    - protocol: TCP
      port: 80
      targetPort: 8080
      nodePort: 31000
  type: NodePort

================================================
FILE: resources/desired_cluster_profile.aws.tfvars
================================================
cluster_name = "dcos-kubernetes"
cluster_name_random_string = true

dcos_version = "1.12.3"
dcos_security = "strict" # valid values are strict, permissive, disabled

num_of_masters = "1"
num_of_private_agents = "4"
num_of_public_agents = "1"

bootstrap_instance_type = "m5.large"
master_instance_type = "m5.2xlarge"
private_agent_instance_type = "m5.2xlarge"
public_agent_instance_type = "m5.2xlarge"

aws_region = "us-west-2"
# ssh_public_key_file = ""
# aws_key_name = "default" # uncomment to use an already defined AWS key
# admin_ips = "0.0.0.0/0" # uncomment to access master from any IP


================================================
FILE: resources/desired_cluster_profile.gcp.tfvars
================================================
cluster_name = "dcos-kubernetes"
cluster_name_random_string = true

dcos_version = "1.12.3"
dcos_security = "strict" # valid values are strict, permissive, disabled

num_of_masters = "1"
num_of_private_agents = "4"
num_of_public_agents = "1"

bootstrap_instance_type = "n1-standard-1"
master_instance_type = "n1-standard-8"
private_agent_instance_type = "n1-standard-8"
public_agent_instance_type = "n1-standard-8"

# admin_ips = "0.0.0.0/0" # uncomment to access master from any IP

gcp_project = "YOUR_GCP_PROJECT"
gcp_region = "us-central1"
ssh_public_key_file = "/PATH/YOUR_GCP_SSH_PUBLIC_KEY.pub"
#
# If you want to use GCP service account key instead of GCP SDK
# uncomment the line below and update it with the path to the key file
# gcp_credentials = "/PATH/YOUR_GCP_SERVICE_ACCOUNT_KEY.json"
#


================================================
FILE: resources/kubeapi-proxy.json
================================================
{
  "id": "/kubeapi-proxy",
  "instances": 1,
  "cpus": 0.001,
  "mem": 16,
  "cmd": "tail -F /dev/null",
  "container": {
    "type": "MESOS"
  },
  "portDefinitions": [
    {
      "protocol": "tcp",
      "port": 0
    }
  ],
  "labels": {
    "HAPROXY_0_MODE": "http",
    "HAPROXY_GROUP": "external",
    "HAPROXY_0_SSL_CERT": "/etc/ssl/cert.pem",
    "HAPROXY_0_PORT": "6443",
    "HAPROXY_0_BACKEND_SERVER_OPTIONS": "  timeout connect 10s\n  timeout client 86400s\n  timeout server 86400s\n  timeout tunnel 86400s\n  server kube-apiserver apiserver.devkubernetes01.l4lb.thisdcos.directory:6443 ssl verify none\n"
  }
}


================================================
FILE: resources/main.aws.tf
================================================
data "http" "whatismyip" {
  url = "http://whatismyip.akamai.com/"
}

locals {
  dcos_admin_ips = "${split(" ", var.admin_ips == "" ? "${data.http.whatismyip.body}/32" : var.admin_ips)}"
}

provider "aws" {
  region = "${var.aws_region}"
}

module "dcos" {
  source  = "dcos-terraform/dcos/aws"
  version = "~> 0.2.0"

  providers = {
    aws = "aws"
  }

  cluster_name = "${var.cluster_name}"
  cluster_name_random_string = "${var.cluster_name_random_string}"

  num_masters = "${var.num_of_masters}"
  num_private_agents = "${var.num_of_private_agents}"
  num_public_agents = "${var.num_of_public_agents}"

  dcos_version = "${var.dcos_version}"
  dcos_variant = "open"
  dcos_security = "${var.dcos_security}"
  dcos_instance_os = "${var.instance_os}"

  bootstrap_instance_type = "${var.bootstrap_instance_type}"
  masters_instance_type = "${var.master_instance_type}"
  private_agents_instance_type = "${var.private_agent_instance_type}"
  public_agents_instance_type = "${var.public_agent_instance_type}"

  admin_ips = "${local.dcos_admin_ips}"
  aws_key_name = "${var.aws_key_name}"
  ssh_public_key_file = "${var.ssh_public_key_file}"

  public_agents_additional_ports = ["6443"]
}

================================================
FILE: resources/main.gcp.tf
================================================
data "http" "whatismyip" {
  url = "http://whatismyip.akamai.com/"
}

locals {
  dcos_admin_ips = "${split(" ", var.admin_ips == "" ? "${data.http.whatismyip.body}/32" : var.admin_ips)}"
}

provider "google" {
  version = "~> 1.18.0"

  credentials = "${var.gcp_credentials}"
  project = "${var.gcp_project}"
  region = "${var.gcp_region}"
  zone = "${var.gcp_zone}"
}

module "dcos" {
  source  = "dcos-terraform/dcos/gcp"
  version = "~> 0.1.0"

  providers = {
    google = "google"
  }

  cluster_name = "${var.cluster_name}"
  cluster_name_random_string = "${var.cluster_name_random_string}"

  num_masters = "${var.num_of_masters}"
  num_private_agents = "${var.num_of_private_agents}"
  num_public_agents = "${var.num_of_public_agents}"

  dcos_version = "${var.dcos_version}"
  dcos_variant = "open"
  dcos_security = "${var.dcos_security}"
  dcos_instance_os = "${var.instance_os}"

  bootstrap_machine_type = "${var.bootstrap_machine_type}"
  masters_machine_type = "${var.master_machine_type}"
  private_agents_machine_type = "${var.private_agent_machine_type}"
  public_agents_machine_type = "${var.public_agent_machine_type}"

  admin_ips = "${local.dcos_admin_ips}"
  ssh_public_key_file = "${var.ssh_public_key_file}"

  public_agents_additional_ports = ["6443"]

  dcos_resolvers = <<EOF
# YAML
  - 169.254.169.254
EOF
}

================================================
FILE: resources/options-ha.json
================================================
{
  "service": {
    "name": "dev/kubernetes01"
  },
  "kubernetes": {
    "authorization_mode": "RBAC",
    "high_availability": true,
    "private_node_count": 3
  }
}


================================================
FILE: resources/options.json
================================================
{
  "service": {
    "name": "dev/kubernetes01"
  },
  "kubernetes": {
    "authorization_mode": "RBAC"
  }
}

================================================
FILE: resources/outputs.tf
================================================

output "masters-ips" {
  value = "${module.dcos.masters-ips}"
}

output "cluster-address" {
  value = "${module.dcos.masters-loadbalancer}"
}

output "public-agents-loadbalancer" {
  value = "${module.dcos.public-agents-loadbalancer}"
}

================================================
FILE: resources/variables.aws.tf
================================================
variable "cluster_name" {
  description = "Name of the DC/OS cluster"
  default     = "dcos-kubernetes"
}

variable "cluster_name_random_string" {
  description = "Add a random string to the cluster name"
  default     = true
}

variable "ssh_public_key_file" {
  description = "Path to SSH public key. This is mandatory."
  default = ""
}

variable "aws_key_name" {
  description = "Specify the aws ssh key to use. We assume its already loaded in your SSH agent."
  default     = "default"
}

variable "aws_region" {
  description = "Region to be used"
  default = "us-west-2"
}

variable "admin_ips" {
  description = "List of CIDR admin IPs (space separated)"
  default     = ""
}

variable "dcos_version" {
  default     = "1.12.3"
  description = "specifies which dcos version instruction to use. Options: `1.9.0`, `1.8.8`, etc. _See [dcos_download_path](https://github.com/dcos/tf_dcos_core/blob/master/download-variables.tf) or [dcos_version](https://github.com/dcos/tf_dcos_core/tree/master/dcos-versions) tree for a full list._"
}

variable "dcos_security" {
  default     = "permissive"
  description = "[Enterprise DC/OS] set the security level of DC/OS. Default is strict. (recommended)"
}

variable "num_of_public_agents" {
  default = "0"
}

variable "num_of_private_agents" {
  default = "4"
}

variable "num_of_masters" {
  default     = "1"
  description = "set the num of master nodes (required with exhibitor_storage_backend set to aws_s3, azure, ZooKeeper)"
}

variable "instance_os" {
  description = "Operating system to use."
  default = "centos_7.5"
}

variable "bootstrap_instance_type" {
  description = "[BOOTSTRAP] Machine type"
  default     = "m5.large"
}

variable "master_instance_type" {
  description = "[MASTERS] Machine type"
  default     = "m5.2xlarge"
}

variable "private_agent_instance_type" {
  description = "[PRIVATE AGENTS] Machine type"
  default     = "m5.2xlarge"
}

variable "public_agent_instance_type" {
  description = "[PUBLIC AGENTS] Machine type"
  default     = "m5.2xlarge"
}


================================================
FILE: resources/variables.gcp.tf
================================================
variable "cluster_name" {
  description = "Name of the DC/OS cluster"
  default     = "dcos-kubernetes"
}

variable "cluster_name_random_string" {
  description = "Add a random string to the cluster name"
  default     = true
}

variable "ssh_public_key_file" {
  description = "Path to SSH public key. This is mandatory."
  default = ""
}

variable "gcp_credentials" {
  description = "Either the path to or the contents of a service account key file in JSON format. You can manage key files using the Cloud Console."
  default = ""
}

variable "gcp_project" {
  description = "The default project to manage resources in. If another project is specified on a resource, it will take precedence."
  default = ""
}

variable "gcp_region" {
  description = "The default region to manage resources in. If another region is specified on a regional resource, it will take precedence."
  default = ""
}

variable "gcp_zone" {
  description = "The default zone to manage resources in. Generally, this zone should be within the default region you specified. If another zone is specified on a zonal resource, it will take precedence."
  default = ""
}

variable "admin_ips" {
  description = "List of CIDR admin IPs (space separated)"
  default     = ""
}

variable "dcos_version" {
  default     = "1.12.3"
  description = "specifies which dcos version instruction to use. Options: `1.9.0`, `1.8.8`, etc. _See [dcos_download_path](https://github.com/dcos/tf_dcos_core/blob/master/download-variables.tf) or [dcos_version](https://github.com/dcos/tf_dcos_core/tree/master/dcos-versions) tree for a full list._"
}

variable "dcos_security" {
  default     = "permissive"
  description = "[Enterprise DC/OS] set the security level of DC/OS. Default is strict. (recommended)"
}

variable "num_of_public_agents" {
  default = "0"
}

variable "num_of_private_agents" {
  default = "4"
}

variable "num_of_masters" {
  default     = "1"
  description = "set the num of master nodes (required with exhibitor_storage_backend set to aws_s3, azure, ZooKeeper)"
}

variable "instance_os" {
  description = "Operating system to use."
  default = "centos_7.5"
}

variable "bootstrap_machine_type" {
  description = "[BOOTSTRAP] Machine type"
  default     = "n1-standard-1"
}

variable "master_machine_type" {
  description = "[MASTERS] Machine type"
  default     = "n1-standard-8"
}

variable "private_agent_machine_type" {
  description = "[PRIVATE AGENTS] Machine type"
  default     = "n1-standard-8"
}

variable "public_agent_machine_type" {
  description = "[PUBLIC AGENTS] Machine type"
  default     = "n1-standard-8"
}


================================================
FILE: scripts/get_cli
================================================
#!/bin/bash

# download dcos and kubectl CLI tools

# check OS version
unamestr=`uname`
if [[ "$unamestr" == "Linux" ]]
then
  OS="linux"
else
  OS="darwin"
fi

#
echo "Downloading dcos v$DCOS_CLI_VERSION cli for $OS"
curl -o dcos https://downloads.dcos.io/binaries/cli/$OS/x86-64/dcos-$DCOS_CLI_VERSION/dcos
chmod +x dcos
echo
#
echo "Downloading kubectl $KUBERNETES_VERSION for $OS"
curl -LO https://storage.googleapis.com/kubernetes-release/release/v$KUBERNETES_VERSION/bin/$OS/amd64/kubectl
chmod +x kubectl
echo "Download has finished !!!"
echo


================================================
FILE: scripts/poll_api.sh
================================================
#!/bin/bash

API_NAME=$1
API_IP=$2
API_PORT=$3

echo " "
echo "Waiting for ${API_NAME} to be ready..."
if [[ "${API_NAME}" = "Kubernetes API" ]]
then
  until curl -o /dev/null -skIf -w "%{http_code}" https://${API_IP}:${API_PORT} | grep -w "401\|403" >/dev/null 2>&1; do sleep 1; done
else
  until curl -o /dev/null -skIf https://${API_IP}:${API_PORT}; do sleep 1; done
fi
echo