Full Code of microsoft/terraform-azure-devops-starter for AI

Repository: microsoft/terraform-azure-devops-starter
Branch: master
Commit: 2f5e14894765
Files: 42
Total size: 61.0 KB

Directory structure:
gitextract_1a44rx17/

├── .gitattributes
├── .gitignore
├── 101-terraform-job/
│   ├── README.md
│   ├── azure-pipelines.yml
│   ├── terraform/
│   │   ├── backend.tf
│   │   ├── main.tf
│   │   ├── outputs.tf
│   │   ├── provider.tf
│   │   └── variables.tf
│   └── terraform-stages-template.yml
├── 201-plan-apply-stages/
│   ├── README.md
│   ├── azure-pipelines.yml
│   ├── terraform/
│   │   ├── main.tf
│   │   ├── outputs.tf
│   │   ├── provider.tf
│   │   ├── sqlserver1_generated_password/
│   │   │   ├── main.tf
│   │   │   ├── outputs.tf
│   │   │   └── variables.tf
│   │   ├── sqlserver2_assigned_password/
│   │   │   ├── main.tf
│   │   │   └── variables.tf
│   │   └── variables.tf
│   ├── terraform-stages-template.yml
│   └── terraform_backend/
│       └── backend.tf
├── 301-deploy-agent-vms/
│   ├── README.md
│   ├── azure-pipelines.yml
│   ├── terraform/
│   │   ├── backend.tf
│   │   ├── devops-agent/
│   │   │   ├── devops_agent_init.sh
│   │   │   ├── main.tf
│   │   │   ├── outputs.tf
│   │   │   ├── shutdown_schedule_arm_template.json
│   │   │   └── variables.tf
│   │   ├── main.tf
│   │   ├── outputs.tf
│   │   ├── provider.tf
│   │   └── variables.tf
│   └── terraform-stages-template.yml
├── CODE_OF_CONDUCT.md
├── LICENSE
├── README.md
├── SECURITY.md
└── infrastructure/
    ├── terraform-init-template.yml
    └── terraform-outputs-template.yml

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitattributes
================================================
# Auto detect text files and perform LF normalization
* text=auto


================================================
FILE: .gitignore
================================================
# Local .terraform directories
**/.terraform/*

# .tfstate files
*.tfstate
*.tfstate.*

# Crash log files
crash.log

# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
# .tfvars files are managed as part of configuration and so should be included in
# version control.
#
# example.tfvars

# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json

# Include override files you do wish to add to version control using negated pattern
#
# !example_override.tf

# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*


================================================
FILE: 101-terraform-job/README.md
================================================
# Basic Terraform job

## About this template

This template includes a multi-stage pipeline that deploys 
an environment from Terraform configuration, and run
a subsequent job configured from Terraform outputs.

[![Watch the video](/docs/images/terraform_starter/201-video.png)](https://youtu.be/GpKdCuv2icY)

The Terraform definition only deploys an empty resource group.
You can extend the definition with your custom infrastructure, such as Web Apps.

## Walkthrough

### Using the template

To use the template, follow the section
[How to use the templates](/README.md#how-to-use-the-templates)
in the main README file.

## Next steps

* The next template, [201-plan-apply-stages](../201-plan-apply-stages) demonstrates
  how to manually review and approve changes before they are applied on an environment.
  It also shows you can structure your project to develop and test locally without an Azure
  backend.


================================================
FILE: 101-terraform-job/azure-pipelines.yml
================================================
pr: none
trigger:
  branches:
    include:
    - master
  paths:
    include:
    - 101-terraform-job/

variables:
- group: terraform-secrets

stages:

- template: terraform-stages-template.yml
  parameters:
    environment: test
    environmentDisplayName: Test
    # Pass variables as environment variables.
    # Terraform recognizes TF_VAR prefixed environment variables.
    TerraformEnvVariables:
      TF_VAR_department: engineering


================================================
FILE: 101-terraform-job/terraform/backend.tf
================================================
#Set the terraform backend
terraform {
  # Backend variables are initialized by Azure DevOps
  backend "azurerm" {}
}


================================================
FILE: 101-terraform-job/terraform/main.tf
================================================
# Deploy a Resource Group with Azure resources.
#
# For suggested naming conventions, refer to:
#   https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-best-practices/naming-and-tagging

# Sample Resource Group

resource "azurerm_resource_group" "main" {
  name     = "rg-${var.appname}-${var.environment}-main"
  location = var.location
  tags     = {
    department = var.department
  }
}

# Add additional modules...


================================================
FILE: 101-terraform-job/terraform/outputs.tf
================================================
output "subscription_id" {
  value = data.azurerm_client_config.current.subscription_id
}

output "resource_group_name" {
  value = azurerm_resource_group.main.name
}


================================================
FILE: 101-terraform-job/terraform/provider.tf
================================================
#Set the terraform required version
terraform {
  required_version = ">= 0.12.6"
}

# Configure the Azure Provider
provider "azurerm" {
  # It is recommended to pin to a given version of the Provider
  version = "=1.44.0"
}

# Data

# Make client_id, tenant_id, subscription_id and object_id variables
data "azurerm_client_config" "current" {}


================================================
FILE: 101-terraform-job/terraform/variables.tf
================================================
variable "appname" {
  type = string
  description = "Application name. Use only lowercase letters and numbers"
  default = "starterterraform"
}

variable "environment" {
  type    = string
  description = "Environment name, e.g. 'dev' or 'stage'"
  default = "dev"
}

variable "location" {
  type    = string
  description = "Azure region where to create resources."
  default = "North Europe"
}

variable "department" {
  type    = string
  description = "A sample variable passed from the build pipeline and used to tag resources."
  default = "Engineering"
}


================================================
FILE: 101-terraform-job/terraform-stages-template.yml
================================================
parameters:
  environment: test
  environmentDisplayName: Test
  TerraformArguments: ''
  TerraformEnvVariables:

stages:
- stage: Terraform_${{ parameters.environment }}
  displayName: Terraform ${{ parameters.environmentDisplayName }}
  pool:
    vmImage: ubuntu-latest
  jobs:

  - job: Terraform
    displayName: Terraform
    # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file.
    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
    steps:

    - template: ../infrastructure/terraform-init-template.yml
      parameters:
        provisionStorage: true
        TerraformDirectory: 101-terraform-job/terraform
        environment: ${{ parameters.environment }}

    # Using bash instead of Terraform extension because of following issues:
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/748
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/725
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/747
    - bash: |
        set -eu
        export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
        terraform apply -input=false -auto-approve -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}
      displayName: Terraform apply
      workingDirectory: 101-terraform-job/terraform
      env:
        ${{ parameters.TerraformEnvVariables }}

- stage: PostTerraform_${{ parameters.environment }}
  displayName: PostTerraform ${{ parameters.environmentDisplayName }}
  pool:
    vmImage: ubuntu-latest
  jobs:
  - job: ReadTerraform
    displayName: Use Terraform outputs
    steps:

    - template: ../infrastructure/terraform-init-template.yml
      parameters:
        TerraformDirectory: 101-terraform-job/terraform
        environment: ${{ parameters.environment }}

    - template: ../infrastructure/terraform-outputs-template.yml
      parameters:
        TerraformDirectory: 101-terraform-job/terraform

    - bash: |
        # Dummy job showing how to consume Terraform outputs
        echo Subscription ID: $(subscription_id)
        echo Resource group: $(resource_group_name)
      displayName: Sample script


================================================
FILE: 201-plan-apply-stages/README.md
================================================
# Separate Plan and Apply stages

## About this template

This template includes a multi-stage pipeline allowing to manually review and approve infrastructure
changes before they are deployed.

[![Watch the video](/docs/images/terraform_starter/201-video.png)](https://youtu.be/PvXonOWOWaM)

The Terraform definition only deploys a resource group and two empty SQL Server instances
(to illustrate two different approaches to managing secrets, in this case the SQL Server
password).
You can extend the definition with your custom infrastructure, such as Web Apps.

The project can be used in local development without a remote Terraform state backend.
This allows quickly iterating while developing the Terraform configuration, and 
good security practices.

When the project is run in Azure DevOps, however, the pipeline adds the
`infrastructure/terraform_backend/backend.tf` to the `infrastructure/terraform` 
directory to enable the Azure Storage shared backend for additional resiliency.
See the Terraform documentation to understand [why a state store is needed](https://www.terraform.io/docs/state/purpose.html).

## Walkthrough

### Using the template

To use the template, follow the section
[How to use the templates](/README.md#how-to-use-the-templates)
in the main README file.

### Manual approvals

As of December 2019, there is no support for stage gates in Azure DevOps multi-stage pipelines, but
*deployment environments* provide a basic mechanism for stage approvals.

Create an environment with no resources. Name it `Staging`.

![create environment](/docs/images/terraform_starter/create_environment.png)

Define environment approvals. If you want to allow anyone out of a group a people to be able to individually approve, add a group.

![create environment_approval1](/docs/images/terraform_starter/create_environment_approval1.png)

![create environment approval2](/docs/images/terraform_starter/create_environment_approval2.png)

![create environment approval3](/docs/images/terraform_starter/create_environment_approval3.png)

![environment approval](/docs/images/terraform_starter/environment_approval.png)

Repeat those steps for an environment named `QA`.

Under Library, create a Variable Group named `terraform-secrets`. Create a secret
named `SQL_PASSWORD` and give it a unique value (e.g. `Strong_Passw0rd!`). Make
the variable secret using the padlock icon.

![environment approval](/docs/images/terraform_starter/variable_group.png)

### Running the pipeline

As you run the pipeline, after running `terraform plan`, the next stage will be waiting for your approval.

![pipeline stage waiting](/docs/images/terraform_starter/pipeline_stage_waiting.png)

Review the detailed plan to ensure no critical resources or data will be lost.

![terraform plan output](/docs/images/terraform_starter/terraform_plan_output.png)

You can also review the plan and terraform configuration files by navigating to Pipeline Artifacts (rightmost column in the table below).

![pipeline artifacts](/docs/images/terraform_starter/pipeline_artifacts.png)

![pipeline artifacts detail](/docs/images/terraform_starter/pipeline_artifacts_detail.png)

Approve or reject the deployment.

![stage approval waiting](/docs/images/terraform_starter/stage_approval_waiting.png)

The pipeline will proceed to `terraform apply`.

At this stage you will have a new resource group deployed named `rg-starterterraform-stage-main`. 

The pipeline will then proceed in the same manner for the `QA` environment.

![pipeline completed](/docs/images/terraform_starter/pipeline_completed.png)

If any changes have been performed on the infrastructure between the Plan and Apply stages, the pipeline will fail.
You can rerun the Plan stage directly in the pipeline view to produce an updated plan.

![plan changed](/docs/images/terraform_starter/plan_changed.png)

## Next steps

* It's not currently possible to skip approval and deployment if there are no
  changes in the Terraform plan, because of limitations in multi-stage
  pipelines (stages cannot be conditioned on the outputs of previous stages).
  You could cancel the pipeline (through the REST API) in that case, but that
  would prevent extending the pipeline to include activities beyond Terraform.
* The next template, [301-deploy-agent-vms](../301-deploy-agent-vms) demonstrates
  how you can use Terraform to manage infrastructure used for the build itself,
  such as build agent VMs.


================================================
FILE: 201-plan-apply-stages/azure-pipelines.yml
================================================
pr: none
trigger:
  branches:
    include:
    - master
  paths:
    include:
    - 201-plan-apply-stages/

variables:
- group: terraform-secrets

stages:
- template: terraform-stages-template.yml
  parameters:
    environment: stage
    environmentDisplayName: Staging
    TerraformArguments: >-
      -var department=Engineering
    # For additional security, pass secret through environment instead of command line.
    # Terraform recognizes TF_VAR prefixed environment variables.
    TerraformEnvVariables:
      TF_VAR_sql2password: $(SQL_PASSWORD)

- template: terraform-stages-template.yml
  parameters:
    environment: qa
    environmentDisplayName: QA
    TerraformArguments: >-
      -var department=QA
    TerraformEnvVariables:
      TF_VAR_sql2password: $(SQL_PASSWORD)


================================================
FILE: 201-plan-apply-stages/terraform/main.tf
================================================
# Deploy a Resource Group with Azure resources.
#
# For suggested naming conventions, refer to:
#   https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-best-practices/naming-and-tagging

# Sample Resource Group

resource "azurerm_resource_group" "main" {
  name     = "rg-${var.appname}-${var.environment}-main"
  location = var.location
  tags     = {
    department = var.department
  }
}

# Sample Resources

module "sqlserver1_generated_password" {
  source = "./sqlserver1_generated_password"
  appname = var.appname
  environment = var.environment
  resource_group = azurerm_resource_group.main.name
  location = azurerm_resource_group.main.location
}

module "sqlserver2_assigned_password" {
  source = "./sqlserver2_assigned_password"
  appname = var.appname
  environment = var.environment
  resource_group = azurerm_resource_group.main.name
  location = azurerm_resource_group.main.location
  sql_password = var.sql2password
}

# Add additional modules...


================================================
FILE: 201-plan-apply-stages/terraform/outputs.tf
================================================
output "subscription_id" {
  value = data.azurerm_client_config.current.subscription_id
}

output "sqlserver1_host" {
  value = module.sqlserver1_generated_password.fully_qualified_domain_name
}

output "sqlserver1_user" {
  value = module.sqlserver1_generated_password.user
}

output "sqlserver1_password" {
  value = module.sqlserver1_generated_password.password
  sensitive = true
}


================================================
FILE: 201-plan-apply-stages/terraform/provider.tf
================================================
#Set the terraform required version
terraform {
  required_version = ">= 0.12.6"
}

# Configure the Azure Provider
provider "azurerm" {
  # It is recommended to pin to a given version of the Provider
  version = "=1.44.0"
}

provider "random" {
  version = "~> 2.2"
}

# Data

# Make client_id, tenant_id, subscription_id and object_id variables
data "azurerm_client_config" "current" {}


================================================
FILE: 201-plan-apply-stages/terraform/sqlserver1_generated_password/main.tf
================================================
resource "random_password" "sql" {
  length = 16
  special = true
  override_special = "!@#$%&*()-_=+[]:?"
  min_upper = 1
  min_lower = 1
  min_numeric = 1
  min_special = 1
}

resource "azurerm_sql_server" "example" {
  name                         = "sqldb-${var.appname}-${var.environment}"
  resource_group_name          = var.resource_group
  location                     = var.location
  version                      = "12.0"
  administrator_login          = "sqladm"
  administrator_login_password = random_password.sql.result
}


================================================
FILE: 201-plan-apply-stages/terraform/sqlserver1_generated_password/outputs.tf
================================================
output "fully_qualified_domain_name" {
  value = azurerm_sql_server.example.fully_qualified_domain_name
}

output "user" {
  value = azurerm_sql_server.example.administrator_login
}

output "password" {
  value = azurerm_sql_server.example.administrator_login_password
  sensitive = true
}


================================================
FILE: 201-plan-apply-stages/terraform/sqlserver1_generated_password/variables.tf
================================================
variable "appname" {
  type = string
}

variable "environment" {
  type    = string
}

variable "resource_group" {
  type    = string
}

variable "location" {
  type    = string
}


================================================
FILE: 201-plan-apply-stages/terraform/sqlserver2_assigned_password/main.tf
================================================
resource "azurerm_sql_server" "example" {
  name                         = "sqldb-${var.appname}-2-${var.environment}"
  resource_group_name          = var.resource_group
  location                     = var.location
  version                      = "12.0"
  administrator_login          = "sqladm"
  administrator_login_password = var.sql_password
}


================================================
FILE: 201-plan-apply-stages/terraform/sqlserver2_assigned_password/variables.tf
================================================
variable "appname" {
  type = string
}

variable "environment" {
  type    = string
}

variable "resource_group" {
  type    = string
}

variable "location" {
  type    = string
}

variable "sql_password" {
  type    = string
}


================================================
FILE: 201-plan-apply-stages/terraform/variables.tf
================================================
variable "appname" {
  type = string
  description = "Application name. Use only lowercase letters and numbers"
  default = "starterterraform"
}

variable "environment" {
  type    = string
  description = "Environment name, e.g. 'dev' or 'stage'"
  default = "dev"
}

variable "location" {
  type    = string
  description = "Azure region where to create resources."
  default = "North Europe"
}

variable "department" {
  type    = string
  description = "A sample variable passed from the build pipeline and used to tag resources."
  default = "Engineering"
}

variable "sql2password" {
  type    = string
  description = "A password for SQL Server #2"
}


================================================
FILE: 201-plan-apply-stages/terraform-stages-template.yml
================================================
parameters:
  environment: stage
  environmentDisplayName: Staging
  TerraformArguments: ''
  TerraformEnvVariables:

stages:
- stage: Terraform_Plan_${{ parameters.environment }}
  displayName: Plan ${{ parameters.environmentDisplayName }}
  jobs:
  - job: Terraform_Plan
    displayName: Plan Terraform
    # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file.
    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
    pool:
      vmImage: ubuntu-latest
    steps:

    - bash: |
        cp terraform_backend/* terraform
      displayName: Configure backend
      workingDirectory: 201-plan-apply-stages

    - template: ../infrastructure/terraform-init-template.yml
      parameters:
        provisionStorage: true
        TerraformDirectory: 201-plan-apply-stages/terraform
        environment: ${{ parameters.environment }}

    # Using bash instead of Terraform extension because of following issues:
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/748
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/725
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/747
    - bash: |
        set -eu
        export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
        terraform plan -input=false -out=tfplan -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}
      displayName: Terraform plan
      workingDirectory: 201-plan-apply-stages/terraform
      env:
        ${{ parameters.TerraformEnvVariables }}

    - bash: |
        # Save a human-friendly version of the plan with passwords hidden
        terraform show -no-color tfplan > plan.txt
        # Remove terraform plan from published artifacts, as it contains clear-text secrets
        rm tfplan
        # Resource providers can be > 100MB large, we don't want them in the published artifacts.
        rm -r .terraform
      displayName: Save plan text
      workingDirectory: 201-plan-apply-stages/terraform

    - task: PublishPipelineArtifact@1
      displayName: Publish plan artifact
      inputs:
        targetPath: 201-plan-apply-stages/terraform
        artifact: terraform_resources_${{ parameters.environment }}

- stage: Terraform_Apply_${{ parameters.environment }}
  displayName: Apply ${{ parameters.environmentDisplayName }}
  jobs:
  - deployment: Apply
    environment: ${{ parameters.environmentDisplayName }}
    displayName: Apply Terraform
    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
    pool:
      vmImage: ubuntu-latest
    strategy:
      runOnce:
        deploy:
          steps:

          - task: DownloadPipelineArtifact@2
            displayName: Download plan
            inputs:
              artifactName: terraform_resources_${{ parameters.environment }}
              targetPath: terraform_resources

          - template: ../infrastructure/terraform-init-template.yml
            parameters:
              TerraformDirectory: terraform_resources
              environment: ${{ parameters.environment }}

          # As the Terraform extension plan task doesn't support -detailed-exitcode
          # (to check if any changes are present), we define an equivalent bash
          # task instead.
          - bash: |
              set -eu
              export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
              # terraform plan -detailed-exitcode exit codes:
              # 0 - Succeeded, diff is empty (no changes)
              # 1 - Errored
              # 2 - Succeeded, there is a diff
              # >2 - unexpected, crash or bug
              if terraform plan -detailed-exitcode -input=false -out=tfplan -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}; then
                echo "Terraform succeeded with no changes"
                # NB terraform apply should still be run, e.g. if new outputs have been created
              else
                terraform_exitcode=$?
                if [ $terraform_exitcode -eq 2 ]; then
                  echo "Terraform succeeded with updates"
                else
                  echo "ERROR: terraform exited with code $terraform_exitcode"
                  exit 1
                fi
              fi
            displayName: Terraform plan
            workingDirectory: terraform_resources
            env:
              ${{ parameters.TerraformEnvVariables }}

          - bash: |
              set -eux  # ensure pipeline stops if terraform fails or diff reports a difference
              terraform show -no-color tfplan > newplan.txt
              diff -u plan.txt newplan.txt
            workingDirectory: terraform_resources
            displayName: Check unchanged plan

          - bash: |
              set -eu
              terraform apply -input=false -auto-approve tfplan
            displayName: Terraform apply
            workingDirectory: terraform_resources
            env:
              ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)

  - job: ReadTerraform
    dependsOn: Apply
    condition: always()
    displayName: Read outputs
    pool:
      vmImage: ubuntu-latest
    steps:

    - bash: |
        cp terraform_backend/* terraform
      displayName: Configure backend
      workingDirectory: 201-plan-apply-stages

    - template: ../infrastructure/terraform-init-template.yml
      parameters:
        TerraformDirectory: 201-plan-apply-stages/terraform
        environment: ${{ parameters.environment }}

    - template: ../infrastructure/terraform-outputs-template.yml
      parameters:
        TerraformDirectory: 201-plan-apply-stages/terraform
 
  - job: DummySampleJob
    displayName: Use Terraform outputs
    dependsOn: ReadTerraform
    variables:
      sqlserver1_host: $[ dependencies.ReadTerraform.outputs['Outputs.sqlserver1_host'] ]
      sqlserver1_user: $[ dependencies.ReadTerraform.outputs['Outputs.sqlserver1_user'] ]
      sqlserver1_password: $[ dependencies.ReadTerraform.outputs['Outputs.sqlserver1_password'] ]
    steps:
      - bash: |
          # Dummy job showing how to consume Terraform outputs
          echo DB_CONN_STRING="User ID=$(sqlserver1_user);Password=$(sqlserver1_password)"
        displayName: Sample script


================================================
FILE: 201-plan-apply-stages/terraform_backend/backend.tf
================================================
#Set the terraform backend
terraform {
  # Backend variables are initialized by Azure DevOps
  backend "azurerm" {}
}


================================================
FILE: 301-deploy-agent-vms/README.md
================================================
# Deploy hosted agent VMs

## About this template

This template shows how to use Terraform to deploy a pool of agent VMs on which a subsequent job is run.

[![Watch the video](/docs/images/terraform_starter/301-video.png)](https://youtu.be/TdXmqqJE-gw)

The Terraform definition does not contain any other resources.
You can extend the definition with your custom infrastructure, such as Web Apps.

### Agent deployment scripts

The template deploys agents at a high density, with (by default) 2 agent VMs with 4 Azure DevOps build agents per VM.
This is efficient when running pipelines that do not require much local computing power, or only sporadically.

## Walkthrough

### Creating an agent pool

In your Azure DevOps project settings, [create an Agent pool](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues).
Name the pool `starterpool` (if you want to use a different name, change the value in [azure-pipelines.yml](azure-pipelines.yml)).

### Creating a PAT token

In Azure DevOps, [create a PAT token](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page).
Click on *Show all scopes* and grant the token *Read and Manage* permissions on *Agent Pools*.

![PAT token](/docs/images/terraform_starter/301-pat-token.png)

Under Library, create a Variable Group named `terraform-secrets`. Create a secret
named `AGENT_POOL_MANAGE_PAT_TOKEN` and paste the token value.
Make the variable secret using the padlock icon.

### Using the template

To use the template, follow the section
[How to use the templates](/README.md#how-to-use-the-templates)
in the main README file.

### Automatic shutdown of agents

The pipeline configures the agent VMs to automatically shutdown daily at 23:00 UTC.
To use a different schedule, change `TF_VAR_az_devops_agent_vm_shutdown_time`
in [azure-pipelines.yml](azure-pipelines.yml),
or remove that line completely to disable automatic shutdown.

The pipeline contains a task to start up the agent VMs again before running the agent job.


================================================
FILE: 301-deploy-agent-vms/azure-pipelines.yml
================================================
pr: none
trigger:
  branches:
    include:
    - master
  paths:
    include:
    - 301-deploy-agent-vms/

variables:
- group: terraform-secrets

stages:

- template: terraform-stages-template.yml
  parameters:
    environment: dev
    environmentDisplayName: Dev
    # Pass variables as environment variables.
    # Terraform recognizes TF_VAR prefixed environment variables.
    TerraformEnvVariables:
      TF_VAR_az_devops_url: $(System.TeamFoundationCollectionUri)
      TF_VAR_az_devops_pat: $(AGENT_POOL_MANAGE_PAT_TOKEN)
      TF_VAR_az_devops_agent_pool: starterpool
      TF_VAR_az_devops_agent_vm_shutdown_time: 2300


================================================
FILE: 301-deploy-agent-vms/terraform/backend.tf
================================================
#Set the terraform backend
terraform {
  # Backend variables are initialized by Azure DevOps
  backend "azurerm" {}
}


================================================
FILE: 301-deploy-agent-vms/terraform/devops-agent/devops_agent_init.sh
================================================
#!/bin/sh


test -n "$1" || { echo "The argument az_devops_url must be provided"; exit 1; }
az_devops_url="$1"
[[ "$az_devops_url" == */ ]] || { echo "The argument az_devops_url must end with /"; exit 1; }
test -n "$2" || { echo "The argument az_devops_pat must be provided"; exit 1; }
az_devops_pat="$2"
test -n "$3" || { echo "The argument az_devops_agent_pool must be provided"; exit 1; }
az_devops_agent_pool="$3"
test -n "$4" || { echo "The argument az_devops_agents_per_vm must be provided"; exit 1; }
az_devops_agents_per_vm="$4"


#strict mode, fail on error
set -euo pipefail


echo "start"

echo "install Ubuntu packages"

# To make it easier for build and release pipelines to run apt-get,
# configure apt to not require confirmation (assume the -y argument by default)
export DEBIAN_FRONTEND=noninteractive
echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90assumeyes
echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/00usepty


apt-get update
apt-get install -y --no-install-recommends \
        ca-certificates \
        jq \
        apt-transport-https \
        docker.io


echo "Allowing agent to run docker"

usermod -aG docker azuredevopsuser

echo "Installing Azure CLI"

curl -sL https://aka.ms/InstallAzureCLIDeb | bash

echo "install VSTS Agent"

cd /home/azuredevopsuser
mkdir -p agent
cd agent

AGENTRELEASE="$(curl -s https://api.github.com/repos/Microsoft/azure-pipelines-agent/releases/latest | grep -oP '"tag_name": "v\K(.*)(?=")')"
AGENTURL="https://vstsagentpackage.azureedge.net/agent/${AGENTRELEASE}/vsts-agent-linux-x64-${AGENTRELEASE}.tar.gz"
echo "Release "${AGENTRELEASE}" appears to be latest" 
echo "Downloading..."
wget -q -O agent_package.tar.gz ${AGENTURL} 

# Generate random prefix for agent names
if ! test -e "host_uuid.txt"; then
  uuidgen > host_uuid.txt.tmp
  mv host_uuid.txt.tmp host_uuid.txt
fi
host_id=$(cat host_uuid.txt)


for agent_num in $(seq 1 $az_devops_agents_per_vm); do
  agent_dir="agent-$agent_num"
  mkdir -p "$agent_dir"
  pushd "$agent_dir"
    agent_id="${agent_num}_${host_id}"
    echo "installing agent $agent_id"
    tar zxf ../agent_package.tar.gz
    chmod -R 777 .
    echo "extracted"
    ./bin/installdependencies.sh
    echo "dependencies installed"

    if test -e .agent; then
      echo "attempting to uninstall agent"
      ./svc.sh stop || true
      ./svc.sh uninstall || true
      sudo -u azuredevopsuser ./config.sh remove --unattended --auth pat --token "$az_devops_pat" || true
    fi

    echo "running installation"
    sudo -u azuredevopsuser ./config.sh --unattended --url "$az_devops_url" --auth pat --token "$az_devops_pat" --pool "$az_devops_agent_pool" --agent "$agent_id" --acceptTeeEula --work ./_work --runAsService
    echo "configuration done"
    ./svc.sh install
    echo "service installed"
    ./svc.sh start
    echo "service started"
    echo "config done"
  popd
done


================================================
FILE: 301-deploy-agent-vms/terraform/devops-agent/main.tf
================================================
resource "azurerm_resource_group" "devops" {
  name   = "rg-${var.appname}-${var.environment}-devops"
  location = var.location
}

# Create virtual network

resource "azurerm_virtual_network" "devops" {
  name                = "vnet-${var.appname}-devops-${var.environment}"
  address_space       = ["10.100.0.0/16"]
  location            = var.location
  resource_group_name = azurerm_resource_group.devops.name
}

resource "azurerm_subnet" "devops" {
  name                 = "agents-subnet"
  resource_group_name  = azurerm_resource_group.devops.name
  virtual_network_name = azurerm_virtual_network.devops.name
  address_prefix       = "10.100.1.0/24"
}

resource "azurerm_storage_account" "devops" {
  name                     = "stado${var.appname}${var.environment}"
  resource_group_name      = azurerm_resource_group.devops.name
  location                 = azurerm_resource_group.devops.location
  account_tier             = "Standard"
  account_replication_type = "LRS"
}

resource "azurerm_storage_container" "devops" {
  name                  = "content"
  storage_account_name  = azurerm_storage_account.devops.name
  container_access_type = "private"
}

resource "azurerm_storage_blob" "devops" {
  name                   = "devops_agent_init-${md5(file("${path.module}/devops_agent_init.sh"))}.sh"
  storage_account_name   = azurerm_storage_account.devops.name
  storage_container_name = azurerm_storage_container.devops.name
  type                   = "Block"
  source                 = "${path.module}/devops_agent_init.sh"
}

data "azurerm_storage_account_blob_container_sas" "devops_agent_init" {
  connection_string = azurerm_storage_account.devops.primary_connection_string
  container_name    = azurerm_storage_container.devops.name
  https_only        = true

  start  = "2000-01-01"
  expiry = "2099-01-01"

  permissions {
    read   = true
    add    = false
    create = false
    write  = false
    delete = false
    list   = false
  }
}


# Create public IPs
resource "azurerm_public_ip" "devops" {
  name                = "pip-${var.appname}-devops-${var.environment}-${format("%03d", count.index + 1)}"
  location            = var.location
  resource_group_name = azurerm_resource_group.devops.name
  allocation_method   = "Dynamic"
  count               = var.az_devops_agent_vm_count
}

# Create network interface
resource "azurerm_network_interface" "devops" {
  name                      = "nic-${var.appname}-devops-${var.environment}-${format("%03d", count.index + 1)}"
  location                  = var.location
  resource_group_name       = azurerm_resource_group.devops.name

  ip_configuration {
    name                          = "AzureDevOpsNicConfiguration"
    subnet_id                     = azurerm_subnet.devops.id
    private_ip_address_allocation = "dynamic"
    public_ip_address_id          = azurerm_public_ip.devops[count.index].id
  }

  count                     = var.az_devops_agent_vm_count
}

# Create virtual machine

resource "random_password" "agent_vms" {
  length = 24
  special = true
  override_special = "!@#$%&*()-_=+[]:?"
  min_upper = 1
  min_lower = 1
  min_numeric = 1
  min_special = 1
}

resource "azurerm_virtual_machine" "devops" {
  name                  = "vm${var.appname}devops${var.environment}-${format("%03d", count.index + 1)}"
  location              = var.location
  resource_group_name   = azurerm_resource_group.devops.name
  network_interface_ids = [azurerm_network_interface.devops[count.index].id]
  vm_size               = var.az_devops_agent_vm_size

  storage_os_disk {
    name              = "osdisk${var.appname}devops${var.environment}${format("%03d", count.index + 1)}"
    caching           = "ReadWrite"
    create_option     = "FromImage"
    managed_disk_type = "Premium_LRS"
  }

  storage_image_reference {
    publisher = "Canonical"
    offer     = "UbuntuServer"
    sku       = "16.04.0-LTS"
    version   = "latest"
  }

  os_profile {
    computer_name  = "AzureDevOps"
    admin_username = "azuredevopsuser"
    admin_password = random_password.agent_vms.result
  }

  os_profile_linux_config {
    disable_password_authentication = false

    dynamic "ssh_keys" {
      for_each = var.az_devops_agent_sshkeys
      content {
        key_data = each.key
        path = "/home/azuredevopsuser/.ssh/authorized_keys"
      }
    }
  }

  boot_diagnostics {
    enabled     = "true"
    storage_uri = azurerm_storage_account.devops.primary_blob_endpoint
  }

  count = var.az_devops_agent_vm_count
}

resource "azurerm_virtual_machine_extension" "devops" {
  name                 = format("install_azure_devops_agent-%03d", count.index + 1)
  virtual_machine_id   = azurerm_virtual_machine.devops[count.index].id
  publisher            = "Microsoft.Azure.Extensions"
  type                 = "CustomScript"
  type_handler_version = "2.0"

  #timestamp: use this field only to trigger a re-run of the script by changing value of this field.
  #           Any integer value is acceptable; it must only be different than the previous value.
  settings = jsonencode({
    "timestamp" : 1
  })
  protected_settings = jsonencode({
  "fileUris": ["${azurerm_storage_blob.devops.url}${data.azurerm_storage_account_blob_container_sas.devops_agent_init.sas}"],
  "commandToExecute": "bash ${azurerm_storage_blob.devops.name} '${var.az_devops_url}' '${var.az_devops_pat}' '${var.az_devops_agent_pool}' '${var.az_devops_agents_per_vm}'"
  })
  count = var.az_devops_agent_vm_count
}

resource "azurerm_template_deployment" "devops_shutdown" {
  name = format("shutdown-vm-%03d", count.index + 1)
  resource_group_name = azurerm_resource_group.devops.name

  template_body = file("${path.module}/shutdown_schedule_arm_template.json")

  parameters = {
    name = "shutdown-computevm-${azurerm_virtual_machine.devops[count.index].name}"
    shutdown_enabled = var.az_devops_agent_vm_shutdown_time != null ? "Enabled" : "Disabled"
    shutdown_time = coalesce(var.az_devops_agent_vm_shutdown_time, "0000")
    vm_id = azurerm_virtual_machine.devops[count.index].id
  }

  depends_on = [
    azurerm_virtual_machine.devops
  ]

  deployment_mode = "Incremental"

  count = var.az_devops_agent_vm_count
}



================================================
FILE: 301-deploy-agent-vms/terraform/devops-agent/outputs.tf
================================================
output "agent_vm_ids" {
  value = azurerm_virtual_machine.devops.*.id
}


================================================
FILE: 301-deploy-agent-vms/terraform/devops-agent/shutdown_schedule_arm_template.json
================================================
{
  "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "name": {
      "type": "string"
    },
    "shutdown_enabled": {
      "type": "string"
    },
    "shutdown_time": {
      "type": "string"
    },
    "vm_id": {
      "type": "string"
    }
  },
  "resources": [
    {
      "type": "microsoft.devtestlab/schedules",
      "apiVersion": "2018-09-15",
      "name": "[parameters('name')]",
      "location": "northeurope",
      "properties": {
        "status": "[parameters('shutdown_enabled')]",
        "taskType": "ComputeVmShutdownTask",
        "dailyRecurrence": {
          "time": "[parameters('shutdown_time')]"
        },
        "timeZoneId": "UTC",
        "notificationSettings": {
          "status": "Disabled",
          "timeInMinutes": 30,
          "notificationLocale": "en"
        },
        "targetResourceId": "[parameters('vm_id')]"
      }
    }
  ]
}


================================================
FILE: 301-deploy-agent-vms/terraform/devops-agent/variables.tf
================================================
variable "appname" {
  type = string
}

variable "environment" {
  type = string
}

variable "location" {
  type = string
}

variable "az_devops_url" {
  type = string
  description = "Specify the Azure DevOps url e.g. https://dev.azure.com/myorg"
}

variable "az_devops_pat" {
  type = string
  description = "Provide a Personal Access Token (PAT) for Azure DevOps. Create it at https://dev.azure.com/[Organization]/_usersSettings/tokens with permission Agent Pools > Read & manage"
}

variable "az_devops_agent_pool" {
  type = string
  description = "Specify the name of the agent pool - must exist before. Create it at https://dev.azure.com/[Organization]/_settings/agentpools"
  default = "pool001"
}

variable "az_devops_agent_sshkeys" {
  type        = list(string)
  description = "Optionally provide ssh public key(s) to logon to the VM"
}

variable "az_devops_agent_vm_size" {
  type    = string
  description = "Specify the size of the VM"
  default = "Standard_D2s_v3"
}

variable "az_devops_agent_vm_count" {
  type    = number
  description = "Number of Azure DevOps agent VMs"
  default = 1
}

variable "az_devops_agent_vm_shutdown_time" {
  type    = string
  description = "UTC Time at which to shutdown the agent VMs daily, for example '2000' for 8 PM"
  default = null
}

variable "az_devops_agents_per_vm" {
  type = number
  description = "Number of Azure DevOps agents spawned per VM. Agents will be named with a random prefix."
  default = 4
}


================================================
FILE: 301-deploy-agent-vms/terraform/main.tf
================================================
# Azure DevOps agent VMs

module "devops-agent" {
  source = "./devops-agent"
  appname = var.appname
  environment = var.environment
  location = var.location
  az_devops_url = var.az_devops_url
  az_devops_pat = var.az_devops_pat
  az_devops_agent_pool = var.az_devops_agent_pool
  az_devops_agents_per_vm = var.az_devops_agents_per_vm
  az_devops_agent_sshkeys = var.az_devops_agent_sshkeys
  az_devops_agent_vm_size = var.az_devops_agent_vm_size
  az_devops_agent_vm_count = var.az_devops_agent_vm_count
  az_devops_agent_vm_shutdown_time = var.az_devops_agent_vm_shutdown_time
}



================================================
FILE: 301-deploy-agent-vms/terraform/outputs.tf
================================================
output "pool_name" {
  value = var.az_devops_agent_pool
}

output "agent_vm_ids" {
  value = module.devops-agent.agent_vm_ids
}


================================================
FILE: 301-deploy-agent-vms/terraform/provider.tf
================================================
#Set the terraform required version
terraform {
  required_version = ">= 0.12.6"
}

# Configure the Azure Provider
provider "azurerm" {
  # It is recommended to pin to a given version of the Provider
  version = "=1.44.0"
}

provider "random" {
  version = "~> 2.2"
}

# Data

# Make client_id, tenant_id, subscription_id and object_id variables
data "azurerm_client_config" "current" {}


================================================
FILE: 301-deploy-agent-vms/terraform/variables.tf
================================================
variable "appname" {
  type = string
  description = "Application name. Use only lowercase letters and numbers"
  default = "starterterraform"
}

variable "environment" {
  type    = string
  description = "Environment name, e.g. 'dev' or 'stage'"
  default = "dev"
}

variable "location" {
  type    = string
  description = "Azure region where to create resources."
  default = "North Europe"
}

variable "department" {
  type    = string
  description = "A sample variable passed from the build pipeline and used to tag resources."
  default = "Engineering"
}

variable "az_devops_url" {
  type = string
  description = "Specify the Azure DevOps url e.g. https://dev.azure.com/myorg"
}

variable "az_devops_pat" {
  type = string
  description = "Provide a Personal Access Token (PAT) for Azure DevOps. Create it at https://dev.azure.com/[Organization]/_usersSettings/tokens with permission Agent Pools > Read & manage"
}

variable "az_devops_agent_pool" {
  type = string
  description = "Specify the name of the agent pool - must exist before. Create it at https://dev.azure.com/[Organization]/_settings/agentpools"
  default = "pool001"
}

variable "az_devops_agent_sshkeys" {
  type        = list(string)
  description = "Optionally provide ssh public key(s) to logon to the VM"
  default     = []
}

variable "az_devops_agent_vm_size" {
  type    = string
  description = "Specify the size of the VM"
  default = "Standard_D2s_v3"
}

variable "az_devops_agent_vm_count" {
  type    = number
  description = "Number of Azure DevOps agent VMs"
  default = 2
}

variable "az_devops_agents_per_vm" {
  type = number
  description = "Number of Azure DevOps agents spawned per VM. Agents will be named with a random prefix."
  default = 4
}

variable "az_devops_agent_vm_shutdown_time" {
  type    = string
  description = "UTC Time at which to shutdown the agent VMs daily, for example '2000' for 8 PM. If null, no shutdown will configured."
  default = null
}


================================================
FILE: 301-deploy-agent-vms/terraform-stages-template.yml
================================================
parameters:
  environment: agents
  TerraformArguments: ''
  TerraformEnvVariables:

stages:
- stage: Terraform_${{ parameters.environment }}
  displayName: Terraform ${{ parameters.environmentDisplayName }}
  pool:
    vmImage: ubuntu-latest
  jobs:

  - job: Terraform
    displayName: Terraform
    # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file.
    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
    steps:

    - template: ../infrastructure/terraform-init-template.yml
      parameters:
        provisionStorage: true
        TerraformDirectory: 301-deploy-agent-vms/terraform
        environment: ${{ parameters.environment }}

    # Using bash instead of Terraform extension because of following issues:
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/748
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/725
    # - https://github.com/microsoft/azure-pipelines-extensions/issues/747
    - bash: |
        set -eu
        export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
        terraform apply -input=false -auto-approve -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}
      displayName: Terraform apply
      workingDirectory: 301-deploy-agent-vms/terraform
      env:
        ${{ parameters.TerraformEnvVariables }}

- stage: PostTerraform_${{ parameters.environment }}
  displayName: PostTerraform ${{ parameters.environmentDisplayName }}
  pool:
    vmImage: ubuntu-latest
  jobs:
  - job: ReadTerraform
    displayName: Read Terraform outputs
    steps:

    - template: ../infrastructure/terraform-init-template.yml
      parameters:
        TerraformDirectory: 301-deploy-agent-vms/terraform
        environment: ${{ parameters.environment }}

    - template: ../infrastructure/terraform-outputs-template.yml
      parameters:
        TerraformDirectory: 301-deploy-agent-vms/terraform

    - bash: env

    - task: AzureCLI@1
      displayName: Start agents
      inputs:
        azureSubscription: Terraform
        scriptLocation: inlineScript
        inlineScript: |
          set -eux  # fail on error
          az vm start --ids $(echo $AGENT_VM_IDS | jq -r '.[]') -o none

  - job: DummySampleJob
    displayName: Run Agent job
    dependsOn: ReadTerraform
    variables:
      pool_name: $[ dependencies.ReadTerraform.outputs['Outputs.pool_name'] ]
    pool: $(pool_name)
    steps:
    - bash: |
        echo This is running on agent
        hostname
      displayName: Sample script


================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Microsoft Open Source Code of Conduct

This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).

Resources:

- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns


================================================
FILE: LICENSE
================================================
    MIT License

    Copyright (c) Microsoft Corporation.

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all
    copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    SOFTWARE


================================================
FILE: README.md
================================================
---
page_type: sample
products:
- devops
description: "Starter project for Azure Pipelines deploying resources on Terraform"
---

# Terraform starter project for Azure Pipelines

<!-- 
Guidelines on README format: https://review.docs.microsoft.com/help/onboard/admin/samples/concepts/readme-template?branch=master

Guidance on onboarding samples to docs.microsoft.com/samples: https://review.docs.microsoft.com/help/onboard/admin/samples/process/onboarding?branch=master

Taxonomies for products and languages: https://review.docs.microsoft.com/new-hope/information-architecture/metadata/taxonomies?branch=master
-->

This project can be used as a starter for Azure Pipelines deploying resources on Terraform.

![pipeline jobs](/docs/images/terraform_starter/pipeline_jobs.png)

## Contents

| File/folder             | Description                                                  |
|-------------------------|--------------------------------------------------------------|
| `infrastructure`        | YAML pipeline templates shared across the samples.           |
| `101-terraform-job`     | Sample YAML pipeline for a simple Terraform job.             |
| `201-plan-apply-stages` | Sample YAML pipeline for manually approving plans.           |
| `301-deploy-agent-vms`  | Sample YAML pipeline for deploying build agent VMs.          |
| `docs`                  | Resources related to documentation.                          |
| `CODE_OF_CONDUCT.md`    | Microsoft Open Source Code of Conduct.                       |
| `LICENSE`               | The license for the sample.                                  |
| `README.md`             | This README file.                                            |
| `SECURITY.md`           | Reporting security issues.                                   |

# Templates

## 101 Basic Terraform job

The first template shows how to build an environment from Terraform configuration, and run
a subsequent job configured from Terraform outputs.

[101-terraform-job: Basic Terraform job](101-terraform-job)

![pipeline job](/docs/images/terraform_starter/101-terraform-job.png)

## 201 Separate Plan and Apply stages

The next template shows how to build a multi-stage pipeline
allowing to manually review and approve infrastructure changes before they are deployed.

[201-plan-apply-stages: Separate Plan and Apply stages](201-plan-apply-stages)

![pipeline jobs](docs/images/terraform_starter/pipeline_stage_waiting.png)

## 301 Deploy hosted agent VMs

The next template shows how to use Terraform to deploy a pool of agent VMs on which to run
subsequent jobs.

[301-deploy-agent-vms: Deploy hosted agent VMs](301-deploy-agent-vms)

![agent job](/docs/images/terraform_starter/301-agent-job.png)

# How to use the templates

## Variables and state management

Variables can be injected using `TF_VAR_` syntax in the `TerraformEnvVariables` parameter or the
`-var key=value` syntax in the `TerraformArguments` parameter.
The pipelines demonstrates this by adding a custom tag named `department` to the
created resource group, with distinct values in staging and QA.

Rather than passing a Terraform plan between stages (which would contain clear-text secrets),
the pipeline in the
[201-plan-apply-stages](201-plan-apply-stages) sample
performs `terraform plan` again before applying changes and verifies that
a textual representation of the plan (not including secrets values) is unchanged.

The Terraform state is managed in a Azure Storage backend. Note that this backend contains
secrets in cleartext.

## Secrets management

### Generate secrets with Terraform

To demonstrate one approach to secrets management, the Terraform configuration
generates a random password (per stage) for the SQL Server 1 instance, stored in
Terraform state.
You can adapt this to suit your lifecycle.

### Manage secrets with Azure DevOps

You might want to read credentials from an externally managed Key Vault
or inject them via pipeline variables. This approach is demonstrated
by defining a password for the SQL Server 2 instance and passing
it to Terraform via an environment variable.

## Getting started

In `infrastructure/terraform/variables.tf`, change the `appname` default value from
`starterterraform` to a globally unique name.

## Azure DevOps pipeline

Install the [Terraform extension for Azure DevOps](https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks).

Create a Service Connection of type Azure Resource Manager at subscription scope. Name the Service Connection `Terraform`.
Allow all pipelines to use the connection.

In `infrastructure/terraform-init-template.yml`, update the `TerraformBackendStorageAccount` name to a globally unique storage account name.
The pipeline will create the storage account.

Create a build pipeline referencing `101-terraform-job/azure-pipelines.yml`.

## Usage on non-master branch

To avoid issues with concurrent access to the Terraform state file, the jobs running Terraform `plan` and `apply` commands
run by default only on the `master` branch. On other branches, they are skipped by default:

![run on non-master branch](/docs/images/terraform_starter/non_master_branch.png)

You can set the `RUN_FLAG_TERRAFORM` variable (to any non-empty value)
when running the pipeline, to trigger Terraform application on a non-`master` branch.

## Local development

In local development, no backend is configured so a local backend is used.

Install Azure CLI and login. Terraform will use your Azure CLI credentials.

```
$ az login -o table
You have logged in. Now let us find all the subscriptions to which you have access...
CloudName    IsDefault    Name                                                  State    TenantId
-----------  -----------  ----------------------------------------------------  -------  ------------------------------------
AzureCloud   True         My Azure subscription                                 Enabled  xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
AzureCloud   False        My other Azure subscription                           Enabled  xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
```

Run `terraform init`.

```
$ terraform init

Initializing the backend...

Initializing provider plugins...
- Checking for available provider plugins...
- Downloading plugin for provider "azurerm" (hashicorp/azurerm) 1.38.0...

Terraform has been successfully initialized!

You may now begin working with Terraform. Try running "terraform plan" to see
any changes that are required for your infrastructure. All Terraform commands
should now work.

If you ever set or change modules or backend configuration for Terraform,
rerun this command to reinitialize your working directory. If you forget, other
commands will detect it and remind you to do so if necessary.
```

Run `terraform plan`.

```
$ terraform plan -out tfplan
Refreshing Terraform state in-memory prior to plan...
The refreshed state will be used to calculate this plan, but will not be
persisted to local or remote state storage.

data.azurerm_client_config.current: Refreshing state...

------------------------------------------------------------------------

An execution plan has been generated and is shown below.
Resource actions are indicated with the following symbols:
  + create

Terraform will perform the following actions:

  # azurerm_resource_group.main will be created
  + resource "azurerm_resource_group" "main" {
      + id       = (known after apply)
      + location = "northeurope"
      + name     = "rg-starterterraform-dev-main"
      + tags     = (known after apply)
    }

Plan: 1 to add, 0 to change, 0 to destroy.
```

Run `terraform apply tfplan`.

```
$ terraform apply tfplan
data.azurerm_client_config.current: Refreshing state...

An execution plan has been generated and is shown below.
Resource actions are indicated with the following symbols:
  + create

Terraform will perform the following actions:

  # azurerm_resource_group.main will be created
  + resource "azurerm_resource_group" "main" {
      + id       = (known after apply)
      + location = "northeurope"
      + name     = "rg-starterterraform-dev-main"
      + tags     = (known after apply)
    }

Plan: 1 to add, 0 to change, 0 to destroy.

Do you want to perform these actions?
  Terraform will perform the actions described above.
  Only 'yes' will be accepted to approve.

  Enter a value: yes

azurerm_resource_group.main: Creating...
azurerm_resource_group.main: Creation complete after 1s [id=/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroups/rg-starterterraform-dev-main]

Apply complete! Resources: 1 added, 0 changed, 0 destroyed.

Outputs:

subscription_id = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
```

At this stage you will have a new resource group deployed named `rg-starterterraform-dev-main`. 

# Using Terraform outputs

The pipeline automatically exports Terraform outputs into pipeline variables.

The pipelines contain a sample job that consumes those variables:

![output variables](/docs/images/terraform_starter/output_variables.png)

- For example, in the template [301-deploy-agent-vms](301-deploy-agent-vms), the Terraform config has an output named [agent_vm_ids](301-deploy-agent-vms/terraform/outputs.tf). In the subsequent task used
we use the bash variable [AGENT_VM_IDS](301-deploy-agent-vms/terraform-stages-template.yml) to pass the list of agent VMs to the `az start` command.

This mechanism is useful for using generated resource names, access keys,
and even [entire kube_config files](https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#kube_config_raw) (for Azure Kubernetes Service)
in downstream testing or continuous delivery jobs.

# Next steps

* You can of course adapt the pipeline to other environments, such as Production.

## Contributing

This project welcomes contributions and suggestions.  Most contributions require you to agree to a
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.

When you submit a pull request, a CLA bot will automatically determine whether you need to provide
a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
provided by the bot. You will only need to do this once across all repos using our CLA.

This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.


================================================
FILE: SECURITY.md
================================================
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.3 BLOCK -->

## Security

Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).

If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)) of a security vulnerability, please report it to us as described below.

## Reporting Security Issues

**Please do not report security vulnerabilities through public GitHub issues.**

Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).

If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).

You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).

Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:

  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
  * Full paths of source file(s) related to the manifestation of the issue
  * The location of the affected source code (tag/branch/commit or direct URL)
  * Any special configuration required to reproduce the issue
  * Step-by-step instructions to reproduce the issue
  * Proof-of-concept or exploit code (if possible)
  * Impact of the issue, including how an attacker might exploit the issue

This information will help us triage your report more quickly.

If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.

## Preferred Languages

We prefer all communications to be in English.

## Policy

Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).

<!-- END MICROSOFT SECURITY.MD BLOCK -->


================================================
FILE: infrastructure/terraform-init-template.yml
================================================
parameters:
  environment: stage
  provisionStorage: false
  TerraformVersion: 0.12.21
  TerraformDirectory:
  TerraformBackendServiceConnection: Terraform
  TerraformEnvironmentServiceConnection: Terraform
  TerraformBackendResourceGroup: terraform
  TerraformBackendStorageAccount: terraformstarterstate
  TerraformBackendStorageContainer: terraformstate
  TerraformBackendLocation: North Europe

steps:

- task: AzureCLI@1
  displayName: Set Terraform backend
  condition: and(succeeded(), ${{ parameters.provisionStorage }})
  inputs:
    azureSubscription: ${{ parameters.TerraformBackendServiceConnection }}
    scriptLocation: inlineScript
    inlineScript: |
      set -eu  # fail on error
      RG='${{ parameters.TerraformBackendResourceGroup }}'
      export AZURE_STORAGE_ACCOUNT='${{ parameters.TerraformBackendStorageAccount }}'
      export AZURE_STORAGE_KEY="$(az storage account keys list -g "$RG" -n "$AZURE_STORAGE_ACCOUNT" --query '[0].value' -o tsv)"
      if test -z "$AZURE_STORAGE_KEY"; then
        az configure --defaults group="$RG" location='${{ parameters.TerraformBackendLocation }}'
        az group create -n "$RG" -o none
        az storage account create -n "$AZURE_STORAGE_ACCOUNT" -o none
        export AZURE_STORAGE_KEY="$(az storage account keys list -g "$RG" -n "$AZURE_STORAGE_ACCOUNT" --query '[0].value' -o tsv)"
      fi

      container='${{ parameters.TerraformBackendStorageContainer }}'
      if ! az storage container show -n "$container" -o none 2>/dev/null; then
        az storage container create -n "$container" -o none
      fi
      blob='${{ parameters.environment }}.tfstate'
      if [[ $(az storage blob exists -c "$container" -n "$blob" --query exists) = "true" ]]; then
        if [[ $(az storage blob show -c "$container" -n "$blob" --query "properties.lease.status=='locked'") = "true" ]]; then
          echo "State is leased"
          lock_jwt=$(az storage blob show -c "$container" -n "$blob" --query metadata.terraformlockid -o tsv)
          if [ "$lock_jwt" != "" ]; then
            lock_json=$(base64 -d <<< "$lock_jwt")
            echo "State is locked"
            jq . <<< "$lock_json"
          fi
          if [ "${TERRAFORM_BREAK_LEASE:-}" != "" ]; then
            az storage blob lease break -c "$container" -b "$blob"
          else
            echo "If you're really sure you want to break the lease, rerun the pipeline with variable TERRAFORM_BREAK_LEASE set to 1."
            exit 1
          fi
        fi
      fi
    addSpnToEnvironment: true

- task: ms-devlabs.custom-terraform-tasks.custom-terraform-installer-task.TerraformInstaller@0
  displayName: Install Terraform
  inputs:
    terraformVersion: ${{ parameters.TerraformVersion }}

- task: AzureCLI@1
  displayName: Terraform credentials
  inputs:
    azureSubscription: ${{ parameters.TerraformEnvironmentServiceConnection }}
    scriptLocation: inlineScript
    inlineScript: |
      set -eu
      subscriptionId=$(az account show --query id -o tsv)
      echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$servicePrincipalId"
      echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET;issecret=true]$servicePrincipalKey"
      echo "##vso[task.setvariable variable=ARM_SUBSCRIPTION_ID]$subscriptionId"
      echo "##vso[task.setvariable variable=ARM_TENANT_ID]$tenantId"
    addSpnToEnvironment: true

# Using bash instead of Terraform extension because of following issue:
# - https://github.com/microsoft/azure-pipelines-extensions/issues/738
- task: AzureCLI@1
  displayName: Terraform init
  inputs:
    azureSubscription: ${{ parameters.TerraformBackendServiceConnection }}
    scriptLocation: inlineScript
    inlineScript: |
      set -eux  # fail on error
      subscriptionId=$(az account show --query id -o tsv)
      terraform init \
        -backend-config=storage_account_name=${{ parameters.TerraformBackendStorageAccount }} \
        -backend-config=container_name=${{ parameters.TerraformBackendStorageContainer }} \
        -backend-config=key=${{ parameters.environment }}.tfstate \
        -backend-config=resource_group_name=${{ parameters.TerraformBackendResourceGroup }} \
        -backend-config=subscription_id=$subscriptionId \
        -backend-config=tenant_id=$tenantId \
        -backend-config=client_id=$servicePrincipalId \
        -backend-config=client_secret="$servicePrincipalKey"
    workingDirectory: ${{ parameters.TerraformDirectory }}
    addSpnToEnvironment: true


================================================
FILE: infrastructure/terraform-outputs-template.yml
================================================
parameters:
  TerraformDirectory:

steps:
- bash: |
    set -eu

    echo "Setting job variables from Terraform outputs:"
    terraform output -json | jq -r '
      . as $in
      | keys[]
      | ["- " + .]
      | @tsv'

    terraform output -json | jq -r '
      . as $in
      | keys[]
      | ($in[.].value | tostring) as $value
      | ($in[.].sensitive | tostring) as $sensitive
      | [
        "- " + . + ": " + if $in[.].sensitive then "(sensitive)" else $value end,  # output name to console
        "##vso[task.setvariable variable=" + . + ";isSecret=" + $sensitive + "]" + $value,  # set as ADO task variable
        "##vso[task.setvariable variable=" + . + ";isOutput=true;isSecret=" + $sensitive + "]" + $value  # also set as ADO job variable
        ]
      | .[]'

  name: Outputs
  displayName: Read Terraform outputs
  workingDirectory: ${{ parameters.TerraformDirectory }}