[ { "path": ".gitignore", "content": "terraform.tfstate\nterraform.tfvars\n.idea/\n*.pem\n*.backup\n.terraform/\n.gcp*\ncluster_bootstrap_state\ngcp-account.json\n*.iml\nid_rsa.pub\npacker-es-manifest.json\npacker-kb-manifest.json\n.terraform.lock.hcl\nplan.*\n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# Deploy Elasticsearch on the cloud easily\n\nThis repository contains a set of tools and scripts to deploy an Elasticsearch cluster on the cloud, using best-practices and state of the art tooling.\n\nNeed to monitor and optimize your cluster after setting it up? Consider using [Pulse](https://pulse.support/).\n\n***Note:*** This branch supports Elasticsearch 7.x only. For other Elasticsearch versions see [feat-8x](https://github.com/BigDataBoutique/elasticsearch-cloud-deploy/tree/feat-8x), [elasticsearch-5.x](https://github.com/BigDataBoutique/elasticsearch-cloud-deploy/tree/elasticsearch-5.x) and [elasticsearch-6.x](https://github.com/BigDataBoutique/elasticsearch-cloud-deploy/tree/elasticsearch-6.x) branches.\n\nYou need to use the latest version of Terraform and Packer for all features to work correctly.\n\nFeatures:\n\n* Deployment of data and master nodes as separate nodes, as well as data-voters\n* Client node with Kibana and authenticated Elasticsearch access\n* Single node cluster support\n* DNS and load-balancing access to client nodes\n* Sealed from external access, only accessible via password-protected external facing client nodes\n* AWS deployment support (under `terraform-aws`)\n* Google Cloud Platform deployment (under `terraform-gcp`)\n* Packer scripts for both GCP and AWS (under `packer`)\n* Azure deployment - not maintained at the moment (under `terraform-azure`)\n\n## Usage\n\nClone this repo to work locally. You might want to fork it in case you need to apply some additional configurations or commit changes to the variables file.\n\nCreate images with Packer (see `packer` folder in this repo), and then go into the terraform folder and run `terraform plan`. See README files in each respective folder. \n\n## tfstate\n\nOnce you run `terraform apply` on any of the terraform folders in this repo, a file `terraform.tfstate` will be created. This file contains the mapping between your cloud elements to the terraform configuration. Make sure to keep this file safe.\n \nSee [this guide](https://blog.gruntwork.io/how-to-manage-terraform-state-28f5697e68fa#.fbb2nalw6) for a discussion on tfstate management and locking between team members.\n" }, { "path": "assets/ec2-role-trust-policy.json", "content": "{\n \"Version\": \"2008-10-17\",\n \"Statement\": [\n {\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": [\"ec2.amazonaws.com\"]\n },\n \"Effect\": \"Allow\"\n }\n ]\n}" }, { "path": "assets/elasticsearch.yml", "content": "bootstrap.memory_lock: true\nnode.name: ${HOSTNAME}\n\naction.destructive_requires_name: true\nindices.fielddata.cache.size: 1% # default is unbounded\n" }, { "path": "assets/node-init.json", "content": "{\n \"Statement\": [\n {\n \"Action\": [\n \"ec2:DescribeInstances\",\n \"ec2:DescribeVolumes\",\n \"ec2:AttachVolume\",\n \"ec2:DescribeTags\",\n \"autoscaling:DescribeAutoScalingGroups\"\n ],\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ]\n }\n ],\n \"Version\": \"2012-10-17\"\n}" }, { "path": "assets/s3-backup.json", "content": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\"s3:ListBucket\"],\n \"Resource\": [\"arn:aws:s3:::${s3_backup_bucket}\"]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:DeleteObject\"\n ],\n \"Resource\": [\"arn:aws:s3:::${s3_backup_bucket}/*\"]\n }\n ]\n}" }, { "path": "assets/scripts/aws/autoattach-disk.sh", "content": "# Required variables\n# - aws_region\n# - es_cluster\n# - elasticsearch_data_dir\n\nAV_ZONE=\"$(ec2metadata --availability-zone)\"\nINSTANCE_ROLE=\"$(aws ec2 describe-tags --region $aws_region --filters Name=resource-id,Values=$(ec2metadata --instance-id) | jq -r '.Tags[] | select(.Key == \"Role\") | .Value')\"\necho \"AV_ZONE: $AV_ZONE\"\necho \"INSTANCE_ROLE: $INSTANCE_ROLE\"\n\nwhile true; do\n UNATTACHED_VOLUME_ID=\"$(aws ec2 describe-volumes --region $aws_region --filters Name=tag:ClusterName,Values=$es_cluster Name=tag:AutoAttachGroup,Values=$INSTANCE_ROLE Name=availability-zone,Values=$AV_ZONE | jq -r '.Volumes[] | select(.Attachments | length == 0) | .VolumeId' | shuf -n 1)\"\n echo \"UNATTACHED_VOLUME_ID: $UNATTACHED_VOLUME_ID\"\n\n aws ec2 attach-volume --device \"/dev/xvdh\" --instance-id=$(ec2metadata --instance-id) --volume-id $UNATTACHED_VOLUME_ID --region \"$aws_region\"\n if [ \"$?\" != \"0\" ]; then\n sleep 10\n continue\n fi\n\n sleep 30\n\n ATTACHMENTS_COUNT=\"$(aws ec2 describe-volumes --region $aws_region --filters Name=volume-id,Values=$UNATTACHED_VOLUME_ID | jq -r '.Volumes[0].Attachments | length')\"\n if [ \"$ATTACHMENTS_COUNT\" != \"0\" ]; then break; fi\ndone\n\necho 'Waiting for 30 seconds for the disk to become mountable...'\nsleep 30\n\nsudo mkdir -p $elasticsearch_data_dir\nexport DEVICE_NAME=$(lsblk -ip | tail -n +2 | awk '{print $1 \" \" ($7? \"MOUNTEDPART\" : \"\") }' | sed ':a;N;$!ba;s/\\n`/ /g' | grep -v MOUNTEDPART)\nif sudo mount -o defaults -t ext4 $DEVICE_NAME $elasticsearch_data_dir; then\n echo 'Successfully mounted existing disk'\nelse\n echo 'Trying to mount a fresh disk'\n sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard $DEVICE_NAME\n sudo mount -o defaults -t ext4 $DEVICE_NAME $elasticsearch_data_dir && echo 'Successfully mounted a fresh disk'\nfi\necho \"$DEVICE_NAME $elasticsearch_data_dir ext4 defaults,nofail 0 2\" | sudo tee -a /etc/fstab\nsudo chown -R elasticsearch:elasticsearch $elasticsearch_data_dir" }, { "path": "assets/scripts/aws/config-bootstrap-node.sh", "content": "# Required variables\n# - aws_region\n# - es_environment\n# - masters_count\n\nwhile true\ndo\n echo \"Fetching masters...\"\n MASTER_INSTANCES=\"$(aws ec2 describe-instances --region=$aws_region --filters Name=instance-state-name,Values=running Name=tag:Role,Values=master,data-voters Name=tag:Cluster,Values=$es_environment | jq -r '.Reservations | map(.Instances[].InstanceId) | .[]' | sort)\"\n COUNT=`echo \"$MASTER_INSTANCES\" | wc -l`\n\n if [ \"$COUNT\" -eq \"$masters_count\" ]; then\n echo \"Masters count is correct... Rechecking in 60 sec\"\n sleep 60\n MASTER_INSTANCES_RECHECK=\"$(aws ec2 describe-instances --region=$aws_region --filters Name=instance-state-name,Values=running Name=tag:Role,Values=master,data-voters Name=tag:Cluster,Values=$es_environment | jq -r '.Reservations | map(.Instances[].InstanceId) | .[]' | sort)\"\n\n if [ \"$MASTER_INSTANCES\" = \"$MASTER_INSTANCES_RECHECK\" ]; then\n break\n fi\n fi\n\n sleep 5\ndone\n\necho \"Fetched masters\"\nMASTER_IPS=\"$(aws ec2 describe-instances --region $aws_region --instance-ids $MASTER_INSTANCES | jq -r '.Reservations[].Instances[].PrivateIpAddress')\"\nSEED_HOSTS=`echo \"$MASTER_IPS\" | paste -sd ',' -`\nINITIAL_MASTER_NODES=`echo \"$MASTER_IPS\" | awk '{print \"ip-\" $0}' | tr . - | paste -sd ',' -`\n\necho \"discovery.seed_hosts: $SEED_HOSTS\" >>/etc/elasticsearch/elasticsearch.yml\necho \"cluster.initial_master_nodes: $(hostname),$INITIAL_MASTER_NODES\" >>/etc/elasticsearch/elasticsearch.yml\n" }, { "path": "assets/scripts/aws/config-cluster.sh", "content": "# Required variables\n# - security_enabled\n# - client_pwd\n# - s3_backup_bucket\n# - ES_HOST\n# - CURL_AUTH\n\nif [ \"${s3_backup_bucket}\" != \"\" ]; then\n curl $CURL_AUTH -k -X PUT \"$ES_HOST/_snapshot/s3_repo\" -H 'Content-Type: application/json' -d'\n {\n \"type\": \"s3\",\n \"settings\": {\n \"bucket\": \"'\"$s3_backup_bucket\"'\"\n }\n }\n '\n sleep 1\n\n curl $CURL_AUTH -k -X POST \"$ES_HOST/_nodes/reload_secure_settings\"\nfi" }, { "path": "assets/scripts/aws/config-es-discovery.sh", "content": "# Required variables\n# - aws_region\n# - security_groups\n# - es_environment\n\ncat <>/etc/elasticsearch/elasticsearch.yml\n\nnetwork.host: _ec2:privateIpv4_,localhost\nplugin.mandatory: discovery-ec2\ncloud.node.auto_attributes: true\ncluster.routing.allocation.awareness.attributes: aws_availability_zone\ndiscovery:\n seed_providers: ec2\n ec2.groups: $security_groups\n ec2.host_type: private_ip\n ec2.tag.Cluster: $es_environment\n ec2.protocol: http # no need in HTTPS for internal AWS calls\n\n # manually set the endpoint because of auto-discovery issues\n # https://github.com/elastic/elasticsearch/issues/27464\n ec2.endpoint: ec2.$aws_region.amazonaws.com\nEOF" }, { "path": "assets/scripts/bootstrap.sh", "content": "#!/bin/bash\nset +e\n\n. /opt/cloud-deploy-scripts/common/env.sh\n. /opt/cloud-deploy-scripts/$cloud_provider/env.sh\n\n/opt/cloud-deploy-scripts/common/config-es.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-bootstrap-node.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh\n\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnode.roles: [ master ]\nEOF\n\n# add bootstrap.password to the keystore, so that config-cluster scripts can run\n# only done on bootstrap and singlenode nodes, before starting ES\nif [ \"${security_enabled}\" == \"true\" ]; then\n echo \"${client_pwd}\" | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin bootstrap.password\nfi\n\n# Start Elasticsearch\nsystemctl daemon-reload\nsystemctl enable elasticsearch.service\nsystemctl start elasticsearch.service\n\nset -e\n/opt/cloud-deploy-scripts/common/config-cluster.sh\nset +e\n/opt/cloud-deploy-scripts/$cloud_provider/config-cluster.sh\n\nwhile true\ndo\n HEALTH=\"$(curl $CURL_AUTH --silent -k \"$ES_HOST/_cluster/health\" | jq -r '.status')\"\n if [ \"$HEALTH\" == \"green\" ]; then\n break\n fi\n sleep 5\ndone\n/opt/cloud-deploy-scripts/$cloud_provider/config-cluster.sh\n\nif [ \"$auto_shut_down_bootstrap_node\" == \"true\" ]\nthen\n if [ \"$cloud_provider\" == \"aws\" ]; then\n \tshutdown -h now\n elif [ \"$cloud_provider\" == \"gcp\" ]; then\n \tgcloud compute instances delete $HOSTNAME --zone $GCP_ZONE --quiet\n fi\nfi\n" }, { "path": "assets/scripts/client.sh", "content": "#!/bin/bash\nset +e\n\n. /opt/cloud-deploy-scripts/common/env.sh\n. /opt/cloud-deploy-scripts/$cloud_provider/env.sh\n\n# It is required to bind to all interfaces for load balancer on GCP to work\nif [ \"$cloud_provider\" == \"gcp\" ]; then\n export BIND_TO_ALL=\"true\"\nfi\n\n/opt/cloud-deploy-scripts/common/config-es.sh\n/opt/cloud-deploy-scripts/common/config-beats.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh\n\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnode.roles: [ ingest, remote_cluster_client ]\nEOF\n\n# Start Elasticsearch\nsystemctl daemon-reload\nsystemctl enable elasticsearch.service\nsystemctl start elasticsearch.service\n\n/opt/cloud-deploy-scripts/common/config-clients.sh\n" }, { "path": "assets/scripts/common/config-beats.sh", "content": "# Required variables\n# - filebeat_monitoring_host\n\nif [ \"${filebeat_monitoring_host}\" != \"\" ]; then\n\n\tcat </etc/filebeat/modules.d/elasticsearch.yml\n# Module: elasticsearch\n# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.6/filebeat-module-elasticsearch.html\n\n- module: elasticsearch\n server:\n enabled: true\n gc:\n enabled: false\n audit:\n enabled: false\n slowlog:\n enabled: true\n deprecation:\n enabled: true\nEOF\n\tcat </etc/filebeat/filebeat.yml\nfilebeat.config.modules.path: /etc/filebeat/modules.d/*.yml\noutput.elasticsearch:\n hosts: [\"$filebeat_monitoring_host\"]\nsetup.ilm.enabled: false\nEOF\n\nsystemctl daemon-reload\n\nfi" }, { "path": "assets/scripts/common/config-clients.sh", "content": "# Required variables\n# - client_user\n# - client_pwd\n# - security_enabled\n# - monitoring_enabled\n# - BIND_TO_ALL\n# - ES_HOST\n# - CURL_AUTH\n# security_encryption_key\n# reporting_encryption_key\n\n# Setup x-pack security also on Kibana configs where applicable\nif [ -f \"/etc/kibana/kibana.yml\" ]; then\n\n if [ \"$BIND_TO_ALL\" == \"true\" ]; then\n echo \"server.host: 0.0.0.0\" | sudo tee -a /etc/kibana/kibana.yml\n else\n echo \"server.host: $(hostname -i)\" | sudo tee -a /etc/kibana/kibana.yml\n fi\n if [ ! -z \"$security_encryption_key\" ]; then\n echo \"$security_encryption_key\" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.security.encryptionKey\n fi\n if [ ! -z \"$reporting_encryption_key\" ]; then\n echo \"$reporting_encryption_key\" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.reporting.encryptionKey\n fi\n echo \"xpack.security.enabled: $security_enabled\" | sudo tee -a /etc/kibana/kibana.yml\n echo \"xpack.monitoring.enabled: $monitoring_enabled\" | sudo tee -a /etc/kibana/kibana.yml\n\n if [ \"$security_enabled\" == \"true\" ]; then\n echo \"elasticsearch.username: \\\"kibana\\\"\" | sudo tee -a /etc/kibana/kibana.yml\n echo \"${client_pwd}\" | /usr/share/kibana/bin/kibana-keystore add --stdin elasticsearch.password\n fi\n\n systemctl daemon-reload\n systemctl enable kibana.service\n sudo service kibana restart\nfi\n" }, { "path": "assets/scripts/common/config-cluster.sh", "content": "# Required variables\n# - security_enabled\n# - client_pwd\n# - ES_HOST\n# - CURL_AUTH\ni=1\nwhile true\ndo\n echo \"Checking cluster health, attempt $i\"\n HEALTH=\"$(curl $CURL_AUTH --silent -k \"$ES_HOST/_cluster/health\" | jq -r '.status')\"\n DATA_NODE_COUNT=\"$(curl $CURL_AUTH --silent -k \"$ES_HOST/_cat/nodes?h=node.role\" | grep 'd\\|h\\|c' | wc -l)\"\n\n if [ \"$HEALTH\" == \"green\" ] && [ \"$DATA_NODE_COUNT\" != \"0\" ]; then\n break\n fi\n\n sleep 5\n i=$((i+1))\ndone\n\n# if any of the below fail, bootstrap failed - exit on error\nset -e\nif [ \"$security_enabled\" == \"true\" ]; then\n curl $CURL_AUTH \\\n -X PUT -H 'Content-Type: application/json' -k \\\n \"$ES_HOST/_security/user/elastic/_password\" -d '{ \"password\": \"'\"$client_pwd\"'\" }'\n\n curl $CURL_AUTH \\\n -X PUT -H 'Content-Type: application/json' -k \\\n \"$ES_HOST/_security/user/kibana/_password\" -d '{ \"password\": \"'\"$client_pwd\"'\" }'\n\n curl $CURL_AUTH \\\n -X PUT -H 'Content-Type: application/json' -k \\\n \"$ES_HOST/_security/user/logstash_system/_password\" -d '{ \"password\": \"'\"$client_pwd\"'\" }'\n\n curl $CURL_AUTH \\\n -X PUT -H 'Content-Type: application/json' -k \\\n \"$ES_HOST/_security/user/remote_monitoring_user/_password\" -d '{ \"password\": \"'\"$client_pwd\"'\" }'\nfi\n" }, { "path": "assets/scripts/common/config-es.sh", "content": "# Required variables\n# - es_cluster\n# - monitoring_enabled\n# - elasticsearch_data_dir\n# - elasticsearch_logs_dir\n# - security_enabled\n# - ca_cert\n# - node_cert\n# - node_key\n# - xpack_monitoring_host\n# - heap_size\n# - use_g1gc\n\n# Configure elasticsearch\ncat <>/etc/elasticsearch/elasticsearch.yml\ncluster.name: $es_cluster\nxpack.monitoring.enabled: $monitoring_enabled\nxpack.monitoring.collection.enabled: $monitoring_enabled\npath.data: $elasticsearch_data_dir\npath.logs: $elasticsearch_logs_dir\nxpack.security.enabled: $security_enabled\nEOF\n\n# Configure log4j retention and level\nsudo sed -i \"21 s,.*,appender.rolling.policies.size.size=${log_size}MB,\" /etc/elasticsearch/log4j2.properties\nsudo sed -i \"55 s,.*,rootLogger.level = $log_level,\" /etc/elasticsearch/log4j2.properties\n\n# If security enabled\nif [ \"$security_enabled\" == \"true\" ]; then\n\n mkdir -p /etc/elasticsearch/config/certs/\n\n echo -n \"$ca_cert\" > /etc/elasticsearch/config/certs/ca.crt\n echo -n \"$node_cert\" > /etc/elasticsearch/config/certs/tls.crt\n echo -n \"$node_key\" > /etc/elasticsearch/config/certs/tls.key\n\n cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nxpack.security.transport.ssl.enabled: true\nxpack.security.transport.ssl.verification_mode: \"certificate\"\nxpack.security.transport.ssl.key: \"/etc/elasticsearch/config/certs/tls.key\"\nxpack.security.transport.ssl.certificate: \"/etc/elasticsearch/config/certs/tls.crt\"\nxpack.security.transport.ssl.certificate_authorities: \"/etc/elasticsearch/config/certs/ca.crt\"\nEOF\nfi\n\nif [ \"$xpack_monitoring_host\" != \"self\" ]; then\ncat <>/etc/elasticsearch/elasticsearch.yml\nxpack.monitoring.exporters.xpack_remote:\n type: http\n host: \"$xpack_monitoring_host\"\nEOF\nfi\n\n\ncat <<'EOF' >>/etc/security/limits.conf\n\n# allow user 'elasticsearch' mlockall\nelasticsearch soft memlock unlimited\nelasticsearch hard memlock unlimited\nEOF\n\nsudo mkdir -p /etc/systemd/system/elasticsearch.service.d\ncat <<'EOF' >>/etc/systemd/system/elasticsearch.service.d/override.conf\n[Service]\nLimitMEMLOCK=infinity\nRestart=always\nRestartSec=10\nEOF\n\n# Setup heap size and memory locking\nsudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/init.d/elasticsearch\nsudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/default/elasticsearch\n\n# Set java heap size\nif [ -d \"/etc/elasticsearch/jvm.options.d\" ]\nthen\n # For versions 7.11 and newer, heap settings are saved in a dedicated file in jvm.options.d\n cat <>/etc/elasticsearch/jvm.options.d/heap.options\n-Xms${heap_size}\n-Xmx${heap_size}\nEOF\n\n# Mitigate log4j lookup exploit\n cat <>/etc/elasticsearch/jvm.options.d/log4j.options\n-Dlog4j2.formatMsgNoLookups=true\n-XX:-HeapDumpOnOutOfMemoryError\nEOF\n\nelse\n # Pre 7.11\n sudo sed -i \"s/^-Xms.*/-Xms$heap_size/\" /etc/elasticsearch/jvm.options\n sudo sed -i \"s/^-Xmx.*/-Xmx$heap_size/\" /etc/elasticsearch/jvm.options\n echo \"-Dlog4j2.formatMsgNoLookups=true\" >> /etc/elasticsearch/jvm.options\n # Disable heap dumps\n echo \"-XX:-HeapDumpOnOutOfMemoryError\" | sudo tee -a /etc/elasticsearch/jvm.options\nfi\n\n# Setup GC\nif [ \"$use_g1gc\" = \"true\" ]; then\n sudo sed -i -re 's/# ([0-9]+-[0-9]+:-XX:-UseConcMarkSweepGC)/\\1/ig' /etc/elasticsearch/jvm.options\n sudo sed -i -re 's/# ([0-9]+-[0-9]+:-XX:-UseCMSInitiatingOccupancyOnly)/\\1/ig' /etc/elasticsearch/jvm.options\n sudo sed -i 's/[0-9]\\+-:-XX:+UseG1GC/10-:-XX:+UseG1GC/ig' /etc/elasticsearch/jvm.options\n sudo sed -i 's/[0-9]\\+-:-XX:G1ReservePercent/10-:-XX:G1ReservePercent/ig' /etc/elasticsearch/jvm.options\n sudo sed -i 's/[0-9]\\+-:-XX:InitiatingHeapOccupancyPercent/10-:-XX:InitiatingHeapOccupancyPercent/ig' /etc/elasticsearch/jvm.options\nfi\n\n\n# Create log and data dirs\nsudo mkdir -p $elasticsearch_logs_dir\nsudo mkdir -p $elasticsearch_data_dir\nsudo chown -R elasticsearch:elasticsearch $elasticsearch_logs_dir\nsudo chown -R elasticsearch:elasticsearch $elasticsearch_data_dir\n" }, { "path": "assets/scripts/common/env.sh", "content": "export ES_HOST=\"http://localhost:9200\"\nif [ \"$https_enabled\" == \"true\" ]; then\n export ES_HOST=\"https://localhost:9200\"\nfi\n\nexport CURL_AUTH=\"\"\nif [ \"$security_enabled\" == \"true\" ]; then\n export CURL_AUTH=\" --user elastic:$client_pwd \"\nfi" }, { "path": "assets/scripts/data.sh", "content": "#!/bin/bash\nset +e\n\n. /opt/cloud-deploy-scripts/common/env.sh\n. /opt/cloud-deploy-scripts/$cloud_provider/env.sh\n\n/opt/cloud-deploy-scripts/$cloud_provider/autoattach-disk.sh\n\n/opt/cloud-deploy-scripts/common/config-es.sh\n/opt/cloud-deploy-scripts/common/config-beats.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh\n\nif [ \"$is_voting_only\" == \"true\" ]\nthen\n cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnode.roles: [ data_hot, data_content, ingest, transform, master, voting_only, remote_cluster_client]\nEOF\nelse\n cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnode.roles: [ data_hot, data_content, ingest, transform, remote_cluster_client ]\nEOF\nfi\n\n# Start Elasticsearch\nsystemctl daemon-reload\nsystemctl enable elasticsearch.service\nsystemctl start elasticsearch.service\n" }, { "path": "assets/scripts/gcp/autoattach-disk.sh", "content": "# Required variables\n# - GCP_ZONE\n# - es_cluster\n# - elasticsearch_data_dir\n\nwhile true; do\n INSTANCE_ROLE=\"$(gcloud compute instances describe $HOSTNAME --zone $GCP_ZONE --format json | jq -r \".labels.role\")\"\n echo \"INSTANCE_ROLE: $INSTANCE_ROLE\"\n UNATTACHED_VOLUME_ID=\"$(gcloud compute disks list --filter=\"zone=$GCP_ZONE AND labels.cluster-name=$es_cluster AND labels.auto-attach-group=$INSTANCE_ROLE\" --format json | jq -r '.[] | .name' | shuf -n 1)\"\n echo \"UNATTACHED_VOLUME_ID: $UNATTACHED_VOLUME_ID\"\n gcloud compute instances attach-disk $HOSTNAME --disk $UNATTACHED_VOLUME_ID --device-name \"espersistent\" --zone $GCP_ZONE\n if [ \"$?\" == \"0\" ]; then\n break\n fi\n\n sleep 30\ndone\n\necho 'Waiting for 30 seconds for the disk to become mountable...'\nsleep 30\n\nsudo mkdir -p $elasticsearch_data_dir\nexport DEVICE_NAME=$(lsblk -ip | tail -n +2 | grep -v \" rom\" | awk '{print $1 \" \" ($7? \"MOUNTEDPART\" : \"\") }' | sed ':a;N;$!ba;s/\\n`/ /g' | sed ':a;N;$!ba;s/\\n|-/ /g' | grep -v MOUNTEDPART)\nif sudo mount -o defaults -t ext4 $DEVICE_NAME $elasticsearch_data_dir; then\n echo 'Successfully mounted existing disk'\nelse\n echo 'Trying to mount a fresh disk'\n sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard $DEVICE_NAME\n sudo mount -o defaults -t ext4 $DEVICE_NAME $elasticsearch_data_dir && echo 'Successfully mounted a fresh disk'\nfi\necho \"$DEVICE_NAME $elasticsearch_data_dir ext4 defaults,nofail 0 2\" | sudo tee -a /etc/fstab\nsudo chown -R elasticsearch:elasticsearch $elasticsearch_data_dir\n" }, { "path": "assets/scripts/gcp/config-bootstrap-node.sh", "content": "# Required variables\n# - es_environment\n# - masters_count\ni=1\nwhile true\ndo\n echo \"Fetching masters...\"\n\n MASTER_INSTANCES=\"$(gcloud compute instances list --filter=\"labels.cluster=$es_environment AND labels.role=(master OR data-voters)\" --format 'get(networkInterfaces[0].networkIP)' | sort)\"\n COUNT=`echo \"$MASTER_INSTANCES\" | wc -l`\n echo \"Found $COUNT instances, expecting $masters_count, attempt $i\"\n if [ \"$COUNT\" -eq \"$masters_count\" ]; then\n echo \"Masters count is correct... Rechecking in 60 sec\"\n sleep 60\n MASTER_INSTANCES_RECHECK=\"$(gcloud compute instances list --filter=\"labels.cluster=$es_environment AND labels.role=(master OR data-voters)\" --format 'get(networkInterfaces[0].networkIP)' | sort)\"\n if [ \"$MASTER_INSTANCES\" = \"$MASTER_INSTANCES_RECHECK\" ]; then\n break\n fi\n fi\n\n sleep 5\n i=$((i+1))\ndone\n\necho \"Fetched masters\"\nMASTER_IPS=\"$MASTER_INSTANCES\"\nSEED_HOSTS=`echo \"$MASTER_IPS\" | paste -sd ',' -`\n\necho \"discovery.seed_hosts: $SEED_HOSTS\" >>/etc/elasticsearch/elasticsearch.yml\necho \"cluster.initial_master_nodes: $(hostname -I),$SEED_HOSTS\" >>/etc/elasticsearch/elasticsearch.yml\n" }, { "path": "assets/scripts/gcp/config-cluster.sh", "content": "# Required variables\n# - security_enabled\n# - client_pwd\n# - gcs_snapshots_bucket\n# - ES_HOST\n# - CURL_AUTH\n\nif [ \"${gcs_snapshots_bucket}\" != \"\" ]; then\n curl $CURL_AUTH -X PUT \"$ES_HOST/_snapshot/gcs_repo\" -H 'Content-Type: application/json' -d'\n {\n \"type\": \"gcs\",\n \"settings\": {\n \"bucket\": \"'$gcs_snapshots_bucket'\"\n }\n }\n '\nfi\n" }, { "path": "assets/scripts/gcp/config-es-discovery.sh", "content": "# Required variables\n# - gcp_zones\n# - gcp_project_id\n# - BIND_TO_ALL\n\ncat <>/etc/elasticsearch/elasticsearch.yml\nplugin.mandatory: discovery-gce\ncloud.gce.project_id: ${gcp_project_id}\ncloud.gce.zone: ${gcp_zones}\ndiscovery.seed_providers: gce\nEOF\n\nif [ \"$BIND_TO_ALL\" == \"true\" ]; then\n\techo \"network.host: 0.0.0.0\" >> /etc/elasticsearch/elasticsearch.yml\nelse\n\techo \"network.host: _gce_,_gce:hostname_,localhost\" >> /etc/elasticsearch/elasticsearch.yml\nfi\n" }, { "path": "assets/scripts/gcp/config-es.sh", "content": "if [ \"${gcs_snapshots_bucket}\" != \"\" ]; then\n\techo \"$gcs_service_account_key\" | base64 -d > /tmp/gcs-snapshots-service-account.json\n\t/usr/share/elasticsearch/bin/elasticsearch-keystore add-file gcs.client.default.credentials_file /tmp/gcs-snapshots-service-account.json\n rm /tmp/gcs-snapshots-service-account.json\nfi" }, { "path": "assets/scripts/gcp/env.sh", "content": "# gcloud cli sometimes fails if you use it right after the instance has started up\n# adding a retry for that case\nwhile true;\ndo\n\texport GCP_ZONE=\"$(gcloud compute instances list --filter=\"name=('\"$HOSTNAME\"')\" --format \"value(zone)\")\"\n\tif [ \"$GCP_ZONE\" != \"\" ]; then\n\t\tbreak\n\tfi\n\techo \"Failed to detect GCP_ZONE. Retrying in 5 seconds...\"\n\tsleep 5\ndone" }, { "path": "assets/scripts/master.sh", "content": "#!/bin/bash\nset +e\n\n. /opt/cloud-deploy-scripts/common/env.sh\n. /opt/cloud-deploy-scripts/$cloud_provider/env.sh\n\n/opt/cloud-deploy-scripts/$cloud_provider/autoattach-disk.sh\n\n/opt/cloud-deploy-scripts/common/config-es.sh\n/opt/cloud-deploy-scripts/common/config-beats.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh\n\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnode.roles: [ master ]\nEOF\n\n# Start Elasticsearch\nsystemctl daemon-reload\nsystemctl enable elasticsearch.service\nsystemctl start elasticsearch.service\n" }, { "path": "assets/scripts/singlenode.sh", "content": "#!/bin/bash\nset +e\n\n. /opt/cloud-deploy-scripts/common/env.sh\n. /opt/cloud-deploy-scripts/$cloud_provider/env.sh\n\n# It is required to bind to all interfaces for load balancer on GCP to work\nif [ \"$cloud_provider\" == \"gcp\" ]; then\n export BIND_TO_ALL=\"true\"\nfi\n\n/opt/cloud-deploy-scripts/$cloud_provider/autoattach-disk.sh\n\n/opt/cloud-deploy-scripts/common/config-es.sh\n/opt/cloud-deploy-scripts/common/config-beats.sh\n\n/opt/cloud-deploy-scripts/$cloud_provider/config-es.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh\n\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnode.roles: [ data_hot, data_content, ingest, transform, master, remote_cluster_client ]\ndiscovery.type: single-node\nEOF\n\n/opt/cloud-deploy-scripts/common/config-clients.sh\n\n# add bootstrap.password to the keystore, so that config-cluster scripts can run\n# only done on bootstrap and singlenode nodes, before starting ES\nif [ \"${security_enabled}\" == \"true\" ]; then\n echo \"${client_pwd}\" | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin bootstrap.password\nfi\n\n# Start Elasticsearch\nsystemctl daemon-reload\nsystemctl enable elasticsearch.service\nsystemctl start elasticsearch.service\n\n/opt/cloud-deploy-scripts/common/config-cluster.sh\n/opt/cloud-deploy-scripts/$cloud_provider/config-cluster.sh\n" }, { "path": "packer/README.md", "content": "# Elasticsearch and Kibana machine images\n\nThis Packer configuration will generate Ubuntu images with Elasticsearch, Kibana and other important tools for deploying and managing Elasticsearch clusters on the cloud.\n\nThe output of running Packer here would be two machine images, as below:\n\n* elasticsearch node image, containing latest Elasticsearch installed (latest version 7.x) and configured with best-practices.\n* kibana node image, based on the elasticsearch node image, and with Kibana (7.x, latest).\n\n## On Amazon Web Services (AWS)\n\nUsing the AWS builder will create the two images and store them as AMIs.\n\nAs a convention the Packer builders will use a dedicated IAM roles, which you will need to have present.\n\n```bash\naws iam create-role --role-name packer --assume-role-policy-document '{\n \"Version\": \"2012-10-17\",\n \"Statement\": {\n \"Effect\": \"Allow\",\n \"Principal\": {\"Service\": \"ec2.amazonaws.com\"},\n \"Action\": \"sts:AssumeRole\",\n \"Sid\": \"\"\n }\n}'\n```\n\nResponse will look something like this:\n\n```json\n{\n \"Role\": {\n \"AssumeRolePolicyDocument\": {\n \"Version\": \"2012-10-17\",\n \"Statement\": {\n \"Action\": \"sts:AssumeRole\",\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n },\n \"RoleId\": \"AROAJ7Q2L7NZJHZBB6JKY\",\n \"CreateDate\": \"2016-12-16T13:22:47.254Z\",\n \"RoleName\": \"packer\",\n \"Path\": \"/\",\n \"Arn\": \"arn:aws:iam::611111111117:role/packer\"\n }\n}\n```\n\nFollow up by execting the following\n\n```bash\naws iam create-instance-profile --instance-profile-name packer\naws iam add-role-to-instance-profile --instance-profile-name packer --role-name packer\n\n```\n\nBy default, AWS builder will pick a subnet from the default VPC for running the builder instance. It is required for that subnet to have Public IPs auto-assignment enabled. Otherwise, packer won't be able to make a SSH connection to the instance and will hang on `Waiting for SSH to become available...` \nIf you don't want to enable public IPs auto-assignment on your default VPC subnets, you can explicitly set the subnet by setting `vpc_id` and `subnet_id` keys in *.packer.json files `amazon-ebs` builder definitions.\n\n## On Microsoft Azure\n\nBefore running Packer for the first time you will need to do a one-time initial setup.\n\nUse PowerShell, and login to AzureRm. See here for more details: https://docs.microsoft.com/en-us/powershell/azure/authenticate-azureps. Once logged in, take note of the subscription and tenant IDs which will be printed out. Alternatively, you can retrieve them by running `Get-AzureRmSubscription` once logged-in.\n\n```Powershell\n$rgName = \"packer-elasticsearch-images\"\n$location = \"East US\"\nNew-AzureRmResourceGroup -Name $rgName -Location $location\n$Password = ([char[]]([char]33..[char]95) + ([char[]]([char]97..[char]126)) + 0..9 | sort {Get-Random})[0..8] -join ''\n\"Password: \" + $Password\n$sp = New-AzureRmADServicePrincipal -DisplayName \"Azure Packer IKF\" -Password $Password\nNew-AzureRmRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $sp.ApplicationId\n$sp.ApplicationId\n```\n\nNote the resource group name, location, password, sp.ApplicationId as used in the script and emitted as output and update `variables.json`.\n\nTo learn more about using Packer on Azure see https://docs.microsoft.com/en-us/azure/virtual-machines/windows/build-image-with-packer\n\nSimilarly, using the Azure CLI is going to look something like below:\n\n```bash\nexport rgName=packer-elasticsearch-images\naz group create -n ${rgName} -l eastus\n\naz ad sp create-for-rbac --query \"{ client_id: appId, client_secret: password, tenant_id: tenant }\"\n# outputs client_id, client_secret and tenant_id\naz account show --query \"{ subscription_id: id }\"\n# outputs subscription_id\n```\n\n## Building\nInstall packer.\nhttps://developer.hashicorp.com/packer/tutorials/docker-get-started/get-started-install-cli\nAlternatively, install pkenv which allows better control over the installed version, and then install packer. \nhttps://github.com/iamhsa/pkenv\n\nInstall the relevant plugin by running one of the following:\n```\npacker plugins install github.com/hashicorp/amazon\npacker plugins install github.com/hashicorp/googlecompute\npacker plugins install github.com/hashicorp/azure\n```\n\nBuilding the AMIs is done using the following commands:\n\n```bash\npacker build -only=aws -var-file=variables.json elasticsearch7-node.packer.json\npacker build -only=aws -var-file=variables.json kibana7-node.packer.json\n```\n\nReplace the `-only` parameter to `azure` to build images for Azure instead of AWS.\n\nFor creating the Kibana image in azure, make sure you update \"azure_elasticsearch_image_name\" in variables.json. You can see the value in the output for the creation of the Elasticsearch image.\n" }, { "path": "packer/elasticsearch7-node.packer.json", "content": "{\n \"description\": \"Elasticsearch Image\",\n \"builders\": [\n {\n \"name\": \"aws\",\n \"type\": \"amazon-ebs\",\n \"ami_name\": \"elasticsearch7-{{isotime | clean_resource_name}}\",\n \"availability_zone\": \"{{user `aws_az`}}\",\n \"iam_instance_profile\": \"packer\",\n \"instance_type\": \"t2.micro\",\n \"region\": \"{{user `aws_region`}}\",\n \"run_tags\": {\n \"role\": \"packer\"\n },\n \"source_ami_filter\": {\n \"filters\": {\n \"virtualization-type\": \"hvm\",\n \"name\": \"*ubuntu-jammy-22.04-amd64-server-*\",\n \"root-device-type\": \"ebs\"\n },\n \"owners\": [\"099720109477\"],\n \"most_recent\": true\n },\n \"ssh_timeout\": \"10m\",\n \"ssh_username\": \"ubuntu\",\n \"tags\": {\n \"ImageType\": \"elasticsearch7-packer-image\"\n }\n },\n {\n \"name\": \"azure\",\n \"type\": \"azure-arm\",\n\n \"client_id\": \"{{user `azure_client_id`}}\",\n \"client_secret\": \"{{user `azure_client_secret`}}\",\n \"tenant_id\": \"{{user `azure_tenant_id`}}\",\n \"subscription_id\": \"{{user `azure_subscription_id`}}\",\n\n \"managed_image_resource_group_name\": \"{{user `azure_resource_group_name`}}\",\n \"managed_image_name\": \"elasticsearch7-{{isotime \\\"2006-01-02T030405\\\"}}\",\n\n \"os_type\": \"Linux\",\n \"image_publisher\": \"Canonical\",\n \"image_offer\": \"UbuntuServer\",\n \"image_sku\": \"18.04-LTS\",\n\n \"location\": \"{{user `azure_location`}}\",\n \"vm_size\": \"Standard_DS2_v2\"\n },\n {\n \"name\": \"gcp\",\n \"type\": \"googlecompute\",\n \"account_file\": \"{{user `gcp_account_file`}}\",\n \"project_id\": \"{{user `gcp_project_id`}}\",\n \"source_image_family\": \"ubuntu-2204-lts\",\n \"zone\": \"{{user `gcp_zone`}}\",\n \"image_family\": \"elasticsearch-7\",\n \"image_name\": \"elasticsearch7-{{isotime | clean_resource_name}}\",\n \"preemptible\": true,\n \"ssh_username\": \"ubuntu\"\n }\n ],\n \"provisioners\": [\n {\n \"type\": \"file\",\n \"source\": \"../assets/scripts\",\n \"destination\": \"/tmp\",\n \"only\": [\"aws\", \"gcp\"]\n },\n {\n \"type\": \"shell\",\n \"inline\": [\n \"sudo mkdir -p /opt/cloud-deploy-scripts\",\n \"sudo mv /tmp/scripts/* /opt/cloud-deploy-scripts\",\n \"sudo chmod +x -R /opt/cloud-deploy-scripts\"\n ],\n \"only\": [\"aws\", \"gcp\"]\n },\n {\n \"type\": \"shell\",\n \"script\": \"update-machine.sh\",\n \"execute_command\": \"echo '' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'\"\n },\n {\n \"type\": \"file\",\n \"source\": \"../assets/elasticsearch.yml\",\n \"destination\": \"elasticsearch.yml\"\n },\n {\n \"type\": \"shell\",\n \"script\": \"install-elasticsearch7.sh\",\n \"environment_vars\": [ \"ES_VERSION={{user `elasticsearch_version`}}\" ],\n \"execute_command\": \"echo '' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'\"\n },\n {\n \"type\": \"shell\",\n \"script\": \"install-cloud-plugin.sh\",\n \"execute_command\": \"echo '' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'\"\n },\n {\n \"type\": \"shell\",\n \"environment_vars\": [ \"ES_VERSION={{user `elasticsearch_version`}}\" ],\n \"script\": \"install-beats.sh\"\n },\n {\n \"type\": \"shell\",\n \"script\": \"install-custom.sh\"\n }\n ],\n \"post-processors\": [\n {\n \"type\": \"manifest\",\n \"output\": \"packer-es-manifest.json\"\n }\n ]\n}\n" }, { "path": "packer/install-beats.sh", "content": "#!/bin/bash\nset -e\n\nES_VERSION=\"${ES_VERSION:-7.9.0}\"\n\ncurl -L -O \"https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-$ES_VERSION-amd64.deb\"\nsudo dpkg -i \"filebeat-$ES_VERSION-amd64.deb\"\nrm \"filebeat-$ES_VERSION-amd64.deb\"\n\ncurl -L -O \"https://artifacts.elastic.co/downloads/beats/heartbeat/heartbeat-$ES_VERSION-amd64.deb\"\nsudo dpkg -i \"heartbeat-$ES_VERSION-amd64.deb\"\nrm \"heartbeat-$ES_VERSION-amd64.deb\"\n\ncurl -L -O \"https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-$ES_VERSION-amd64.deb\"\nsudo dpkg -i \"metricbeat-$ES_VERSION-amd64.deb\"\nrm \"metricbeat-$ES_VERSION-amd64.deb\"" }, { "path": "packer/install-cloud-plugin.sh", "content": "#!/bin/bash\nset -e\n\ncd /usr/share/elasticsearch/\n\nif [[ $PACKER_BUILD_NAME == \"aws\" ]]; then\n sudo bin/elasticsearch-plugin install --batch discovery-ec2\n sudo bin/elasticsearch-plugin install --batch repository-s3\nelif [[ $PACKER_BUILD_NAME == \"azure\" ]]; then\n sudo bin/elasticsearch-plugin install --batch repository-azure\nelif [[ $PACKER_BUILD_NAME == \"gcp\" ]]; then\n sudo bin/elasticsearch-plugin install --batch discovery-gce\n sudo bin/elasticsearch-plugin install --batch repository-gcs\nfi\n" }, { "path": "packer/install-cloudwatch-agent.sh", "content": "curl https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py -O\n\nchmod +x awslogs-agent-setup.py\nsudo mv awslogs-agent-setup.py /usr/bin\n\n#sudo python ./awslogs-agent-setup.py --region us-east-2\n" }, { "path": "packer/install-custom.sh", "content": "#!/bin/bash\nset -e\n\n" }, { "path": "packer/install-elasticsearch7.sh", "content": "#!/bin/bash\nset -e\n\n# Get the PGP Key\nwget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -\necho \"deb https://artifacts.elastic.co/packages/7.x/apt stable main\" | tee -a /etc/apt/sources.list.d/elastic-7.x.list\n\napt-get update\nif [ -z \"$ES_VERSION\" ]; then\n echo \"Installing the latest Elasticsearch version\"\n apt-get install elasticsearch\nelse\n echo \"Installing Elasticsearch version $ES_VERSION\"\n apt-get install elasticsearch=$ES_VERSION\nfi\n\nmkdir /usr/share/elasticsearch/logs\nmkdir /usr/share/elasticsearch/data\nchown elasticsearch:elasticsearch /usr/share/elasticsearch/logs\nchown elasticsearch:elasticsearch /usr/share/elasticsearch/data\n\nmv elasticsearch.yml /etc/elasticsearch/elasticsearch.yml\nchown elasticsearch:elasticsearch /etc/elasticsearch/elasticsearch.yml" }, { "path": "packer/install-kibana7.sh", "content": "#!/bin/bash\nset -e\n\n# Get the PGP Key\n# wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -\n\n# echo \"deb https://artifacts.elastic.co/packages/7.x/apt stable main\" | tee -a /etc/apt/sources.list.d/elastic-7.x.list\n\n# Fix for the apt lock issue\nsleep 100\n\napt-get update\nif [ -z \"$ES_VERSION\" ]; then\n echo \"Installing latest Kibana version\"\n apt-get install kibana\nelse\n echo \"Installing Kibana version $ES_VERSION\"\n apt-get install kibana=$ES_VERSION\nfi\n\n# This needs to be here explicitly because of a long first-initialization time of Kibana\nsystemctl daemon-reload\nsystemctl enable kibana.service\nsudo service kibana start\n" }, { "path": "packer/kibana7-node.packer.json", "content": "{\n \"description\": \"Kibana Image\",\n \"builders\": [\n {\n\n \"name\": \"aws\",\n \"type\": \"amazon-ebs\",\n \"ami_name\": \"kibana7-{{isotime | clean_resource_name}}\",\n \"availability_zone\": \"{{user `aws_az`}}\",\n \"iam_instance_profile\": \"packer\",\n \"instance_type\": \"t2.medium\",\n \"region\": \"{{user `aws_region`}}\",\n \"run_tags\": {\n \"role\": \"packer\"\n },\n \"source_ami_filter\": {\n \"filters\": {\n \"virtualization-type\": \"hvm\",\n \"name\": \"elasticsearch7-*\",\n \"root-device-type\": \"ebs\"\n },\n \"owners\": [\n \"self\"\n ],\n \"most_recent\": true\n },\n \"ssh_timeout\": \"10m\",\n \"ssh_username\": \"ubuntu\",\n \"tags\": {\n \"ImageType\": \"kibana7-packer-image\"\n }\n },\n {\n \"name\": \"azure\",\n \"type\": \"azure-arm\",\n\n \"client_id\": \"{{user `azure_client_id`}}\",\n \"client_secret\": \"{{user `azure_client_secret`}}\",\n \"tenant_id\": \"{{user `azure_tenant_id`}}\",\n \"subscription_id\": \"{{user `azure_subscription_id`}}\",\n\n \"managed_image_resource_group_name\": \"{{user `azure_resource_group_name`}}\",\n \"managed_image_name\": \"kibana7-{{isotime \\\"2006-01-02T030405\\\"}}\",\n\n \"os_type\": \"Linux\",\n \"custom_managed_image_name\": \"{{user `azure_elasticsearch_image_name`}}\",\n \"custom_managed_image_resource_group_name\":\"{{user `azure_resource_group_name`}}\",\n\n \"location\": \"{{user `azure_location`}}\",\n \"vm_size\": \"Standard_DS2_v2\"\n },\n {\n \"name\": \"gcp\",\n \"type\": \"googlecompute\",\n \"account_file\": \"{{user `gcp_account_file`}}\",\n \"project_id\": \"{{user `gcp_project_id`}}\",\n \"source_image_family\": \"elasticsearch-7\",\n \"zone\": \"{{user `gcp_zone`}}\",\n \"image_family\": \"kibana-7\",\n \"image_name\": \"kibana7-{{isotime | clean_resource_name}}\",\n \"preemptible\": true,\n \"ssh_username\": \"ubuntu\"\n }\n ],\n \"provisioners\": [\n {\n \"type\": \"shell\",\n \"script\": \"install-kibana7.sh\",\n \"environment_vars\": [ \"ES_VERSION={{user `elasticsearch_version`}}\" ],\n \"execute_command\": \"echo '' | {{ .Vars }} sudo -E -S sh '{{ .Path }}'\"\n }\n ],\n \"post-processors\": [\n {\n \"type\": \"manifest\",\n \"output\": \"packer-kb-manifest.json\"\n }\n ]\n}\n" }, { "path": "packer/update-machine.sh", "content": "#!/bin/bash\n\nexport DEBIAN_FRONTEND=noninteractive\n\nsudo rm /boot/grub/menu.lst\n\n# https://github.com/hashicorp/packer/issues/2639\necho \"Waiting 100 seconds for cloud-init to finish...\"\nsleep 100\n\nsudo apt-get update\nsudo -E apt-get upgrade -y\nsudo -E apt-get install -y software-properties-common git python3-dev htop ntp jq apt-transport-https unzip\n\nif [[ $PACKER_BUILD_NAME == \"aws\" ]]; then\n\tsudo -E apt-get install -y awscli\nfi\n\n# Disable daily apt unattended updates.\necho 'APT::Periodic::Enable \"0\";' >> /etc/apt/apt.conf.d/10periodic\n" }, { "path": "packer/variables.json", "content": "{\n \"elasticsearch_version\": \"\",\n\n \"aws_region\": \"us-east-1\",\n \"aws_az\": \"us-east-1d\",\n\n \"azure_client_id\": \"\",\n \"azure_client_secret\": \"\",\n \"azure_subscription_id\": \"\",\n \"azure_tenant_id\": \"\",\n\n \"gcp_zone\": \"us-central1-a\",\n \"gcp_project_id\": \"elasticsearch\",\n \"gcp_account_file\": \".gcp_account.json\",\n\n \"azure_location\": \"East US\",\n \"azure_resource_group_name\": \"packer-elasticsearch-images\",\n \"azure_elasticsearch_image_name\": \"\"\n}\n" }, { "path": "templates/aws_user_data.sh", "content": "#!/bin/bash\n\nexec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n\nif [ \"${DEV_MODE_scripts_s3_bucket}\" != \"\" ]; then\n\tsudo aws s3 cp --recursive \"s3://${DEV_MODE_scripts_s3_bucket}\" /opt/cloud-deploy-scripts/\n\tsudo chmod -R +x /opt/cloud-deploy-scripts\nfi\n\nexport cloud_provider=\"${cloud_provider}\"\nexport elasticsearch_data_dir=\"${elasticsearch_data_dir}\"\nexport elasticsearch_logs_dir=\"${elasticsearch_logs_dir}\"\nexport heap_size=\"${heap_size}\"\nexport is_voting_only=\"${is_voting_only}\"\nexport es_cluster=\"${es_cluster}\"\nexport es_environment=\"${es_environment}\"\nexport security_groups=\"${security_groups}\"\nexport aws_region=\"${aws_region}\"\nexport use_g1gc=\"${use_g1gc}\"\nexport security_enabled=\"${security_enabled}\"\nexport monitoring_enabled=\"${monitoring_enabled}\"\nexport masters_count=\"${masters_count}\"\nexport client_user=\"${client_user}\"\nexport s3_backup_bucket=\"${s3_backup_bucket}\"\nexport xpack_monitoring_host=\"${xpack_monitoring_host}\"\nexport filebeat_monitoring_host=\"${filebeat_monitoring_host}\"\nexport client_pwd=\"${client_pwd}\"\nexport master=\"${master}\"\nexport data=\"${data}\"\nexport bootstrap_node=\"${bootstrap_node}\"\nexport ca_cert=\"${ca_cert}\"\nexport node_cert=\"${node_cert}\"\nexport node_key=\"${node_key}\"\nexport log_level=\"${log_level}\"\nexport log_size=\"${log_size}\"\nexport security_encryption_key=\"${security_encryption_key}\"\nexport reporting_encryption_key=\"${reporting_encryption_key}\"\nexport auto_shut_down_bootstrap_node=\"${auto_shut_down_bootstrap_node}\"\n\n/opt/cloud-deploy-scripts/${startup_script}\n" }, { "path": "templates/gcp_user_data.sh", "content": "#!/bin/bash\n\nexec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n\nif [ \"${DEV_MODE_scripts_gcs_bucket}\" != \"\" ]; then\n\tsudo gsutil cp -r \"gs://${DEV_MODE_scripts_gcs_bucket}/*\" /opt/cloud-deploy-scripts/\n\tsudo chmod -R +x /opt/cloud-deploy-scripts\nfi\n\nexport cloud_provider=\"${cloud_provider}\"\nexport gcp_zones=\"${gcp_zones}\"\nexport gcp_project_id=\"${gcp_project_id}\"\nexport gcs_snapshots_bucket=\"${gcs_snapshots_bucket}\"\nexport gcs_service_account_key=\"${gcs_service_account_key}\"\nexport elasticsearch_data_dir=\"${elasticsearch_data_dir}\"\nexport elasticsearch_logs_dir=\"${elasticsearch_logs_dir}\"\nexport heap_size=\"${heap_size}\"\nexport is_voting_only=\"${is_voting_only}\"\nexport es_cluster=\"${es_cluster}\"\nexport es_environment=\"${es_environment}\"\nexport use_g1gc=\"${use_g1gc}\"\nexport security_enabled=\"${security_enabled}\"\nexport monitoring_enabled=\"${monitoring_enabled}\"\nexport masters_count=\"${masters_count}\"\nexport client_user=\"${client_user}\"\nexport xpack_monitoring_host=\"${xpack_monitoring_host}\"\nexport filebeat_monitoring_host=\"${filebeat_monitoring_host}\"\nexport client_pwd=\"${client_pwd}\"\nexport master=\"${master}\"\nexport data=\"${data}\"\nexport bootstrap_node=\"${bootstrap_node}\"\nexport ca_cert=\"${ca_cert}\"\nexport node_cert=\"${node_cert}\"\nexport node_key=\"${node_key}\"\nexport log_level=\"${log_level}\"\nexport log_size=\"${log_size}\"\nexport security_encryption_key=\"${security_encryption_key}\"\nexport reporting_encryption_key=\"${reporting_encryption_key}\"\nexport auto_shut_down_bootstrap_node=\"${auto_shut_down_bootstrap_node}\"\n/opt/cloud-deploy-scripts/${startup_script}\n" }, { "path": "templates/user_data.sh", "content": "#!/bin/bash\n\nexec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n\nfunction fetch_master_nodes_ips() {\n if [ \"${cloud_provider}\" == \"aws\" ]; then\n local master_instance_ids=\"$(aws ec2 describe-instances --region=${aws_region} --filters Name=instance-state-name,Values=running Name=tag:Role,Values=master Name=tag:Cluster,Values=${es_environment} | jq -r '.Reservations | map(.Instances[].InstanceId) | .[]' | sort)\"\n echo \"$(aws ec2 describe-instances --region ${aws_region} --instance-ids $master_instance_ids | jq -r '.Reservations[].Instances[].PrivateIpAddress' | sort)\"\n fi\n\n if [ \"${cloud_provider}\" == \"gcp\" ]; then\n echo \"$(gcloud compute instances list --filter 'tags.items=es-master-node AND tags.items=${es_cluster}' --format 'get(networkInterfaces[0].networkIP)' | sort)\"\n fi\n}\n\nif [ \"${cloud_provider}\" == \"azure\" ] || [ \"${cloud_provider}\" == \"gcp\" ]; then\n # Change node name to AWS-like hostname\n sudo sed -i -e \"s/node.name: .*$/node.name: ip-$(hostname -I | tr . -)/ig\" /etc/elasticsearch/elasticsearch.yml\nfi\n\nif [ \"${bootstrap_node}\" == \"true\" ]; then\n while true\n do\n echo \"Fetching masters...\"\n MASTER_INSTANCES=\"$(fetch_master_nodes_ips)\"\n COUNT=`echo \"$MASTER_INSTANCES\" | wc -l`\n\n if [ \"$COUNT\" -eq \"${masters_count}\" ]; then\n echo \"Masters count is correct... Rechecking in 60 sec\"\n sleep 60\n MASTER_INSTANCES_RECHECK=\"$(fetch_master_nodes_ips)\"\n\n if [ \"$MASTER_INSTANCES\" = \"$MASTER_INSTANCES_RECHECK\" ]; then\n break\n fi\n fi\n\n sleep 5\n done\n\n echo \"Fetched masters\"\n MASTER_IPS=\"$MASTER_INSTANCES\"\n SEED_HOSTS=`echo \"$MASTER_IPS\" | paste -sd ',' -`\n INITIAL_MASTER_NODES=`echo \"$MASTER_IPS\" | awk '{print \"ip-\" $0}' | tr . - | paste -sd ',' -`\nfi\n\n# Configure elasticsearch\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\ncluster.name: ${es_cluster}\n\n# only data nodes should have ingest and http capabilities\nnode.master: ${master}\nnode.data: ${data}\nnode.ingest: ${data}\nxpack.security.enabled: ${security_enabled}\nxpack.monitoring.enabled: false\npath.data: ${elasticsearch_data_dir}\npath.logs: ${elasticsearch_logs_dir}\nEOF\n\nif [ \"${bootstrap_node}\" == \"true\" ]; then\n echo \"discovery.seed_hosts: $SEED_HOSTS\" >>/etc/elasticsearch/elasticsearch.yml\n echo \"cluster.initial_master_nodes: $(hostname -I),$INITIAL_MASTER_NODES\" >>/etc/elasticsearch/elasticsearch.yml\nfi\n\nif [ \"${master}\" == \"true\" ] && [ \"${data}\" == \"true\" ]; then\n echo \"discovery.type: single-node\" >>/etc/elasticsearch/elasticsearch.yml\nfi\n\nif [ \"${monitoring_enabled}\" == \"true\" ]; then\n cat <<'EOF' >/etc/metricbeat/metricbeat.yml\nmetricbeat.modules:\n- module: elasticsearch\n period: 10s\n hosts: [\"http://localhost:9200\"]\n #username: \"elastic\"\n #password: \"changeme\"\n #ssl.certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n\n # Set to true to send data collected by module to X-Pack\n # Monitoring instead of metricbeat-* indices.\n xpack.enabled: true\n- module: system\n metricsets:\n - cpu # CPU usage\n - load # CPU load averages\n - memory # Memory usage\n - network # Network IO\n #- process # Per process metrics\n #- process_summary # Process summary\n #- uptime # System Uptime\n - socket_summary # Socket summary\n #- core # Per CPU core usage\n - diskio # Disk IO\n #- filesystem # File system usage for each mountpoint\n #- fsstat # File system summary metrics\n #- raid # Raid\n #- socket # Sockets and connection info (linux only)\n #- service # systemd service information\n enabled: true\n period: 10s\n processes: ['.*']\n\noutput.elasticsearch:\n enabled: true\n\n # Array of hosts to connect to.\n # Scheme and port can be left out and will be set to the default (http and 9200)\n # In case you specify and additional path, the scheme is required: http://localhost:9200/path\n # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200\n hosts: [\"${xpack_monitoring_host}\"]\nEOF\nfi\n\nif [ \"${cloud_provider}\" == \"aws\" ]; then\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\n\nnetwork.host: _ec2:privateIpv4_,localhost\nplugin.mandatory: discovery-ec2\ncloud.node.auto_attributes: true\ncluster.routing.allocation.awareness.attributes: aws_availability_zone\ndiscovery:\n seed_providers: ec2\n ec2.groups: ${security_groups}\n ec2.host_type: private_ip\n ec2.tag.Cluster: ${es_environment}\n ec2.availability_zones: ${availability_zones}\n ec2.protocol: http # no need in HTTPS for internal AWS calls\n\n # manually set the endpoint because of auto-discovery issues\n # https://github.com/elastic/elasticsearch/issues/27464\n ec2.endpoint: ec2.${aws_region}.amazonaws.com\nEOF\nfi\n\nif [ \"${cloud_provider}\" == \"gcp\" ]; then\ncat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\n\nnetwork.host: _gce_,localhost\nplugin.mandatory: discovery-gce\ncloud.gce.project_id: ${gcp_project_id}\ncloud.gce.zone: ${gcp_zone}\ndiscovery.seed_providers: gce\nEOF\nfi\n\n# Azure doesn't have a proper discovery plugin, hence we are going old-school and relying on scaleset name prefixes\nif [ \"${cloud_provider}\" == \"azure\" ]; then\n cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\nnetwork.host: _site_,localhost\n\n# For discovery we are using predictable hostnames (thanks for the computer name prefix), but could just as well use the\n# predictable subnet addresses starting at 10.1.0.5.\nEOF\n\n # avoiding discovery noise in single-node scenario\n if [ \"${master}\" == \"true\" ] && [ \"${data}\" == \"true\" ]; then\n cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\ndiscovery.seed_hosts: [\"${es_cluster}-master000000\", \"${es_cluster}-data000000\"]\nEOF\n else\n cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml\ndiscovery.seed_hosts: [\"${es_cluster}-master000000\", \"${es_cluster}-master000001\", \"${es_cluster}-master000002\", \"${es_cluster}-data000000\", \"${es_cluster}-data000001\"]\nEOF\n fi\nfi\n\ncat <<'EOF' >>/etc/security/limits.conf\n\n# allow user 'elasticsearch' mlockall\nelasticsearch soft memlock unlimited\nelasticsearch hard memlock unlimited\nEOF\n\nsudo mkdir -p /etc/systemd/system/elasticsearch.service.d\ncat <<'EOF' >>/etc/systemd/system/elasticsearch.service.d/override.conf\n[Service]\nLimitMEMLOCK=infinity\nRestart=always\nRestartSec=10\nEOF\n\n# Setup heap size and memory locking\nsudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/init.d/elasticsearch\nsudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/default/elasticsearch\nsudo sed -i \"s/^-Xms.*/-Xms${heap_size}/\" /etc/elasticsearch/jvm.options\nsudo sed -i \"s/^-Xmx.*/-Xmx${heap_size}/\" /etc/elasticsearch/jvm.options\n\n# Setup GC\nsudo sed -i \"s/^-XX:+UseConcMarkSweepGC/-XX:+UseG1GC/\" /etc/elasticsearch/jvm.options\n\n# Storage\nsudo mkdir -p ${elasticsearch_logs_dir}\nsudo chown -R elasticsearch:elasticsearch ${elasticsearch_logs_dir}\n\n# # we are assuming volume is declared and attached when data_dir is passed to the script\nif { [ \"${master}\" == \"true\" ] || [ \"${data}\" == \"true\" ]; } && [ \"${bootstrap_node}\" != \"true\" ]; then\n sudo mkdir -p ${elasticsearch_data_dir}\n\n export DEVICE_NAME=$(lsblk -ip | tail -n +2 | grep -v \" rom\" | awk '{print $1 \" \" ($7? \"MOUNTEDPART\" : \"\") }' | sed ':a;N;$!ba;s/\\n`/ /g' | sed ':a;N;$!ba;s/\\n|-/ /g' | grep -v MOUNTEDPART)\n if sudo mount -o defaults -t ext4 $DEVICE_NAME ${elasticsearch_data_dir}; then\n echo 'Successfully mounted existing disk'\n else\n echo 'Trying to mount a fresh disk'\n sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard $DEVICE_NAME\n sudo mount -o defaults -t ext4 $DEVICE_NAME ${elasticsearch_data_dir} && echo 'Successfully mounted a fresh disk'\n fi\n echo \"$DEVICE_NAME ${elasticsearch_data_dir} ext4 defaults,nofail 0 2\" | sudo tee -a /etc/fstab\n sudo chown -R elasticsearch:elasticsearch ${elasticsearch_data_dir}\nfi\n\nif [ -f \"/etc/nginx/nginx.conf\" ]; then\n # Setup basic auth for nginx web front and start the service if exists\n sudo htpasswd -bc /etc/nginx/conf.d/search.htpasswd ${client_user} \"${client_pwd}\"\n sudo service nginx start\nfi\n\n# Start Elasticsearch\nsystemctl daemon-reload\nsystemctl enable elasticsearch.service\nsystemctl start elasticsearch.service\n\nif [ \"${bootstrap_node}\" == \"true\" ]; then\n while true\n do\n echo \"Checking cluster health\"\n HEALTH=\"$(curl --silent http://localhost:9200/_cluster/health | jq -r '.status')\"\n if [ \"$HEALTH\" = \"green\" ]; then\n break\n fi\n sleep 5\n done\n\n if [ \"${cloud_provider}\" == \"aws\" ]; then\n # AWS instance is set to terminate after shutdown automatically\n shutdown -h now\n fi\n\n if [ \"${cloud_provider}\" == \"gcp\" ]; then\n INSTANCE_NAME=\"$(gcloud compute instances list --filter 'tags.items=es-bootstrap-node AND tags.items=${es_cluster}' --format 'get(name)')\"\n gcloud compute instances delete $INSTANCE_NAME --zone ${gcp_zone} --quiet\n fi\nelse\n # Setup x-pack security also on Kibana configs where applicable\n if [ -f \"/etc/kibana/kibana.yml\" ]; then\n echo \"xpack.security.enabled: ${security_enabled}\" | sudo tee -a /etc/kibana/kibana.yml\n echo \"xpack.monitoring.enabled: ${monitoring_enabled}\" | sudo tee -a /etc/kibana/kibana.yml\n systemctl daemon-reload\n systemctl enable kibana.service\n sudo service kibana restart\n fi\n\n if [ -f \"/etc/nginx/nginx.conf\" ]; then\n sudo rm /etc/grafana/grafana.ini\n cat <<'EOF' >>/etc/grafana/grafana.ini\n[security]\nadmin_user = ${client_user}\nadmin_password = ${client_pwd}\nEOF\n sudo /bin/systemctl daemon-reload\n sudo /bin/systemctl enable grafana-server.service\n sudo service grafana-server start\n fi\n\n sleep 60\n if [ `systemctl is-failed elasticsearch.service` == 'failed' ];\n then\n echo \"Elasticsearch unit failed to start\"\n exit 1\n fi\nfi\n" }, { "path": "terraform-aws/README.md", "content": "# AWS deployment\n\n## Create the AMIs with Packer\n\nGo to the packer folder and see the README there. Once you have the AMI IDs, return here and continue with the next steps.\n\n## Create key-pair\n\n```bash\naws ec2 create-key-pair --key-name elasticsearch --query 'KeyMaterial' --output text > elasticsearch.pem\n```\n\n## VPC\n\nCreate a VPC, or use existing. You will need the VPC ID we will use the available subnets within it.\n\n## Configurations\n\nEdit `terraform.tfvars` (syntax `var_name = value` per line) to specify the following:\n\n* `aws_region` - the region where to launch the cluster in.\n* `availability_zones` - at least 2 availability zones in that region.\n* `es_cluster` - the name of the Elasticsearch cluster to launch.\n* `key_name` - the name of the key to use - that key needs to be handy so you can access the machines if needed.\n* `vpc_id` - the ID of the VPC to launch the cluster in.\n\nThe rest of the configurations are mostly around cluster topology and machine types and sizes.\n\n* define clients_subnet_ids/cluster_subnet_ids with a map of availability zones to a list of subnets:\n```\ncluster_subnet_ids = {us-east-1b=[\"subnet-xxxxxxxx\",\"subnet-yyyyyyyy\"]}\n```\n\n* define instance counts with a map of availability zones to counts:\n```\nmasters_count = {\n \"us-east-1a\" = 2,\n \"us-east-1b\" = 1\n}\ndatas_count = {\n \"us-east-1a\" = 1\n}\nclients_count = {\n \"us-east-1a\" = 1\n}\n```\n\nsingle nodes are created by having empty maps for all counts (the default)\n\n* an example for the single node availability zone:\n```\nsinglenode_az = \"us-east-1b\"\n```\n\n### Cluster topology\n\nTwo modes of deployment are supported:\n\n* A recommended configuration, with dedicated master-eligible nodes, data nodes, and client nodes. This is a production-ready and best-practice configuration. See more details in the [official documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html).\n* Single node mode - mostly useful for experimentation\n\nAt this point we consider the role `ingest` as unanimous with `data`, so all data nodes are also ingest nodes.\n\nThe default mode is the single-node mode. To change it to the recommended configuration, edit `terraform.tfvars` and set number of master nodes to 3, data nodes to at least 2, and client nodes to at least 1.\n\nAll nodes with the `client` role will be attached to an ALB, so access to all client nodes can be done via the DNS it exposes.\n\n### Cluster bootstrap\nDeploying a cluster in non single-node mode requires [bootstrapping the cluster](https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-discovery-bootstrap-cluster.html). \nWe do this automatically, by spinning up a special bootstrap node, and terminating it once finished. This only happens once, first time you deploy the cluster. State information on whether cluster is bootstrapped or not is kept in a local file `cluster_bootstrap_state` which is used on later `terraform apply` runs (use `printf 0 > cluster_bootstrap_state` to reset it)\nAfter the bootstrap node has terminated, you can start using the cluster.\n\n### Security groups\n\nBy default we create two security groups - one for the internal cluster nodes (data and master), and one for the client nodes. Your applications need to be in the latter only, and communicate with the cluster via the client nodes only.\n\nIf you prefer using a security group of your own, you can add it to `additional_security_groups` in `terraform.tfvars`.\n\n## Launch the cluster with Terraform\n\nOn first usage, you will need to execute `terraform init` to initialize the terraform providers used.\n\nTo deploy the cluster, or apply any changes to an existing cluster deployed using this project, run:\n\n```bash\nterraform plan\nterraform apply\n```\n\nWhen terraform is done, you should see a lot of output ending with something like this:\n\n```\nApply complete! Resources: 14 added, 0 changed, 0 destroyed.\n\nThe state of your infrastructure has been saved to the path\nbelow. This state is required to modify and destroy your\ninfrastructure, so keep it safe. To inspect the complete state\nuse the `terraform show` command.\n\nState path: terraform.tfstate\n\nOutputs:\n\nclients_dns = internal-es-test-client-lb-963348710.eu-central-1.elb.amazonaws.com\n```\n\nNote `clients_dns` - that's your entry point to the cluster.\n\n### Look around\n\nThe client nodes are the ones exposed to external networks. They provide Kibana, Grafana, Cerebro and direct Elasticsearch access. Client nodes are accessible via their public IPs (depending on your security group / VPC settings) and the DNS of the ALB they are attached to (see above).\n\nClient nodes listen on port 9200 and are password protected depending on your choice (variable `security_enabled`). user is defined in the variable `client_user`, make note of the password as you run terraform.\n\nOn client nodes you will find:\n\n* Kibana access is direct on port 5601 (http://host:5601)\n* [Cerebro](https://github.com/lmenezes/cerebro) (a cluster management UI) is available on http://host:9000/cerebro/\n* For direct Elasticsearch access, go to host:9200\n\nYou can pull the list of instances by their state and role using aws-cli:\n\n```bash\naws ec2 describe-instances --filters Name=instance-state-name,Values=running\naws ec2 describe-instances --filters Name=instance-state-name,Values=running,Name=tag:Role,Values=client\n```\n\nTo login to one of the instances:\n\n```bash\nssh -i elasticsearch.pem ubuntu@{public IP / DNS of the instance}\n```\n\n### Changing cluster size after deployment\n\nTerraform is smart enough to make the least amount of changes possible and resize resources when possible instead of destroying them.\n\nWhen you want to change the cluster configuration (e.g. add more client nodes, data nodes, resize disk or instances, etc) just edit `terraform.tfvars` and run `terraform plan` followed by `terraform apply`.\n" }, { "path": "terraform-aws/alb.tf", "content": "resource \"aws_security_group\" \"elasticsearch-alb-sg\" {\n name = \"${var.es_cluster}-alb-sg\"\n description = \"ElasticSearch Ports for ALB Access\"\n vpc_id = var.vpc_id\n}\n\n# allow ES port access\nresource \"aws_security_group_rule\" \"elasticsearch-alb-sg-ingress-rule-es\" {\n type = \"ingress\"\n protocol = \"tcp\"\n cidr_blocks = [\"0.0.0.0/0\"]\n from_port = 9200\n to_port = 9200\n\n security_group_id = aws_security_group.elasticsearch-alb-sg.id\n}\n\n# allow egress\nresource \"aws_security_group_rule\" \"elasticsearch-alb-sg-egress-rule-all\" {\n type = \"egress\"\n protocol = \"-1\"\n cidr_blocks = [\"0.0.0.0/0\"]\n from_port = 0\n to_port = 0\n\n security_group_id = aws_security_group.elasticsearch-alb-sg.id\n}\n\n# allow Kibana port access\nresource \"aws_security_group_rule\" \"elasticsearch-alb-sg-ingress-rule-kibana\" {\n count = length(keys(var.clients_count)) > 0 || local.singlenode_mode ? 1 : 0\n type = \"ingress\"\n protocol = \"tcp\"\n cidr_blocks = [\"0.0.0.0/0\"]\n from_port = 5601\n to_port = 5601\n\n security_group_id = aws_security_group.elasticsearch-alb-sg.id\n}\n\n# Target Groups\n#-----------------------------------------------------\n\nresource \"aws_lb_target_group\" \"esearch-p9200-tg\" {\n name = \"${var.es_cluster}-p9200-tg\"\n port = 9200\n protocol = \"HTTP\"\n vpc_id = var.vpc_id\n\n health_check {\n healthy_threshold = 5\n unhealthy_threshold = 2\n timeout = 5\n path = \"/\"\n port = 9200\n interval = 15\n matcher = \"401\"\n }\n}\n\nresource \"aws_lb_target_group\" \"kibana-p5601-tg\" {\n count = length(keys(var.clients_count)) > 0 || local.singlenode_mode ? 1 : 0\n name = \"${var.es_cluster}-p5601-tg\"\n port = 5601\n protocol = \"HTTP\"\n vpc_id = var.vpc_id\n\n health_check {\n healthy_threshold = 5\n unhealthy_threshold = 2\n timeout = 5\n path = \"/\"\n port = 5601\n interval = 15\n matcher = \"302\"\n }\n}\n\nresource \"aws_lb\" \"elasticsearch-alb\" {\n name = \"${var.es_cluster}-alb\"\n internal = ! var.public_facing\n load_balancer_type = \"application\"\n security_groups = [aws_security_group.elasticsearch-alb-sg.id]\n subnets = coalescelist(var.alb_subnets, tolist(data.aws_subnets.all-subnets.ids))\n\n enable_deletion_protection = false\n}\n\n#-----------------------------------------------------\n\n# ALB Listeners and Listener Rules\n#-----------------------------------------------------\n\nresource \"aws_lb_listener\" \"esearch\" {\n load_balancer_arn = aws_lb.elasticsearch-alb.arn\n port = \"9200\"\n protocol = \"HTTP\"\n\n default_action {\n type = \"forward\"\n target_group_arn = aws_lb_target_group.esearch-p9200-tg.arn\n }\n}\n\nresource \"aws_lb_listener\" \"kibana\" {\n count = length(keys(var.clients_count)) > 0 || local.singlenode_mode ? 1 : 0\n load_balancer_arn = aws_lb.elasticsearch-alb.arn\n port = \"5601\"\n protocol = \"HTTP\"\n\n default_action {\n type = \"forward\"\n target_group_arn = aws_lb_target_group.kibana-p5601-tg[0].arn\n }\n}\n" }, { "path": "terraform-aws/ami.tf", "content": "// Find the latest available AMI for Elasticsearch\ndata \"aws_ami\" \"elasticsearch\" {\n filter {\n name = \"state\"\n values = [\"available\"]\n }\n filter {\n name = \"tag:ImageType\"\n values = [var.elasticsearch_packer_image]\n }\n most_recent = true\n owners = [\"self\"]\n}\n\n// Find the latest available AMI for the Kibana client node\ndata \"aws_ami\" \"kibana_client\" {\n filter {\n name = \"state\"\n values = [\"available\"]\n }\n filter {\n name = \"tag:ImageType\"\n values = [var.kibana_packer_image]\n }\n most_recent = true\n owners = [\"self\"]\n}\n\n" }, { "path": "terraform-aws/certs.tf", "content": "locals {\n cert_common_name = \"elasticsearch-cloud-deploy autogenerated CA\"\n validity_period_hours = 365 * 24\n early_renewal_hours = 30 * 24\n}\n\nresource \"tls_private_key\" \"ca\" {\n count = var.security_enabled ? 1 : 0\n\n algorithm = \"RSA\"\n}\n\nresource \"tls_self_signed_cert\" \"ca\" {\n count = var.security_enabled ? 1 : 0\n\n #key_algorithm = \"RSA\"\n private_key_pem = join(\"\", tls_private_key.ca[*].private_key_pem)\n\n subject {\n common_name = local.cert_common_name\n }\n\n validity_period_hours = local.validity_period_hours\n early_renewal_hours = local.early_renewal_hours\n is_ca_certificate = true\n\n allowed_uses = [\n \"server_auth\",\n \"cert_signing\",\n \"crl_signing\",\n \"client_auth\"\n ]\n}\n\nresource \"tls_private_key\" \"node\" {\n count = var.security_enabled ? 1 : 0\n\n algorithm = \"RSA\"\n}\n\nresource \"tls_cert_request\" \"node\" {\n count = var.security_enabled ? 1 : 0\n\n #key_algorithm = \"RSA\"\n private_key_pem = join(\"\", tls_private_key.node[*].private_key_pem)\n\n subject {\n common_name = local.cert_common_name\n }\n}\n\nresource \"tls_locally_signed_cert\" \"node\" {\n count = var.security_enabled ? 1 : 0\n\n #ca_key_algorithm = \"RSA\"\n cert_request_pem = join(\"\", tls_cert_request.node[*].cert_request_pem)\n ca_private_key_pem = join(\"\", tls_private_key.ca[*].private_key_pem)\n ca_cert_pem = join(\"\", tls_self_signed_cert.ca[*].cert_pem)\n\n validity_period_hours = local.validity_period_hours\n early_renewal_hours = local.early_renewal_hours\n\n allowed_uses = [\n \"key_encipherment\",\n \"digital_signature\",\n \"server_auth\",\n \"client_auth\"\n ]\n}\n" }, { "path": "terraform-aws/client.tf", "content": "data \"template_file\" \"client_userdata_script\" {\n template = file(\"${path.module}/../templates/aws_user_data.sh\")\n vars = merge(local.user_data_common, {\n startup_script = \"client.sh\",\n heap_size = var.client_heap_size\n })\n}\n\nresource \"aws_launch_template\" \"client\" {\n name_prefix = \"elasticsearch-${var.es_cluster}-client-nodes\"\n image_id = data.aws_ami.kibana_client.id\n instance_type = var.master_instance_type\n user_data = base64encode(data.template_file.client_userdata_script.rendered)\n key_name = var.key_name\n\n iam_instance_profile {\n arn = aws_iam_instance_profile.elasticsearch.arn\n }\n\n network_interfaces {\n delete_on_termination = true\n associate_public_ip_address = false\n security_groups = concat(\n [aws_security_group.elasticsearch_security_group.id],\n [aws_security_group.elasticsearch_clients_security_group.id],\n var.additional_security_groups,\n )\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n\nresource \"aws_autoscaling_group\" \"client_nodes\" {\n count = length(keys(var.clients_count))\n\n name = \"elasticsearch-${var.es_cluster}-client-nodes-${keys(var.clients_count)[count.index]}\"\n max_size = var.clients_count[keys(var.clients_count)[count.index]]\n min_size = var.clients_count[keys(var.clients_count)[count.index]]\n desired_capacity = var.clients_count[keys(var.clients_count)[count.index]]\n default_cooldown = 30\n force_delete = true\n\n vpc_zone_identifier = local.clients_subnet_ids[keys(var.clients_count)[count.index]]\n\n target_group_arns = [\n aws_lb_target_group.esearch-p9200-tg.arn,\n aws_lb_target_group.kibana-p5601-tg[0].arn,\n ]\n\n launch_template {\n id = aws_launch_template.client.id\n version = \"$Latest\"\n }\n\n tag {\n key = \"Name\"\n value = format(\"%s-client-node\", var.es_cluster)\n propagate_at_launch = true\n }\n\n tag {\n key = \"Environment\"\n value = var.environment\n propagate_at_launch = true\n }\n\n tag {\n key = \"Cluster\"\n value = \"${var.environment}-${var.es_cluster}\"\n propagate_at_launch = true\n }\n\n tag {\n key = \"Role\"\n value = \"client\"\n propagate_at_launch = true\n }\n\n tag {\n key = \"AutoAttachDiskDisabled\"\n value = \"true\"\n propagate_at_launch = true\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n" }, { "path": "terraform-aws/datas-voters.tf", "content": "data \"template_file\" \"data_voters_userdata_script\" {\n template = file(\"${path.module}/../templates/aws_user_data.sh\")\n vars = merge(local.user_data_common, {\n heap_size = var.data_heap_size\n is_voting_only = \"true\"\n startup_script = \"data.sh\"\n })\n}\n\nresource \"aws_launch_template\" \"data_voters\" {\n name_prefix = \"elasticsearch-${var.es_cluster}-data-voters-nodes\"\n image_id = data.aws_ami.elasticsearch.id\n instance_type = var.data_instance_type\n user_data = base64encode(data.template_file.data_voters_userdata_script.rendered)\n key_name = var.key_name\n\n ebs_optimized = var.ebs_optimized\n\n iam_instance_profile {\n arn = aws_iam_instance_profile.elasticsearch.arn\n }\n\n network_interfaces {\n delete_on_termination = true\n associate_public_ip_address = false\n security_groups = concat(\n [aws_security_group.elasticsearch_security_group.id],\n var.additional_security_groups,\n )\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n\nresource \"aws_autoscaling_group\" \"data_voters_nodes\" {\n count = length(keys(var.data_voters_count))\n\n name = \"elasticsearch-${var.es_cluster}-data-voters-nodes-${keys(var.data_voters_count)[count.index]}\"\n max_size = var.data_voters_count[keys(var.data_voters_count)[count.index]]\n min_size = var.data_voters_count[keys(var.data_voters_count)[count.index]]\n desired_capacity = var.data_voters_count[keys(var.data_voters_count)[count.index]]\n default_cooldown = 30\n force_delete = true\n\n vpc_zone_identifier = local.cluster_subnet_ids[keys(var.data_voters_count)[count.index]]\n\n depends_on = [\n aws_autoscaling_group.master_nodes,\n aws_ebs_volume.data-voter\n ]\n\n target_group_arns = [\n aws_lb_target_group.esearch-p9200-tg.arn,\n ]\n\n launch_template {\n id = aws_launch_template.data_voters.id\n version = \"$Latest\"\n }\n\n tag {\n key = \"Name\"\n value = format(\"%s-data-voter-node\", var.es_cluster)\n propagate_at_launch = true\n }\n\n tag {\n key = \"Environment\"\n value = var.environment\n propagate_at_launch = true\n }\n\n tag {\n key = \"Cluster\"\n value = \"${var.environment}-${var.es_cluster}\"\n propagate_at_launch = true\n }\n\n tag {\n key = \"Role\"\n value = \"data-voters\"\n propagate_at_launch = true\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n" }, { "path": "terraform-aws/datas.tf", "content": "data \"template_file\" \"data_userdata_script\" {\n template = file(\"${path.module}/../templates/aws_user_data.sh\")\n vars = merge(local.user_data_common, {\n startup_script = \"data.sh\",\n heap_size = var.data_heap_size\n })\n}\n\nresource \"aws_launch_template\" \"data\" {\n name_prefix = \"elasticsearch-${var.es_cluster}-data-nodes\"\n image_id = data.aws_ami.elasticsearch.id\n instance_type = var.data_instance_type\n user_data = base64encode(data.template_file.data_userdata_script.rendered)\n key_name = var.key_name\n\n ebs_optimized = var.ebs_optimized\n\n iam_instance_profile {\n arn = aws_iam_instance_profile.elasticsearch.arn\n }\n\n network_interfaces {\n delete_on_termination = true\n associate_public_ip_address = false\n security_groups = concat(\n [aws_security_group.elasticsearch_security_group.id],\n var.additional_security_groups,\n )\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n\nresource \"aws_autoscaling_group\" \"data_nodes\" {\n count = length(keys(var.datas_count))\n\n name = \"elasticsearch-${var.es_cluster}-data-nodes-${keys(var.datas_count)[count.index]}\"\n max_size = var.datas_count[keys(var.datas_count)[count.index]]\n min_size = var.datas_count[keys(var.datas_count)[count.index]]\n desired_capacity = var.datas_count[keys(var.datas_count)[count.index]]\n default_cooldown = 30\n force_delete = true\n\n vpc_zone_identifier = local.cluster_subnet_ids[keys(var.datas_count)[count.index]]\n\n depends_on = [\n aws_autoscaling_group.master_nodes,\n aws_ebs_volume.data\n ]\n\n target_group_arns = [\n aws_lb_target_group.esearch-p9200-tg.arn,\n ]\n\n launch_template {\n id = aws_launch_template.data.id\n version = \"$Latest\"\n }\n\n tag {\n key = \"Name\"\n value = format(\"%s-data-node\", var.es_cluster)\n propagate_at_launch = true\n }\n\n tag {\n key = \"Environment\"\n value = var.environment\n propagate_at_launch = true\n }\n\n tag {\n key = \"Cluster\"\n value = \"${var.environment}-${var.es_cluster}\"\n propagate_at_launch = true\n }\n\n tag {\n key = \"Role\"\n value = \"data\"\n propagate_at_launch = true\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n" }, { "path": "terraform-aws/dev.tf", "content": "# data \"template_file\" \"dev-s3\" {\n# template = file(\"${path.module}/../assets/s3-backup.json\")\n\n# vars = {\n# s3_backup_bucket = var.DEV_MODE_scripts_s3_bucket\n# }\n# }\n\n# resource \"aws_s3_bucket\" \"dev\" {\n# count = var.DEV_MODE_scripts_s3_bucket == \"\" ? 0 : 1\n\n# bucket = \"${var.DEV_MODE_scripts_s3_bucket}\"\n# region = var.aws_region\n# acl = \"private\"\n# }\n\n# resource \"aws_iam_role_policy\" \"dev-s3\" {\n# count = var.DEV_MODE_scripts_s3_bucket != \"\" ? 1 : 0\n# name = \"${var.es_cluster}-elasticsearch-s3-devmode-policy\"\n# role = aws_iam_role.elasticsearch.id\n# policy = data.template_file.dev-s3.rendered\n# }\n" }, { "path": "terraform-aws/disks.tf", "content": "locals {\n master_az_flattened = toset(flatten([\n for az, count in var.masters_count : [\n for i in range(0, count) : jsonencode({ \"az\" = az, \"index\" = i, \"name\" = \"${az}-${i}\" })\n ]\n ]))\n\n data_az_flattened = toset(flatten([\n for az, count in var.datas_count : [\n for i in range(0, count) : jsonencode({ \"az\" = az, \"index\" = i, \"name\" = \"${az}-${i}\" })\n ]\n ]))\n\n data_voters_az_flattened = toset(flatten([\n for az, count in var.data_voters_count : [\n for i in range(0, count) : jsonencode({ \"az\" = az, \"index\" = i, \"name\" = \"${az}-${i}\" })\n ]\n ]))\n}\n\nresource \"aws_ebs_volume\" \"master\" {\n for_each = local.master_az_flattened\n\n availability_zone = jsondecode(each.value)[\"az\"]\n size = 10\n type = var.disk_type\n encrypted = var.volume_encryption\n\n tags = {\n Name = \"elasticsearch-${var.es_cluster}-master-${jsondecode(each.value)[\"name\"]}\"\n ClusterName = var.es_cluster\n VolumeIndex = jsondecode(each.value)[\"index\"]\n AutoAttachGroup = \"master\"\n }\n}\n\nresource \"aws_ebs_volume\" \"data\" {\n for_each = local.data_az_flattened\n\n availability_zone = jsondecode(each.value)[\"az\"]\n size = var.elasticsearch_volume_size\n type = var.disk_type\n encrypted = var.volume_encryption\n\n tags = {\n Name = \"elasticsearch-${var.es_cluster}-data-${jsondecode(each.value)[\"name\"]}\"\n ClusterName = var.es_cluster\n VolumeIndex = jsondecode(each.value)[\"index\"]\n AutoAttachGroup = \"data\"\n }\n}\n\nresource \"aws_ebs_volume\" \"data-voter\" {\n for_each = local.data_voters_az_flattened\n\n availability_zone = jsondecode(each.value)[\"az\"]\n size = var.elasticsearch_volume_size\n type = var.disk_type\n encrypted = var.volume_encryption\n\n tags = {\n Name = \"elasticsearch-${var.es_cluster}-data-voters-${jsondecode(each.value)[\"name\"]}\"\n ClusterName = var.es_cluster\n VolumeIndex = jsondecode(each.value)[\"index\"]\n AutoAttachGroup = \"data-voters\"\n }\n}\n\n\nresource \"aws_ebs_volume\" \"singlenode\" {\n count = local.singlenode_mode ? 1 : 0\n\n availability_zone = var.singlenode_az\n size = var.elasticsearch_volume_size\n type = var.disk_type\n encrypted = var.volume_encryption\n\n tags = {\n Name = \"elasticsearch-${var.es_cluster}-singlenode\"\n ClusterName = var.es_cluster\n VolumeIndex = \"0\"\n AutoAttachGroup = \"singlenode\"\n }\n}\n" }, { "path": "terraform-aws/iam.tf", "content": "data \"template_file\" \"data_s3_backup\" {\n template = file(\"${path.module}/../assets/s3-backup.json\")\n\n vars = {\n s3_backup_bucket = var.s3_backup_bucket\n }\n}\n\nresource \"aws_iam_role\" \"elasticsearch\" {\n name = \"${var.es_cluster}-elasticsearch-discovery-role\"\n assume_role_policy = file(\"${path.module}/../assets/ec2-role-trust-policy.json\")\n}\n\nresource \"aws_iam_role_policy\" \"elasticsearch\" {\n name = \"${var.es_cluster}-elasticsearch-node-init-policy\"\n policy = file(\n \"${path.module}/../assets/node-init.json\",\n )\n role = aws_iam_role.elasticsearch.id\n}\n\nresource \"aws_iam_role_policy\" \"s3_backup\" {\n count = var.s3_backup_bucket != \"\" ? 1 : 0\n name = \"${var.es_cluster}-elasticsearch-backup-policy\"\n policy = data.template_file.data_s3_backup.rendered\n role = aws_iam_role.elasticsearch.id\n}\n\nresource \"aws_iam_instance_profile\" \"elasticsearch\" {\n name = \"${var.es_cluster}-elasticsearch-discovery-profile\"\n path = \"/\"\n role = aws_iam_role.elasticsearch.name\n}\n\n" }, { "path": "terraform-aws/main.tf", "content": "provider \"aws\" {\n region = var.aws_region\n}\n\nresource \"random_string\" \"vm-login-password\" {\n length = 16\n special = false\n}\n\nresource \"random_string\" \"security-encryption-key\" {\n length = 32\n special = false\n}\nresource \"random_string\" \"reporting-encryption-key\" {\n length = 32\n special = false\n}\n\nlocals {\n\n all_availability_zones = compact(tolist(setunion(\n keys(var.masters_count),\n keys(var.datas_count),\n keys(var.clients_count),\n keys(var.data_voters_count),\n toset([var.singlenode_az])\n )))\n\n cluster_subnet_ids = {\n for i, az in local.all_availability_zones : az => lookup(var.cluster_subnet_ids, az, element(data.aws_subnets.subnets-per-az.*.ids, i))\n }\n\n clients_subnet_ids = {\n for i, az in local.all_availability_zones : az => lookup(var.clients_subnet_ids, az, element(data.aws_subnets.subnets-per-az.*.ids, i))\n }\n\n flat_cluster_subnet_ids = flatten(values(local.cluster_subnet_ids))\n flat_clients_subnet_ids = flatten(values(local.clients_subnet_ids))\n\n bootstrap_node_subnet_id = var.bootstrap_node_subnet_id != \"\" ? var.bootstrap_node_subnet_id : coalescelist(local.flat_cluster_subnet_ids, [\"\"])[0]\n\n singlenode_mode = (length(keys(var.masters_count)) + length(keys(var.datas_count)) + length(keys(var.data_voters_count)) + length(keys(var.clients_count))) == 0\n singlenode_subnet_id = local.singlenode_mode ? local.cluster_subnet_ids[var.singlenode_az][0] : \"\"\n\n masters_count = local.singlenode_mode ? 0 : sum(concat(values(var.masters_count), values(var.data_voters_count)))\n is_cluster_bootstrapped = data.local_file.cluster_bootstrap_state.content == \"1\" || !var.requires_bootstrapping\n\n user_data_common = {\n cloud_provider = \"aws\"\n elasticsearch_data_dir = var.elasticsearch_data_dir\n elasticsearch_logs_dir = var.elasticsearch_logs_dir\n es_cluster = var.es_cluster\n es_environment = \"${var.environment}-${var.es_cluster}\"\n security_groups = aws_security_group.elasticsearch_security_group.id\n aws_region = var.aws_region\n security_enabled = var.security_enabled\n monitoring_enabled = var.monitoring_enabled\n masters_count = local.masters_count\n client_user = var.client_user\n xpack_monitoring_host = var.xpack_monitoring_host\n filebeat_monitoring_host = var.filebeat_monitoring_host\n s3_backup_bucket = var.s3_backup_bucket\n use_g1gc = var.use_g1gc\n client_pwd = random_string.vm-login-password.result\n master = false\n data = false\n bootstrap_node = false\n log_level = var.log_level\n log_size = var.log_size\n is_voting_only = false\n\n ca_cert = var.security_enabled ? join(\"\", tls_self_signed_cert.ca[*].cert_pem) : \"\"\n node_cert = var.security_enabled ? join(\"\", tls_locally_signed_cert.node[*].cert_pem) : \"\"\n node_key = var.security_enabled ? join(\"\", tls_private_key.node[*].private_key_pem) : \"\"\n\n DEV_MODE_scripts_s3_bucket = var.DEV_MODE_scripts_s3_bucket\n\n security_encryption_key = random_string.security-encryption-key.result\n reporting_encryption_key = random_string.reporting-encryption-key.result\n auto_shut_down_bootstrap_node = var.auto_shut_down_bootstrap_node\n }\n}\n\n##############################################################################\n# Elasticsearch\n##############################################################################\n\nresource \"aws_security_group\" \"elasticsearch_security_group\" {\n name = \"elasticsearch-${var.es_cluster}-security-group\"\n description = \"Elasticsearch ports with ssh\"\n vpc_id = var.vpc_id\n\n tags = {\n Name = \"${var.es_cluster}-elasticsearch\"\n cluster = var.es_cluster\n }\n\n # ssh access from everywhere\n ingress {\n from_port = 22\n to_port = 22\n protocol = \"tcp\"\n cidr_blocks = [\"0.0.0.0/0\"]\n }\n\n # inter-cluster communication over ports 9200-9400\n ingress {\n from_port = 9200\n to_port = 9400\n protocol = \"tcp\"\n self = true\n }\n\n # allow inter-cluster ping\n ingress {\n from_port = 8\n to_port = 0\n protocol = \"icmp\"\n self = true\n }\n\n # allow alb sg access\n ingress {\n from_port = 9200\n to_port = 9200\n protocol = \"tcp\"\n security_groups = [aws_security_group.elasticsearch-alb-sg.id]\n }\n\n egress {\n from_port = 0\n to_port = 0\n protocol = \"-1\"\n cidr_blocks = [\"0.0.0.0/0\"]\n }\n}\n\nresource \"aws_security_group\" \"elasticsearch_clients_security_group\" {\n name = \"elasticsearch-${var.es_cluster}-clients-security-group\"\n description = \"Kibana HTTP access from outside\"\n vpc_id = var.vpc_id\n\n tags = {\n Name = \"${var.es_cluster}-kibana\"\n cluster = var.es_cluster\n }\n\n # allow alb sg access\n ingress {\n from_port = 9200\n to_port = 9200\n protocol = \"tcp\"\n security_groups = [aws_security_group.elasticsearch-alb-sg.id]\n }\n ingress {\n from_port = 5601\n to_port = 5601\n protocol = \"tcp\"\n security_groups = [aws_security_group.elasticsearch-alb-sg.id]\n }\n\n egress {\n from_port = 0\n to_port = 0\n protocol = \"-1\"\n cidr_blocks = [\"0.0.0.0/0\"]\n }\n}\n" }, { "path": "terraform-aws/masters.tf", "content": "data \"local_file\" \"cluster_bootstrap_state\" {\n filename = \"${path.module}/cluster_bootstrap_state\"\n}\n\ndata \"template_file\" \"master_userdata_script\" {\n template = file(\"${path.module}/../templates/aws_user_data.sh\")\n vars = merge(local.user_data_common, {\n startup_script = \"master.sh\",\n heap_size = var.master_heap_size\n })\n}\n\ndata \"template_file\" \"bootstrap_userdata_script\" {\n template = file(\"${path.module}/../templates/aws_user_data.sh\")\n vars = merge(local.user_data_common, {\n startup_script = \"bootstrap.sh\",\n heap_size = var.master_heap_size\n })\n}\n\nresource \"aws_launch_template\" \"master\" {\n name_prefix = \"elasticsearch-${var.es_cluster}-master-nodes\"\n image_id = data.aws_ami.elasticsearch.id\n instance_type = var.master_instance_type\n user_data = base64encode(data.template_file.master_userdata_script.rendered)\n key_name = var.key_name\n\n iam_instance_profile {\n arn = aws_iam_instance_profile.elasticsearch.arn\n }\n\n network_interfaces {\n delete_on_termination = true\n associate_public_ip_address = false\n security_groups = concat(\n [aws_security_group.elasticsearch_security_group.id],\n var.additional_security_groups,\n )\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n\nresource \"aws_autoscaling_group\" \"master_nodes\" {\n count = length(keys(var.masters_count))\n\n name = \"elasticsearch-${var.es_cluster}-master-nodes-${keys(var.masters_count)[count.index]}\"\n max_size = var.masters_count[keys(var.masters_count)[count.index]]\n min_size = var.masters_count[keys(var.masters_count)[count.index]]\n desired_capacity = var.masters_count[keys(var.masters_count)[count.index]]\n default_cooldown = 30\n force_delete = true\n\n vpc_zone_identifier = local.cluster_subnet_ids[keys(var.masters_count)[count.index]]\n\n launch_template {\n id = aws_launch_template.master.id\n version = \"$Latest\"\n }\n\n tag {\n key = \"Name\"\n value = format(\"%s-master-node\", var.es_cluster)\n propagate_at_launch = true\n }\n\n tag {\n key = \"Environment\"\n value = var.environment\n propagate_at_launch = true\n }\n\n tag {\n key = \"Cluster\"\n value = \"${var.environment}-${var.es_cluster}\"\n propagate_at_launch = true\n }\n\n tag {\n key = \"Role\"\n value = \"master\"\n propagate_at_launch = true\n }\n\n lifecycle {\n create_before_destroy = true\n }\n\n depends_on = [aws_ebs_volume.master]\n}\n\nresource \"aws_instance\" \"bootstrap_node\" {\n count = local.singlenode_mode || local.is_cluster_bootstrapped ? 0 : 1\n\n ami = data.aws_ami.elasticsearch.id\n instance_type = var.master_instance_type\n instance_initiated_shutdown_behavior = \"terminate\"\n\n vpc_security_group_ids = concat(\n [aws_security_group.elasticsearch_security_group.id],\n var.additional_security_groups,\n )\n iam_instance_profile = aws_iam_instance_profile.elasticsearch.id\n user_data = data.template_file.bootstrap_userdata_script.rendered\n key_name = var.key_name\n subnet_id = local.bootstrap_node_subnet_id\n\n associate_public_ip_address = false\n\n tags = {\n Name = \"${var.es_cluster}-bootstrap-node\"\n Environment = var.environment\n Cluster = \"${var.environment}-${var.es_cluster}\"\n Role = \"bootstrap\"\n }\n}\n\nresource \"null_resource\" \"cluster_bootstrap_state\" {\n provisioner \"local-exec\" {\n command = \"printf 1 > ${path.module}/cluster_bootstrap_state\"\n }\n provisioner \"local-exec\" {\n when = destroy\n command = \"printf 0 > ${path.module}/cluster_bootstrap_state\"\n }\n\n depends_on = [aws_instance.bootstrap_node]\n}\n" }, { "path": "terraform-aws/outputs.tf", "content": "output \"clients_dns\" {\n value = aws_lb.elasticsearch-alb.*.dns_name\n}\n\noutput \"vm_password\" {\n value = random_string.vm-login-password.result\n}" }, { "path": "terraform-aws/singlenode.tf", "content": "data \"template_file\" \"singlenode_userdata_script\" {\n template = file(\"${path.module}/../templates/aws_user_data.sh\")\n vars = merge(local.user_data_common, {\n startup_script = \"singlenode.sh\",\n heap_size = var.master_heap_size\n })\n}\n\nresource \"aws_launch_template\" \"single_node\" {\n name_prefix = \"elasticsearch-${var.es_cluster}-single-node\"\n image_id = data.aws_ami.kibana_client.id\n instance_type = var.data_instance_type\n user_data = base64encode(data.template_file.singlenode_userdata_script.rendered)\n key_name = var.key_name\n\n ebs_optimized = var.ebs_optimized\n\n iam_instance_profile {\n arn = aws_iam_instance_profile.elasticsearch.arn\n }\n\n network_interfaces {\n delete_on_termination = true\n associate_public_ip_address = false\n security_groups = [aws_security_group.elasticsearch_security_group.id, aws_security_group.elasticsearch_clients_security_group.id]\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n\nresource \"aws_autoscaling_group\" \"singlenode\" {\n count = local.singlenode_mode ? 1 : 0\n\n name = \"elasticsearch-${var.es_cluster}-singlenode\"\n min_size = 1\n max_size = 1\n desired_capacity = 1\n default_cooldown = 30\n force_delete = true\n\n vpc_zone_identifier = [local.singlenode_subnet_id]\n\n target_group_arns = [\n aws_lb_target_group.esearch-p9200-tg.arn,\n aws_lb_target_group.kibana-p5601-tg[0].arn,\n ]\n\n launch_template {\n id = aws_launch_template.single_node.id\n version = \"$Latest\"\n }\n\n tag {\n key = \"Name\"\n value = format(\"%s-elasticsearch\", var.es_cluster)\n propagate_at_launch = true\n }\n tag {\n key = \"Environment\"\n value = var.environment\n propagate_at_launch = true\n }\n tag {\n key = \"Cluster\"\n value = \"${var.environment}-${var.es_cluster}\"\n propagate_at_launch = true\n }\n tag {\n key = \"Role\"\n value = \"singlenode\"\n propagate_at_launch = true\n }\n\n lifecycle {\n create_before_destroy = true\n }\n\n depends_on = [aws_ebs_volume.singlenode]\n}\n" }, { "path": "terraform-aws/terraform.tfvars.example", "content": "aws_region=\"us-east-1\"\nes_cluster=\"elastic-cluster\"\nvpc_id=\"vpc-somevpc\"\nkey_name=\"keyname\"\nmasters_count = {\n \"us-east-1a\" = 3\n}\ndatas_count = {\n \"us-east-1a\" = 1\n}\nclients_count = {\n \"us-east-1a\" = 1\n}\ndata_voters_count = {\n \"us-east-1a\" = 2\n}\n\nsecurity_enabled = true\nmonitoring_enabled = false\nclient_user = \"someuser\"\npublic_facing = false\nauto_shut_down_bootstrap_node = true\n" }, { "path": "terraform-aws/variables.tf", "content": "### MANDATORY ###\nvariable \"es_cluster\" {\n description = \"Name of the elasticsearch cluster, used in node discovery\"\n}\n\nvariable \"aws_region\" {\n type = string\n}\n\nvariable \"vpc_id\" {\n description = \"VPC ID to create the Elasticsearch cluster in\"\n type = string\n}\n\nvariable \"clients_subnet_ids\" {\n description = \"Subnets to run client nodes in, defined as avalabilityZone -> subnets mapping. Will autofill to all available subnets in AZ when left empty.\"\n type = map(list(string))\n default = {}\n}\n\nvariable \"cluster_subnet_ids\" {\n description = \"Subnets to run cluster nodes in, defined as avalabilityZone -> subnets mapping. Will autofill to all available subnets in AZ when left empty.\"\n type = map(list(string))\n default = {}\n}\n\nvariable \"key_name\" {\n description = \"Key name to be used with the launched EC2 instances.\"\n default = \"elasticsearch\"\n}\n\nvariable \"environment\" {\n default = \"default\"\n}\n\nvariable \"data_instance_type\" {\n type = string\n default = \"c5.2xlarge\"\n}\n\nvariable \"master_instance_type\" {\n type = string\n default = \"c5.large\"\n}\n\nvariable \"elasticsearch_volume_size\" {\n type = string\n default = \"100\" # gb\n}\n\nvariable \"volume_encryption\" {\n default = true\n}\n\nvariable \"elasticsearch_data_dir\" {\n default = \"/opt/elasticsearch/data\"\n}\n\nvariable \"elasticsearch_logs_dir\" {\n default = \"/var/log/elasticsearch\"\n}\n\n# default elasticsearch heap size\nvariable \"data_heap_size\" {\n type = string\n default = \"8g\"\n}\n\nvariable \"master_heap_size\" {\n type = string\n default = \"2g\"\n}\n\nvariable \"client_heap_size\" {\n type = string\n default = \"1g\"\n}\n\nvariable \"masters_count\" {\n type = map(number)\n default = {}\n description = \"Master nodes count per avalabilityZone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"datas_count\" {\n type = map(number)\n default = {}\n description = \"Data nodes count per avalabilityZone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"data_voters_count\" {\n type = map(number)\n default = {}\n description = \"Data voter nodes count per avalabilityZone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"clients_count\" {\n type = map(number)\n default = {}\n description = \"Client nodes count per avalabilityZone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"security_enabled\" {\n description = \"Whether or not to enable x-pack security on the cluster\"\n default = false\n}\n\nvariable \"monitoring_enabled\" {\n description = \"Whether or not to enable x-pack monitoring on the cluster\"\n default = false\n}\n\nvariable \"client_user\" {\n description = \"The username to use when setting up basic auth on Grafana and Cerebro.\"\n default = \"elastic\"\n}\n\nvariable \"public_facing\" {\n description = \"Whether or not the created cluster should be accessible from the public internet\"\n type = bool\n default = true\n}\n\n# the ability to add additional existing security groups. In our case\n# we have consul running as agents on the box\nvariable \"additional_security_groups\" {\n type = list(string)\n default = []\n}\n\nvariable \"ebs_optimized\" {\n description = \"Whether data instances are EBS optimized or not\"\n default = \"true\"\n}\n\nvariable \"xpack_monitoring_host\" {\n description = \"ES host to send monitoring data\"\n default = \"http://localhost:9200\"\n}\n\nvariable \"filebeat_monitoring_host\" {\n description = \"ES host to send filebeat data\"\n default = false\n}\n\nvariable \"s3_backup_bucket\" {\n description = \"S3 bucket for backups\"\n default = \"\"\n}\n\nvariable \"alb_subnets\" {\n description = \"Subnets to run the ALB in. Defaults to all VPC subnets.\"\n default = []\n}\n\nvariable \"singlenode_az\" {\n description = \"This variable is required when running in singlenode mode. Singlenode mode is enabled when masters_count, datas_count and clients_count are all empty,\"\n default = \"\"\n}\n\nvariable \"bootstrap_node_subnet_id\" {\n description = \"Use to override which subnet the bootstrap node is created in.\"\n default = \"\"\n}\n\nvariable \"use_g1gc\" {\n description = \"Whether or not to enable G1GC in jvm.options ES config. Left in for backwards compatibility, deployments with Elasticsearch 7.7 and above should not use this.\"\n default = false\n}\n\nvariable \"DEV_MODE_scripts_s3_bucket\" {\n description = \"S3 bucket to override init scripts from. Should not be used on production.\"\n default = \"\"\n}\n\nvariable \"requires_bootstrapping\" {\n description = \"Overrides cluster bootstrap state\"\n default = true\n}\n\nvariable \"elasticsearch_packer_image\" {\n description = \"The name of the image family for elasticsearch\"\n default = \"elasticsearch7-packer-image\"\n}\n\nvariable \"kibana_packer_image\" {\n description = \"The name of the image family for kibana\"\n default = \"kibana7-packer-image\"\n}\n\nvariable \"ec2_vpc_endpoint_id\" {\n description = \"Use to skip creation of ec2 VPC endpoint and reference your own\"\n default = \"\"\n}\n\nvariable \"s3_vpc_endpoint_id\" {\n description = \"Use to skip creation of s3 VPC endpoint and reference your own\"\n default = \"\"\n}\n\nvariable \"autoscaling_vpc_endpoint_id\" {\n description = \"Use to skip creation of autoscaling VPC endpoint and reference your own\"\n default = \"\"\n}\n\nvariable \"log_size\" {\n description = \"Retained log4j log size in MB\"\n default = \"128\"\n}\n\nvariable \"log_level\" {\n description = \"log4j log level\"\n default = \"INFO\"\n}\n\nvariable \"auto_shut_down_bootstrap_node\" {\n description = \"disable to prevent bootstrap node from shutting down\"\n default = true\n}\n\nvariable \"disk_type\" {\n description = \"disk type\"\n default = \"gp2\"\n}\n" }, { "path": "terraform-aws/versions.tf", "content": "\nterraform {\n required_version = \">= 0.12\"\n}\n" }, { "path": "terraform-aws/vpc.tf", "content": "data \"aws_vpc\" \"selected\" {\n id = var.vpc_id\n}\n\ndata \"aws_subnets\" \"all-subnets\" {\n filter {\n name = \"vpc-id\"\n values = [var.vpc_id]\n }\n}\n\ndata \"aws_route_tables\" \"vpc_route_tables\" {\n vpc_id = var.vpc_id\n}\n\ndata \"aws_subnets\" \"subnets-per-az\" {\n count = length(local.all_availability_zones)\n\n filter {\n name = \"availability-zone\"\n values = [local.all_availability_zones[count.index]]\n }\n filter {\n name = \"vpc-id\"\n values = [var.vpc_id]\n }\n\n}\n\nresource \"aws_security_group\" \"vpc-endpoint\" {\n vpc_id = var.vpc_id\n\n ingress {\n from_port = 80\n to_port = 80\n protocol = \"tcp\"\n cidr_blocks = [\"0.0.0.0/0\"]\n }\n\n\n ingress {\n from_port = 443\n to_port = 443\n protocol = \"tcp\"\n cidr_blocks = [\"0.0.0.0/0\"]\n }\n\n egress {\n from_port = 0\n to_port = 0\n protocol = \"-1\"\n cidr_blocks = [\"0.0.0.0/0\"]\n }\n}\n\nresource \"aws_vpc_endpoint\" \"ec2\" {\n vpc_id = var.vpc_id\n count = var.ec2_vpc_endpoint_id == \"\" ? 1 : 0\n service_name = \"com.amazonaws.${var.aws_region}.ec2\"\n vpc_endpoint_type = \"Interface\"\n private_dns_enabled = true\n\n security_group_ids = [aws_security_group.vpc-endpoint.id]\n subnet_ids = compact(setunion(\n local.flat_cluster_subnet_ids,\n local.flat_clients_subnet_ids,\n [local.singlenode_subnet_id],\n [local.bootstrap_node_subnet_id]\n ))\n}\n\nresource \"aws_vpc_endpoint\" \"autoscaling\" {\n vpc_id = var.vpc_id\n count = var.autoscaling_vpc_endpoint_id == \"\" ? 1 : 0\n service_name = \"com.amazonaws.${var.aws_region}.autoscaling\"\n vpc_endpoint_type = \"Interface\"\n private_dns_enabled = true\n\n security_group_ids = [aws_security_group.vpc-endpoint.id]\n subnet_ids = compact(setunion(\n local.flat_cluster_subnet_ids,\n local.flat_clients_subnet_ids,\n [local.singlenode_subnet_id],\n [local.bootstrap_node_subnet_id]\n ))\n}\n\nresource \"aws_vpc_endpoint\" \"s3\" {\n vpc_id = var.vpc_id\n count = var.s3_vpc_endpoint_id == \"\" ? 1 : 0\n service_name = \"com.amazonaws.${var.aws_region}.s3\"\n vpc_endpoint_type = \"Gateway\"\n route_table_ids = data.aws_route_tables.vpc_route_tables.ids\n}\n" }, { "path": "terraform-azure/README.md", "content": "# Azure deployment\n\n## Create the machine images with Packer\n\nGo to the packer folder and see the README there. Once you have the machine image IDs, return here and continue with the next steps.\n\n## Create key-pair or use your own\n\nThis deployment is configured to use your default SSH keys as machine credentials. If you want to use other keys, change the path to the keys you want to use (look for `key_path` in variables.tf). Use [this guide](https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/) to generate new keys if needed.\n\n## Configurations\n\nEdit `variables.tf` to specify the following:\n\n* `azure_location` - the Azure location where to launch the cluster in.\n* `azure_subscription_id`, `azure_client_id`, `azure_client_secret`, `azure_tenant_id` - the same credentials used in the Packer step. See the README there for instructions on how to retrieve them.\n* `es_cluster` - the name of the Elasticsearch cluster to launch.\n* `key_path` - the filesystem path to the SSH key to use as virtual machines login credentials.\n* `data_instance_type`, `master_instance_type`, `client_instance_type` - Azure machine instance types to use for each machine type in the cluster.\n* `security_enabled`, `monitoring_enabled` - whether to enable X-Pack Security and Monitoring features, respectively.\n* `client_user` - the username to use for HTTP basic authentication that is enabled on the client nodes. Password is generated automatically and can be accessed after deployment by running `terraform output`.\n\nThe rest of the configurations are mostly around cluster topology and machine types and sizes.\n\n### Cluster topology\n\nTwo modes of deployment are supported:\n\n* A recommended configuration, with dedicated master-eligible nodes, data nodes, and client nodes. This is a production-ready and best-practice configuration. See more details in the [official documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html).\n* Single node mode - mostly useful for experimentation\n\nAt this point we consider the role `ingest` as unanimous with `data`, so all data nodes are also ingest nodes.\n\nThe default mode is the single-node mode. To change it to the recommended configuration, edit `variables.tf` and set number of master nodes to 3, data nodes to at least 2, and client nodes to at least 1.\n\nAll nodes with the `client` role will be attached to an Azure load balancer, so access to all client nodes can be done via the DNS it exposes.\n\n## Launch the cluster with Terraform\n\n```bash\nterraform plan\nterraform apply\n```\n\nWhen terraform is done, you should see a lot of output ending with something like this:\n\n```\nApply complete! Resources: 14 added, 0 changed, 0 destroyed.\n\nThe state of your infrastructure has been saved to the path\nbelow. This state is required to modify and destroy your\ninfrastructure, so keep it safe. To inspect the complete state\nuse the `terraform show` command.\n\nState path: terraform.tfstate\n\nOutputs:\n\npublic_dns = elasticsearch-cluster-foo.eastus.cloudapp.azure.com\nvm_password = rBTKoLsf7x8ODZVd\n```\n\nNote `clients_lb_public_ipaddress` and `vm-password` - that's your entry point to the cluster and the password for the `exampleuser` default user.\n\n### Look around\n\nThe client nodes are the ones exposed to external networks. They provide endpoints for Kibana, Grafana, Cerebro and direct Elasticsearch access. By default client nodes are accessible via their public IPs and the DNS of the load balancer they are attached to (see above).\n\nClient nodes listen on port 8080 and are password protected. Access is managed by nginx which is expecting a username and password pair. Default user name is exampleuser and the password is generated automatically when deploying. You can change those defaults by editing [this file](https://github.com/synhershko/elasticsearch-cloud-deploy/blob/master/packer/install-nginx.sh) and running Packer again.\n\nOn client nodes you will find:\n\n* Kibana access is direct on port 80 of the load balancer host (http://host)\n* [Cerebro](https://github.com/lmenezes/cerebro) (a cluster management UI) is available on http://host/cerebro/\n* For direct Elasticsearch access, go to http://host/es/\n* In the single-node deployment mode, the default port is 8080 and the host is the machine host (not the load balancer)\n* Grafana is accessible on port 3000 - http://host:3000/\n\nThe default credentials are `exampleuser` as username, and password as generated by Terraform during the deployment (will show up as `vm-password` after deployment when you run `terraform output`).\n\nElastic's X-Pack is deployed on the cluster out of the box with monitoring enabled but security disabled - you should enable and setup X-Pack Security for any production deployment.\n\nTo ssh to one of the instances:\n\n```bash\nssh ubuntu@{public IP / DNS of the instance or load balancer}\n```\n\n## Backups\n\nThe Azure repository plugin is installed on the cluster and ready to be used for index snapshots and (should you ever need) a restore. Official documentation is available here: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure-usage.html\n\n### Auto- and manual- scale out\n\nThe entire stack is deployed using Azure scale-sets, which are easy to scale up and down manually (from the Azure portal, from the command line, or using the same Terraform scripts), or automatically based on host metrics and application metrics using [Azure scale-set features](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-autoscale-overview).\n\n## Elastic Discovery on Azure\n\nUnfortunately, the story of cluster discovery on Azure is practically non-existent. There is an Azure \"Classic\" discovery plugin that has been deprecated since circa 5.0 and Elastic are yet to release a properly working discovery plugin (there is [a PR for one](https://github.com/elastic/elasticsearch/pull/22679) which is open for over a year now if you want to track it).\n\nA discovery plugin on a public cloud is important because it takes a lot of complexity off you, and manages the initial cluster nodes discovery using the available cloud APIs.\n\nHaving none available, I defaulted to using vnet and naming conventions. Another viable option is using file-based discovery, which is a file describing your cluster you can upload to the images and use as a seed.\n" }, { "path": "terraform-azure/clients.tf", "content": "data \"template_file\" \"client_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/user_data.sh\")}\"\n\n vars {\n cloud_provider = \"azure\"\n volume_name = \"\"\n elasticsearch_data_dir = \"/var/lib/elasticsearch\"\n elasticsearch_logs_dir = \"${var.elasticsearch_logs_dir}\"\n heap_size = \"1g\"\n es_cluster = \"${var.es_cluster}\"\n es_environment = \"${var.environment}-${var.es_cluster}\"\n security_groups = \"\"\n availability_zones = \"\"\n minimum_master_nodes = \"${format(\"%d\", var.masters_count / 2 + 1)}\"\n master = \"false\"\n data = \"false\"\n http_enabled = \"true\"\n security_enabled = \"${var.security_enabled}\"\n monitoring_enabled = \"${var.monitoring_enabled}\"\n client_user = \"${var.client_user}\"\n client_pwd = \"${random_string.vm-login-password.result}\"\n }\n}\n\nresource \"azurerm_virtual_machine_scale_set\" \"client-nodes\" {\n count = \"${var.clients_count == \"0\" ? \"0\" : \"1\"}\"\n\n name = \"es-${var.es_cluster}-client-nodes\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n location = \"${var.azure_location}\"\n \"sku\" {\n name = \"${var.client_instance_type}\"\n tier = \"Standard\"\n capacity = \"${var.clients_count}\"\n }\n upgrade_policy_mode = \"Manual\"\n overprovision = false\n\n \"os_profile\" {\n computer_name_prefix = \"${var.es_cluster}-client\"\n admin_username = \"ubuntu\"\n admin_password = \"${random_string.vm-login-password.result}\"\n custom_data = \"${data.template_file.client_userdata_script.rendered}\"\n }\n\n \"network_profile\" {\n name = \"es-${var.es_cluster}-net-profile\"\n primary = true\n\n \"ip_configuration\" {\n name = \"es-${var.es_cluster}-ip-profile\"\n subnet_id = \"${azurerm_subnet.elasticsearch_subnet.id}\"\n load_balancer_backend_address_pool_ids = [\"${azurerm_lb_backend_address_pool.clients-lb-backend.id}\"]\n }\n }\n\n storage_profile_image_reference {\n id = \"${data.azurerm_image.kibana.id}\"\n }\n\n \"storage_profile_os_disk\" {\n caching = \"ReadWrite\"\n create_option = \"FromImage\"\n managed_disk_type = \"Standard_LRS\"\n }\n\n os_profile_linux_config {\n disable_password_authentication = true\n ssh_keys {\n path = \"/home/ubuntu/.ssh/authorized_keys\"\n key_data = \"${file(var.key_path)}\"\n }\n }\n}" }, { "path": "terraform-azure/datas.tf", "content": "data \"template_file\" \"data_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/user_data.sh\")}\"\n\n vars {\n cloud_provider = \"azure\"\n volume_name = \"\"\n elasticsearch_data_dir = \"${var.elasticsearch_data_dir}\"\n elasticsearch_logs_dir = \"${var.elasticsearch_logs_dir}\"\n heap_size = \"${var.data_heap_size}\"\n es_cluster = \"${var.es_cluster}\"\n es_environment = \"${var.environment}-${var.es_cluster}\"\n security_groups = \"\"\n availability_zones = \"\"\n minimum_master_nodes = \"${format(\"%d\", var.masters_count / 2 + 1)}\"\n master = \"false\"\n data = \"true\"\n http_enabled = \"true\"\n security_enabled = \"${var.security_enabled}\"\n monitoring_enabled = \"${var.monitoring_enabled}\"\n client_user = \"\"\n client_pwd = \"\"\n }\n}\n\nresource \"azurerm_virtual_machine_scale_set\" \"data-nodes\" {\n count = \"${var.datas_count == \"0\" ? \"0\" : \"1\"}\"\n\n name = \"es-${var.es_cluster}-data-nodes\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n location = \"${var.azure_location}\"\n \"sku\" {\n name = \"${var.data_instance_type}\"\n tier = \"Standard\"\n capacity = \"${var.datas_count}\"\n }\n upgrade_policy_mode = \"Manual\"\n overprovision = false\n\n \"os_profile\" {\n computer_name_prefix = \"${var.es_cluster}-data\"\n admin_username = \"ubuntu\"\n admin_password = \"${random_string.vm-login-password.result}\"\n custom_data = \"${data.template_file.data_userdata_script.rendered}\"\n }\n\n \"network_profile\" {\n name = \"es-${var.es_cluster}-net-profile\"\n primary = true\n accelerated_networking = true\n\n \"ip_configuration\" {\n name = \"es-${var.es_cluster}-ip-profile\"\n subnet_id = \"${azurerm_subnet.elasticsearch_subnet.id}\"\n }\n }\n\n storage_profile_image_reference {\n id = \"${data.azurerm_image.elasticsearch.id}\"\n }\n\n \"storage_profile_os_disk\" {\n caching = \"ReadWrite\"\n create_option = \"FromImage\"\n managed_disk_type = \"Standard_LRS\"\n }\n\n os_profile_linux_config {\n disable_password_authentication = true\n ssh_keys {\n path = \"/home/ubuntu/.ssh/authorized_keys\"\n key_data = \"${file(var.key_path)}\"\n }\n }\n\n\n// storage_profile_data_disk {\n// lun = 0\n// caching = \"ReadWrite\"\n// create_option = \"Empty\"\n// disk_size_gb = \"${var.elasticsearch_volume_size}\"\n// managed_disk_type = \"Standard_LRS\"\n// }\n}" }, { "path": "terraform-azure/images.tf", "content": "data \"azurerm_image\" \"elasticsearch\" {\n resource_group_name = \"packer-elasticsearch-images\"\n name_regex = \"^elasticsearch6-\\\\d{4,4}-\\\\d{2,2}-\\\\d{2,2}T\\\\d{6,6}\"\n sort_descending = true\n}\n\ndata \"azurerm_image\" \"kibana\" {\n resource_group_name = \"packer-elasticsearch-images\"\n name_regex = \"^kibana6-\\\\d{4,4}-\\\\d{2,2}-\\\\d{2,2}T\\\\d{6,6}\"\n sort_descending = true\n}\n" }, { "path": "terraform-azure/lb.tf", "content": "resource \"azurerm_public_ip\" \"clients\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-public-ip\"\n location = \"${var.azure_location}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n public_ip_address_allocation = \"static\"\n domain_name_label = \"${azurerm_resource_group.elasticsearch.name}\"\n}\n\nresource \"azurerm_lb\" \"clients\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n\n location = \"${var.azure_location}\"\n name = \"es-${var.es_cluster}-clients-lb\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n\n frontend_ip_configuration {\n name = \"es-${var.es_cluster}-ip\"\n subnet_id = \"${azurerm_subnet.elasticsearch_subnet.id}\"\n private_ip_address_allocation = \"dynamic\"\n }\n}\n\nresource \"azurerm_lb\" \"clients-public\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n\n location = \"${var.azure_location}\"\n name = \"es-${var.es_cluster}-clients-public-lb\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n\n frontend_ip_configuration {\n name = \"es-${var.es_cluster}-public-ip\"\n public_ip_address_id = \"${azurerm_public_ip.clients.id}\"\n }\n}\n\nresource \"azurerm_lb_backend_address_pool\" \"clients-lb-backend\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-clients-lb-backend\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n loadbalancer_id = \"${var.associate_public_ip == true ? azurerm_lb.clients-public.id : azurerm_lb.clients.id}\"\n}\n\nresource \"azurerm_lb_probe\" \"clients-httpprobe\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-clients-lb-probe\"\n port = 8080\n protocol = \"Http\"\n request_path = \"/status\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n loadbalancer_id = \"${var.associate_public_ip == true ? azurerm_lb.clients-public.id : azurerm_lb.clients.id}\"\n}\n\n// Kibana, Cerebro and Elasticsearch access - protected by default by the nginx proxy\nresource \"azurerm_lb_rule\" \"clients-lb-rule\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-clients-lb-rule\"\n backend_port = 8080\n frontend_port = 80\n frontend_ip_configuration_name = \"${var.associate_public_ip == true ? \"es-${var.es_cluster}-public-ip\" : \"es-${var.es_cluster}-ip\"}\"\n backend_address_pool_id = \"${azurerm_lb_backend_address_pool.clients-lb-backend.id}\"\n protocol = \"Tcp\"\n loadbalancer_id = \"${var.associate_public_ip == true ? azurerm_lb.clients-public.id : azurerm_lb.clients.id}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n}\n\n// Grafana instance, protected by default by their own login screen\nresource \"azurerm_lb_rule\" \"clients-lb-rule2\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-clients-lb-rule2\"\n backend_port = 3000\n frontend_port = 3000\n frontend_ip_configuration_name = \"${var.associate_public_ip == true ? \"es-${var.es_cluster}-public-ip\" : \"es-${var.es_cluster}-ip\"}\"\n backend_address_pool_id = \"${azurerm_lb_backend_address_pool.clients-lb-backend.id}\"\n protocol = \"Tcp\"\n loadbalancer_id = \"${var.associate_public_ip == true ? azurerm_lb.clients-public.id : azurerm_lb.clients.id}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n}\n\n// SSH access\nresource \"azurerm_lb_rule\" \"clients-lb-rule-ssh\" {\n count = \"${var.associate_public_ip == \"true\" && var.clients_count != \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-clients-lb-rule-ssh\"\n backend_port = 22\n frontend_port = 22\n frontend_ip_configuration_name = \"${var.associate_public_ip == true ? \"es-${var.es_cluster}-public-ip\" : \"es-${var.es_cluster}-ip\"}\"\n backend_address_pool_id = \"${azurerm_lb_backend_address_pool.clients-lb-backend.id}\"\n protocol = \"Tcp\"\n loadbalancer_id = \"${var.associate_public_ip == true ? azurerm_lb.clients-public.id : azurerm_lb.clients.id}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n}" }, { "path": "terraform-azure/main.tf", "content": "provider \"azurerm\" {\n subscription_id = \"${var.azure_subscription_id}\"\n client_id = \"${var.azure_client_id}\"\n client_secret = \"${var.azure_client_secret}\"\n tenant_id = \"${var.azure_tenant_id}\"\n}\n\nresource \"random_string\" \"vm-login-password\" {\n length = 16\n special = true\n override_special = \"!@#%&-_\"\n}\n\nresource \"azurerm_resource_group\" \"elasticsearch\" {\n location = \"${var.azure_location}\"\n name = \"elasticsearch-cluster-${var.es_cluster}\"\n}\n\nresource \"azurerm_virtual_network\" \"elasticsearch_vnet\" {\n name = \"es-${var.es_cluster}-vnet\"\n location = \"${var.azure_location}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n address_space = [\"10.1.0.0/24\"]\n}\n\nresource \"azurerm_subnet\" \"elasticsearch_subnet\" {\n name = \"es-${var.es_cluster}-subnet\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n virtual_network_name = \"${azurerm_virtual_network.elasticsearch_vnet.name}\"\n address_prefix = \"10.1.0.0/24\"\n}" }, { "path": "terraform-azure/masters.tf", "content": "data \"template_file\" \"master_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/user_data.sh\")}\"\n\n vars {\n cloud_provider = \"azure\"\n volume_name = \"\"\n elasticsearch_data_dir = \"/var/lib/elasticsearch\"\n elasticsearch_logs_dir = \"${var.elasticsearch_logs_dir}\"\n heap_size = \"${var.master_heap_size}\"\n es_cluster = \"${var.es_cluster}\"\n es_environment = \"${var.environment}-${var.es_cluster}\"\n security_groups = \"\"\n availability_zones = \"\"\n minimum_master_nodes = \"${format(\"%d\", var.masters_count / 2 + 1)}\"\n master = \"true\"\n data = \"false\"\n http_enabled = \"false\"\n security_enabled = \"${var.security_enabled}\"\n monitoring_enabled = \"${var.monitoring_enabled}\"\n client_user = \"\"\n client_pwd = \"\"\n }\n}\n\nresource \"azurerm_virtual_machine_scale_set\" \"master-nodes\" {\n count = \"${var.masters_count == \"0\" ? \"0\" : \"1\"}\"\n\n name = \"es-${var.es_cluster}-master-nodes\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n location = \"${var.azure_location}\"\n \"sku\" {\n name = \"${var.master_instance_type}\"\n tier = \"Standard\"\n capacity = \"${var.masters_count}\"\n }\n upgrade_policy_mode = \"Manual\"\n overprovision = false\n\n \"os_profile\" {\n computer_name_prefix = \"${var.es_cluster}-master\"\n admin_username = \"ubuntu\"\n admin_password = \"${random_string.vm-login-password.result}\"\n custom_data = \"${data.template_file.master_userdata_script.rendered}\"\n }\n\n \"network_profile\" {\n name = \"es-${var.es_cluster}-net-profile\"\n primary = true\n\n \"ip_configuration\" {\n name = \"es-${var.es_cluster}-ip-profile\"\n subnet_id = \"${azurerm_subnet.elasticsearch_subnet.id}\"\n }\n }\n\n storage_profile_image_reference {\n id = \"${data.azurerm_image.elasticsearch.id}\"\n }\n\n \"storage_profile_os_disk\" {\n caching = \"ReadWrite\"\n create_option = \"FromImage\"\n managed_disk_type = \"Standard_LRS\"\n }\n\n os_profile_linux_config {\n disable_password_authentication = true\n ssh_keys {\n path = \"/home/ubuntu/.ssh/authorized_keys\"\n key_data = \"${file(var.key_path)}\"\n }\n }\n}" }, { "path": "terraform-azure/outputs.tf", "content": "output \"es_image_id\" {\n value = \"${data.azurerm_image.elasticsearch.name}\"\n}\n\noutput \"kibana_image_id\" {\n value = \"${data.azurerm_image.kibana.name}\"\n}\n\noutput \"clients_public_dns\" {\n value = \"${azurerm_public_ip.clients.*.fqdn}\"\n}\n\noutput \"clients_public_ip_address\" {\n value = \"${azurerm_public_ip.clients.*.ip_address}\"\n}\n\noutput \"public_dns\" {\n value = \"${azurerm_public_ip.single-node.*.fqdn}\"\n}\n\noutput \"public_ip_address\" {\n value = \"${azurerm_public_ip.single-node.*.ip_address}\"\n}\n\noutput \"vm_password\" {\n value = \"${random_string.vm-login-password.result}\"\n}" }, { "path": "terraform-azure/single-node.tf", "content": "data \"template_file\" \"singlenode_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/user_data.sh\")}\"\n\n vars {\n cloud_provider = \"azure\"\n volume_name = \"\"\n elasticsearch_data_dir = \"${var.elasticsearch_data_dir}\"\n elasticsearch_logs_dir = \"${var.elasticsearch_logs_dir}\"\n heap_size = \"${var.data_heap_size}\"\n es_cluster = \"${var.es_cluster}\"\n es_environment = \"${var.environment}-${var.es_cluster}\"\n security_groups = \"\"\n availability_zones = \"\"\n minimum_master_nodes = \"${format(\"%d\", var.masters_count / 2 + 1)}\"\n master = \"true\"\n data = \"true\"\n http_enabled = \"true\"\n security_enabled = \"${var.security_enabled}\"\n monitoring_enabled = \"${var.monitoring_enabled}\"\n client_user = \"${var.client_user}\"\n client_pwd = \"${random_string.vm-login-password.result}\"\n }\n}\n\nresource \"azurerm_public_ip\" \"single-node\" {\n count = \"${var.masters_count == \"0\" && var.datas_count == \"0\" ? \"1\" : \"0\"}\"\n name = \"es-${var.es_cluster}-single-node-public-ip\"\n location = \"${var.azure_location}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n public_ip_address_allocation = \"static\"\n domain_name_label = \"${azurerm_resource_group.elasticsearch.name}\"\n}\n\nresource \"azurerm_network_interface\" \"single-node\" {\n // Only create if it's a single-node configuration\n count = \"${var.masters_count == \"0\" && var.datas_count == \"0\" ? \"1\" : \"0\"}\"\n\n name = \"es-${var.es_cluster}-singlenode-nic\"\n location = \"${var.azure_location}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n\n ip_configuration {\n name = \"es-${var.es_cluster}-singlenode-ip\"\n subnet_id = \"${azurerm_subnet.elasticsearch_subnet.id}\"\n private_ip_address_allocation = \"dynamic\"\n public_ip_address_id = \"${azurerm_public_ip.single-node.id}\"\n }\n}\n\nresource \"azurerm_virtual_machine\" \"single-node\" {\n // Only create if it's a single-node configuration\n count = \"${var.masters_count == \"0\" && var.datas_count == \"0\" ? \"1\" : \"0\"}\"\n\n name = \"es-${var.es_cluster}-singlenode\"\n location = \"${var.azure_location}\"\n resource_group_name = \"${azurerm_resource_group.elasticsearch.name}\"\n network_interface_ids = [\"${azurerm_network_interface.single-node.id}\"]\n vm_size = \"${var.data_instance_type}\"\n\n storage_image_reference {\n id = \"${data.azurerm_image.kibana.id}\"\n }\n\n storage_os_disk {\n name = \"es-${var.es_cluster}-singlenode-osdisk\"\n caching = \"ReadWrite\"\n create_option = \"FromImage\"\n managed_disk_type = \"Standard_LRS\"\n }\n\n \"os_profile\" {\n computer_name = \"es-${var.es_cluster}-singlenode\"\n admin_username = \"ubuntu\"\n admin_password = \"${random_string.vm-login-password.result}\"\n custom_data = \"${data.template_file.singlenode_userdata_script.rendered}\"\n }\n\n os_profile_linux_config {\n disable_password_authentication = true\n\n ssh_keys {\n path = \"/home/ubuntu/.ssh/authorized_keys\"\n key_data = \"${file(var.key_path)}\"\n }\n }\n}" }, { "path": "terraform-azure/variables.tf", "content": "variable \"azure_location\" {\n type = \"string\"\n default = \"East US\"\n}\n\nvariable \"azure_client_id\" {\n type = \"string\"\n}\n\nvariable \"azure_client_secret\" {\n type = \"string\"\n}\n\nvariable \"azure_subscription_id\" {\n type = \"string\"\n}\n\nvariable \"azure_tenant_id\" {\n type = \"string\"\n}\n\nvariable \"es_cluster\" {\n description = \"Name of the elasticsearch cluster, used in node discovery\"\n default = \"my-cluster\"\n}\n\nvariable \"key_path\" {\n description = \"Key name to be used with the launched EC2 instances.\"\n default = \"~/.ssh/id_rsa.pub\"\n}\n\nvariable \"environment\" {\n default = \"default\"\n}\n\nvariable \"data_instance_type\" {\n type = \"string\"\n default = \"Standard_D12_v2\"\n}\n\nvariable \"master_instance_type\" {\n type = \"string\"\n default = \"Standard_A2_v2\"\n}\n\nvariable \"client_instance_type\" {\n type = \"string\"\n default = \"Standard_A2_v2\"\n}\n\nvariable \"elasticsearch_volume_size\" {\n type = \"string\"\n default = \"100\" # gb\n}\n\nvariable \"use_instance_storage\" {\n default = \"true\"\n}\n\nvariable \"associate_public_ip\" {\n default = \"true\"\n}\n\nvariable \"elasticsearch_data_dir\" {\n default = \"/mnt/elasticsearch/data\"\n}\n\nvariable \"elasticsearch_logs_dir\" {\n default = \"/var/log/elasticsearch\"\n}\n\n# default elasticsearch heap size\nvariable \"data_heap_size\" {\n type = \"string\"\n default = \"7g\"\n}\n\nvariable \"master_heap_size\" {\n type = \"string\"\n default = \"2g\"\n}\n\nvariable \"masters_count\" {\n default = \"1\"\n}\n\nvariable \"datas_count\" {\n default = \"1\"\n}\n\nvariable \"clients_count\" {\n default = \"1\"\n}\n\n# whether or not to enable x-pack security on the cluster\nvariable \"security_enabled\" {\n default = \"false\"\n}\n\n# whether or not to enable x-pack monitoring on the cluster\nvariable \"monitoring_enabled\" {\n default = \"true\"\n}\n\n# client nodes have nginx installed on them, these credentials are used for basic auth\nvariable \"client_user\" {\n default = \"exampleuser\"\n}" }, { "path": "terraform-gcp/certs.tf", "content": "locals {\n cert_common_name = \"elasticsearch-cloud-deploy autogenerated CA\"\n validity_period_hours = 365 * 24\n early_renewal_hours = 30 * 24\n}\n\nresource \"tls_private_key\" \"ca\" {\n count = var.security_enabled ? 1 : 0\n\n algorithm = \"RSA\"\n}\n\nresource \"tls_self_signed_cert\" \"ca\" {\n count = var.security_enabled ? 1 : 0\n\n key_algorithm = \"RSA\"\n private_key_pem = join(\"\", tls_private_key.ca[*].private_key_pem)\n\n subject {\n common_name = local.cert_common_name\n }\n\n validity_period_hours = local.validity_period_hours\n early_renewal_hours = local.early_renewal_hours\n is_ca_certificate = true\n\n allowed_uses = [\n \"server_auth\",\n \"cert_signing\",\n \"crl_signing\",\n \"client_auth\"\n ]\n}\n\nresource \"tls_private_key\" \"node\" {\n count = var.security_enabled ? 1 : 0\n\n algorithm = \"RSA\"\n}\n\nresource \"tls_cert_request\" \"node\" {\n count = var.security_enabled ? 1 : 0\n\n key_algorithm = \"RSA\"\n private_key_pem = join(\"\", tls_private_key.node[*].private_key_pem)\n\n subject {\n common_name = local.cert_common_name\n }\n}\n\nresource \"tls_locally_signed_cert\" \"node\" {\n count = var.security_enabled ? 1 : 0\n\n ca_key_algorithm = \"RSA\"\n cert_request_pem = join(\"\", tls_cert_request.node[*].cert_request_pem)\n ca_private_key_pem = join(\"\", tls_private_key.ca[*].private_key_pem)\n ca_cert_pem = join(\"\", tls_self_signed_cert.ca[*].cert_pem)\n\n validity_period_hours = local.validity_period_hours\n early_renewal_hours = local.early_renewal_hours\n\n allowed_uses = [\n \"key_encipherment\",\n \"digital_signature\",\n \"server_auth\",\n \"client_auth\"\n ]\n}\n" }, { "path": "terraform-gcp/client.tf", "content": "data \"template_file\" \"client_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/gcp_user_data.sh\")}\"\n vars = merge(local.user_data_common, {\n heap_size = \"${var.client_heap_size}\"\n startup_script = \"client.sh\"\n })\n}\n\n\nresource \"google_compute_target_pool\" \"client\" {\n name = \"${var.es_cluster}-client-targetpool\"\n}\n\n\n\nresource \"google_compute_instance_group_manager\" \"client\" {\n for_each = toset(keys(var.clients_count))\n\n provider = google\n name = \"${var.es_cluster}-igm-client-${each.value}\"\n project = \"${var.gcp_project_id}\"\n zone = each.value\n\n named_port {\n name = \"nginx\"\n port = 8080\n }\n\n named_port {\n name = \"es\"\n port = 9200\n }\n\n version {\n instance_template = google_compute_instance_template.client.self_link\n name = \"primary\"\n }\n\n base_instance_name = \"${var.es_cluster}-client\"\n target_pools = [google_compute_target_pool.client.self_link]\n}\n\nresource \"google_compute_autoscaler\" \"client\" {\n for_each = toset(keys(var.clients_count))\n\n name = \"${var.es_cluster}-autoscaler-client-${each.value}\"\n zone = each.value\n target = google_compute_instance_group_manager.client[each.value].self_link\n\n autoscaling_policy {\n max_replicas = var.clients_count[each.value]\n min_replicas = var.clients_count[each.value]\n cooldown_period = 60\n }\n}\n\nresource \"google_compute_instance_template\" \"client\" {\n provider = google\n name_prefix = \"${var.es_cluster}-instance-template-client\"\n project = \"${var.gcp_project_id}\"\n machine_type = \"${var.master_machine_type}\"\n can_ip_forward = true\n\n tags = [\n \"${var.es_cluster}\",\n \"es-client-node\",\n \"http-server\",\n \"https-server\"\n ]\n\n metadata_startup_script = \"${data.template_file.client_userdata_script.rendered}\"\n\n labels = {\n environment = var.environment\n cluster = \"${var.environment}-${var.es_cluster}\"\n role = \"client\"\n }\n\n disk {\n source_image = data.google_compute_image.kibana.self_link\n boot = true\n }\n\n network_interface {\n network = var.cluster_network\n }\n\n service_account {\n scopes = [\"userinfo-email\", \"compute-rw\", \"storage-ro\"]\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n" }, { "path": "terraform-gcp/datas-voters.tf", "content": "data \"template_file\" \"data_voting_userdata_script\" {\n template = file(\"${path.module}/../templates/gcp_user_data.sh\")\n vars = merge(local.user_data_common, {\n heap_size = \"${var.data_heap_size}\"\n is_voting_only = \"true\"\n startup_script = \"data.sh\"\n })\n}\n\nresource \"google_compute_instance_group_manager\" \"data-voters\" {\n for_each = toset(keys(var.data_voters_count))\n\n provider = google\n name = \"${var.es_cluster}-igm-data-voters-${each.value}\"\n project = var.gcp_project_id\n zone = each.value\n\n version {\n instance_template = google_compute_instance_template.data-voters.self_link\n name = \"primary\"\n }\n\n named_port {\n name = \"es\"\n port = 9200\n }\n\n base_instance_name = \"${var.es_cluster}-data-voting\"\n target_pools = var.enable_direct_data_access ? [google_compute_target_pool.client.self_link] : []\n\n}\n\nresource \"google_compute_autoscaler\" \"data-voters\" {\n for_each = toset(keys(var.data_voters_count))\n\n name = \"${var.es_cluster}-autoscaler-data-voters-${each.value}\"\n zone = each.value\n target = google_compute_instance_group_manager.data-voters[each.value].self_link\n\n autoscaling_policy {\n max_replicas = var.data_voters_count[each.value]\n min_replicas = var.data_voters_count[each.value]\n cooldown_period = 60\n }\n}\n\nresource \"google_compute_instance_template\" \"data-voters\" {\n provider = google\n name_prefix = \"${var.es_cluster}-instance-template-data-voters\"\n project = var.gcp_project_id\n machine_type = var.data_machine_type\n can_ip_forward = false\n\n tags = [\"${var.es_cluster}\", \"es-data-node\", \"es-master-node\"]\n\n metadata_startup_script = data.template_file.data_voting_userdata_script.rendered\n\n labels = {\n environment = var.environment\n cluster = \"${var.environment}-${var.es_cluster}\"\n role = \"data-voters\"\n }\n\n disk {\n source_image = data.google_compute_image.elasticsearch.self_link\n boot = true\n }\n\n network_interface {\n network = var.cluster_network\n }\n\n service_account {\n scopes = [\"userinfo-email\", \"compute-rw\", \"storage-ro\"]\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n" }, { "path": "terraform-gcp/datas.tf", "content": "data \"template_file\" \"data_userdata_script\" {\n template = file(\"${path.module}/../templates/gcp_user_data.sh\")\n vars = merge(local.user_data_common, {\n heap_size = \"${var.data_heap_size}\"\n startup_script = \"data.sh\"\n })\n}\n\nresource \"google_compute_instance_group_manager\" \"data\" {\n for_each = toset(keys(var.datas_count))\n\n provider = google\n name = \"${var.es_cluster}-igm-data-${each.value}\"\n project = var.gcp_project_id\n zone = each.value\n\n version {\n instance_template = google_compute_instance_template.data.self_link\n name = \"primary\"\n }\n\n named_port {\n name = \"es\"\n port = 9200\n }\n\n base_instance_name = \"${var.es_cluster}-data\"\n target_pools = var.enable_direct_data_access ? [google_compute_target_pool.client.self_link] : []\n\n}\n\nresource \"google_compute_autoscaler\" \"data\" {\n for_each = toset(keys(var.datas_count))\n\n name = \"${var.es_cluster}-autoscaler-data-${each.value}\"\n zone = each.value\n target = google_compute_instance_group_manager.data[each.value].self_link\n\n autoscaling_policy {\n max_replicas = var.datas_count[each.value]\n min_replicas = var.datas_count[each.value]\n cooldown_period = 60\n }\n}\n\nresource \"google_compute_instance_template\" \"data\" {\n provider = google\n name_prefix = \"${var.es_cluster}-instance-template-data\"\n project = var.gcp_project_id\n machine_type = var.data_machine_type\n can_ip_forward = false\n\n tags = [\"${var.es_cluster}\", \"es-data-node\"]\n\n metadata_startup_script = data.template_file.data_userdata_script.rendered\n\n labels = {\n environment = var.environment\n cluster = \"${var.environment}-${var.es_cluster}\"\n role = \"data\"\n }\n\n disk {\n source_image = data.google_compute_image.elasticsearch.self_link\n boot = true\n }\n\n network_interface {\n network = var.cluster_network\n }\n\n service_account {\n scopes = [\"userinfo-email\", \"compute-rw\", \"storage-ro\"]\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n" }, { "path": "terraform-gcp/dev.tf", "content": "resource \"google_storage_bucket\" \"dev\" {\n count = var.DEV_MODE_scripts_gcs_bucket != \"\" ? 1 : 0\n name = var.DEV_MODE_scripts_gcs_bucket\n location = var.gcp_region\n force_destroy = true\n}" }, { "path": "terraform-gcp/disks.tf", "content": "locals {\n master_zone_flattened = toset(flatten([\n for zone, count in var.masters_count : [\n for i in range(0, count) : jsonencode({\n \"zone\" = zone,\n \"index\" = i,\n \"name\" = \"${zone}-${i}\"\n })\n ]\n ]))\n\n data_voters_zone_flattened = toset(flatten([\n for zone, count in var.data_voters_count : [\n for i in range(0, count) : jsonencode({\n \"zone\" = zone,\n \"index\" = i,\n \"name\" = \"${zone}-${i}\"\n })\n ]\n ]))\n\n data_zone_flattened = toset(flatten([\n for zone, count in var.datas_count : [\n for i in range(0, count) : jsonencode({\n \"zone\" = zone,\n \"index\" = i,\n \"name\" = \"${zone}-${i}\"\n })\n ]\n ]))\n}\n\nresource \"google_compute_disk\" \"master\" {\n for_each = local.master_zone_flattened\n\n name = \"elasticsearch-${var.es_cluster}-master-${jsondecode(each.value)[\"name\"]}\"\n zone = jsondecode(each.value)[\"zone\"]\n size = 10\n\n labels = {\n cluster-name = \"${var.es_cluster}\"\n volume-index = jsondecode(each.value)[\"index\"]\n auto-attach-group = \"master\"\n }\n}\n\nresource \"google_compute_disk\" \"data\" {\n for_each = local.data_zone_flattened\n\n name = \"elasticsearch-${var.es_cluster}-data-${jsondecode(each.value)[\"name\"]}\"\n zone = jsondecode(each.value)[\"zone\"]\n size = var.elasticsearch_volume_size\n\n labels = {\n cluster-name = \"${var.es_cluster}\"\n volume-index = jsondecode(each.value)[\"index\"]\n auto-attach-group = \"data\"\n }\n}\n\nresource \"google_compute_disk\" \"data_voters\" {\n for_each = local.data_voters_zone_flattened\n\n name = \"elasticsearch-${var.es_cluster}-data-voters-${jsondecode(each.value)[\"name\"]}\"\n zone = jsondecode(each.value)[\"zone\"]\n size = var.elasticsearch_volume_size\n\n labels = {\n cluster-name = \"${var.es_cluster}\"\n volume-index = jsondecode(each.value)[\"index\"]\n auto-attach-group = \"data-voters\"\n }\n}\n\nresource \"google_compute_disk\" \"singlenode\" {\n count = local.singlenode_mode ? 1 : 0\n\n name = \"elasticsearch-${var.es_cluster}-singlenode\"\n zone = var.singlenode_zone\n size = var.elasticsearch_volume_size\n\n labels = {\n cluster-name = \"${var.es_cluster}\"\n volume-index = \"0\"\n auto-attach-group = \"singlenode\"\n }\n}\n" }, { "path": "terraform-gcp/image.tf", "content": "data \"google_compute_image\" \"elasticsearch\" {\n family = \"elasticsearch-7\"\n}\n\ndata \"google_compute_image\" \"kibana\" {\n family = \"kibana-7\"\n}\n" }, { "path": "terraform-gcp/lb.tf", "content": "# Public LB\n\nlocals {\n external_ports = var.public_facing ? toset([\"9200\", \"5601\"]) : toset([])\n\n load_balance_data_nodes = !local.singlenode_mode && var.enable_direct_data_access\n load_balance_client_nodes = !local.singlenode_mode && length(var.clients_count) > 0\n\n}\n\n## Address\nresource \"google_compute_address\" \"external-lb\" {\n count = var.public_facing ? 1 : 0\n name = \"${var.es_cluster}-external-lb\"\n}\n\n## Single node mode\nresource \"google_compute_forwarding_rule\" \"singlenode\" {\n for_each = local.singlenode_mode ? local.external_ports : []\n\n ip_address = join(\"\", google_compute_address.external-lb[*].address)\n name = \"${var.es_cluster}-external-singlenode-${each.value}\"\n target = google_compute_target_pool.singlenode.self_link\n port_range = each.value\n}\n\n## cluster mode\nresource \"google_compute_forwarding_rule\" \"client\" {\n for_each = (local.load_balance_client_nodes) ? local.external_ports : []\n\n ip_address = join(\"\", google_compute_address.external-lb[*].address)\n name = \"${var.es_cluster}-external-client-${each.value}\"\n target = google_compute_target_pool.client.self_link\n port_range = each.value\n}\n\n\n# Internal LB\n\n## Healthcheck\nresource \"google_compute_health_check\" \"internal\" {\n name = \"${var.es_cluster}-internal-healthcheck\"\n\n timeout_sec = 1\n check_interval_sec = 1\n\n tcp_health_check {\n port = \"9200\"\n port_specification = \"USE_FIXED_PORT\"\n }\n\n log_config {\n enable = true\n }\n}\n\n\n## Single node\nresource \"google_compute_region_backend_service\" \"internal-singlenode\" {\n count = local.singlenode_mode ? 1 : 0\n\n name = \"${var.es_cluster}-internal-singlenode\"\n region = var.gcp_region\n health_checks = [google_compute_health_check.internal.self_link]\n protocol = \"TCP\"\n\n backend {\n group = google_compute_instance_group_manager.singlenode.instance_group\n }\n}\n\n\nresource \"google_compute_forwarding_rule\" \"internal-singlenode\" {\n count = local.singlenode_mode ? 1 : 0\n\n name = \"${var.es_cluster}-internal-singlenode\"\n region = var.gcp_region\n service_label = \"${var.es_cluster}-internal\"\n load_balancing_scheme = \"INTERNAL\"\n backend_service = join(\"\", google_compute_region_backend_service.internal-singlenode[*].self_link)\n all_ports = true\n}\n\n\n## Client nodes\nresource \"google_compute_region_backend_service\" \"internal-client\" {\n count = local.load_balance_client_nodes || local.load_balance_data_nodes ? 1 : 0\n\n name = \"${var.es_cluster}-internal-client\"\n region = var.gcp_region\n health_checks = [google_compute_health_check.internal.self_link]\n protocol = \"TCP\"\n\n dynamic \"backend\" {\n for_each = local.load_balance_client_nodes ? toset(keys(var.clients_count)) : []\n content {\n group = google_compute_instance_group_manager.client[backend.value].instance_group\n }\n }\n\n dynamic \"backend\" {\n for_each = local.load_balance_data_nodes ? toset(keys(var.datas_count)) : []\n content {\n group = google_compute_instance_group_manager.data[backend.value].instance_group\n }\n }\n\n dynamic \"backend\" {\n for_each = local.load_balance_data_nodes ? toset(keys(var.data_voters_count)) : []\n content {\n group = google_compute_instance_group_manager.data-voters[backend.value].instance_group\n }\n }\n}\n\n## forwarding rule\nresource \"google_compute_forwarding_rule\" \"internal-client\" {\n count = !local.singlenode_mode ? 1 : 0\n\n name = \"${var.es_cluster}-internal-client\"\n region = var.gcp_region\n service_label = \"${var.es_cluster}-internal\"\n load_balancing_scheme = \"INTERNAL\"\n backend_service = join(\"\", google_compute_region_backend_service.internal-client[*].self_link)\n all_ports = true\n}\n" }, { "path": "terraform-gcp/main.tf", "content": "terraform {\n required_providers {\n tls = {\n source = \"hashicorp/tls\"\n version = \"3.1.0\"\n }\n }\n}\n\nprovider \"google\" {\n # comment out to use environment credentials\n credentials = var.gcp_credentials_path\n project = var.gcp_project_id\n region = var.gcp_region\n zone = var.gcp_zone\n}\n\nresource \"random_string\" \"vm-login-password\" {\n length = 16\n special = false\n}\n\nresource \"random_string\" \"security-encryption-key\" {\n length = 32\n special = false\n}\nresource \"random_string\" \"reporting-encryption-key\" {\n length = 32\n special = false\n}\n\nresource \"google_compute_firewall\" \"internode\" {\n name = \"${var.es_cluster}-firewall-allow-internode\"\n network = var.cluster_network\n\n allow {\n protocol = \"tcp\"\n ports = [\"9200-9400\"]\n }\n\n source_tags = [var.es_cluster]\n}\n\nresource \"google_compute_firewall\" \"external\" {\n count = var.public_facing ? 1 : 0\n\n name = \"${var.es_cluster}-firewall-allow-external\"\n network = var.cluster_network\n\n allow {\n protocol = \"tcp\"\n ports = [\"9200\", \"5601\"]\n }\n}\n\nresource \"google_compute_router\" \"router\" {\n name = \"${var.es_cluster}-router\"\n network = var.cluster_network\n}\n\nresource \"google_compute_router_nat\" \"nat\" {\n name = \"${var.es_cluster}-router-nat\"\n router = google_compute_router.router.name\n nat_ip_allocate_option = \"AUTO_ONLY\"\n source_subnetwork_ip_ranges_to_nat = \"ALL_SUBNETWORKS_ALL_IP_RANGES\"\n}\n\nresource \"google_service_account\" \"gcs\" {\n account_id = \"${var.es_cluster}-gcs\"\n display_name = \"${var.es_cluster}-gcs-service-account\"\n}\n\nresource \"google_service_account_key\" \"gcs\" {\n service_account_id = google_service_account.gcs.name\n public_key_type = \"TYPE_X509_PEM_FILE\"\n}\n\nresource \"google_storage_bucket\" \"snapshots\" {\n count = var.gcs_snapshots_bucket != \"\" ? 1 : 0\n name = var.gcs_snapshots_bucket\n location = var.gcp_region\n}\n\nresource \"google_storage_bucket_iam_member\" \"legacy-bucket-reader\" {\n count = var.gcs_snapshots_bucket != \"\" ? 1 : 0\n bucket = join(\"\", google_storage_bucket.snapshots[*].name)\n role = \"roles/storage.legacyBucketReader\"\n member = \"serviceAccount:${google_service_account.gcs.email}\"\n}\n\nresource \"google_storage_bucket_iam_member\" \"object-admin\" {\n count = var.gcs_snapshots_bucket != \"\" ? 1 : 0\n bucket = join(\"\", google_storage_bucket.snapshots[*].name)\n role = \"roles/storage.objectAdmin\"\n member = \"serviceAccount:${google_service_account.gcs.email}\"\n}\n\nlocals {\n masters_count = sum(concat(values(var.masters_count), values(var.data_voters_count)))\n\n all_zones = compact(tolist(setunion(\n keys(var.masters_count),\n keys(var.datas_count),\n keys(var.data_voters_count),\n keys(var.clients_count),\n toset([var.singlenode_zone])\n )))\n\n singlenode_mode = (length(keys(var.masters_count)) + length(keys(var.datas_count)) + length(keys(var.clients_count))) == 0\n is_cluster_bootstrapped = data.local_file.cluster_bootstrap_state.content == \"1\" || !var.requires_bootstrapping\n\n user_data_common = {\n cloud_provider = \"gcp\"\n gcs_snapshots_bucket = var.gcs_snapshots_bucket\n elasticsearch_data_dir = var.elasticsearch_data_dir\n elasticsearch_logs_dir = var.elasticsearch_logs_dir\n es_cluster = var.es_cluster\n gcp_project_id = var.gcp_project_id\n gcp_zones = join(\",\", tolist(local.all_zones))\n es_environment = \"${var.environment}-${var.es_cluster}\"\n security_enabled = var.security_enabled\n monitoring_enabled = var.monitoring_enabled\n masters_count = local.masters_count\n client_user = var.client_user\n xpack_monitoring_host = var.xpack_monitoring_host\n filebeat_monitoring_host = var.filebeat_monitoring_host\n use_g1gc = var.use_g1gc\n client_pwd = random_string.vm-login-password.result\n master = false\n data = false\n bootstrap_node = false\n log_level = var.log_level\n log_size = var.log_size\n is_voting_only = false\n gcs_service_account_key = join(\"\", google_service_account_key.gcs[*].private_key)\n ca_cert = var.security_enabled ? join(\"\", tls_self_signed_cert.ca[*].cert_pem) : \"\"\n node_cert = var.security_enabled ? join(\"\", tls_locally_signed_cert.node[*].cert_pem) : \"\"\n node_key = var.security_enabled ? join(\"\", tls_private_key.node[*].private_key_pem) : \"\"\n DEV_MODE_scripts_gcs_bucket = var.DEV_MODE_scripts_gcs_bucket\n security_encryption_key = random_string.security-encryption-key.result\n reporting_encryption_key = random_string.reporting-encryption-key.result\n auto_shut_down_bootstrap_node = var.auto_shut_down_bootstrap_node\n }\n}\n" }, { "path": "terraform-gcp/masters.tf", "content": "data \"local_file\" \"cluster_bootstrap_state\" {\n filename = \"${path.module}/cluster_bootstrap_state\"\n}\n\ndata \"template_file\" \"master_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/gcp_user_data.sh\")}\"\n vars = merge(local.user_data_common, {\n heap_size = \"${var.master_heap_size}\"\n startup_script = \"master.sh\"\n })\n}\n\ndata \"template_file\" \"bootstrap_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/gcp_user_data.sh\")}\"\n vars = merge(local.user_data_common, {\n heap_size = \"${var.master_heap_size}\"\n startup_script = \"bootstrap.sh\"\n })\n}\n\nresource \"google_compute_instance_group_manager\" \"master\" {\n for_each = toset(keys(var.masters_count))\n\n provider = google-beta\n name = \"${var.es_cluster}-igm-master-${each.value}\"\n project = \"${var.gcp_project_id}\"\n zone = each.value\n\n version {\n instance_template = google_compute_instance_template.master.self_link\n name = \"primary\"\n }\n\n base_instance_name = \"${var.es_cluster}-master\"\n}\n\nresource \"google_compute_autoscaler\" \"master\" {\n for_each = toset(keys(var.masters_count))\n\n name = \"${var.es_cluster}-autoscaler-master-${each.value}\"\n zone = each.value\n target = google_compute_instance_group_manager.master[each.value].self_link\n\n autoscaling_policy {\n max_replicas = var.masters_count[each.value]\n min_replicas = var.masters_count[each.value]\n cooldown_period = 60\n }\n}\n\nresource \"google_compute_instance\" \"bootstrap_node\" {\n count = local.singlenode_mode || local.is_cluster_bootstrapped ? 0 : 1\n\n name = \"${var.es_cluster}-bootstrap-node\"\n machine_type = \"${var.master_machine_type}\"\n zone = \"${var.gcp_zone}\"\n\n tags = [\"${var.es_cluster}\", \"es-bootstrap-node\"]\n\n boot_disk {\n initialize_params {\n image = data.google_compute_image.elasticsearch.self_link\n }\n }\n\n network_interface {\n network = var.cluster_network\n }\n\n metadata_startup_script = \"${data.template_file.bootstrap_userdata_script.rendered}\"\n\n service_account {\n scopes = [\"userinfo-email\", \"compute-rw\", \"storage-ro\"]\n }\n}\n\nresource \"google_compute_instance_template\" \"master\" {\n provider = google-beta\n name_prefix = \"${var.es_cluster}-instance-template-master\"\n project = \"${var.gcp_project_id}\"\n machine_type = \"${var.master_machine_type}\"\n can_ip_forward = false\n\n tags = [\"${var.es_cluster}\", \"es-master-node\"]\n\n metadata_startup_script = \"${data.template_file.master_userdata_script.rendered}\"\n\n labels = {\n environment = var.environment\n cluster = \"${var.environment}-${var.es_cluster}\"\n role = \"master\"\n }\n\n disk {\n source_image = data.google_compute_image.elasticsearch.self_link\n boot = true\n }\n\n network_interface {\n network = var.cluster_network\n }\n\n service_account {\n scopes = [\"userinfo-email\", \"compute-rw\", \"storage-ro\"]\n }\n\n lifecycle {\n create_before_destroy = true\n }\n}\n\n\nresource \"null_resource\" \"cluster_bootstrap_state\" {\n provisioner \"local-exec\" {\n command = \"printf 1 > ${path.module}/cluster_bootstrap_state\"\n }\n provisioner \"local-exec\" {\n when = destroy\n command = \"printf 0 > ${path.module}/cluster_bootstrap_state\"\n }\n\n depends_on = [google_compute_instance.bootstrap_node]\n}" }, { "path": "terraform-gcp/outputs.tf", "content": "output \"external_lb\" {\n value = var.public_facing ? join(\"\", google_compute_address.external-lb[*].address) : \"\"\n}\n\noutput \"internal_lb\" {\n value = local.singlenode_mode ? join(\"\", google_compute_forwarding_rule.internal-singlenode[*].service_name) : join(\"\", google_compute_forwarding_rule.internal-client[*].service_name)\n}\n\noutput \"vm_password\" {\n value = \"${random_string.vm-login-password.result}\"\n}" }, { "path": "terraform-gcp/singlenode.tf", "content": "data \"template_file\" \"singlenode_userdata_script\" {\n template = \"${file(\"${path.module}/../templates/gcp_user_data.sh\")}\"\n vars = merge(local.user_data_common, {\n heap_size = \"${var.master_heap_size}\"\n startup_script = \"singlenode.sh\"\n })\n}\n\nresource \"google_compute_target_pool\" \"singlenode\" {\n name = \"${var.es_cluster}-singlenode-targetpool\"\n}\n\nresource \"google_compute_instance_group_manager\" \"singlenode\" {\n provider = google\n\n name = \"${var.es_cluster}-igm-singlenode\"\n project = \"${var.gcp_project_id}\"\n zone = \"${var.singlenode_zone}\"\n\n version {\n instance_template = google_compute_instance_template.singlenode.self_link\n name = \"primary\"\n }\n\n base_instance_name = \"${var.es_cluster}-singlenode\"\n target_pools = [google_compute_target_pool.singlenode.self_link]\n}\n\nresource \"google_compute_autoscaler\" \"singlenode\" {\n count = local.singlenode_mode ? 1 : 0\n\n name = \"${var.es_cluster}-autoscaler-singlenode\"\n zone = \"${var.singlenode_zone}\"\n target = google_compute_instance_group_manager.singlenode.self_link\n\n autoscaling_policy {\n max_replicas = 1\n min_replicas = 1\n cooldown_period = 60\n }\n}\n\nresource \"google_compute_instance_template\" \"singlenode\" {\n provider = google\n name_prefix = \"${var.es_cluster}-instance-template-single\"\n\n project = \"${var.gcp_project_id}\"\n machine_type = \"${var.data_machine_type}\"\n\n tags = [\"${var.es_cluster}\", \"es-singlenode-node\", \"http-server\", \"https-server\"]\n\n metadata = {\n sshKeys = \"ubuntu:${file(var.gcp_ssh_pub_key_file)}\"\n }\n metadata_startup_script = \"${data.template_file.singlenode_userdata_script.rendered}\"\n\n labels = {\n environment = var.environment\n cluster = \"${var.environment}-${var.es_cluster}\"\n role = \"singlenode\"\n }\n\n disk {\n source_image = data.google_compute_image.kibana.self_link\n boot = true\n }\n\n network_interface {\n network = var.cluster_network\n }\n\n service_account {\n scopes = [\"userinfo-email\", \"compute-rw\", \"storage-ro\"]\n }\n\n lifecycle {\n create_before_destroy = true\n }\n\n}\n" }, { "path": "terraform-gcp/terraform.tfvars.example", "content": "es_cluster = \"elastic-cluster\"\ngcp_project_id = \"some_project\"\n# see main.tf for using environment credentials\ngcp_credentials_path = \"credentials.json\"\ngcp_zone = \"us-east1-b\"\ngcp_region = \"us-east1\"\nenvironment = \"test\"\nmasters_count = {\n \"us-east1-b\" = 1\n}\ndatas_count = {\n \"us-east1-b\" = 2\n}\ndata_voters_count = {\n \"us-east1-b\" = 2\n}\nclients_count = {\n \"us-east1-b\" = 1\n}\nsecurity_enabled = true\nmonitoring_enabled = false\nclient_user = \"someuser\"\npublic_facing = false\ndata_machine_type = \"n2-highmem-2\"\nelasticsearch_volume_size = \"200\"\ndata_heap_size = \"8g\"\ngcp_ssh_pub_key_file = \"id_rsa.pub\"\nenable_direct_data_access = true\n" }, { "path": "terraform-gcp/variables.tf", "content": "### MANDATORY ###\nvariable \"es_cluster\" {\n description = \"Name of the elasticsearch cluster, used in node discovery\"\n}\n\nvariable \"gcp_project_id\" {\n type = string\n}\n\nvariable \"gcp_credentials_path\" {\n type = string\n default = \"\"\n}\n\nvariable \"gcp_zone\" {\n type = string\n default = \"us-central1-a\"\n}\n\nvariable \"gcp_region\" {\n type = string\n default = \"us-central1\"\n}\n\nvariable \"environment\" {\n default = \"default\"\n}\n\nvariable \"masters_count\" {\n type = map(number)\n default = {}\n description = \"Master nodes count per GCP zone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"datas_count\" {\n type = map(number)\n default = {}\n description = \"Data nodes count per GCP zone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"data_voters_count\" {\n type = map(number)\n default = {}\n description = \"Data nodes count per GCP zone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"clients_count\" {\n type = map(number)\n default = {}\n description = \"Client nodes count per GCP zone. If all node counts are empty, will run in singlenode mode.\"\n}\n\nvariable \"security_enabled\" {\n description = \"Whether or not to enable x-pack security on the cluster\"\n default = true\n}\n\nvariable \"singlenode_zone\" {\n description = \"This variable is required when running in singlenode mode. Singlenode mode is enabled when masters_count, datas_count and clients_count are all empty,\"\n default = \"\"\n}\n\nvariable \"monitoring_enabled\" {\n description = \"Whether or not to enable x-pack monitoring on the cluster\"\n default = \"true\"\n}\n\nvariable \"client_user\" {\n description = \"The username to use when setting up basic auth on Grafana and Cerebro.\"\n default = \"elastic\"\n}\n\nvariable \"public_facing\" {\n description = \"Whether or not the created cluster should be accessible from the public internet\"\n type = bool\n default = true\n}\n\nvariable \"gcs_snapshots_bucket\" {\n description = \"GCS bucket for backups\"\n default = \"\"\n}\n\nvariable \"cluster_network\" {\n default = \"default\"\n}\n\nvariable \"master_machine_type\" {\n default = \"n1-standard-1\"\n}\n\nvariable \"data_machine_type\" {\n default = \"n1-standard-4\"\n}\n\nvariable \"elasticsearch_volume_size\" {\n type = string\n default = \"100\" # gb\n}\n\nvariable \"elasticsearch_data_dir\" {\n default = \"/opt/elasticsearch/data\"\n}\n\nvariable \"elasticsearch_logs_dir\" {\n default = \"/var/log/elasticsearch\"\n}\n\nvariable \"data_heap_size\" {\n type = string\n default = \"8g\"\n}\n\nvariable \"master_heap_size\" {\n type = string\n default = \"2g\"\n}\n\nvariable \"client_heap_size\" {\n type = string\n default = \"1g\"\n}\n\nvariable \"xpack_monitoring_host\" {\n description = \"ES host to send monitoring data\"\n default = \"http://localhost:9200\"\n}\n\nvariable \"filebeat_monitoring_host\" {\n description = \"ES host to send filebeat data\"\n default = \"\"\n}\n\nvariable \"use_g1gc\" {\n description = \"Whether or not to enable G1GC in jvm.options ES config. . Left in for backwards compatibility, deployments with Elasticsearch 7.7 and above should not use this.\"\n default = false\n}\n\nvariable \"DEV_MODE_scripts_gcs_bucket\" {\n description = \"GCS bucket to override init scripts from. Should not be used on production.\"\n default = \"\"\n}\n\nvariable \"gcp_ssh_pub_key_file\" {\n default = \"id_rsa.pub\"\n}\n\n\nvariable \"enable_direct_data_access\" {\n default = false\n description = \"Enable attaching load balancer directly to data nodes\"\n}\n\nvariable \"requires_bootstrapping\" {\n description = \"Overrides cluster bootstrap state\"\n default = true\n}\n\nvariable \"log_size\" {\n description = \"Retained log4j log size in MB\"\n default = \"128\"\n}\n\nvariable \"log_level\" {\n description = \"log4j log level\"\n default = \"INFO\"\n}\n\nvariable \"auto_shut_down_bootstrap_node\" {\n description = \"disable to prevent bootstrap node from shutting down\"\n default = true\n}\n" } ]