[ { "path": ".gitattributes", "content": "# Declare files that will always have LF line endings on checkout.\n*.sh text eol=lf" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false\ncontact_links:\n- name: Discuss the Elastic Stack\n url: https://discuss.elastic.co\n about: Please ask questions related to the usage of Elastic products in those forums.\n- name: Docker Community Forums\n url: https://forums.docker.com\n about: Please ask questions related to the usage of Docker products in those forums.\n- name: docker-elk Gitter chat room\n url: https://app.gitter.im/#/room/#deviantony_docker-elk:gitter.im\n about: General questions regarding this project can also be asked in the chat.\n" }, { "path": ".github/ISSUE_TEMPLATE/issue_report.md", "content": "---\nname: Issue report\nabout: Report a problem with the docker-elk integration or its documentation.\n---\n\n\n\n\n### Problem description\n\n\n\n### Extra information\n\n#### Stack configuration\n\n\n\n#### Docker setup\n\n\n\n```console\n$ docker version\n\n[OUTPUT HERE]\n```\n\n\n\n```console\n$ docker compose version\n\n[OUTPUT HERE]\n```\n\n#### Container logs\n\n\n\n```console\n$ docker compose logs\n\n[OUTPUT HERE]\n```\n" }, { "path": ".github/dependabot.yml", "content": "# Dependabot configuration\n#\n# For more information, please refer to:\n# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates\n\nversion: 2\n\nupdates:\n\n# Maintain dependencies for GitHub Actions\n- package-ecosystem: github-actions\n directory: /\n schedule:\n interval: weekly\n" }, { "path": ".github/workflows/ci.yml", "content": "name: CI\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\njobs:\n\n test:\n name: Test suite\n # List of supported runners:\n # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners\n runs-on: ubuntu-22.04\n\n env:\n COMPOSE_PROJECT_NAME: docker-elk\n\n steps:\n - uses: actions/checkout@v6\n\n ########################################################\n # #\n # Ensure §\"Initial setup\" of the README remains valid. #\n # #\n ########################################################\n\n - name: Set password of every built-in user to 'testpasswd'\n run: >-\n sed -i\n -e 's/\\(ELASTIC_PASSWORD=\\)'\\''changeme'\\''/\\1testpasswd/g'\n -e 's/\\(LOGSTASH_INTERNAL_PASSWORD=\\)'\\''changeme'\\''/\\1testpasswd/g'\n -e 's/\\(KIBANA_SYSTEM_PASSWORD=\\)'\\''changeme'\\''/\\1testpasswd/g'\n -e 's/\\(METRICBEAT_INTERNAL_PASSWORD=\\)'\\'\\''/\\1testpasswd/g'\n -e 's/\\(FILEBEAT_INTERNAL_PASSWORD=\\)'\\'\\''/\\1testpasswd/g'\n -e 's/\\(HEARTBEAT_INTERNAL_PASSWORD=\\)'\\'\\''/\\1testpasswd/g'\n -e 's/\\(MONITORING_INTERNAL_PASSWORD=\\)'\\'\\''/\\1testpasswd/g'\n -e 's/\\(BEATS_SYSTEM_PASSWORD=\\)'\\'\\''/\\1testpasswd/g'\n .env\n\n # Elasticsearch's high disk watermark gets regularly exceeded on GitHub Actions runners.\n # https://www.elastic.co/guide/en/elasticsearch/reference/current/fix-watermark-errors.html\n - name: Disable Elasticsearch disk allocation decider\n run: |\n docker compose up -d elasticsearch\n .github/workflows/scripts/disable-disk-alloc-decider.sh\n\n - name: Pre-build container images\n run: >-\n docker compose\n -f docker-compose.yml\n -f extensions/fleet/fleet-compose.yml\n -f extensions/fleet/agent-apmserver-compose.yml\n -f extensions/metricbeat/metricbeat-compose.yml\n -f extensions/filebeat/filebeat-compose.yml\n -f extensions/heartbeat/heartbeat-compose.yml\n build\n\n - name: Generate Kibana encryption keys\n run: docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml\n\n - name: Set up users and roles\n run: docker compose up --exit-code-from=setup setup\n\n #############################\n # #\n # Test core and extensions. #\n # #\n #############################\n\n #\n # Core components: Elasticsearch, Logstash, Kibana\n #\n\n - name: Execute core test suite\n run: |\n docker compose up -d\n .github/workflows/scripts/run-tests-core.sh\n # next steps don't need Logstash\n docker compose stop logstash\n\n #\n # Fleet\n #\n\n - name: Execute Fleet test suite\n run: |\n docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml -f extensions/fleet/agent-apmserver-compose.yml up --remove-orphans -d fleet-server apm-server\n .github/workflows/scripts/run-tests-fleet.sh\n\n #\n # Metricbeat\n #\n\n - name: Execute Metricbeat test suite\n run: |\n docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up --remove-orphans -d metricbeat\n .github/workflows/scripts/run-tests-metricbeat.sh\n\n #\n # Filebeat\n #\n\n - name: Execute Filebeat test suite\n run: |\n docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up --remove-orphans -d filebeat\n .github/workflows/scripts/run-tests-filebeat.sh\n\n #\n # Heartbeat\n #\n\n - name: Execute Heartbeat test suite\n run: |\n docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat\n .github/workflows/scripts/run-tests-heartbeat.sh\n\n - name: Collect troubleshooting data\n id: debug-data\n if: failure()\n run: |\n declare debug_data_dir=\"$(mktemp -d)\"\n\n docker compose \\\n -f docker-compose.yml \\\n -f extensions/fleet/fleet-compose.yml \\\n -f extensions/fleet/agent-apmserver-compose.yml \\\n -f extensions/metricbeat/metricbeat-compose.yml \\\n -f extensions/filebeat/filebeat-compose.yml \\\n -f extensions/heartbeat/heartbeat-compose.yml \\\n ps >\"$debug_data_dir\"/docker_ps.log\n\n docker compose \\\n -f docker-compose.yml \\\n -f extensions/fleet/fleet-compose.yml \\\n -f extensions/fleet/agent-apmserver-compose.yml \\\n -f extensions/metricbeat/metricbeat-compose.yml \\\n -f extensions/filebeat/filebeat-compose.yml \\\n -f extensions/heartbeat/heartbeat-compose.yml \\\n logs >\"$debug_data_dir\"/docker_logs.log\n\n echo \"path=${debug_data_dir}\" >>\"$GITHUB_OUTPUT\"\n\n - name: Upload collected troubleshooting data\n if: always() && steps.debug-data.outputs.path\n uses: actions/upload-artifact@v7\n with:\n name: debug-data\n path: ${{ steps.debug-data.outputs.path }}/*.*\n\n ##############\n # #\n # Tear down. #\n # #\n ##############\n\n - name: Terminate all components\n if: always()\n run: >-\n docker compose\n -f docker-compose.yml\n -f extensions/fleet/fleet-compose.yml\n -f extensions/fleet/agent-apmserver-compose.yml\n -f extensions/metricbeat/metricbeat-compose.yml\n -f extensions/filebeat/filebeat-compose.yml\n -f extensions/heartbeat/heartbeat-compose.yml\n down -v\n" }, { "path": ".github/workflows/docs.yml", "content": "name: Documentation\n\non:\n schedule:\n - cron: '0 0 * * 0' # At 00:00 every Sunday\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\njobs:\n\n markdown-check:\n name: Check Markdown\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v6\n\n - name: Check links\n uses: UmbrellaDocs/action-linkspector@v1\n with:\n config_file: .github/workflows/lint/linkspector.yml\n fail_on_error: 'true'\n filter_mode: nofilter\n\n - name: Lint\n uses: DavidAnson/markdownlint-cli2-action@v22\n with:\n config: .github/workflows/lint/rules.markdownlint.yml\n globs: |\n **/*.md\n !.github/ISSUE_TEMPLATE/*.md\n" }, { "path": ".github/workflows/lint/linkspector.yml", "content": "dirs: [ . ]\nignorePatterns:\n - pattern: '^http:\\/\\/localhost:'\n" }, { "path": ".github/workflows/lint/rules.markdownlint.yml", "content": "default: false # includes/excludes all rules by default\n\n# Heading levels should only increment by one level at a time \nMD001: true\n\n# Heading style \nMD003: true\n\n# Unordered list style \nMD004: true\n\n# Inconsistent indentation for list items at the same level \nMD005: true\n\n# Consider starting bulleted lists at the beginning of the line \nMD006: true\n\n# Unordered list indentation \nMD007: true\n\n# Trailing spaces \nMD009: true\n\n# Hard tabs \nMD010: true\n\n# Reversed link syntax \nMD011: true\n\n# Multiple consecutive blank lines \nMD012: true\n\n# Line length \nMD013:\n line_length: 120\n code_blocks: false\n\n# Dollar signs used before commands without showing output \nMD014: false\n\n# No space after hash on atx style heading \nMD018: true\n\n# Multiple spaces after hash on atx style heading \nMD019: true\n\n# No space inside hashes on closed atx style heading \nMD020: true\n\n# Multiple spaces inside hashes on closed atx style heading \nMD021: true\n\n# Headings should be surrounded by blank lines \nMD022: true\n\n# Headings must start at the beginning of the line \nMD023: true\n\n# Multiple headings with the same content \nMD024:\n allow_different_nesting: true\n\n# Multiple top level headings in the same document \nMD025: true\n\n# Trailing punctuation in heading \nMD026: true\n\n# Multiple spaces after blockquote symbol \nMD027: true\n\n# Blank line inside blockquote \nMD028: false\n\n# Ordered list item prefix \nMD029:\n style: 'one'\n\n# Spaces after list markers \nMD030: true\n\n# Fenced code blocks should be surrounded by blank lines \nMD031: true\n\n# Lists should be surrounded by blank lines \nMD032: true\n\n# Inline HTML \nMD033: true\n\n# Bare URL used \nMD034: true\n\n# Horizontal rule style \nMD035:\n style: '---'\n\n# Emphasis used instead of a heading \nMD036: true\n\n# Spaces inside emphasis markers \nMD037: true\n\n# Spaces inside code span elements \nMD038: true\n\n# Spaces inside link text \nMD039: true\n\n# Fenced code blocks should have a language specified \nMD040: true\n\n# First line in file should be a top level heading \nMD041: true\n\n# No empty links \nMD042: true\n\n# Required heading structure \nMD043: false\n\n# Proper names should have the correct capitalization \nMD044:\n names:\n - docker-elk\n - Elasticsearch\n - Logstash\n - Kibana\n - Docker\n - Compose\n - macOS\n code_blocks: false\n\n# Images should have alternate text (alt text) \nMD045: true\n\n# Code block style \nMD046:\n style: fenced\n\n# Files should end with a single newline character \nMD047: true\n\n# Code fence style \nMD048:\n style: 'backtick'\n" }, { "path": ".github/workflows/scripts/disable-disk-alloc-decider.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib/testing.sh\n\n\ncid_es=\"$(container_id elasticsearch)\"\nip_es=\"$(service_ip elasticsearch)\"\n\ngrouplog 'Wait for readiness of Elasticsearch'\npoll_ready \"$cid_es\" 'http://elasticsearch:9200/' --resolve \"elasticsearch:9200:${ip_es}\" -u 'elastic:testpasswd'\nendgroup\n\nlog 'Disabling disk allocation decider'\n\ndeclare -a put_args=( '-X' 'PUT' '--fail-with-body' '-s' '-u' 'elastic:testpasswd'\n\t'-H' 'Content-Type: application/json'\n\t'http://elasticsearch:9200/_cluster/settings?pretty'\n\t'--resolve' \"elasticsearch:9200:${ip_es}\"\n\t'-d' '{\"persistent\":{\"cluster.routing.allocation.disk.threshold_enabled\":false}}'\n)\ndeclare response\ndeclare -i exit_code=0\n\nresponse=$(curl \"${put_args[@]}\") || exit_code=$?\necho \"$response\"\n\nexit $exit_code\n" }, { "path": ".github/workflows/scripts/lib/testing.sh", "content": "#!/usr/bin/env bash\n\n# Log a message.\nfunction log {\n\techo -e \"\\n[+] $1\\n\"\n}\n\n# Log an error.\nfunction err {\n\techo -e \"\\n[x] $1\\n\" >&2\n}\n\n# Start an expandable group in the GitHub Action log.\n# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines\nfunction grouplog {\n\techo \"::group::$1\"\n}\n\n# End the current expandable group in the GitHub Action log.\nfunction endgroup {\n\techo '::endgroup::'\n}\n\n# Return the ID of the container running the given service.\nfunction container_id {\n\tlocal svc=$1\n\n\tlocal label=\"com.docker.compose.service=${svc}\"\n\n\tlocal cid\n\n\tlocal -i was_retried=0\n\n\t# retry for max 60s (30*2s)\n\tfor _ in $(seq 1 30); do\n\t\tcid=\"$(docker container ls -aq -f label=\"$label\")\"\n\t\tif [ -n \"$cid\" ]; then\n\t\t\tbreak\n\t\tfi\n\n\t\twas_retried=1\n\t\techo -n '.' >&2\n\t\tsleep 2\n\tdone\n\tif ((was_retried)); then\n\t\t# flush stderr, important in non-interactive environments (CI)\n\t\techo >&2\n\tfi\n\n\tif [ -z \"${cid:-}\" ]; then\n\t\terr \"Timed out waiting for creation of container with label ${label}\"\n\t\treturn 1\n\tfi\n\n\techo \"$cid\"\n}\n\n# Return the IP address at which a service can be reached.\n# In Compose mode, returns the container's IP.\nfunction service_ip {\n\tlocal svc=$1\n\n\tlocal ip\n\n\tlocal cid\n\tcid=\"$(container_id \"$svc\")\"\n\n\tlocal ip\n\n\tlocal -i was_retried=0\n\n\t# retry for max 10s (5*2s)\n\tfor _ in $(seq 1 5); do\n\t\tip=\"$(docker container inspect \"$cid\" --format '{{ (index .NetworkSettings.Networks \"docker-elk_elk\").IPAddress }}')\"\n\t\tif [ -n \"$ip\" ]; then\n\t\t\tbreak\n\t\tfi\n\n\t\twas_retried=1\n\t\techo -n '.' >&2\n\t\tsleep 2\n\tdone\n\tif ((was_retried)); then\n\t\t# flush stderr, important in non-interactive environments (CI)\n\t\techo >&2\n\tfi\n\n\tif [ -z \"${ip:-}\" ]; then\n\t\terr \"Container ${cid} has no IP address\"\n\t\treturn 1\n\tfi\n\n\techo \"$ip\"\n}\n\n# Poll the given service at the given port:/path until it responds with HTTP code 200.\nfunction poll_ready {\n\tlocal cid=$1\n\tlocal url=$2\n\n\tlocal -a args=( '-s' '-D-' '-m3' '-w' '%{http_code}' \"$url\" )\n\tif [ \"$#\" -ge 3 ]; then\n\t\targs+=( ${@:3} )\n\tfi\n\n\techo \"curl arguments: ${args[*]}\"\n\n\tlocal -i result=1\n\tlocal output\n\n\tlocal -i was_retried=0\n\n\t# retry for max 300s (60*5s)\n\tfor _ in $(seq 1 60); do\n\t\tif [[ $(docker container inspect \"$cid\" --format '{{ .State.Status}}') == 'exited' ]]; then\n\t\t\terr \"Container exited ($(docker container inspect \"$cid\" --format '{{ .Name }}'))\"\n\t\t\treturn 1\n\t\tfi\n\n\t\toutput=\"$(curl \"${args[@]}\" || true)\"\n\t\tif [ \"${output: -3}\" -eq 200 ]; then\n\t\t\tresult=0\n\t\t\tbreak\n\t\tfi\n\n\t\twas_retried=1\n\t\techo -n 'x' >&2\n\t\tsleep 5\n\tdone\n\tif ((was_retried)); then\n\t\t# flush stderr, important in non-interactive environments (CI)\n\t\techo >&2\n\tfi\n\n\techo -e \"\\n${output::-3}\"\n\n\treturn $result\n}\n" }, { "path": ".github/workflows/scripts/run-tests-core.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib/testing.sh\n\n\ncid_es=\"$(container_id elasticsearch)\"\ncid_ls=\"$(container_id logstash)\"\ncid_kb=\"$(container_id kibana)\"\n\nip_es=\"$(service_ip elasticsearch)\"\nip_ls=\"$(service_ip logstash)\"\nip_kb=\"$(service_ip kibana)\"\n\ngrouplog 'Wait for readiness of Elasticsearch'\npoll_ready \"$cid_es\" 'http://elasticsearch:9200/' --resolve \"elasticsearch:9200:${ip_es}\" -u 'elastic:testpasswd'\nendgroup\n\ngrouplog 'Wait for readiness of Logstash'\npoll_ready \"$cid_ls\" 'http://logstash:9600/_node/pipelines/main?pretty' --resolve \"logstash:9600:${ip_ls}\"\nendgroup\n\ngrouplog 'Wait for readiness of Kibana'\npoll_ready \"$cid_kb\" 'http://kibana:5601/api/status' --resolve \"kibana:5601:${ip_kb}\" -u 'kibana_system:testpasswd'\nendgroup\n\nlog 'Sending message to Logstash TCP input'\n\ndeclare -i was_retried=0\n\n# retry for max 10s (5*2s)\nfor _ in $(seq 1 5); do\n\tif echo 'dockerelk' | nc -q0 \"$ip_ls\" 50000; then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 2\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\ndeclare -a refresh_args=( '-X' 'POST' '-s' '-w' '%{http_code}' '-u' 'elastic:testpasswd'\n\t'http://elasticsearch:9200/logs-generic-default/_refresh'\n\t'--resolve' \"elasticsearch:9200:${ip_es}\"\n)\n\necho \"curl arguments: ${refresh_args[*]}\"\n\n# It might take a few seconds before the indices and alias are created, so we\n# need to be resilient here.\nwas_retried=0\n\n# retry for max 10s (10*1s)\nfor _ in $(seq 1 10); do\n\toutput=\"$(curl \"${refresh_args[@]}\")\"\n\tif [ \"${output: -3}\" -eq 200 ]; then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 1\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\nlog 'Searching message in Elasticsearch'\n\nquery=$( (IFS= read -r -d '' data || echo \"$data\" | jq -c) <&2\n\tsleep 1\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\necho \"$response\"\nif (( count != 1 )); then\n\techo \"Expected 1 document, got ${count}\"\n\texit 1\nfi\n" }, { "path": ".github/workflows/scripts/run-tests-filebeat.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib/testing.sh\n\n\ncid_es=\"$(container_id elasticsearch)\"\ncid_fb=\"$(container_id filebeat)\"\n\nip_es=\"$(service_ip elasticsearch)\"\nip_fb=\"$(service_ip filebeat)\"\n\ngrouplog 'Wait for readiness of Elasticsearch'\npoll_ready \"$cid_es\" 'http://elasticsearch:9200/' --resolve \"elasticsearch:9200:${ip_es}\" -u 'elastic:testpasswd'\nendgroup\n\ngrouplog 'Wait for readiness of Filebeat'\npoll_ready \"$cid_fb\" 'http://filebeat:5066/?pretty' --resolve \"filebeat:5066:${ip_fb}\"\nendgroup\n\nlog 'Searching documents generated by Filebeat'\n\nquery=$( (IFS= read -r -d '' data || echo \"$data\" | jq -c) < 0 )); then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 2\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\necho \"$response\"\nif (( count == 0 )); then\n\techo 'Expected at least 1 document'\n\texit 1\nfi\n" }, { "path": ".github/workflows/scripts/run-tests-fleet.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib/testing.sh\n\n\ncid_es=\"$(container_id elasticsearch)\"\ncid_fl=\"$(container_id fleet-server)\"\ncid_apm=\"$(container_id apm-server)\"\n\nip_es=\"$(service_ip elasticsearch)\"\nip_fl=\"$(service_ip fleet-server)\"\nip_apm=\"$(service_ip apm-server)\"\n\ngrouplog 'Wait for readiness of Elasticsearch'\npoll_ready \"$cid_es\" 'http://elasticsearch:9200/' --resolve \"elasticsearch:9200:${ip_es}\" -u 'elastic:testpasswd'\nendgroup\n\ngrouplog 'Wait for readiness of Fleet Server'\npoll_ready \"$cid_fl\" 'http://fleet-server:8220/api/status' --resolve \"fleet-server:8220:${ip_fl}\"\nendgroup\n\ngrouplog 'Wait for readiness of APM Server'\npoll_ready \"$cid_apm\" 'http://apm-server:8200/' --resolve \"apm-server:8200:${ip_apm}\"\nendgroup\n\nlog 'Searching a system document generated by Fleet Server'\n\nquery=$( (IFS= read -r -d '' data || echo \"$data\" | jq -c) < 0 )); then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 2\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\necho \"$response\"\n# Elastic Agent buffers metrics until Elasticsearch becomes ready, so we\n# tolerate multiple results\nif (( count == 0 )); then\n\techo 'Expected at least 1 document'\n\texit 1\nfi\n\nlog 'Searching a container document generated by Fleet Server'\n\nquery=$( (IFS= read -r -d '' data || echo \"$data\" | jq -c) < 0 )); then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 2\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\necho \"$response\"\nif (( count == 0 )); then\n\techo 'Expected at least 1 document'\n\texit 1\nfi\n" }, { "path": ".github/workflows/scripts/run-tests-heartbeat.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib/testing.sh\n\n\ncid_es=\"$(container_id elasticsearch)\"\ncid_hb=\"$(container_id heartbeat)\"\n\nip_es=\"$(service_ip elasticsearch)\"\nip_hb=\"$(service_ip heartbeat)\"\n\ngrouplog 'Wait for readiness of Elasticsearch'\npoll_ready \"$cid_es\" 'http://elasticsearch:9200/' --resolve \"elasticsearch:9200:${ip_es}\" -u 'elastic:testpasswd'\nendgroup\n\ngrouplog 'Wait for readiness of Heartbeat'\npoll_ready \"$cid_hb\" 'http://heartbeat:5066/?pretty' --resolve \"heartbeat:5066:${ip_hb}\"\nendgroup\n\nlog 'Searching a document generated by Heartbeat'\n\nquery=$( (IFS= read -r -d '' data || echo \"$data\" | jq -c) < 0 )); then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 2\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\necho \"$response\"\nif (( count == 0 )); then\n\techo 'Expected at least 1 document'\n\texit 1\nfi\n" }, { "path": ".github/workflows/scripts/run-tests-metricbeat.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib/testing.sh\n\n\ncid_es=\"$(container_id elasticsearch)\"\ncid_mb=\"$(container_id metricbeat)\"\n\nip_es=\"$(service_ip elasticsearch)\"\nip_mb=\"$(service_ip metricbeat)\"\n\ngrouplog 'Wait for readiness of Elasticsearch'\npoll_ready \"$cid_es\" 'http://elasticsearch:9200/' --resolve \"elasticsearch:9200:${ip_es}\" -u 'elastic:testpasswd'\nendgroup\n\ngrouplog 'Wait for readiness of Metricbeat'\npoll_ready \"$cid_mb\" 'http://metricbeat:5066/?pretty' --resolve \"metricbeat:5066:${ip_mb}\"\nendgroup\n\nlog 'Searching a document generated by Metricbeat'\n\nquery=$( (IFS= read -r -d '' data || echo \"$data\" | jq -c) < 0 )); then\n\t\tbreak\n\tfi\n\n\twas_retried=1\n\techo -n 'x' >&2\n\tsleep 2\ndone\nif ((was_retried)); then\n\t# flush stderr, important in non-interactive environments (CI)\n\techo >&2\nfi\n\necho \"$response\"\n# Metricbeat buffers metrics until Elasticsearch becomes ready, so we tolerate\n# multiple results\nif (( count == 0 )); then\n\techo 'Expected at least 1 document'\n\texit 1\nfi\n" }, { "path": ".github/workflows/spam-issue-close.yml", "content": "name: Close issues without context\n\npermissions:\n issues: write\n\non:\n issues:\n types: [ labeled ]\n\njobs:\n\n close-lock:\n name: Close and lock issues\n if: contains(github.event.issue.labels.*.name, 'bot:close') && github.event.issue.state == 'open'\n runs-on: ubuntu-latest\n\n steps:\n - name: Close\n id: close\n uses: actions/stale@v10.2.0\n with:\n days-before-issue-stale: -1\n days-before-issue-close: 0\n stale-issue-label: bot:close\n close-issue-label: insufficient information\n close-issue-message: >-\n This description omits all, or critical parts of the information requested by maintainers to be able to\n reproduce the issue:\n\n\n - the **complete** log history of your Elastic components, including `setup`.\n - any change(s) performed to the docker-elk configuration.\n - details about the runtime environment, for both Docker and Compose.\n\n\n Therefore, this issue will now be **closed**. Please open a new issue and fill in the template. It saves\n everyone's efforts, and allows maintainers to provide you with a solution in as few round trips as possible.\n\n Thank you for your understanding. :pray:\n\n # Due to eventual consistency, listing closed issues immediately after a\n # close does not always yield the expected results. A sleep is a simple\n # enough remediation to this issue.\n - name: Pause\n if: fromJson(steps.close.outputs.closed-issues-prs)[0]\n run: sleep 5\n\n - name: Lock\n uses: dessant/lock-threads@v6\n if: fromJson(steps.close.outputs.closed-issues-prs)[0]\n with:\n process-only: issues\n issue-inactive-days: 0\n include-any-issue-labels: bot:close\n remove-issue-labels: bot:close\n issue-lock-reason: spam\n log-output: true\n" }, { "path": ".github/workflows/update-merge.yml", "content": "name: Merge Elastic updates\n\non:\n workflow_run:\n workflows: [ CI ]\n types:\n - completed\n branches:\n - update/main\n - update/tls\n - update/release-8.x\n\njobs:\n\n merge:\n name: Merge pull request\n if: github.event.workflow_run.conclusion == 'success'\n runs-on: ubuntu-latest\n\n steps:\n - name: Impersonate update bot\n uses: actions/create-github-app-token@v3\n id: generate-token\n with:\n app-id: ${{ secrets.APP_ID }}\n private-key: ${{ secrets.APP_PRIVATE_KEY }}\n\n - name: Approve and merge\n uses: ridedott/merge-me-action@v2\n with:\n GITHUB_LOGIN: docker-elk-updater\n GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}\n\n - name: Delete branch\n uses: actions/github-script@v8\n with:\n script: |\n await github.request('DELETE /repos/{owner}/{repo}/git/refs/{ref}', {\n owner: '${{ github.event.workflow_run.repository.owner.login }}',\n repo: '${{ github.event.workflow_run.repository.name }}',\n ref: 'heads/${{ github.event.workflow_run.head_branch }}'\n })\n" }, { "path": ".github/workflows/update.yml", "content": "name: Update Elastic release\n\non:\n schedule:\n - cron: '0 0 * * 0' # At 00:00 every Sunday\n\njobs:\n\n check-and-update:\n name: Check and update Elastic release\n runs-on: ubuntu-latest\n strategy:\n matrix:\n branch:\n - main\n - tls\n - release-8.x\n\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ matrix.branch }}\n sparse-checkout-cone-mode: false\n sparse-checkout: /.env\n - name: Read current stack version\n id: current-release\n run: |\n source .env\n : ${ELASTIC_VERSION:?unset}\n echo \"version=${ELASTIC_VERSION}\" >>\"$GITHUB_OUTPUT\"\n\n - uses: actions/setup-node@v6\n - run: npm install semver\n\n - name: Get latest release version\n uses: actions/github-script@v8\n id: get-latest-release\n with:\n script: |\n const semver = require('semver')\n\n const latestVersion = await github.\n paginate(github.rest.repos.listReleases, {\n owner: 'elastic',\n repo: 'elasticsearch'\n })\n .then(releases => {\n for (const release of releases) {\n // Results are returned sorted by created_at, so it is safe to assume\n // that the first encountered match is also the series' latest release.\n\n const version=semver.clean(release.tag_name)\n\n if (semver.satisfies(version, '^${{ steps.current-release.outputs.version }}')) {\n return version\n }\n }\n });\n\n if (latestVersion) {\n // Return an object so that the result can be handled as structured data\n // instead of a quoted string in subsequent steps.\n return { version: latestVersion }\n }\n\n # Subsequent executions of actions/checkout omit to revert this setting to 'false',\n # even if sparse-checkout is later disabled (see actions/checkout#2034).\n - name: Disable sparse checkout\n run: git config core.sparseCheckout false\n # Removes untracked files created by npm (node_modules/, package.json, ...).\n # Disables previous sparse checkout.\n - name: Clean checkout\n uses: actions/checkout@v6\n if: steps.get-latest-release.outputs.result && fromJson(steps.get-latest-release.outputs.result).version != steps.current-release.outputs.version\n with:\n ref: ${{ matrix.branch }}\n\n - name: Update stack version\n id: update-files\n if: steps.get-latest-release.outputs.result && fromJson(steps.get-latest-release.outputs.result).version != steps.current-release.outputs.version\n run: |\n cur_ver=${{ steps.current-release.outputs.version }}\n new_ver=${{ fromJson(steps.get-latest-release.outputs.result).version }}\n\n # Escape period characters so sed interprets them literally\n cur_ver=\"${cur_ver//./\\\\.}\"\n\n declare -a upd_files=( .env README.md */Dockerfile extensions/*/Dockerfile )\n if [ -f tls/README.md ]; then\n upd_files+=( tls/README.md )\n fi\n\n sed -i \"s/${cur_ver}/${new_ver}/g\" \"${upd_files[@]}\"\n\n git_status=\"$(git status --porcelain)\"\n if [[ ${git_status} ]]; then\n echo -e 'Changes to be committed:\\n'\n echo \"${git_status}\"\n echo 'has-changes=true' >>\"$GITHUB_OUTPUT\"\n fi\n\n - name: Impersonate update bot\n uses: actions/create-github-app-token@v3\n id: generate-token\n if: steps.update-files.outputs.has-changes\n with:\n app-id: ${{ secrets.APP_ID }}\n private-key: ${{ secrets.APP_PRIVATE_KEY }}\n\n - name: Send pull request to update to new version\n if: steps.update-files.outputs.has-changes\n uses: peter-evans/create-pull-request@v8\n with:\n token: ${{ steps.generate-token.outputs.token }}\n branch: update/${{ matrix.branch }}\n commit-message: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }}\n title: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }}\n delete-branch: true\n" }, { "path": "LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2015 Anthony Lapenna\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# Elastic stack (ELK) on Docker\n\n[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-9.3.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases)\n[![Build Status](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml?query=branch%3Amain)\n\nRun the latest version of the [Elastic stack][elk-stack] with Docker and Docker Compose.\n\nIt gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and\nthe visualization power of Kibana.\n\nBased on the [official Docker images][elastic-docker] from Elastic:\n\n* [Elasticsearch](https://github.com/elastic/elasticsearch/tree/main/distribution/docker)\n* [Logstash](https://github.com/elastic/logstash/tree/main/docker)\n* [Kibana](https://github.com/elastic/kibana/tree/main/src/dev/build/tasks/os_packages/docker_generator)\n\nOther available stack variants:\n\n* [`tls`](https://github.com/deviantony/docker-elk/tree/tls): TLS encryption enabled in Elasticsearch, Kibana (opt in),\n and Fleet\n\n> [!IMPORTANT]\n> [Platinum][subscriptions] features are enabled by default for a [trial][license-mngmt] duration of **30 days**. After\n> this evaluation period, you will retain access to all the free features included in the Open Basic license seamlessly,\n> without manual intervention required, and without losing any data. Refer to the [How to disable paid\n> features](#how-to-disable-paid-features) section to opt out of this behaviour.\n\n---\n\n## tl;dr\n\n```sh\ndocker compose up setup\n```\n\n```sh\ndocker compose up\n```\n\n\n \n \"Animated\n\n\n---\n\n## Philosophy\n\nThe main goal of docker-elk is to make the Elastic stack as easy as possible to get into. It is **not a blueprint for a\nproduction-ready deployment**, but rather a _template_ that promotes tweaking and exploration.\n\nThe authors believe in good documentation over elaborate automation. The project's default configuration is purposely\nminimal and unopinionated. The initial setup does not rely on any external dependency, and uses as little scripting as\nnecessary to get things up and running.\n\n---\n\n## Contents\n\n1. [Requirements](#requirements)\n * [Host setup](#host-setup)\n * [Docker Desktop](#docker-desktop)\n * [Windows](#windows)\n * [macOS](#macos)\n1. [Usage](#usage)\n * [Bringing up the stack](#bringing-up-the-stack)\n * [Initial setup](#initial-setup)\n * [Setting up user authentication](#setting-up-user-authentication)\n * [Injecting data](#injecting-data)\n * [Cleanup](#cleanup)\n * [Version selection](#version-selection)\n1. [Configuration](#configuration)\n * [How to configure Elasticsearch](#how-to-configure-elasticsearch)\n * [How to configure Kibana](#how-to-configure-kibana)\n * [How to configure Logstash](#how-to-configure-logstash)\n * [How to disable paid features](#how-to-disable-paid-features)\n * [How to scale out the Elasticsearch cluster](#how-to-scale-out-the-elasticsearch-cluster)\n * [How to re-execute the setup](#how-to-re-execute-the-setup)\n * [How to reset a password programmatically](#how-to-reset-a-password-programmatically)\n1. [Extensibility](#extensibility)\n * [How to add plugins](#how-to-add-plugins)\n * [How to enable the provided extensions](#how-to-enable-the-provided-extensions)\n1. [JVM tuning](#jvm-tuning)\n * [How to specify the amount of memory used by a service](#how-to-specify-the-amount-of-memory-used-by-a-service)\n * [How to enable a remote JMX connection to a service](#how-to-enable-a-remote-jmx-connection-to-a-service)\n1. [Going further](#going-further)\n * [Plugins and integrations](#plugins-and-integrations)\n\n## Requirements\n\n### Host setup\n\n* [Docker Engine][docker-install] version **18.06.0** or newer\n* [Docker Compose][compose-install] version **2.0.0** or newer\n* 1.5 GB of RAM\n\n> [!NOTE]\n> Especially on Linux, make sure your user has the [required permissions][linux-postinstall] to interact with the Docker\n> daemon.\n\nBy default, the stack exposes the following ports:\n\n* 5044: Logstash Beats input\n* 50000: Logstash TCP input\n* 9600: Logstash monitoring API\n* 9200: Elasticsearch HTTP\n* 9300: Elasticsearch TCP transport\n* 5601: Kibana\n\n> [!WARNING]\n> Elasticsearch's [bootstrap checks][bootstrap-checks] were purposely disabled to facilitate the setup of the Elastic\n> stack in development environments. For production setups, we recommend users to set up their host according to the\n> instructions from the Elasticsearch documentation: [Important System Configuration][es-sys-config].\n\n### Docker Desktop\n\n#### Windows\n\nIf you are using the legacy Hyper-V mode of _Docker Desktop for Windows_, ensure that [File\nSharing][desktop-filesharing] is enabled for the `C:` drive.\n\n#### macOS\n\nThe default configuration of _Docker Desktop for Mac_ allows mounting files from `/Users/`, `/Volume/`, `/private/`,\n`/tmp` and `/var/folders` exclusively. Make sure the repository is cloned in one of those locations or follow the\ninstructions from the [documentation][desktop-filesharing] to add more locations.\n\n## Usage\n\n> [!WARNING]\n> You must rebuild the stack images with `docker compose build` whenever you switch branch or update the\n> [version](#version-selection) of an already existing stack.\n\n### Bringing up the stack\n\nClone this repository onto the Docker host that will run the stack with the command below:\n\n```sh\ngit clone https://github.com/deviantony/docker-elk.git\n```\n\nThen, initialize the Elasticsearch users and groups required by docker-elk by executing the command:\n\n```sh\ndocker compose up setup\n```\n\nOptionally (but highly recommended), generate encryption keys for Kibana using the following command and copy its output\nto the Kibana configuration file (`kibana/config/kibana.yml`):\n\n```sh\ndocker compose up kibana-genkeys\n```\n\nIf everything went well and the setup completed without error, start the other stack components:\n\n```sh\ndocker compose up\n```\n\n> [!NOTE]\n> You can also run all services in the background (detached mode) by appending the `-d` flag to the above command.\n\nGive Kibana about a minute to initialize, then access the Kibana web UI by opening in a web\nbrowser and use the following (default) credentials to log in:\n\n* user: *elastic*\n* password: *changeme*\n\n> [!NOTE]\n> Upon the initial startup, the `elastic`, `logstash_internal` and `kibana_system` Elasticsearch users are initialized\n> with the values of the passwords defined in the [`.env`](.env) file (_\"changeme\"_ by default). The first one is the\n> [built-in superuser][builtin-users], the other two are used by Kibana and Logstash respectively to communicate with\n> Elasticsearch. This task is only performed during the _initial_ startup of the stack. To change users' passwords\n> _after_ they have been initialized, please refer to the instructions in the next section.\n\n### Initial setup\n\n#### Setting up user authentication\n\n> [!NOTE]\n> Refer to [Security settings in Elasticsearch][es-security] to disable authentication.\n\n> [!WARNING]\n> Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstrapped privileged `elastic` user.\n\nThe _\"changeme\"_ password set by default for all aforementioned users is **unsecure**. For increased security, we will\nreset the passwords of all aforementioned Elasticsearch users to random secrets.\n\n1. Reset passwords for default users\n\n The commands below reset the passwords of the `elastic`, `logstash_internal` and `kibana_system` users. Take note\n of them.\n\n ```sh\n docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user elastic\n ```\n\n ```sh\n docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user logstash_internal\n ```\n\n ```sh\n docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system\n ```\n\n If the need for it arises (e.g. if you want to [collect monitoring information][ls-monitoring] through Beats and\n other components), feel free to repeat this operation at any time for the rest of the [built-in\n users][builtin-users].\n\n1. Replace usernames and passwords in configuration files\n\n Replace the password of the `elastic` user inside the `.env` file with the password generated in the previous step.\n Its value isn't used by any core component, but [extensions](#how-to-enable-the-provided-extensions) use it to\n connect to Elasticsearch.\n\n > [!NOTE]\n > In case you don't plan on using any of the provided [extensions](#how-to-enable-the-provided-extensions), or\n > prefer to create your own roles and users to authenticate these services, it is safe to remove the\n > `ELASTIC_PASSWORD` entry from the `.env` file altogether after the stack has been initialized.\n\n Replace the password of the `logstash_internal` user inside the `.env` file with the password generated in the\n previous step. Its value is referenced inside the Logstash pipeline file (`logstash/pipeline/logstash.conf`).\n\n Replace the password of the `kibana_system` user inside the `.env` file with the password generated in the previous\n step. Its value is referenced inside the Kibana configuration file (`kibana/config/kibana.yml`).\n\n See the [Configuration](#configuration) section below for more information about these configuration files.\n\n1. Restart Logstash and Kibana to re-connect to Elasticsearch using the new passwords\n\n ```sh\n docker compose up -d logstash kibana\n ```\n\n> [!NOTE]\n> Learn more about the security of the Elastic stack at [Secure the Elastic Stack][sec-cluster].\n\n#### Injecting data\n\nLaunch the Kibana web UI by opening in a web browser, and use the following credentials to log\nin:\n\n* user: *elastic*\n* password: *\\*\n\nNow that the stack is fully configured, you can go ahead and inject some log entries.\n\nThe shipped Logstash configuration allows you to send data over the TCP port 50000. For example, you can use one of the\nfollowing commands — depending on your installed version of `nc` (Netcat) — to ingest the content of the log file\n`/path/to/logfile.log` in Elasticsearch, via Logstash:\n\n```sh\n# Execute `nc -h` to determine your `nc` version\n\ncat /path/to/logfile.log | nc -q0 localhost 50000 # BSD\ncat /path/to/logfile.log | nc -c localhost 50000 # GNU\ncat /path/to/logfile.log | nc --send-only localhost 50000 # nmap\n```\n\nYou can also load the sample data provided by your Kibana installation.\n\n### Cleanup\n\nElasticsearch data is persisted inside a volume by default.\n\nIn order to entirely shutdown the stack and remove all persisted data, use the following Docker Compose command:\n\n```sh\ndocker compose --profile=setup down -v\n```\n\n### Version selection\n\nThis repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major\nversion (9.x).\n\nTo use a different version of the core Elastic components, simply change the version number inside the [`.env`](.env)\nfile. If you are upgrading an existing stack, remember to rebuild all container images using the `docker compose build`\ncommand.\n\n> [!IMPORTANT]\n> Always pay attention to the [official upgrade instructions][upgrade] for each individual component before performing a\n> stack upgrade.\n\nOlder major versions are also supported on separate branches:\n\n* [`release-8.x`](https://github.com/deviantony/docker-elk/tree/release-8.x): 8.x series\n* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series (End-of-Life)\n* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life)\n* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life)\n\n## Configuration\n\n> [!IMPORTANT]\n> Configuration is not dynamically reloaded, you will need to restart individual components after any configuration\n> change.\n\n### How to configure Elasticsearch\n\nThe Elasticsearch configuration is stored in [`elasticsearch/config/elasticsearch.yml`][config-es].\n\nYou can also specify the options you want to override by setting environment variables inside the Compose file:\n\n```yml\nelasticsearch:\n\n environment:\n network.host: _non_loopback_\n cluster.name: my-cluster\n```\n\nPlease refer to the following documentation page for more details about how to configure Elasticsearch inside Docker\ncontainers: [Install Elasticsearch with Docker][es-docker].\n\n### How to configure Kibana\n\nThe Kibana default configuration is stored in [`kibana/config/kibana.yml`][config-kbn].\n\nYou can also specify the options you want to override by setting environment variables inside the Compose file:\n\n```yml\nkibana:\n\n environment:\n SERVER_NAME: kibana.example.org\n```\n\nPlease refer to the following documentation page for more details about how to configure Kibana inside Docker\ncontainers: [Install Kibana with Docker][kbn-docker].\n\n### How to configure Logstash\n\nThe Logstash configuration is stored in [`logstash/config/logstash.yml`][config-ls].\n\nYou can also specify the options you want to override by setting environment variables inside the Compose file:\n\n```yml\nlogstash:\n\n environment:\n LOG_LEVEL: debug\n```\n\nPlease refer to the following documentation page for more details about how to configure Logstash inside Docker\ncontainers: [Configuring Logstash for Docker][ls-docker].\n\n### How to disable paid features\n\nYou can cancel an ongoing trial before its expiry date — and thus revert to a basic license — either from the [License\nManagement][license-mngmt] panel of Kibana, or using Elasticsearch's `start_basic` [Licensing API][license-apis]. Please\nnote that the second option is the only way to recover access to Kibana if the license isn't either switched to `basic`\nor upgraded before the trial's expiry date.\n\nChanging the license type by switching the value of Elasticsearch's `xpack.license.self_generated.type` setting from\n`trial` to `basic` (see [License settings][license-settings]) will only work **if done prior to the initial setup.**\nAfter a trial has been started, the loss of features from `trial` to `basic` _must_ be acknowledged using one of the two\nmethods described in the first paragraph.\n\n### How to scale out the Elasticsearch cluster\n\nFollow the instructions from the Wiki: [Scaling out Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster)\n\n### How to re-execute the setup\n\nTo run the setup container again and re-initialize all users for which a password was defined inside the `.env` file,\nsimply \"up\" the `setup` Compose service again:\n\n```console\n$ docker compose up setup\n ⠿ Container docker-elk-elasticsearch-1 Running\n ⠿ Container docker-elk-setup-1 Created\nAttaching to docker-elk-setup-1\n...\ndocker-elk-setup-1 | [+] User 'monitoring_internal'\ndocker-elk-setup-1 | ⠿ User does not exist, creating\ndocker-elk-setup-1 | [+] User 'beats_system'\ndocker-elk-setup-1 | ⠿ User exists, setting password\ndocker-elk-setup-1 exited with code 0\n```\n\n### How to reset a password programmatically\n\nIf for any reason your are unable to use Kibana to change the password of your users (including [built-in\nusers][builtin-users]), you can use the Elasticsearch API instead and achieve the same result.\n\nIn the example below, we reset the password of the `elastic` user (notice \"/user/elastic\" in the URL):\n\n```sh\ncurl -XPOST -D- 'http://localhost:9200/_security/user/elastic/_password' \\\n -H 'Content-Type: application/json' \\\n -u elastic: \\\n -d '{\"password\" : \"\"}'\n```\n\n## Extensibility\n\n### How to add plugins\n\nTo add plugins to any ELK component you have to:\n\n1. Add a `RUN` statement to the corresponding `Dockerfile` (eg. `RUN logstash-plugin install logstash-filter-json`)\n1. Add the associated plugin code configuration to the service configuration (eg. Logstash input/output)\n1. Rebuild the images using the `docker compose build` command\n\n### How to enable the provided extensions\n\nA few extensions are available inside the [`extensions`](extensions) directory. These extensions provide features which\nare not part of the standard Elastic stack, but can be used to enrich it with extra integrations.\n\nThe documentation for these extensions is provided inside each individual subdirectory, on a per-extension basis. Some\nof them require manual changes to the default ELK configuration.\n\n## JVM tuning\n\n### How to specify the amount of memory used by a service\n\nThe startup scripts for Elasticsearch and Logstash can append extra JVM options from the value of an environment\nvariable, allowing the user to adjust the amount of memory that can be used by each component:\n\n| Service | Environment variable |\n|---------------|----------------------|\n| Elasticsearch | ES_JAVA_OPTS |\n| Logstash | LS_JAVA_OPTS |\n\nTo accommodate environments where memory is scarce (Docker Desktop for Mac has only 2 GB available by default), the Heap\nSize allocation is capped by default in the `docker-compose.yml` file to 512 MB for Elasticsearch and 256 MB for\nLogstash. If you want to override the default JVM configuration, edit the matching environment variable(s) in the\n`docker-compose.yml` file.\n\nFor example, to increase the maximum JVM Heap Size for Logstash:\n\n```yml\nlogstash:\n\n environment:\n LS_JAVA_OPTS: -Xms1g -Xmx1g\n```\n\nWhen these options are not set:\n\n* Elasticsearch starts with a JVM Heap Size that is [determined automatically][es-heap].\n* Logstash starts with a fixed JVM Heap Size of 1 GB.\n\n### How to enable a remote JMX connection to a service\n\nAs for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the Docker\nhost.\n\nUpdate the `{ES,LS}_JAVA_OPTS` environment variable with the following content (I've mapped the JMX service on the port\n18080, you can change that). Do not forget to update the `-Djava.rmi.server.hostname` option with the IP address of your\nDocker host (replace **DOCKER_HOST_IP**):\n\n```yml\nlogstash:\n\n environment:\n LS_JAVA_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false\n```\n\n## Going further\n\n### Plugins and integrations\n\nSee the following Wiki pages:\n\n* [External applications](https://github.com/deviantony/docker-elk/wiki/External-applications)\n* [Popular integrations](https://github.com/deviantony/docker-elk/wiki/Popular-integrations)\n\n[elk-stack]: https://www.elastic.co/elastic-stack/\n[elastic-docker]: https://www.docker.elastic.co/\n[subscriptions]: https://www.elastic.co/subscriptions\n[es-security]: https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/security-settings\n[license-settings]: https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/license-settings\n[license-mngmt]: https://www.elastic.co/docs/deploy-manage/license/manage-your-license-in-self-managed-cluster\n[license-apis]: https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-license\n\n[docker-install]: https://docs.docker.com/get-started/get-docker/\n[compose-install]: https://docs.docker.com/compose/install/\n[linux-postinstall]: https://docs.docker.com/engine/install/linux-postinstall/\n[desktop-filesharing]: https://docs.docker.com/desktop/settings-and-maintenance/settings/#file-sharing\n\n[bootstrap-checks]: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/bootstrap-checks\n[es-sys-config]: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/important-system-configuration\n[es-heap]: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/important-settings-configuration#heap-size-settings\n\n[builtin-users]: https://www.elastic.co/docs/deploy-manage/users-roles/cluster-or-deployment-auth/built-in-users\n[ls-monitoring]: https://www.elastic.co/docs/reference/logstash/monitoring-with-metricbeat\n[sec-cluster]: https://www.elastic.co/docs/deploy-manage/security#cluster-or-deployment-security-features\n\n[config-es]: ./elasticsearch/config/elasticsearch.yml\n[config-kbn]: ./kibana/config/kibana.yml\n[config-ls]: ./logstash/config/logstash.yml\n\n[es-docker]: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/install-elasticsearch-with-docker\n[kbn-docker]: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/install-kibana-with-docker\n[ls-docker]: https://www.elastic.co/docs/reference/logstash/docker-config\n\n[upgrade]: https://www.elastic.co/docs/deploy-manage/upgrade/deployment-or-cluster/self-managed\n\n\n" }, { "path": "demo.tape", "content": "# VHS - terminal recorder\n# https://github.com/charmbracelet/vhs\n\nOutput demo.gif\n\nRequire docker\n\nSet Shell bash\nSet FontSize 24\nSet Width 1500\nSet Height 1200\nSet WindowBar Rings\nSet TypingSpeed 75ms\nSet Theme \"iTerm2 Light Background\"\n#Set Theme \"iTerm2 Dark Background\"\n\nHide\n\n# BEGIN - Reset to clean state\nType \"docker compose down setup\"\nEnter\nSleep 2s\nType \"docker compose down -v\"\nEnter\nSleep 6s\nType \"docker compose up elasticsearch -d\"\nEnter\nSleep 20s\nType \"docker compose down\"\nEnter\nSleep 6s\nCtrl+L\n# END - Reset to clean state\n\nShow\n\nType \"docker compose up setup\"\nSleep 3s\nEnter\nSleep 20s\n\nEnter 2\nType \"docker compose up -d\"\nSleep 3s\nEnter\nSleep 6s\n\nEnter 2\nType \"curl -s http://localhost:9200 -u elastic:changeme\"\nSleep 3s\nEnter\nSleep 12s\n\nEnter 2\nType \"👉 Open your web browser at http://localhost:5601 🚀📈\"\nSleep 15s\n" }, { "path": "docker-compose.yml", "content": "services:\n\n # The 'setup' service runs a one-off script which initializes users inside\n # Elasticsearch — such as 'logstash_internal' and 'kibana_system' — with the\n # values of the passwords defined in the '.env' file. It also creates the\n # roles required by some of these users.\n #\n # This task only needs to be performed once, during the *initial* startup of\n # the stack. Any subsequent run will reset the passwords of existing users to\n # the values defined inside the '.env' file, and the built-in roles to their\n # default permissions.\n #\n # By default, it is excluded from the services started by 'docker compose up'\n # due to the non-default profile it belongs to. To run it, either provide the\n # '--profile=setup' CLI flag to Compose commands, or \"up\" the service by name\n # such as 'docker compose up setup'.\n setup:\n profiles:\n - setup\n build:\n context: setup/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n init: true\n volumes:\n - ./setup/entrypoint.sh:/entrypoint.sh:ro,Z\n - ./setup/lib.sh:/lib.sh:ro,Z\n - ./setup/roles:/roles:ro,Z\n environment:\n ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}\n LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}\n KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}\n METRICBEAT_INTERNAL_PASSWORD: ${METRICBEAT_INTERNAL_PASSWORD:-}\n FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}\n HEARTBEAT_INTERNAL_PASSWORD: ${HEARTBEAT_INTERNAL_PASSWORD:-}\n MONITORING_INTERNAL_PASSWORD: ${MONITORING_INTERNAL_PASSWORD:-}\n BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}\n network_mode: service:elasticsearch\n depends_on:\n - elasticsearch\n\n # The 'kibana-genkeys' service runs a one-off command which generates and\n # prints encryption keys for Kibana.\n #\n # By default, it is excluded from the services started by 'docker compose up'\n # due to the non-default profile it belongs to. To run it, either provide the\n # '--profile=setup' CLI flag to Compose commands, or \"up\" the service by name\n # such as 'docker compose up kibana-genkeys'.\n kibana-genkeys:\n profiles:\n - setup\n build:\n context: kibana/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n command:\n - bin/kibana-encryption-keys\n - generate\n network_mode: none\n\n elasticsearch:\n build:\n context: elasticsearch/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n volumes:\n - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,Z\n - elasticsearch:/usr/share/elasticsearch/data:Z\n ports:\n - 9200:9200\n - 9300:9300\n environment:\n node.name: elasticsearch\n ES_JAVA_OPTS: -Xms512m -Xmx512m\n # Bootstrap password.\n # Used to initialize the keystore during the initial startup of\n # Elasticsearch. Ignored on subsequent runs.\n ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}\n # Use single node discovery in order to disable production mode and avoid bootstrap checks.\n # see: https://www.elastic.co/docs/deploy-manage/deploy/self-managed/bootstrap-checks\n discovery.type: single-node\n networks:\n - elk\n restart: unless-stopped\n\n logstash:\n build:\n context: logstash/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n volumes:\n - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z\n - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z\n ports:\n - 5044:5044\n - 50000:50000/tcp\n - 50000:50000/udp\n - 9600:9600\n environment:\n LS_JAVA_OPTS: -Xms256m -Xmx256m\n LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}\n networks:\n - elk\n depends_on:\n - elasticsearch\n restart: unless-stopped\n\n kibana:\n build:\n context: kibana/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n volumes:\n - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z\n ports:\n - 5601:5601\n environment:\n KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}\n networks:\n - elk\n depends_on:\n - elasticsearch\n restart: unless-stopped\n\nnetworks:\n elk:\n driver: bridge\n\nvolumes:\n elasticsearch:\n" }, { "path": "elasticsearch/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "elasticsearch/Dockerfile", "content": "ARG ELASTIC_VERSION\n\n# https://www.docker.elastic.co/\nFROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.3.1}\n\n# Add your elasticsearch plugins setup here\n# Example: RUN elasticsearch-plugin install analysis-icu\n" }, { "path": "elasticsearch/config/elasticsearch.yml", "content": "---\n## Default Elasticsearch configuration from Elasticsearch base image.\n## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml\n#\ncluster.name: docker-cluster\nnetwork.host: 0.0.0.0\n\nxpack.license.self_generated.type: trial\nxpack.security.enabled: true\n" }, { "path": "extensions/README.md", "content": "# Extensions\n\nThird-party extensions that enable extra integrations with the Elastic stack.\n" }, { "path": "extensions/curator/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "extensions/curator/Dockerfile", "content": "FROM untergeek/curator:8.0.21\n\nUSER root\n\nRUN >>/var/spool/cron/crontabs/nobody \\\n echo '* * * * * /curator/curator /.curator/delete_log_files_curator.yml'\n\nENTRYPOINT [\"crond\"]\nCMD [\"-f\", \"-d8\"]\n" }, { "path": "extensions/curator/README.md", "content": "# Curator\n\nElasticsearch Curator helps you curate or manage your indices.\n\n## Usage\n\nIf you want to include the Curator extension, run Docker Compose from the root of the repository with an additional\ncommand line argument referencing the `curator-compose.yml` file:\n\n```bash\n$ docker compose -f docker-compose.yml -f extensions/curator/curator-compose.yml up\n```\n\nThis sample setup demonstrates how to run `curator` every minute using `cron`.\n\nAll configuration files are available in the `config/` directory.\n\n## Documentation\n\n[Curator Reference](https://www.elastic.co/docs/reference/elasticsearch/curator)\n" }, { "path": "extensions/curator/config/curator.yml", "content": "# Curator configuration\n# https://www.elastic.co/docs/reference/elasticsearch/curator/configfile\n\nelasticsearch:\n client:\n hosts: [ http://elasticsearch:9200 ]\n other_settings:\n username: elastic\n password: ${ELASTIC_PASSWORD}\n\nlogging:\n loglevel: INFO\n logformat: default\n" }, { "path": "extensions/curator/config/delete_log_files_curator.yml", "content": "actions:\n 1:\n action: delete_indices\n description: >-\n Delete indices. Find which to delete by first limiting the list to\n logstash- prefixed indices. Then further filter those to prevent deletion\n of anything less than the number of days specified by unit_count.\n Ignore the error if the filter does not result in an actionable list of\n indices (ignore_empty_list) and exit cleanly.\n options:\n ignore_empty_list: True\n disable_action: False\n filters:\n - filtertype: pattern\n kind: prefix\n value: logstash-\n - filtertype: age\n source: creation_date\n direction: older\n unit: days\n unit_count: 2\n" }, { "path": "extensions/curator/curator-compose.yml", "content": "services:\n curator:\n build:\n context: extensions/curator/\n init: true\n volumes:\n - ./extensions/curator/config/curator.yml:/.curator/curator.yml:ro,Z\n - ./extensions/curator/config/delete_log_files_curator.yml:/.curator/delete_log_files_curator.yml:ro,Z\n environment:\n ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}\n networks:\n - elk\n depends_on:\n - elasticsearch\n" }, { "path": "extensions/filebeat/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "extensions/filebeat/Dockerfile", "content": "ARG ELASTIC_VERSION\n\nFROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-9.3.1}\n" }, { "path": "extensions/filebeat/README.md", "content": "# Filebeat\n\nFilebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers,\nFilebeat monitors the log files or locations that you specify, collects log events, and forwards them either to\nElasticsearch or Logstash for indexing.\n\n## Usage\n\n**This extension requires the `filebeat_internal` and `beats_system` users to be created and initialized with a\npassword.** In case you haven't done that during the initial startup of the stack, please refer to [How to re-execute\nthe setup][setup] to run the setup container again and initialize these users.\n\nTo include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line\nargument referencing the `filebeat-compose.yml` file:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up\n```\n\n## Configuring Filebeat\n\nThe Filebeat configuration is stored in [`config/filebeat.yml`](./config/filebeat.yml). You can modify this file with\nthe help of the [Configuration reference][filebeat-config].\n\nAny change to the Filebeat configuration requires a restart of the Filebeat container:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat\n```\n\nPlease refer to the following documentation page for more details about how to configure Filebeat inside a Docker\ncontainer: [Run Filebeat on Docker][filebeat-docker].\n\n## See also\n\n[Filebeat documentation][filebeat-doc]\n\n[filebeat-config]: https://www.elastic.co/docs/reference/beats/filebeat/filebeat-reference-yml\n[filebeat-docker]: https://www.elastic.co/docs/reference/beats/filebeat/running-on-docker\n[filebeat-doc]: https://www.elastic.co/docs/reference/beats/filebeat\n\n[setup]: ../../README.md#how-to-re-execute-the-setup\n" }, { "path": "extensions/filebeat/config/filebeat.yml", "content": "## Filebeat configuration\n## https://github.com/elastic/beats/blob/main/deploy/docker/filebeat.docker.yml\n#\n\nname: filebeat\n\nfilebeat.config:\n modules:\n path: ${path.config}/modules.d/*.yml\n reload.enabled: false\n\nfilebeat.autodiscover:\n providers:\n # The Docker autodiscover provider automatically retrieves logs from Docker\n # containers as they start and stop.\n - type: docker\n hints.enabled: true\n hints.default_config:\n type: container\n paths:\n - /var/lib/docker/containers/${data.container.id}/*-json.log\n templates:\n - condition:\n contains:\n docker.container.image: elasticsearch\n config:\n - module: elasticsearch\n server:\n input:\n type: container\n paths:\n - /var/lib/docker/containers/${data.container.id}/*-json.log\n\nprocessors:\n - add_cloud_metadata: ~\n\nmonitoring:\n enabled: true\n elasticsearch:\n username: beats_system\n password: ${BEATS_SYSTEM_PASSWORD}\n\noutput.elasticsearch:\n hosts: [ http://elasticsearch:9200 ]\n username: filebeat_internal\n password: ${FILEBEAT_INTERNAL_PASSWORD}\n\n## HTTP endpoint for health checking\n## https://www.elastic.co/docs/reference/beats/filebeat/http-endpoint\n#\n\nhttp:\n enabled: true\n host: 0.0.0.0\n" }, { "path": "extensions/filebeat/filebeat-compose.yml", "content": "services:\n filebeat:\n build:\n context: extensions/filebeat/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading\n # 'docker.sock' and the host's filesystem.\n user: root\n command:\n # Log to stderr.\n - -e\n # Disable config file permissions checks. Allows mounting\n # 'config/filebeat.yml' even if it's not owned by root.\n # see: https://www.elastic.co/docs/reference/beats/libbeat/config-file-permissions\n - --strict.perms=false\n volumes:\n - ./extensions/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,Z\n - type: bind\n source: /var/lib/docker/containers\n target: /var/lib/docker/containers\n read_only: true\n - type: bind\n source: /var/run/docker.sock\n target: /var/run/docker.sock\n read_only: true\n environment:\n FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}\n BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}\n networks:\n - elk\n depends_on:\n - elasticsearch\n" }, { "path": "extensions/fleet/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "extensions/fleet/Dockerfile", "content": "ARG ELASTIC_VERSION\n\nFROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-9.3.1}\n\n# Ensure the 'state' directory exists and is owned by the 'elastic-agent' user,\n# otherwise mounting a named volume in that location creates a directory owned\n# by root:root which the 'elastic-agent' user isn't allowed to write to.\nRUN mkdir state\n" }, { "path": "extensions/fleet/README.md", "content": "# Fleet Server\n\n> [!WARNING]\n> This extension currently exists for preview purposes and should be considered **EXPERIMENTAL**. Expect regular changes\n> to the default Fleet settings, both in the Elastic Agent and Kibana.\n>\n> See [Known Issues](#known-issues) for a list of issues that need to be addressed before this extension can be\n> considered functional.\n\nFleet provides central management capabilities for [Elastic Agents][fleet-doc] via an API and web UI served by Kibana,\nwith Elasticsearch acting as the communication layer.\nFleet Server is the central component which allows connecting Elastic Agents to the Fleet.\n\n## Requirements\n\nThe Fleet Server exposes the TCP port `8220` for Agent to Server communications.\n\n## Usage\n\nTo include Fleet Server in the stack, run Docker Compose from the root of the repository with an additional command line\nargument referencing the `fleet-compose.yml` file:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/fleet/fleet-compose.yml up\n```\n\n## Configuring Fleet Server\n\nFleet Server — like any Elastic Agent — is configured via [Agent Policies][fleet-pol] which can be either managed\nthrough the Fleet management UI in Kibana, or statically pre-configured inside the Kibana configuration file.\n\nTo ease the enrollment of Fleet Server in this extension, docker-elk comes with a pre-configured Agent Policy for Fleet\nServer defined inside [`kibana/config/kibana.yml`][config-kbn].\n\nPlease refer to the following documentation page for more details about configuring Fleet Server through the Fleet\nmanagement UI: [Fleet UI Settings][fleet-cfg].\n\n## Known Issues\n\n- The Elastic Agent auto-enrolls using the `elastic` super-user. With this approach, you do not need to generate a\n service token — either using the Fleet management UI or [CLI utility][es-svc-token] — prior to starting this\n extension. However convenient that is, this approach _does not follow security best practices_, and we recommend\n generating a service token for Fleet Server instead.\n\n## See also\n\n[Fleet and Elastic Agent Guide][fleet-doc]\n\n## Screenshots\n\n![fleet-agents](https://user-images.githubusercontent.com/3299086/202701399-27518fe4-17b7-49d1-aefb-868dffeaa68a.png\n\"Fleet Agents\")\n![elastic-agent-dashboard](https://user-images.githubusercontent.com/3299086/202701404-958f8d80-a7a0-4044-bbf9-bf73f3bdd17a.png\n\"Elastic Agent Dashboard\")\n\n[fleet-doc]: https://www.elastic.co/docs/reference/fleet\n[fleet-pol]: https://www.elastic.co/docs/reference/fleet/agent-policy\n[fleet-cfg]: https://www.elastic.co/docs/reference/fleet/fleet-settings\n\n[config-kbn]: ../../kibana/config/kibana.yml\n\n[es-svc-token]: https://www.elastic.co/docs/reference/elasticsearch/command-line-tools/service-tokens-command\n" }, { "path": "extensions/fleet/agent-apmserver-compose.yml", "content": "# Example of Fleet-enrolled Elastic Agent pre-configured with an agent policy\n# for running the APM Server integration (see kibana.yml).\n#\n# Run with\n# docker compose \\\n# -f docker-compose.yml \\\n# -f extensions/fleet/fleet-compose.yml \\\n# -f extensions/fleet/agent-apmserver-compose.yml \\\n# up\n\nservices:\n apm-server:\n build:\n context: extensions/fleet/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n volumes:\n - apm-server:/usr/share/elastic-agent/state:Z\n environment:\n FLEET_ENROLL: '1'\n FLEET_TOKEN_POLICY_NAME: Agent Policy APM Server\n FLEET_INSECURE: '1'\n FLEET_URL: http://fleet-server:8220\n # Enrollment.\n # (a) Auto-enroll using basic authentication\n ELASTICSEARCH_USERNAME: elastic\n ELASTICSEARCH_PASSWORD: ${ELASTIC_PASSWORD:-}\n # (b) Enroll using a pre-generated enrollment token\n #FLEET_ENROLLMENT_TOKEN: \n ports:\n - 8200:8200\n hostname: apm-server\n # Elastic Agent does not retry failed connections to Kibana upon the initial enrollment phase.\n restart: on-failure\n networks:\n - elk\n depends_on:\n - elasticsearch\n - kibana\n - fleet-server\n\nvolumes:\n apm-server:\n" }, { "path": "extensions/fleet/fleet-compose.yml", "content": "services:\n fleet-server:\n build:\n context: extensions/fleet/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n # Run as 'root' instead of 'elastic-agent' (uid 1000) to allow reading\n # 'docker.sock' and the host's filesystem.\n user: root\n volumes:\n - fleet-server:/usr/share/elastic-agent/state:Z\n - type: bind\n source: /var/lib/docker/containers\n target: /var/lib/docker/containers\n read_only: true\n - type: bind\n source: /var/run/docker.sock\n target: /var/run/docker.sock\n read_only: true\n environment:\n FLEET_SERVER_ENABLE: '1'\n FLEET_SERVER_INSECURE_HTTP: '1'\n FLEET_SERVER_HOST: 0.0.0.0\n FLEET_SERVER_POLICY_ID: fleet-server-policy\n # Fleet plugin in Kibana\n KIBANA_FLEET_SETUP: '1'\n # Enrollment.\n # (a) Auto-enroll using basic authentication\n ELASTICSEARCH_USERNAME: elastic\n ELASTICSEARCH_PASSWORD: ${ELASTIC_PASSWORD:-}\n # (b) Enroll using a pre-generated service token\n #FLEET_SERVER_SERVICE_TOKEN: \n ports:\n - 8220:8220\n hostname: fleet-server\n # Elastic Agent does not retry failed connections to Kibana upon the initial enrollment phase.\n restart: on-failure\n networks:\n - elk\n depends_on:\n - elasticsearch\n - kibana\n\nvolumes:\n fleet-server:\n" }, { "path": "extensions/heartbeat/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "extensions/heartbeat/Dockerfile", "content": "ARG ELASTIC_VERSION\n\nFROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-9.3.1}\n" }, { "path": "extensions/heartbeat/README.md", "content": "# Heartbeat\n\nHeartbeat is a lightweight daemon that periodically checks the status of your services and determines whether they are\navailable.\n\n## Usage\n\n**This extension requires the `heartbeat_internal` and `beats_system` users to be created and initialized with a\npassword.** In case you haven't done that during the initial startup of the stack, please refer to [How to re-execute\nthe setup][setup] to run the setup container again and initialize these users.\n\nTo include Heartbeat in the stack, run Docker Compose from the root of the repository with an additional command line\nargument referencing the `heartbeat-compose.yml` file:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up\n```\n\n## Configuring Heartbeat\n\nThe Heartbeat configuration is stored in [`config/heartbeat.yml`](./config/heartbeat.yml). You can modify this file\nwith the help of the [Configuration reference][heartbeat-config].\n\nAny change to the Heartbeat configuration requires a restart of the Heartbeat container:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml restart heartbeat\n```\n\nPlease refer to the following documentation page for more details about how to configure Heartbeat inside a\nDocker container: [Run Heartbeat on Docker][heartbeat-docker].\n\n## See also\n\n[Heartbeat documentation][heartbeat-doc]\n\n[heartbeat-config]: https://www.elastic.co/docs/reference/beats/heartbeat/heartbeat-reference-yml\n[heartbeat-docker]: https://www.elastic.co/docs/reference/beats/heartbeat/running-on-docker\n[heartbeat-doc]: https://www.elastic.co/docs/reference/beats/heartbeat\n\n[setup]: ../../README.md#how-to-re-execute-the-setup\n" }, { "path": "extensions/heartbeat/config/heartbeat.yml", "content": "## Heartbeat configuration\n## https://github.com/elastic/beats/blob/main/deploy/docker/heartbeat.docker.yml\n#\n\nname: heartbeat\n\nheartbeat.monitors:\n- type: http\n schedule: '@every 5s'\n urls:\n - http://elasticsearch:9200\n username: heartbeat_internal\n password: ${HEARTBEAT_INTERNAL_PASSWORD}\n\n- type: icmp\n schedule: '@every 5s'\n hosts:\n - elasticsearch\n\nprocessors:\n- add_cloud_metadata: ~\n\nmonitoring:\n enabled: true\n elasticsearch:\n username: beats_system\n password: ${BEATS_SYSTEM_PASSWORD}\n\noutput.elasticsearch:\n hosts: [ http://elasticsearch:9200 ]\n username: heartbeat_internal\n password: ${HEARTBEAT_INTERNAL_PASSWORD}\n\n## HTTP endpoint for health checking\n## https://www.elastic.co/docs/reference/beats/heartbeat/http-endpoint\n#\n\nhttp:\n enabled: true\n host: 0.0.0.0\n" }, { "path": "extensions/heartbeat/heartbeat-compose.yml", "content": "services:\n heartbeat:\n build:\n context: extensions/heartbeat/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n command:\n # Log to stderr.\n - -e\n # Disable config file permissions checks. Allows mounting\n # 'config/heartbeat.yml' even if it's not owned by root.\n # see: https://www.elastic.co/docs/reference/beats/libbeat/config-file-permissions\n - --strict.perms=false\n volumes:\n - ./extensions/heartbeat/config/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro,Z\n environment:\n HEARTBEAT_INTERNAL_PASSWORD: ${HEARTBEAT_INTERNAL_PASSWORD:-}\n BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}\n networks:\n - elk\n depends_on:\n - elasticsearch\n" }, { "path": "extensions/metricbeat/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "extensions/metricbeat/Dockerfile", "content": "ARG ELASTIC_VERSION\n\nFROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-9.3.1}\n" }, { "path": "extensions/metricbeat/README.md", "content": "# Metricbeat\n\nMetricbeat is a lightweight shipper that you can install on your servers to periodically collect metrics from the\noperating system and from services running on the server. Metricbeat takes the metrics and statistics that it collects\nand ships them to the output that you specify, such as Elasticsearch or Logstash.\n\n## Usage\n\n**This extension requires the `metricbeat_internal`, `monitoring_internal` and `beats_system` users to be created and\ninitialized with a password.** In case you haven't done that during the initial startup of the stack, please refer to\n[How to re-execute the setup][setup] to run the setup container again and initialize these users.\n\nTo include Metricbeat in the stack, run Docker Compose from the root of the repository with an additional command line\nargument referencing the `metricbeat-compose.yml` file:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml up\n```\n\n## Configuring Metricbeat\n\nThe Metricbeat configuration is stored in [`config/metricbeat.yml`](./config/metricbeat.yml). You can modify this file\nwith the help of the [Configuration reference][metricbeat-config].\n\nAny change to the Metricbeat configuration requires a restart of the Metricbeat container:\n\n```console\n$ docker compose -f docker-compose.yml -f extensions/metricbeat/metricbeat-compose.yml restart metricbeat\n```\n\nPlease refer to the following documentation page for more details about how to configure Metricbeat inside a\nDocker container: [Run Metricbeat on Docker][metricbeat-docker].\n\n## See also\n\n[Metricbeat documentation][metricbeat-doc]\n\n## Screenshots\n\n![stack-monitoring](https://user-images.githubusercontent.com/3299086/202710574-32a3d419-86ea-4334-b6f7-62d7826df18d.png\n\"Stack Monitoring\")\n![host-dashboard](https://user-images.githubusercontent.com/3299086/202710594-0deccf40-3a9a-4e63-8411-2e0d9cc6ad3a.png\n\"Host Overview Dashboard\")\n\n[metricbeat-config]: https://www.elastic.co/docs/reference/beats/metricbeat/metricbeat-reference-yml\n[metricbeat-docker]: https://www.elastic.co/docs/reference/beats/metricbeat/running-on-docker\n[metricbeat-doc]: https://www.elastic.co/docs/reference/beats/metricbeat\n\n[setup]: ../../README.md#how-to-re-execute-the-setup\n" }, { "path": "extensions/metricbeat/config/metricbeat.yml", "content": "## Metricbeat configuration\n## https://github.com/elastic/beats/blob/main/deploy/docker/metricbeat.docker.yml\n#\n\nname: metricbeat\n\nmetricbeat.config:\n modules:\n path: ${path.config}/modules.d/*.yml\n # Reload module configs as they change:\n reload.enabled: false\n\nmetricbeat.autodiscover:\n providers:\n - type: docker\n hints.enabled: true\n\nmetricbeat.modules:\n- module: elasticsearch\n hosts: [ http://elasticsearch:9200 ]\n username: monitoring_internal\n password: ${MONITORING_INTERNAL_PASSWORD}\n xpack.enabled: true\n period: 10s\n enabled: true\n- module: logstash\n hosts: [ http://logstash:9600 ]\n xpack.enabled: true\n period: 10s\n enabled: true\n- module: kibana\n hosts: [ http://kibana:5601 ]\n username: monitoring_internal\n password: ${MONITORING_INTERNAL_PASSWORD}\n xpack.enabled: true\n period: 10s\n enabled: true\n- module: docker\n metricsets:\n - container\n - cpu\n - diskio\n - healthcheck\n - info\n #- image\n - memory\n - network\n hosts: [ unix:///var/run/docker.sock ]\n period: 10s\n enabled: true\n\nprocessors:\n - add_cloud_metadata: ~\n\nmonitoring:\n enabled: true\n elasticsearch:\n username: beats_system\n password: ${BEATS_SYSTEM_PASSWORD}\n\noutput.elasticsearch:\n hosts: [ http://elasticsearch:9200 ]\n username: metricbeat_internal\n password: ${METRICBEAT_INTERNAL_PASSWORD}\n\n## HTTP endpoint for health checking\n## https://www.elastic.co/docs/reference/beats/metricbeat/http-endpoint\n#\n\nhttp:\n enabled: true\n host: 0.0.0.0\n" }, { "path": "extensions/metricbeat/metricbeat-compose.yml", "content": "services:\n metricbeat:\n build:\n context: extensions/metricbeat/\n args:\n ELASTIC_VERSION: ${ELASTIC_VERSION}\n # Run as 'root' instead of 'metricbeat' (uid 1000) to allow reading\n # 'docker.sock' and the host's filesystem.\n user: root\n # If AppArmor or SELinux are enabled on the host, some permissions may be\n # denied unless the container is running with escalated privileges.\n # Use with extreme care.\n #privileged: true\n cap_add:\n # Allows collecting I/O metrics for host processes.\n - SYS_PTRACE\n command:\n # Log to stderr.\n - -e\n # Disable config file permissions checks. Allows mounting\n # 'config/metricbeat.yml' even if it's not owned by root.\n # see: https://www.elastic.co/docs/reference/beats/libbeat/config-file-permissions\n - --strict.perms=false\n # Mount point of the host’s filesystem. Required to monitor the host\n # from within a container.\n - --system.hostfs=/hostfs\n volumes:\n - ./extensions/metricbeat/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro,Z\n - type: bind\n source: /\n target: /hostfs\n read_only: true\n - type: bind\n source: /sys/fs/cgroup\n target: /hostfs/sys/fs/cgroup\n read_only: true\n - type: bind\n source: /proc\n target: /hostfs/proc\n read_only: true\n - type: bind\n source: /var/run/docker.sock\n target: /var/run/docker.sock\n read_only: true\n environment:\n METRICBEAT_INTERNAL_PASSWORD: ${METRICBEAT_INTERNAL_PASSWORD:-}\n MONITORING_INTERNAL_PASSWORD: ${MONITORING_INTERNAL_PASSWORD:-}\n BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}\n cgroup: host\n networks:\n - elk\n depends_on:\n - elasticsearch\n" }, { "path": "kibana/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "kibana/Dockerfile", "content": "ARG ELASTIC_VERSION\n\n# https://www.docker.elastic.co/\nFROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-9.3.1}\n\n# Add your kibana plugins setup here\n# Example: RUN kibana-plugin install \n" }, { "path": "kibana/config/kibana.yml", "content": "---\n## Default Kibana configuration from Kibana base image.\n## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts\n#\nserver.name: kibana\nserver.host: 0.0.0.0\nelasticsearch.hosts: [ http://elasticsearch:9200 ]\n\nmonitoring.ui.container.elasticsearch.enabled: true\nmonitoring.ui.container.logstash.enabled: true\n\n## Security credentials\n#\nelasticsearch.username: kibana_system\nelasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}\n\n## Encryption keys (optional but highly recommended)\n##\n## Generate with either\n## $ docker compose up kibana-genkeys\n## $ openssl rand -hex 32\n##\n## https://www.elastic.co/docs/deploy-manage/security/self-setup\n## https://www.elastic.co/docs/reference/kibana/commands/kibana-encryption-keys\n#\n#xpack.security.encryptionKey:\n#xpack.encryptedSavedObjects.encryptionKey:\n#xpack.reporting.encryptionKey:\n\n## Fleet\n## https://www.elastic.co/docs/reference/kibana/configuration-reference/fleet-settings\n#\nxpack.fleet.agents.fleet_server.hosts: [ http://fleet-server:8220 ]\n\nxpack.fleet.outputs:\n - id: fleet-default-output\n name: default\n type: elasticsearch\n hosts: [ http://elasticsearch:9200 ]\n is_default: true\n is_default_monitoring: true\n\nxpack.fleet.packages:\n - name: fleet_server\n version: latest\n - name: system\n version: latest\n - name: elastic_agent\n version: latest\n - name: docker\n version: latest\n - name: apm\n version: latest\n\nxpack.fleet.agentPolicies:\n - name: Fleet Server Policy\n id: fleet-server-policy\n description: Static agent policy for Fleet Server\n monitoring_enabled:\n - logs\n - metrics\n package_policies:\n - name: fleet_server-1\n package:\n name: fleet_server\n - name: system-1\n package:\n name: system\n - name: elastic_agent-1\n package:\n name: elastic_agent\n - name: docker-1\n package:\n name: docker\n - name: Agent Policy APM Server\n id: agent-policy-apm-server\n description: Static agent policy for the APM Server integration\n monitoring_enabled:\n - logs\n - metrics\n package_policies:\n - name: system-1\n package:\n name: system\n - name: elastic_agent-1\n package:\n name: elastic_agent\n - name: apm-1\n package:\n name: apm\n # See the APM package manifest for a list of possible inputs.\n # https://github.com/elastic/integrations/blob/d62b684/packages/apm/manifest.yml#L44-L174\n inputs:\n - type: apm\n vars:\n - name: host\n value: 0.0.0.0:8200\n - name: url\n value: http://apm-server:8200\n" }, { "path": "logstash/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n" }, { "path": "logstash/Dockerfile", "content": "ARG ELASTIC_VERSION\n\n# https://www.docker.elastic.co/\nFROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-9.3.1}\n\n# Add your logstash plugins setup here\n# Example: RUN logstash-plugin install logstash-filter-json\n" }, { "path": "logstash/config/logstash.yml", "content": "---\n## Default Logstash configuration from Logstash base image.\n## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml\n#\napi.http.host: 0.0.0.0\n\nnode.name: logstash\n" }, { "path": "logstash/pipeline/logstash.conf", "content": "input {\n\tbeats {\n\t\tport => 5044\n\t}\n\n\ttcp {\n\t\tport => 50000\n\t}\n}\n\n## Add your filters / logstash plugins configuration here\n\noutput {\n\telasticsearch {\n\t\thosts => \"elasticsearch:9200\"\n\t\tuser => \"logstash_internal\"\n\t\tpassword => \"${LOGSTASH_INTERNAL_PASSWORD}\"\n\t}\n}\n" }, { "path": "setup/.dockerignore", "content": "# Ignore Docker build files\nDockerfile\n.dockerignore\n\n# Ignore OS artifacts\n**/.DS_Store\n\n# Ignore Git files\n.gitignore\n" }, { "path": "setup/Dockerfile", "content": "ARG ELASTIC_VERSION\n\n# https://www.docker.elastic.co/\nFROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.3.1}\n\nENTRYPOINT [\"/entrypoint.sh\"]\n" }, { "path": "setup/entrypoint.sh", "content": "#!/usr/bin/env bash\n\nset -eu\nset -o pipefail\n\nsource \"${BASH_SOURCE[0]%/*}\"/lib.sh\n\n\n# --------------------------------------------------------\n# Users declarations\n\ndeclare -A users_passwords\nusers_passwords=(\n\t[logstash_internal]=\"${LOGSTASH_INTERNAL_PASSWORD:-}\"\n\t[kibana_system]=\"${KIBANA_SYSTEM_PASSWORD:-}\"\n\t[metricbeat_internal]=\"${METRICBEAT_INTERNAL_PASSWORD:-}\"\n\t[filebeat_internal]=\"${FILEBEAT_INTERNAL_PASSWORD:-}\"\n\t[heartbeat_internal]=\"${HEARTBEAT_INTERNAL_PASSWORD:-}\"\n\t[monitoring_internal]=\"${MONITORING_INTERNAL_PASSWORD:-}\"\n\t[beats_system]=\"${BEATS_SYSTEM_PASSWORD:-}\"\n)\n\ndeclare -A users_roles\nusers_roles=(\n\t[logstash_internal]='logstash_writer'\n\t[metricbeat_internal]='metricbeat_writer'\n\t[filebeat_internal]='filebeat_writer'\n\t[heartbeat_internal]='heartbeat_writer'\n\t[monitoring_internal]='remote_monitoring_collector'\n)\n\n# --------------------------------------------------------\n# Roles declarations\n\ndeclare -A roles_files\nroles_files=(\n\t[logstash_writer]='logstash_writer.json'\n\t[metricbeat_writer]='metricbeat_writer.json'\n\t[filebeat_writer]='filebeat_writer.json'\n\t[heartbeat_writer]='heartbeat_writer.json'\n)\n\n# --------------------------------------------------------\n\n\nlog 'Waiting for availability of Elasticsearch. This can take several minutes.'\n\ndeclare -i exit_code=0\nwait_for_elasticsearch || exit_code=$?\n\nif ((exit_code)); then\n\tcase $exit_code in\n\t\t6)\n\t\t\tsuberr 'Could not resolve host. Is Elasticsearch running?'\n\t\t\t;;\n\t\t7)\n\t\t\tsuberr 'Failed to connect to host. Is Elasticsearch healthy?'\n\t\t\t;;\n\t\t28)\n\t\t\tsuberr 'Timeout connecting to host. Is Elasticsearch healthy?'\n\t\t\t;;\n\t\t*)\n\t\t\tsuberr \"Connection to Elasticsearch failed. Exit code: ${exit_code}\"\n\t\t\t;;\n\tesac\n\n\texit $exit_code\nfi\n\nsublog 'Elasticsearch is running'\n\nlog 'Waiting for initialization of built-in users'\n\nwait_for_builtin_users || exit_code=$?\n\nif ((exit_code)); then\n\tsuberr 'Timed out waiting for condition'\n\texit $exit_code\nfi\n\nsublog 'Built-in users were initialized'\n\nfor role in \"${!roles_files[@]}\"; do\n\tlog \"Role '$role'\"\n\n\tdeclare body_file\n\tbody_file=\"${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}\"\n\tif [[ ! -f \"${body_file:-}\" ]]; then\n\t\tsublog \"No role body found at '${body_file}', skipping\"\n\t\tcontinue\n\tfi\n\n\tsublog 'Creating/updating'\n\tensure_role \"$role\" \"$(<\"${body_file}\")\"\ndone\n\nfor user in \"${!users_passwords[@]}\"; do\n\tlog \"User '$user'\"\n\tif [[ -z \"${users_passwords[$user]:-}\" ]]; then\n\t\tsublog 'No password defined, skipping'\n\t\tcontinue\n\tfi\n\n\tdeclare -i user_exists=0\n\tuser_exists=\"$(check_user_exists \"$user\")\"\n\n\tif ((user_exists)); then\n\t\tsublog 'User exists, setting password'\n\t\tset_user_password \"$user\" \"${users_passwords[$user]}\"\n\telse\n\t\tif [[ -z \"${users_roles[$user]:-}\" ]]; then\n\t\t\tsuberr ' No role defined, skipping creation'\n\t\t\tcontinue\n\t\tfi\n\n\t\tsublog 'User does not exist, creating'\n\t\tcreate_user \"$user\" \"${users_passwords[$user]}\" \"${users_roles[$user]}\"\n\tfi\ndone\n" }, { "path": "setup/lib.sh", "content": "#!/usr/bin/env bash\n\n# Log a message.\nfunction log {\n\techo \"[+] $1\"\n}\n\n# Log a message at a sub-level.\nfunction sublog {\n\techo \" ⠿ $1\"\n}\n\n# Log an error.\nfunction err {\n\techo \"[x] $1\" >&2\n}\n\n# Log an error at a sub-level.\nfunction suberr {\n\techo \" ⠍ $1\" >&2\n}\n\n# Inject common arguments to curl commands based on the environment.\nfunction augment_curl_args {\n\tlocal args_var_name=$1\n\tlocal -n args_ref=\"${args_var_name}\"\n\tif [[ -n \"${ELASTIC_PASSWORD:-}\" ]]; then\n\t\targs_ref+=( '-u' \"elastic:${ELASTIC_PASSWORD}\" )\n\tfi\n}\n\n# Poll the 'elasticsearch' service until it responds with HTTP code 200.\nfunction wait_for_elasticsearch {\n\tlocal elasticsearch_host=\"${ELASTICSEARCH_HOST:-elasticsearch}\"\n\n\tlocal -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' \"http://${elasticsearch_host}:9200/\" )\n\n\taugment_curl_args args\n\n\tlocal -i result=1\n\tlocal output\n\n\t# retry for max 300s (60*5s)\n\tfor _ in $(seq 1 60); do\n\t\tlocal -i exit_code=0\n\t\toutput=\"$(curl \"${args[@]}\")\" || exit_code=$?\n\n\t\tif ((exit_code)); then\n\t\t\tresult=$exit_code\n\t\tfi\n\n\t\tif [[ \"${output: -3}\" -eq 200 ]]; then\n\t\t\tresult=0\n\t\t\tbreak\n\t\tfi\n\n\t\tsleep 5\n\tdone\n\n\tif ((result)) && [[ \"${output: -3}\" -ne 000 ]]; then\n\t\techo -e \"\\n${output::-3}\"\n\tfi\n\n\treturn $result\n}\n\n# Poll the Elasticsearch users API until it returns users.\nfunction wait_for_builtin_users {\n\tlocal elasticsearch_host=\"${ELASTICSEARCH_HOST:-elasticsearch}\"\n\n\tlocal -a args=( '-s' '-D-' '-m15' \"http://${elasticsearch_host}:9200/_security/user?pretty\" )\n\n\taugment_curl_args args\n\n\tlocal -i result=1\n\n\tlocal line\n\tlocal -i exit_code\n\tlocal -i num_users\n\n\t# retry for max 30s (30*1s)\n\tfor _ in $(seq 1 30); do\n\t\tnum_users=0\n\n\t\t# read exits with a non-zero code if the last read input doesn't end\n\t\t# with a newline character. The printf without newline that follows the\n\t\t# curl command ensures that the final input not only contains curl's\n\t\t# exit code, but causes read to fail so we can capture the return value.\n\t\t# Ref. https://unix.stackexchange.com/a/176703/152409\n\t\twhile IFS= read -r line || ! exit_code=\"$line\"; do\n\t\t\tif [[ \"$line\" =~ _reserved.+true ]]; then\n\t\t\t\t(( num_users++ ))\n\t\t\tfi\n\t\tdone < <(curl \"${args[@]}\"; printf '%s' \"$?\")\n\n\t\tif ((exit_code)); then\n\t\t\tresult=$exit_code\n\t\tfi\n\n\t\t# we expect more than just the 'elastic' user in the result\n\t\tif (( num_users > 1 )); then\n\t\t\tresult=0\n\t\t\tbreak\n\t\tfi\n\n\t\tsleep 1\n\tdone\n\n\treturn $result\n}\n\n# Verify that the given Elasticsearch user exists.\nfunction check_user_exists {\n\tlocal username=$1\n\n\tlocal elasticsearch_host=\"${ELASTICSEARCH_HOST:-elasticsearch}\"\n\n\tlocal -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'\n\t\t\"http://${elasticsearch_host}:9200/_security/user/${username}\"\n\t\t)\n\n\taugment_curl_args args\n\n\tlocal -i result=1\n\tlocal -i exists=0\n\tlocal output\n\n\toutput=\"$(curl \"${args[@]}\")\"\n\tif [[ \"${output: -3}\" -eq 200 || \"${output: -3}\" -eq 404 ]]; then\n\t\tresult=0\n\tfi\n\tif [[ \"${output: -3}\" -eq 200 ]]; then\n\t\texists=1\n\tfi\n\n\tif ((result)); then\n\t\techo -e \"\\n${output::-3}\"\n\telse\n\t\techo \"$exists\"\n\tfi\n\n\treturn $result\n}\n\n# Set password of a given Elasticsearch user.\nfunction set_user_password {\n\tlocal username=$1\n\tlocal password=$2\n\n\tlocal elasticsearch_host=\"${ELASTICSEARCH_HOST:-elasticsearch}\"\n\n\tlocal -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'\n\t\t\"http://${elasticsearch_host}:9200/_security/user/${username}/_password\"\n\t\t'-X' 'POST'\n\t\t'-H' 'Content-Type: application/json'\n\t\t'-d' \"{\\\"password\\\" : \\\"${password}\\\"}\"\n\t\t)\n\n\taugment_curl_args args\n\n\tlocal -i result=1\n\tlocal output\n\n\toutput=\"$(curl \"${args[@]}\")\"\n\tif [[ \"${output: -3}\" -eq 200 ]]; then\n\t\tresult=0\n\tfi\n\n\tif ((result)); then\n\t\techo -e \"\\n${output::-3}\\n\"\n\tfi\n\n\treturn $result\n}\n\n# Create the given Elasticsearch user.\nfunction create_user {\n\tlocal username=$1\n\tlocal password=$2\n\tlocal role=$3\n\n\tlocal elasticsearch_host=\"${ELASTICSEARCH_HOST:-elasticsearch}\"\n\n\tlocal -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'\n\t\t\"http://${elasticsearch_host}:9200/_security/user/${username}\"\n\t\t'-X' 'POST'\n\t\t'-H' 'Content-Type: application/json'\n\t\t'-d' \"{\\\"password\\\":\\\"${password}\\\",\\\"roles\\\":[\\\"${role}\\\"]}\"\n\t\t)\n\n\taugment_curl_args args\n\n\tlocal -i result=1\n\tlocal output\n\n\toutput=\"$(curl \"${args[@]}\")\"\n\tif [[ \"${output: -3}\" -eq 200 ]]; then\n\t\tresult=0\n\tfi\n\n\tif ((result)); then\n\t\techo -e \"\\n${output::-3}\\n\"\n\tfi\n\n\treturn $result\n}\n\n# Ensure that the given Elasticsearch role is up-to-date, create it if required.\nfunction ensure_role {\n\tlocal name=$1\n\tlocal body=$2\n\n\tlocal elasticsearch_host=\"${ELASTICSEARCH_HOST:-elasticsearch}\"\n\n\tlocal -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'\n\t\t\"http://${elasticsearch_host}:9200/_security/role/${name}\"\n\t\t'-X' 'POST'\n\t\t'-H' 'Content-Type: application/json'\n\t\t'-d' \"$body\"\n\t\t)\n\n\taugment_curl_args args\n\n\tlocal -i result=1\n\tlocal output\n\n\toutput=\"$(curl \"${args[@]}\")\"\n\tif [[ \"${output: -3}\" -eq 200 ]]; then\n\t\tresult=0\n\tfi\n\n\tif ((result)); then\n\t\techo -e \"\\n${output::-3}\\n\"\n\tfi\n\n\treturn $result\n}\n" }, { "path": "setup/roles/filebeat_writer.json", "content": "{\n \"cluster\": [\n \"manage_ilm\",\n \"manage_index_templates\",\n \"manage_ingest_pipelines\",\n \"monitor\",\n \"read_pipeline\"\n ],\n \"indices\": [\n {\n \"names\": [\n \"filebeat-*\"\n ],\n \"privileges\": [\n \"create_doc\",\n \"manage\"\n ]\n }\n ]\n}\n" }, { "path": "setup/roles/heartbeat_writer.json", "content": "{\n \"cluster\": [\n \"manage_ilm\",\n \"manage_index_templates\",\n \"monitor\"\n ],\n \"indices\": [\n {\n \"names\": [\n \"heartbeat-*\"\n ],\n \"privileges\": [\n \"create_doc\",\n \"manage\"\n ]\n }\n ]\n}\n" }, { "path": "setup/roles/logstash_writer.json", "content": "{\n \"cluster\": [\n \"manage_index_templates\",\n \"monitor\",\n \"manage_ilm\"\n ],\n \"indices\": [\n {\n \"names\": [\n \"logs-generic-default\",\n \"logstash-*\",\n \"ecs-logstash-*\"\n ],\n \"privileges\": [\n \"write\",\n \"create\",\n \"create_index\",\n \"manage\",\n \"manage_ilm\"\n ]\n },\n {\n \"names\": [\n \"logstash\",\n \"ecs-logstash\"\n ],\n \"privileges\": [\n \"write\",\n \"manage\"\n ]\n }\n ]\n}\n" }, { "path": "setup/roles/metricbeat_writer.json", "content": "{\n \"cluster\": [\n \"manage_ilm\",\n \"manage_index_templates\",\n \"monitor\"\n ],\n \"indices\": [\n {\n \"names\": [\n \".monitoring-*-mb\",\n \"metricbeat-*\"\n ],\n \"privileges\": [\n \"create_doc\",\n \"manage\"\n ]\n }\n ]\n}\n" } ]