[ { "path": ".editorconfig", "content": "root = true\n\n[*]\nend_of_line = lf\ninsert_final_newline = true\ncharset = utf-8\n\n[*.py]\nindent_style = space\nindent_size = 4\n\n[Makefile]\nindent_style = tab\n\n[{*.json,*.yml,*.yaml}]\nindent_style = space\nindent_size = 2\n" }, { "path": ".gitignore", "content": "config.yaml\n.tox/\n.coverage\n.idea/*\n.cache/\n__pycache__/\n*.pyc\nvirtualenv_run/\n*.egg-info/\ndist/\nvenv/\nenv/\ndocs/build/\nbuild/\n.pytest_cache/\nmy_rules\n*.swp\n*~\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n- repo: git://github.com/pre-commit/pre-commit-hooks\n sha: v1.1.1\n hooks:\n - id: trailing-whitespace\n - id: end-of-file-fixer\n - id: autopep8-wrapper\n args:\n - -i\n - --ignore=E265,E309,E501\n - id: flake8\n - id: check-yaml\n - id: debug-statements\n - id: requirements-txt-fixer\n - id: name-tests-test\n- repo: git://github.com/asottile/reorder_python_imports\n sha: v0.3.5\n hooks:\n - id: reorder-python-imports\n- repo: git://github.com/Yelp/detect-secrets\n sha: 0.9.1\n hooks:\n - id: detect-secrets\n args: ['--baseline', '.secrets.baseline']\n exclude: .*tests/.*|.*yelp/testing/.*|\\.pre-commit-config\\.yaml\n" }, { "path": ".secrets.baseline", "content": "{\n \"exclude_regex\": \".*tests/.*|.*yelp/testing/.*|\\\\.pre-commit-config\\\\.yaml\",\n \"generated_at\": \"2018-07-06T22:54:22Z\",\n \"plugins_used\": [\n {\n \"base64_limit\": 4.5,\n \"name\": \"Base64HighEntropyString\"\n },\n {\n \"hex_limit\": 3,\n \"name\": \"HexHighEntropyString\"\n },\n {\n \"name\": \"PrivateKeyDetector\"\n }\n ],\n \"results\": {\n \".travis.yml\": [\n {\n \"hashed_secret\": \"4f7a1ea04dafcbfee994ee1d08857b8aaedf8065\",\n \"line_number\": 14,\n \"type\": \"Base64 High Entropy String\"\n }\n ]\n },\n \"version\": \"0.9.1\"\n}\n" }, { "path": ".travis.yml", "content": "language: python\npython:\n- '3.6'\nenv:\n- TOXENV=docs\n- TOXENV=py36\ninstall:\n- pip install tox\n- >\n if [[ -n \"${ES_VERSION}\" ]] ; then\n wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz\n mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1\n ./elasticsearch-${ES_VERSION}/bin/elasticsearch &\n fi\nscript:\n- >\n if [[ -n \"${ES_VERSION}\" ]] ; then\n wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200\n make test-elasticsearch\n else\n make test\n fi\njobs:\n include:\n - stage: 'Elasticsearch test'\n env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64\n - env: TOXENV=py36 ES_VERSION=6.6.2\n - env: TOXENV=py36 ES_VERSION=6.3.2\n - env: TOXENV=py36 ES_VERSION=6.2.4\n - env: TOXENV=py36 ES_VERSION=6.0.1\n - env: TOXENV=py36 ES_VERSION=5.6.16\n\ndeploy:\n provider: pypi\n user: yelplabs\n password:\n secure: TpSTlFu89tciZzboIfitHhU5NhAB1L1/rI35eQTXstiqzYg2mweOuip+MPNx9AlX3Swg7MhaFYnSUvRqPljuoLjLD0EQ7BHLVSBFl92ukkAMTeKvM6LbB9HnGOwzmAvTR5coegk8IHiegudODWvnhIj4hp7/0EA+gVX7E55kEAw=\n on:\n tags: true\n distributions: sdist bdist_wheel\n repo: Yelp/elastalert\n branch: master\n" }, { "path": "Dockerfile-test", "content": "FROM ubuntu:latest\n\nRUN apt-get update && apt-get upgrade -y\nRUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git\n\nWORKDIR /home/elastalert\n\nADD requirements*.txt ./\nRUN pip3 install -r requirements-dev.txt\n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n" }, { "path": "Makefile", "content": ".PHONY: all production test docs clean\n\nall: production\n\nproduction:\n\t@true\n\ndocs:\n\ttox -e docs\n\ndev: $(LOCAL_CONFIG_DIR) $(LOGS_DIR) install-hooks\n\ninstall-hooks:\n\tpre-commit install -f --install-hooks\n\ntest:\n\ttox\n\ntest-elasticsearch:\n\ttox -- --runelasticsearch\n\ntest-docker:\n\tdocker-compose --project-name elastalert build tox\n\tdocker-compose --project-name elastalert run tox\n\nclean:\n\tmake -C docs clean\n\tfind . -name '*.pyc' -delete\n\tfind . -name '__pycache__' -delete\n\trm -rf virtualenv_run .tox .coverage *.egg-info build\n" }, { "path": "README.md", "content": "**ElastAlert is no longer maintained. Please use [ElastAlert2](https://github.com/jertel/elastalert2) instead.**\n\n\n[![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert)\n[![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)\n\n## ElastAlert - [Read the Docs](http://elastalert.readthedocs.org).\n### Easy & Flexible Alerting With Elasticsearch\n\nElastAlert is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch.\n\nElastAlert works with all versions of Elasticsearch.\n\nAt Yelp, we use Elasticsearch, Logstash and Kibana for managing our ever increasing amount of data and logs.\nKibana is great for visualizing and querying data, but we quickly realized that it needed a companion tool for alerting\non inconsistencies in our data. Out of this need, ElastAlert was created.\n\nIf you have data being written into Elasticsearch in near real time and want to be alerted when that data matches certain patterns, ElastAlert is the tool for you. If you can see it in Kibana, ElastAlert can alert on it.\n\n## Overview\n\nWe designed ElastAlert to be reliable, highly modular, and easy to set up and configure.\n\nIt works by combining Elasticsearch with two types of components, rule types and alerts.\nElasticsearch is periodically queried and the data is passed to the rule type, which determines when\na match is found. When a match occurs, it is given to one or more alerts, which take action based on the match.\n\nThis is configured by a set of rules, each of which defines a query, a rule type, and a set of alerts.\n\nSeveral rule types with common monitoring paradigms are included with ElastAlert:\n\n- Match where there are at least X events in Y time\" (``frequency`` type)\n- Match when the rate of events increases or decreases\" (``spike`` type)\n- Match when there are less than X events in Y time\" (``flatline`` type)\n- Match when a certain field matches a blacklist/whitelist\" (``blacklist`` and ``whitelist`` type)\n- Match on any event matching a given filter\" (``any`` type)\n- Match when a field has two different values within some time\" (``change`` type)\n- Match when a never before seen term appears in a field\" (``new_term`` type)\n- Match when the number of unique values for a field is above or below a threshold (``cardinality`` type)\n\nCurrently, we have built-in support for the following alert types:\n\n- Email\n- JIRA\n- OpsGenie\n- Commands\n- HipChat\n- MS Teams\n- Slack\n- Telegram\n- GoogleChat\n- AWS SNS\n- VictorOps\n- PagerDuty\n- PagerTree\n- Exotel\n- Twilio\n- Gitter\n- Line Notify\n- Zabbix\n\nAdditional rule types and alerts can be easily imported or written.\n\nIn addition to this basic usage, there are many other features that make alerts more useful:\n\n- Alerts link to Kibana dashboards\n- Aggregate counts for arbitrary fields\n- Combine alerts into periodic reports\n- Separate alerts by using a unique key field\n- Intercept and enhance match data\n\nTo get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org).\n\n## Running ElastAlert\nYou can either install the latest released version of ElastAlert using pip:\n\n```pip install elastalert```\n\nor you can clone the ElastAlert repository for the most recent changes:\n\n```git clone https://github.com/Yelp/elastalert.git```\n\nInstall the module:\n\n```pip install \"setuptools>=11.3\"```\n\n```python setup.py install```\n\nThe following invocation can be used to run ElastAlert after installing\n\n``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]``\n\n``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`.\n\n``--verbose`` will print additional information without suppressing alerts. Not compatible with `--debug.`\n\n``--start`` will begin querying at the given timestamp. By default, ElastAlert will begin querying from the present.\nTimestamp format is ``YYYY-MM-DDTHH-MM-SS[-/+HH:MM]`` (Note the T between date and hour).\nEg: ``--start 2014-09-26T12:00:00`` (UTC) or ``--start 2014-10-01T07:30:00-05:00``\n\n``--end`` will cause ElastAlert to stop querying at the given timestamp. By default, ElastAlert will continue\nto query indefinitely.\n\n``--rule`` will allow you to run only one rule. It must still be in the rules folder.\nEg: ``--rule this_rule.yaml``\n\n``--config`` allows you to specify the location of the configuration. By default, it is will look for config.yaml in the current directory.\n\n## Third Party Tools And Extras\n### Kibana plugin\n![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif)\nAvailable at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin).\n\n### Docker\nA [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`.\n\n```bash\ngit clone https://github.com/bitsensor/elastalert.git; cd elastalert\ndocker run -d -p 3030:3030 \\\n -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \\\n -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \\\n -v `pwd`/rules:/opt/elastalert/rules \\\n -v `pwd`/rule_templates:/opt/elastalert/rule_templates \\\n --net=\"host\" \\\n --name elastalert bitsensor/elastalert:latest\n```\n\n## Documentation\n\nRead the documentation at [Read the Docs](http://elastalert.readthedocs.org).\n\nTo build a html version of the docs locally\n\n```\npip install sphinx_rtd_theme sphinx\ncd docs\nmake html\n```\n\nView in browser at build/html/index.html\n\n## Configuration\n\nSee config.yaml.example for details on configuration.\n\n## Example rules\n\nExamples of different types of rules can be found in example_rules/.\n\n- ``example_spike.yaml`` is an example of the \"spike\" rule type, which allows you to alert when the rate of events, averaged over a time period,\nincreases by a given factor. This example will send an email alert when there are 3 times more events matching a filter occurring within the\nlast 2 hours than the number of events in the previous 2 hours.\n\n- ``example_frequency.yaml`` is an example of the \"frequency\" rule type, which will alert when there are a given number of events occuring\nwithin a time period. This example will send an email when 50 documents matching a given filter occur within a 4 hour timeframe.\n\n- ``example_change.yaml`` is an example of the \"change\" rule type, which will alert when a certain field in two documents changes. In this example,\nthe alert email is sent when two documents with the same 'username' field but a different value of the 'country_name' field occur within 24 hours\nof each other.\n\n- ``example_new_term.yaml`` is an example of the \"new term\" rule type, which alerts when a new value appears in a field or fields. In this example,\nan email is sent when a new value of (\"username\", \"computer\") is encountered in example login logs.\n\n## Frequently Asked Questions\n\n### My rule is not getting any hits?\n\nSo you've managed to set up ElastAlert, write a rule, and run it, but nothing happens, or it says ``0 query hits``. First of all, we recommend using the command ``elastalert-test-rule rule.yaml`` to debug. It will show you how many documents match your filters for the last 24 hours (or more, see ``--help``), and then shows you if any alerts would have fired. If you have a filter in your rule, remove it and try again. This will show you if the index is correct and that you have at least some documents. If you have a filter in Kibana and want to recreate it in ElastAlert, you probably want to use a query string. Your filter will look like\n\n```\nfilter:\n- query:\n query_string:\n query: \"foo: bar AND baz: abc*\"\n```\nIf you receive an error that Elasticsearch is unable to parse it, it's likely the YAML is not spaced correctly, and the filter is not in the right format. If you are using other types of filters, like ``term``, a common pitfall is not realizing that you may need to use the analyzed token. This is the default if you are using Logstash. For example,\n\n```\nfilter:\n- term:\n foo: \"Test Document\"\n```\n\nwill not match even if the original value for ``foo`` was exactly \"Test Document\". Instead, you want to use ``foo.raw``. If you are still having trouble troubleshooting why your documents do not match, try running ElastAlert with ``--es_debug_trace /path/to/file.log``. This will log the queries made to Elasticsearch in full so that you can see exactly what is happening.\n\n### I got hits, why didn't I get an alert?\n\nIf you got logs that had ``X query hits, 0 matches, 0 alerts sent``, it depends on the ``type`` why you didn't get any alerts. If ``type: any``, a match will occur for every hit. If you are using ``type: frequency``, ``num_events`` must occur within ``timeframe`` of each other for a match to occur. Different rules apply for different rule types.\n\nIf you see ``X matches, 0 alerts sent``, this may occur for several reasons. If you set ``aggregation``, the alert will not be sent until after that time has elapsed. If you have gotten an alert for this same rule before, that rule may be silenced for a period of time. The default is one minute between alerts. If a rule is silenced, you will see ``Ignoring match for silenced rule`` in the logs.\n\nIf you see ``X alerts sent`` but didn't get any alert, it's probably related to the alert configuration. If you are using the ``--debug`` flag, you will not receive any alerts. Instead, the alert text will be written to the console. Use ``--verbose`` to achieve the same affects without preventing alerts. If you are using email alert, make sure you have it configured for an SMTP server. By default, it will connect to localhost on port 25. It will also use the word \"elastalert\" as the \"From:\" address. Some SMTP servers will reject this because it does not have a domain while others will add their own domain automatically. See the email section in the documentation for how to configure this.\n\n### Why did I only get one alert when I expected to get several?\n\nThere is a setting called ``realert`` which is the minimum time between two alerts for the same rule. Any alert that occurs within this time will simply be dropped. The default value for this is one minute. If you want to receive an alert for every single match, even if they occur right after each other, use\n\n```\nrealert:\n minutes: 0\n```\n\nYou can of course set it higher as well.\n\n### How can I prevent duplicate alerts?\n\nBy setting ``realert``, you will prevent the same rule from alerting twice in an amount of time.\n\n```\nrealert:\n days: 1\n```\n\nYou can also prevent duplicates based on a certain field by using ``query_key``. For example, to prevent multiple alerts for the same user, you might use\n\n```\nrealert:\n hours: 8\nquery_key: user\n```\n\nNote that this will also affect the way many rule types work. If you are using ``type: frequency`` for example, ``num_events`` for a single value of ``query_key`` must occur before an alert will be sent. You can also use a compound of multiple fields for this key. For example, if you only wanted to receieve an alert once for a specific error and hostname, you could use\n\n```\nquery_key: [error, hostname]\n```\n\nInternally, this works by creating a new field for each document called ``field1,field2`` with a value of ``value1,value2`` and using that as the ``query_key``.\n\nThe data for when an alert will fire again is stored in Elasticsearch in the ``elastalert_status`` index, with a ``_type`` of ``silence`` and also cached in memory.\n\n### How can I change what's in the alert?\n\nYou can use the field ``alert_text`` to add custom text to an alert. By setting ``alert_text_type: alert_text_only``, it will be the entirety of the alert. You can also add different fields from the alert by using Python style string formatting and ``alert_text_args``. For example\n\n```\nalert_text: \"Something happened with {0} at {1}\"\nalert_text_type: alert_text_only\nalert_text_args: [\"username\", \"@timestamp\"]\n```\n\nYou can also limit the alert to only containing certain fields from the document by using ``include``.\n\n```\ninclude: [\"ip_address\", \"hostname\", \"status\"]\n```\n\n### My alert only contains data for one event, how can I see more?\n\nIf you are using ``type: frequency``, you can set the option ``attach_related: true`` and every document will be included in the alert. An alternative, which works for every type, is ``top_count_keys``. This will show the top counts for each value for certain fields. For example, if you have\n\n```\ntop_count_keys: [\"ip_address\", \"status\"]\n```\n\nand 10 documents matched your alert, it may contain something like\n\n```\nip_address:\n127.0.0.1: 7\n10.0.0.1: 2\n192.168.0.1: 1\n\nstatus:\n200: 9\n500: 1\n```\n\n### How can I make the alert come at a certain time?\n\nThe ``aggregation`` feature will take every alert that has occured over a period of time and send them together in one alert. You can use cron style syntax to send all alerts that have occured since the last once by using\n\n```\naggregation:\n schedule: '2 4 * * mon,fri'\n```\n\n### I have lots of documents and it's really slow, how can I speed it up?\n\nThere are several ways to potentially speed up queries. If you are using ``index: logstash-*``, Elasticsearch will query all shards, even if they do not possibly contain data with the correct timestamp. Instead, you can use Python time format strings and set ``use_strftime_index``\n\n```\nindex: logstash-%Y.%m\nuse_strftime_index: true\n```\n\nAnother thing you could change is ``buffer_time``. By default, ElastAlert will query large overlapping windows in order to ensure that it does not miss any events, even if they are indexed in real time. In config.yaml, you can adjust ``buffer_time`` to a smaller number to only query the most recent few minutes.\n\n```\nbuffer_time:\n minutes: 5\n```\n\nBy default, ElastAlert will download every document in full before processing them. Instead, you can have ElastAlert simply get a count of the number of documents that have occured in between each query. To do this, set ``use_count_query: true``. This cannot be used if you use ``query_key``, because ElastAlert will not know the contents of each documents, just the total number of them. This also reduces the precision of alerts, because all events that occur between each query will be rounded to a single timestamp.\n\nIf you are using ``query_key`` (a single key, not multiple keys) you can use ``use_terms_query``. This will make ElastAlert perform a terms aggregation to get the counts for each value of a certain field. Both ``use_terms_query`` and ``use_count_query`` also require ``doc_type`` to be set to the ``_type`` of the documents. They may not be compatible with all rule types.\n\n### Can I perform aggregations?\n\nThe only aggregation supported currently is a terms aggregation, by setting ``use_terms_query``.\n\n### I'm not using @timestamp, what do I do?\n\nYou can use ``timestamp_field`` to change which field ElastAlert will use as the timestamp. You can use ``timestamp_type`` to change it between ISO 8601 and unix timestamps. You must have some kind of timestamp for ElastAlert to work. If your events are not in real time, you can use ``query_delay`` and ``buffer_time`` to adjust when ElastAlert will look for documents.\n\n### I'm using flatline but I don't see any alerts\n\nWhen using ``type: flatline``, ElastAlert must see at least one document before it will alert you that it has stopped seeing them.\n\n### How can I get a \"resolve\" event?\n\nElastAlert does not currently support stateful alerts or resolve events.\n\n### Can I set a warning threshold?\n\nCurrently, the only way to set a warning threshold is by creating a second rule with a lower threshold.\n\n## License\n\nElastAlert is licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0\n\n### Read the documentation at [Read the Docs](http://elastalert.readthedocs.org).\n\n### Questions? Drop by #elastalert on Freenode IRC.\n" }, { "path": "changelog.md", "content": "# Change Log\n\n# v0.2.4\n\n### Added\n- Added back customFields support for The Hive\n\n# v0.2.3\n\n### Added\n- Added back TheHive alerter without TheHive4py library\n\n# v0.2.2\n\n### Added\n- Integration with Kibana Discover app\n- Addied ability to specify opsgenie alert detailsĀ \n\n### Fixed\n- Fix some encoding issues with command alerter\n- Better error messages for missing config file\n- Fixed an issue with run_every not applying per-rule\n- Fixed an issue with rules not being removed\n- Fixed an issue with top count keys and nested query keys\n- Various documentation fixes\n- Fixed an issue with not being able to use spike aggregation\n\n### Removed\n- Remove The Hive alerter\n\n# v0.2.1\n\n### Fixed\n- Fixed an AttributeError introduced in 0.2.0\n\n# v0.2.0\n\n- Switched to Python 3\n\n### Added\n- Add rule loader class for customized rule loading\n- Added thread based rules and limit_execution\n- Run_every can now be customized per rule\n\n### Fixed\n- Various small fixes\n\n# v0.1.39\n\n### Added\n- Added spike alerts for metric aggregations\n- Allow SSL connections for Stomp\n- Allow limits on alert text length\n- Add optional min doc count for terms queries\n- Add ability to index into arrays for alert_text_args, etc\n\n### Fixed\n- Fixed bug involving --config flag with create-index\n- Fixed some settings not being inherited from the config properly\n- Some fixes for Hive alerter\n- Close SMTP connections properly\n- Fix timestamps in Pagerduty v2 payload\n- Fixed an bug causing aggregated alerts to mix up\n\n# v0.1.38\n\n### Added\n- Added PagerTree alerter\n- Added Line alerter\n- Added more customizable logging\n- Added new logic in test-rule to detemine the default timeframe\n\n### Fixed\n- Fixed an issue causing buffer_time to sometimes be ignored\n\n# v0.1.37\n\n### Added\n- Added more options for Opsgenie alerter\n- Added more pagerduty options\n- Added ability to add metadata to elastalert logs\n\n### Fixed\n- Fixed some documentation to be more clear\n- Stop requiring doc_type for metric aggregations\n- No longer puts quotes around regex terms in blacklists or whitelists\n\n# v0.1.36\n\n### Added\n- Added a prefix \"metric_\" to the key used for metric aggregations to avoid possible conflicts\n- Added option to skip Alerta certificate validation\n\n### Fixed\n- Fixed a typo in the documentation for spike rule\n\n# v0.1.35\n\n### Fixed\n- Fixed an issue preventing new term rule from working with terms query\n\n# v0.1.34\n\n### Added\n- Added prefix/suffix support for summary table\n- Added support for ignoring SSL validation in Slack\n- More visible exceptions during query parse failures\n\n### Fixed\n- Fixed top_count_keys when using compound query_key\n- Fixed num_hits sometimes being reported too low\n- Fixed an issue with setting ES_USERNAME via env\n- Fixed an issue when using test script with custom timestamps\n- Fixed a unicode error when using Telegram\n- Fixed an issue with jsonschema version conflict\n- Fixed an issue with nested timestamps in cardinality type\n\n# v0.1.33\n\n### Added\n- Added ability to pipe alert text to a command\n- Add --start and --end support for elastalert-test-rule\n- Added ability to turn blacklist/whitelist files into queries for better performance\n- Allow setting of OpsGenie priority\n- Add ability to query the adjacent index if timestamp_field not used for index timestamping\n- Add support for pagerduty v2\n- Add option to turn off .raw/.keyword field postfixing in new term rule\n- Added --use-downloaded feature for elastalert-test-rule\n\n### Fixed\n- Fixed a bug that caused num_hits in matches to sometimes be erroneously small\n- Fixed an issue with HTTP Post alerter that could cause it to hang indefinitely\n- Fixed some issues with string formatting for various alerters\n- Fixed a couple of incorrect parts of the documentation\n\n# v0.1.32\n\n### Added\n- Add support for setting ES url prefix via environment var\n- Add support for using native Slack fields in alerts\n\n### Fixed\n- Fixed a bug that would could scrolling queries to sometimes terminate early\n\n# v0.1.31\n\n### Added\n- Added ability to add start date to new term rule\n\n### Fixed\n- Fixed a bug in create_index which would try to delete a nonexistent index\n- Apply filters to new term rule all terms query\n- Support Elasticsearch 6 for new term rule\n- Fixed is_enabled not working on rule changes\n\n\n# v0.1.30\n\n### Added\n- Alerta alerter\n- Added support for transitioning JIRA issues\n- Option to recreate index in elastalert-create-index\n\n### Fixed\n- Update jira_ custom fields before each alert if they were modified\n- Use json instead of simplejson\n- Allow for relative path for smtp_auth_file\n- Fixed some grammar issues\n- Better code formatting of index mappings\n- Better formatting and size limit for HipChat HTML\n- Fixed gif link in readme for kibana plugin\n- Fixed elastalert-test-rule with Elasticsearch > 4\n- Added documentation for is_enabled option\n\n## v0.1.29\n\n### Added\n- Added a feature forget_keys to prevent realerting when using flatline with query_key\n- Added a new alert_text_type, aggregation_summary_only\n\n### Fixed\n- Fixed incorrect documentation about es_conn_timeout default\n\n## v0.1.28\n\n### Added\n- Added support for Stride formatting of simple HTML tags\n- Added support for custom titles in Opsgenie alerts\n- Added a denominator to percentage match based alerts\n\n### Fixed\n- Fixed a bug with Stomp alerter connections\n- Removed escaping of some characaters in Slack messages\n\n## v0.1.27\n\n# Added\n- Added support for a value other than in formatted alerts\n\n### Fixed\n- Fixed a failed creation of elastalert indicies when using Elasticsearch 6\n- Truncate Telegram alerts to avoid API errors\n\n## v0.1.26\n\n### Added\n- Added support for Elasticsearch 6\n- Added support for mentions in Hipchat\n\n### Fixed\n- Fixed an issue where a nested field lookup would crash if one of the intermediate fields was null\n\n## v0.1.25\n\n### Fixed\n- Fixed a bug causing new term rule to break unless you passed a start time\n- Add a slight clarification on the localhost:9200 reported in es_debug_trace\n\n## v0.1.24\n\n### Fixed\n- Pinned pytest\n- create-index reads index name from config.yaml\n- top_count_keys now works for context on a flatline rule type\n- Fixed JIRA behavior for issues with statuses that have spaces in the name\n\n## v0.1.22\n\n### Added\n- Added Stride alerter\n- Allow custom string formatters for aggregation percentage\n- Added a field to disable rules from config\n- Added support for subaggregations for the metric rule type\n\n### Fixed\n- Fixed a bug causing create-index to fail if missing config.yaml\n- Fixed a bug when using ES5 with query_key and top_count_keys\n- Allow enhancements to set and clear arbitrary JIRA fields\n- Fixed a bug causing timestamps to be formatted in scientific notation\n- Stop attempting to initialize alerters in debug mode\n- Changed default alert ordering so that JIRA tickets end up in other alerts\n- Fixed a bug when using Stomp alerter with complex query_key\n- Fixed a bug preventing hipchat room ID from being an integer\n- Fixed a bug causing duplicate alerts when using spike with alert_on_new_data\n- Minor fixes to summary table formatting\n- Fixed elastalert-test-rule when using new term rule type\n\n## v0.1.21\n\n### Fixed\n- Fixed an incomplete bug fix for preventing duplicate enhancement runs\n\n## v0.1.20\n\n### Added\n- Added support for client TLS keys\n\n### Fixed\n- Fixed the formatting of summary tables in Slack\n- Fixed ES_USE_SSL env variable\n- Fixed the unique value count printed by new_term rule type\n- Jira alerter no longer uses the non-existent json code formatter\n\n## v0.1.19\n\n### Added\n- Added support for populating JIRA fields via fields in the match\n- Added support for using a TLS certificate file for SMTP connections\n- Allow a custom suffix for non-analyzed Elasticsearch fields, like \".raw\" or \".keyword\"\n- Added match_time to Elastalert alert documents in Elasticsearch\n\n### Fixed\n- Fixed an error in the documentation for rule importing\n- Prevent enhancements from re-running on retried alerts\n- Fixed a bug when using custom timestamp formats and new term rule\n- Lowered jira_bump_after_inactivity default to 0 days\n\n## v0.1.18\n\n### Added\n- Added a new alerter \"post\" based on \"simple\" which makes POSTS JSON to HTTP endpoints\n- Added an option jira_bump_after_inacitivty to prevent ElastAlert commenting on active JIRA tickets\n\n### Removed\n- Removed \"simple\" alerter, replaced by \"post\"\n\n## v0.1.17\n\n### Added\n- Added a --patience flag to allow Elastalert to wait for Elasticsearch to become available\n- Allow custom PagerDuty alert titles via alert_subject\n\n## v0.1.16\n\n### Fixed\n- Fixed a bug where JIRA titles might not use query_key values\n- Fixed a bug where flatline alerts don't respect query_key for realert\n- Fixed a typo \"twilio_accout_sid\"\n\n### Added\n- Added support for env variables in kibana4 dashboard links\n- Added ca_certs option for custom CA support\n\n## v0.1.15\n\n### Fixed\n- Fixed a bug where Elastalert would crash on connection error during startup\n- Fixed some typos in documentation\n- Fixed a bug in metric bucket offset calculation\n- Fixed a TypeError in Service Now alerter\n\n### Added\n- Added support for compound compare key in change rules\n- Added support for absolute paths in rule config imports\n- Added Microsoft Teams alerter\n- Added support for markdown in Slack alerts\n- Added error codes to test script\n- Added support for lists in email_from_field\n\n\n## v0.1.14 - 2017-05-11\n\n### Fixed\n- Twilio alerter uses the from number appropriately\n- Fixed a TypeError in SNS alerter\n- Some changes to requirements.txt and setup.py\n- Fixed a TypeError in new term rule\n\n### Added\n- Set a custom pagerduty incident key\n- Preserve traceback in most exceptions\n\n## v0.1.12 - 2017-04-21\n\n### Fixed\n- Fixed a bug causing filters to be ignored when using Elasticsearch 5\n\n\n## v0.1.11 - 2017-04-19\n\n### Fixed\n- Fixed an issue that would cause filters starting with \"query\" to sometimes throw errors in ES5\n- Fixed a bug with multiple versions of ES on different rules\n- Fixed a possible KeyError when using use_terms_query with ES5\n\n## v0.1.10 - 2017-04-17\n\n### Fixed\n- Fixed an AttributeError occuring with older versions of Elasticsearch library\n- Made example rules more consistent and with unique names\n- Fixed an error caused by a typo when es_username is used\n\n## v0.1.9 - 2017-04-14\n\n### Added\n- Added a changelog\n- Added metric aggregation rule type\n- Added percentage match rule type\n- Added default doc style and improved the instructions\n- Rule names will default to the filename\n- Added import keyword in rules to include sections from other files\n- Added email_from_field option to derive the recipient from a field in the match\n- Added simple HTTP alerter\n- Added Exotel SMS alerter\n- Added a readme link to third party Kibana plugin\n- Added option to use env variables to configure some settings\n- Added duplicate hits count in log line\n\n### Fixed\n- Fixed a bug in change rule where a boolean false would be ignored\n- Clarify documentation on format of alert_text_args and alert_text_kw\n- Fixed a bug preventing new silence stashes from being loaded after a rule has previous alerted\n- Changed the default es_host in elastalert-test-rule to localhost\n- Fixed a bug preventing ES <5.0 formatted queries working in elastalert-test-rule\n- Fixed top_count_keys adding .raw on ES >5.0, uses .keyword instead\n- Fixed a bug causing compound aggregation keys not to work\n- Better error reporting for the Jira alerter\n- AWS request signing now refreshes credentials, uses boto3\n- Support multiple ES versions on different rules\n- Added documentation for percentage match rule type\n\n### Removed\n- Removed a feature that would disable writeback_es on errors, causing various issues\n" }, { "path": "config.yaml.example", "content": "# This is the folder that contains the rule yaml files\n# Any .yaml file will be loaded as a rule\nrules_folder: example_rules\n\n# How often ElastAlert will query Elasticsearch\n# The unit can be anything from weeks to seconds\nrun_every:\n minutes: 1\n\n# ElastAlert will buffer results from the most recent\n# period of time, in case some log sources are not in real time\nbuffer_time:\n minutes: 15\n\n# The Elasticsearch hostname for metadata writeback\n# Note that every rule can have its own Elasticsearch host\nes_host: elasticsearch.example.com\n\n# The Elasticsearch port\nes_port: 9200\n\n# The AWS region to use. Set this when using AWS-managed elasticsearch\n#aws_region: us-east-1\n\n# The AWS profile to use. Use this if you are using an aws-cli profile.\n# See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html\n# for details\n#profile: test\n\n# Optional URL prefix for Elasticsearch\n#es_url_prefix: elasticsearch\n\n# Connect with TLS to Elasticsearch\n#use_ssl: True\n\n# Verify TLS certificates\n#verify_certs: True\n\n# GET request with body is the default option for Elasticsearch.\n# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.\n# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport\n# for details\n#es_send_get_body_as: GET\n\n# Option basic-auth username and password for Elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# Use SSL authentication with client certificates client_cert must be\n# a pem file containing both cert and key for client\n#verify_certs: True\n#ca_certs: /path/to/cacert.pem\n#client_cert: /path/to/client_cert.pem\n#client_key: /path/to/client_key.key\n\n# The index on es_host which is used for metadata storage\n# This can be a unmapped index, but it is recommended that you run\n# elastalert-create-index to set a mapping\nwriteback_index: elastalert_status\nwriteback_alias: elastalert_alerts\n\n# If an alert fails for some reason, ElastAlert will retry\n# sending the alert until this time period has elapsed\nalert_time_limit:\n days: 2\n\n# Custom logging configuration\n# If you want to setup your own logging configuration to log into\n# files as well or to Logstash and/or modify log levels, use\n# the configuration below and adjust to your needs.\n# Note: if you run ElastAlert with --verbose/--debug, the log level of\n# the \"elastalert\" logger is changed to INFO, if not already INFO/DEBUG.\n#logging:\n# version: 1\n# incremental: false\n# disable_existing_loggers: false\n# formatters:\n# logline:\n# format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'\n#\n# handlers:\n# console:\n# class: logging.StreamHandler\n# formatter: logline\n# level: DEBUG\n# stream: ext://sys.stderr\n#\n# file:\n# class : logging.FileHandler\n# formatter: logline\n# level: DEBUG\n# filename: elastalert.log\n#\n# loggers:\n# elastalert:\n# level: WARN\n# handlers: []\n# propagate: true\n#\n# elasticsearch:\n# level: WARN\n# handlers: []\n# propagate: true\n#\n# elasticsearch.trace:\n# level: WARN\n# handlers: []\n# propagate: true\n#\n# '': # root logger\n# level: WARN\n# handlers:\n# - console\n# - file\n# propagate: false\n" }, { "path": "docker-compose.yml", "content": "version: '2'\nservices:\n tox:\n build:\n context: ./\n dockerfile: Dockerfile-test\n command: tox\n container_name: elastalert_tox\n working_dir: /home/elastalert\n volumes:\n - ./:/home/elastalert/\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = build\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source\n\n.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest\n\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\nclean:\n\t-rm -rf $(BUILDDIR)/*\n\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/monitor.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/monitor.qhc\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make all-pdf' or \\`make all-ps' in that directory to\" \\\n\t \"run these through (pdf)latex.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n" }, { "path": "docs/source/_static/.gitkeep", "content": "" }, { "path": "docs/source/conf.py", "content": "import sphinx_rtd_theme\n\n# -*- coding: utf-8 -*-\n#\n# ElastAlert documentation build configuration file, created by\n# sphinx-quickstart on Thu Jul 11 15:45:31 2013.\n#\n# This file is execfile()d with the current directory set to its containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n# sys.path.append(os.path.abspath('.'))\n# -- General configuration -----------------------------------------------------\n# Add any Sphinx extension module names here, as strings. They can be extensions\n# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.\nextensions = []\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix of source filenames.\nsource_suffix = '.rst'\n\n# The encoding of source files.\n# source_encoding = 'utf-8'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = u'ElastAlert'\ncopyright = u'2014, Yelp'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = '0.0.1'\n# The full version, including alpha/beta/rc tags.\nrelease = '0.0.1'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n# language = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n# today = ''\n# Else, today_fmt is used as the format for a strftime call.\n# today_fmt = '%B %d, %Y'\n\n# List of documents that shouldn't be included in the build.\n# unused_docs = []\n\n# List of directories, relative to source directory, that shouldn't be searched\n# for source files.\nexclude_trees = []\n\n# The reST default role (used for this markup: `text`) to use for all documents.\n# default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n# show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n\n# -- Options for HTML output ---------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. Major themes that come with\n# Sphinx are currently 'default' and 'sphinxdoc'.\nhtml_theme = 'sphinx_rtd_theme'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n# html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\nhtml_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n# html_theme_path = []\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n# html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n# html_logo = None\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n# html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\n# html_last_updated_fmt = '%b %d, %Y'\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n# html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n# html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n# html_use_modindex = True\n\n# If false, no index is generated.\n# html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n# html_show_sourcelink = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n# html_use_opensearch = ''\n\n# If nonempty, this is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = ''\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'elastalertdoc'\n\n\n# -- Options for LaTeX output --------------------------------------------------\n\n# The paper size ('letter' or 'a4').\n# latex_paper_size = 'letter'\n\n# The font size ('10pt', '11pt' or '12pt').\n# latex_font_size = '10pt'\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title, author, documentclass [howto/manual]).\nlatex_documents = [\n ('index', 'elastalert.tex', u'ElastAlert Documentation',\n u'Quentin Long', 'manual'),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n# latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n# latex_use_parts = False\n\n# Additional stuff for the LaTeX preamble.\n# latex_preamble = ''\n\n# Documents to append as an appendix to all manuals.\n# latex_appendices = []\n\n# If false, no module index is generated.\n# latex_use_modindex = True\n" }, { "path": "docs/source/elastalert.rst", "content": "ElastAlert - Easy & Flexible Alerting With Elasticsearch\n********************************************************\n\nElastAlert is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch.\n\nAt Yelp, we use Elasticsearch, Logstash and Kibana for managing our ever increasing amount of data and logs.\nKibana is great for visualizing and querying data, but we quickly realized that it needed a companion tool for alerting\non inconsistencies in our data. Out of this need, ElastAlert was created.\n\nIf you have data being written into Elasticsearch in near real time and want to be alerted when that data matches certain patterns, ElastAlert is the tool for you.\n\nOverview\n========\n\nWe designed ElastAlert to be :ref:`reliable `, highly :ref:`modular `, and easy to :ref:`set up ` and :ref:`configure `.\n\nIt works by combining Elasticsearch with two types of components, rule types and alerts.\nElasticsearch is periodically queried and the data is passed to the rule type, which determines when\na match is found. When a match occurs, it is given to one or more alerts, which take action based on the match.\n\nThis is configured by a set of rules, each of which defines a query, a rule type, and a set of alerts.\n\nSeveral rule types with common monitoring paradigms are included with ElastAlert:\n\n- \"Match where there are X events in Y time\" (``frequency`` type)\n- \"Match when the rate of events increases or decreases\" (``spike`` type)\n- \"Match when there are less than X events in Y time\" (``flatline`` type)\n- \"Match when a certain field matches a blacklist/whitelist\" (``blacklist`` and ``whitelist`` type)\n- \"Match on any event matching a given filter\" (``any`` type)\n- \"Match when a field has two different values within some time\" (``change`` type)\n\nCurrently, we have support built in for these alert types:\n\n- Command\n- Email\n- JIRA\n- OpsGenie\n- SNS\n- HipChat\n- Slack\n- Telegram\n- GoogleChat\n- Debug\n- Stomp\n- TheHive\n\nAdditional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `)\n\nIn addition to this basic usage, there are many other features that make alerts more useful:\n\n- Alerts link to Kibana dashboards\n- Aggregate counts for arbitrary fields\n- Combine alerts into periodic reports\n- Separate alerts by using a unique key field\n- Intercept and enhance match data\n\nTo get started, check out :ref:`Running ElastAlert For The First Time `.\n\n.. _reliability:\n\nReliability\n===========\n\nElastAlert has several features to make it more reliable in the event of restarts or Elasticsearch unavailability:\n\n- ElastAlert :ref:`saves its state to Elasticsearch ` and, when started, will resume where previously stopped\n- If Elasticsearch is unresponsive, ElastAlert will wait until it recovers before continuing\n- Alerts which throw errors may be automatically retried for a period of time\n\n.. _modularity:\n\nModularity\n==========\n\nElastAlert has three main components that may be imported as a module or customized:\n\nRule types\n----------\n\nThe rule type is responsible for processing the data returned from Elasticsearch. It is initialized with the rule configuration, passed data\nthat is returned from querying Elasticsearch with the rule's filters, and outputs matches based on this data. See :ref:`Writing rule types `\nfor more information.\n\nAlerts\n------\n\nAlerts are responsible for taking action based on a match. A match is generally a dictionary containing values from a document in Elasticsearch,\nbut may contain arbitrary data added by the rule type. See :ref:`Writing alerts ` for more information.\n\nEnhancements\n------------\n\nEnhancements are a way of intercepting an alert and modifying or enhancing it in some way. They are passed the match dictionary before it is given\nto the alerter. See :ref:`Enhancements` for more information.\n\n.. _configuration:\n\nConfiguration\n=============\n\nElastAlert has a global configuration file, ``config.yaml``, which defines several aspects of its operation:\n\n``buffer_time``: ElastAlert will continuously query against a window from the present to ``buffer_time`` ago.\nThis way, logs can be back filled up to a certain extent and ElastAlert will still process the events. This\nmay be overridden by individual rules. This option is ignored for rules where ``use_count_query`` or ``use_terms_query``\nis set to true. Note that back filled data may not always trigger count based alerts as if it was queried in real time.\n\n``es_host``: The host name of the Elasticsearch cluster where ElastAlert records metadata about its searches.\nWhen ElastAlert is started, it will query for information about the time that it was last run. This way,\neven if ElastAlert is stopped and restarted, it will never miss data or look at the same events twice. It will also specify the default cluster for each rule to run on.\nThe environment variable ``ES_HOST`` will override this field.\n\n``es_port``: The port corresponding to ``es_host``. The environment variable ``ES_PORT`` will override this field.\n\n``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``.\nThe environment variable ``ES_USE_SSL`` will override this field.\n\n``verify_certs``: Optional; whether or not to verify TLS certificates; set to ``True`` or ``False``. The default is ``True``.\n\n``client_cert``: Optional; path to a PEM certificate to use as the client certificate.\n\n``client_key``: Optional; path to a private key file to use as the client key.\n\n``ca_certs``: Optional; path to a CA cert bundle to use to verify SSL connections\n\n``es_username``: Optional; basic-auth username for connecting to ``es_host``. The environment variable ``ES_USERNAME`` will override this field.\n\n``es_password``: Optional; basic-auth password for connecting to ``es_host``. The environment variable ``ES_PASSWORD`` will override this field.\n\n``es_url_prefix``: Optional; URL prefix for the Elasticsearch endpoint. The environment variable ``ES_URL_PREFIX`` will override this field.\n\n``es_send_get_body_as``: Optional; Method for querying Elasticsearch - ``GET``, ``POST`` or ``source``. The default is ``GET``\n\n``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``.\n\n``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes.\nDefaults to ``FileRulesLoader`` if not set.\n\n``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all\nfiles in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload\nor remove rules based on their respective config files. (only required when using ``FileRulesLoader``).\n\n``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true``\n\n``run_every``: How often ElastAlert should query Elasticsearch. ElastAlert will remember the last time\nit ran the query for a given rule, and periodically query from that time until the present. The format of\nthis field is a nested unit of time, such as ``minutes: 5``. This is how time is defined in every ElastAlert\nconfiguration.\n\n``writeback_index``: The index on ``es_host`` to use.\n\n``max_query_size``: The maximum number of documents that will be downloaded from Elasticsearch in a single query. The\ndefault is 10,000, and if you expect to get near this number, consider using ``use_count_query`` for the rule. If this\nlimit is reached, ElastAlert will `scroll `_\nusing the size of ``max_query_size`` through the set amount of pages, when ``max_scrolling_count`` is set or until processing all results.\n\n``max_scrolling_count``: The maximum amount of pages to scroll through. The default is ``0``, which means the scrolling has no limit.\nFor example if this value is set to ``5`` and the ``max_query_size`` is set to ``10000`` then ``50000`` documents will be downloaded at most.\n\n``scroll_keepalive``: The maximum time (formatted in `Time Units `_) the scrolling context should be kept alive. Avoid using high values as it abuses resources in Elasticsearch, but be mindful to allow sufficient time to finish processing all the results.\n\n``max_aggregation``: The maximum number of alerts to aggregate together. If a rule has ``aggregation`` set, all\nalerts occuring within a timeframe will be sent together. The default is 10,000.\n\n``old_query_limit``: The maximum time between queries for ElastAlert to start at the most recently run query.\nWhen ElastAlert starts, for each rule, it will search ``elastalert_metadata`` for the most recently run query and start\nfrom that time, unless it is older than ``old_query_limit``, in which case it will start from the present time. The default is one week.\n\n``disable_rules_on_error``: If true, ElastAlert will disable rules which throw uncaught (not EAException) exceptions. It\nwill upload a traceback message to ``elastalert_metadata`` and if ``notify_email`` is set, send an email notification. The\nrule will no longer be run until either ElastAlert restarts or the rule file has been modified. This defaults to True.\n\n``show_disabled_rules``: If true, ElastAlert show the disable rules' list when finishes the execution. This defaults to True.\n\n``notify_email``: An email address, or list of email addresses, to which notification emails will be sent. Currently,\nonly an uncaught exception will send a notification email. The from address, SMTP host, and reply-to header can be set\nusing ``from_addr``, ``smtp_host``, and ``email_reply_to`` options, respectively. By default, no emails will be sent.\n\n``from_addr``: The address to use as the from header in email notifications.\nThis value will be used for email alerts as well, unless overwritten in the rule config. The default value\nis \"ElastAlert\".\n\n``smtp_host``: The SMTP host used to send email notifications. This value will be used for email alerts as well,\nunless overwritten in the rule config. The default is \"localhost\".\n\n``email_reply_to``: This sets the Reply-To header in emails. The default is the recipient address.\n\n``aws_region``: This makes ElastAlert to sign HTTP requests when using Amazon Elasticsearch Service. It'll use instance role keys to sign the requests.\nThe environment variable ``AWS_DEFAULT_REGION`` will override this field.\n\n``boto_profile``: Deprecated! Boto profile to use when signing requests to Amazon Elasticsearch Service, if you don't want to use the instance role keys.\n\n``profile``: AWS profile to use when signing requests to Amazon Elasticsearch Service, if you don't want to use the instance role keys.\nThe environment variable ``AWS_DEFAULT_PROFILE`` will override this field.\n\n``replace_dots_in_field_names``: If ``True``, ElastAlert replaces any dots in field names with an underscore before writing documents to Elasticsearch.\nThe default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in field names.\n\n``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch.\nThe default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5.\n\n``add_metadata_alert``: If set, alerts will include metadata described in rules (``category``, ``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``.\n\n``skip_invalid``: If ``True``, skip invalid files instead of exiting.\n\nLogging\n-------\n\nBy default, ElastAlert uses a simple basic logging configuration to print log messages to standard error.\nYou can change the log level to ``INFO`` messages by using the ``--verbose`` or ``--debug`` command line options.\n\nIf you need a more sophisticated logging configuration, you can provide a full logging configuration\nin the config file. This way you can also configure logging to a file, to Logstash and\nadjust the logging format.\n\nFor details, see the end of ``config.yaml.example`` where you can find an example logging\nconfiguration.\n\n\n.. _runningelastalert:\n\nRunning ElastAlert\n==================\n\n``$ python elastalert/elastalert.py``\n\nSeveral arguments are available when running ElastAlert:\n\n``--config`` will specify the configuration file to use. The default is ``config.yaml``.\n\n``--debug`` will run ElastAlert in debug mode. This will increase the logging verboseness, change\nall alerts to ``DebugAlerter``, which prints alerts and suppresses their normal action, and skips writing\nsearch and alert metadata back to Elasticsearch. Not compatible with `--verbose`.\n\n``--verbose`` will increase the logging verboseness, which allows you to see information about the state\nof queries. Not compatible with `--debug`.\n\n``--start `` will force ElastAlert to begin querying from the given time, instead of the default,\nquerying from the present. The timestamp should be ISO8601, e.g. ``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone\n``YYYY-MM-DDTHH:MM:SS-08:00`` (PST). Note that if querying over a large date range, no alerts will be\nsent until that rule has finished querying over the entire time period. To force querying from the current time, use \"NOW\".\n\n``--end `` will cause ElastAlert to stop querying at the specified timestamp. By default, ElastAlert\nwill periodically query until the present indefinitely.\n\n``--rule `` will only run the given rule. The rule file may be a complete file path or a filename in ``rules_folder``\nor its subdirectories.\n\n``--silence =`` will silence the alerts for a given rule for a period of time. The rule must be specified using\n``--rule``. is one of days, weeks, hours, minutes or seconds. is an integer. For example,\n``--rule noisy_rule.yaml --silence hours=4`` will stop noisy_rule from generating any alerts for 4 hours.\n\n``--es_debug`` will enable logging for all queries made to Elasticsearch.\n\n``--es_debug_trace `` will enable logging curl commands for all queries made to Elasticsearch to the\nspecified log file. ``--es_debug_trace`` is passed through to `elasticsearch.py\n`_ which logs `localhost:9200`\ninstead of the actual ``es_host``:``es_port``.\n\n``--end `` will force ElastAlert to stop querying after the given time, instead of the default,\nquerying to the present time. This really only makes sense when running standalone. The timestamp is formatted\nas ``YYYY-MM-DDTHH:MM:SS`` (UTC) or with timezone ``YYYY-MM-DDTHH:MM:SS-XX:00`` (UTC-XX).\n\n``--pin_rules`` will stop ElastAlert from loading, reloading or removing rules based on changes to their config files.\n" }, { "path": "docs/source/elastalert_status.rst", "content": ".. _metadata:\n\nElastAlert Metadata Index\n=========================\n\nElastAlert uses Elasticsearch to store various information about its state. This not only allows for some\nlevel of auditing and debugging of ElastAlert's operation, but also to avoid loss of data or duplication of alerts\nwhen ElastAlert is shut down, restarted, or crashes. This cluster and index information is defined\nin the global config file with ``es_host``, ``es_port`` and ``writeback_index``. ElastAlert must be able\nto write to this index. The script, ``elastalert-create-index`` will create the index with the correct mapping\nfor you, and optionally copy the documents from an existing ElastAlert writeback index. Run it and it will\nprompt you for the cluster information.\n\nElastAlert will create three different types of documents in the writeback index:\n\nelastalert_status\n~~~~~~~~~~~~~~~~~\n\n``elastalert_status`` is a log of the queries performed for a given rule and contains:\n\n- ``@timestamp``: The time when the document was uploaded to Elasticsearch. This is after a query has been run and the results have been processed.\n- ``rule_name``: The name of the corresponding rule.\n- ``starttime``: The beginning of the timestamp range the query searched.\n- ``endtime``: The end of the timestamp range the query searched.\n- ``hits``: The number of results from the query.\n- ``matches``: The number of matches that the rule returned after processing the hits. Note that this does not necessarily mean that alerts were triggered.\n- ``time_taken``: The number of seconds it took for this query to run.\n\n``elastalert_status`` is what ElastAlert will use to determine what time range to query when it first starts to avoid duplicating queries.\nFor each rule, it will start querying from the most recent endtime. If ElastAlert is running in debug mode, it will still attempt to base\nits start time by looking for the most recent search performed, but it will not write the results of any query back to Elasticsearch.\n\nelastalert\n~~~~~~~~~~\n\n``elastalert`` is a log of information about every alert triggered and contains:\n\n- ``@timestamp``: The time when the document was uploaded to Elasticsearch. This is not the same as when the alert was sent, but rather when the rule outputs a match.\n- ``rule_name``: The name of the corresponding rule.\n- ``alert_info``: This contains the output of Alert.get_info, a function that alerts implement to give some relevant context to the alert type. This may contain alert_info.type, alert_info.recipient, or any number of other sub fields.\n- ``alert_sent``: A boolean value as to whether this alert was actually sent or not. It may be false in the case of an exception or if it is part of an aggregated alert.\n- ``alert_time``: The time that the alert was or will be sent. Usually, this is the same as @timestamp, but may be some time in the future, indicating when an aggregated alert will be sent.\n- ``match_body``: This is the contents of the match dictionary that is used to create the alert. The subfields may include a number of things containing information about the alert.\n- ``alert_exception``: This field is only present when the alert failed because of an exception occurring, and will contain the exception information.\n- ``aggregate_id``: This field is only present when the rule is configured to use aggregation. The first alert of the aggregation period will contain an alert_time set to the aggregation time into the future, and subsequent alerts will contain the document ID of the first. When the alert_time is reached, all alerts with that aggregate_id will be sent together.\n\nelastalert_error\n~~~~~~~~~~~~~~~~\n\nWhen an error occurs in ElastAlert, it is written to both Elasticsearch and to stderr. The ``elastalert_error`` type contains:\n\n- ``@timestamp``: The time when the error occurred.\n- ``message``: The error or exception message.\n- ``traceback``: The traceback from when the error occurred.\n- ``data``: Extra information about the error. This often contains the name of the rule which caused the error.\n\nsilence\n~~~~~~~\n\n``silence`` is a record of when alerts for a given rule will be suppressed, either because of a ``realert`` setting or from using --silence. When\nan alert with ``realert`` is triggered, a ``silence`` record will be written with ``until`` set to the alert time plus ``realert``.\n\n- ``@timestamp``: The time when the document was uploaded to Elasticsearch.\n- ``rule_name``: The name of the corresponding rule.\n- ``until``: The timestamp when alerts will begin being sent again.\n- ``exponent``: The exponential factor which multiplies ``realert``. The length of this silence is equal to ``realert`` * 2**exponent. This will\n be 0 unless ``exponential_realert`` is set.\n\nWhenever an alert is triggered, ElastAlert will check for a matching ``silence`` document, and if the ``until`` timestamp is in the future, it will ignore\nthe alert completely. See the :ref:`Running ElastAlert ` section for information on how to silence an alert.\n" }, { "path": "docs/source/index.rst", "content": ".. ElastAlert documentation master file, created by\n sphinx-quickstart on Thu Jul 11 15:45:31 2013.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\nElastAlert - Easy & Flexible Alerting With Elasticsearch\n========================================================\n\nContents:\n\n.. toctree::\n :maxdepth: 2\n\n elastalert\n running_elastalert\n ruletypes\n elastalert_status\n recipes/adding_rules\n recipes/adding_alerts\n recipes/writing_filters\n recipes/adding_enhancements\n recipes/adding_loaders\n recipes/signing_requests\n\nIndices and Tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "docs/source/recipes/adding_alerts.rst", "content": ".. _writingalerts:\n\nAdding a New Alerter\n====================\n\nAlerters are subclasses of ``Alerter``, found in ``elastalert/alerts.py``. They are given matches\nand perform some action based on that. Your alerter needs to implement two member functions, and will look\nsomething like this:\n\n.. code-block:: python\n\n class AwesomeNewAlerter(Alerter):\n required_options = set(['some_config_option'])\n def alert(self, matches):\n ...\n def get_info(self):\n ...\n\nYou can import alert types by specifying the type as ``module.file.AlertName``, where module is the name of a python module,\nand file is the name of the python file containing a ``Alerter`` subclass named ``AlertName``.\n\nBasics\n------\n\nThe alerter class will be instantiated when ElastAlert starts, and be periodically passed\nmatches through the ``alert`` method. ElastAlert also writes back info about the alert into\nElasticsearch that it obtains through ``get_info``. Several important member properties:\n\n``self.required_options``: This is a set containing names of configuration options that must be\npresent. ElastAlert will not instantiate the alert if any are missing.\n\n``self.rule``: The dictionary containing the rule configuration. All options specific to the alert\nshould be in the rule configuration file and can be accessed here.\n\n``self.pipeline``: This is a dictionary object that serves to transfer information between alerts. When an alert is triggered,\na new empty pipeline object will be created and each alerter can add or receive information from it. Note that alerters\nare called in the order they are defined in the rule file. For example, the JIRA alerter will add its ticket number\nto the pipeline and the email alerter will add that link if it's present in the pipeline.\n\nalert(self, match):\n-------------------\n\nElastAlert will call this function to send an alert. ``matches`` is a list of dictionary objects with\ninformation about the match. You can get a nice string representation of the match by calling\n``self.rule['type'].get_match_str(match, self.rule)``. If this method raises an exception, it will\nbe caught by ElastAlert and the alert will be marked as unsent and saved for later.\n\nget_info(self):\n---------------\n\nThis function is called to get information about the alert to save back to Elasticsearch. It should\nreturn a dictionary, which is uploaded directly to Elasticsearch, and should contain useful information\nabout the alert such as the type, recipients, parameters, etc.\n\nTutorial\n--------\n\nLet's create a new alert that will write alerts to a local output file. First,\ncreate a modules folder in the base ElastAlert folder:\n\n.. code-block:: console\n\n $ mkdir elastalert_modules\n $ cd elastalert_modules\n $ touch __init__.py\n\nNow, in a file named ``my_alerts.py``, add\n\n.. code-block:: python\n\n from elastalert.alerts import Alerter, BasicMatchString\n\n class AwesomeNewAlerter(Alerter):\n\n # By setting required_options to a set of strings\n # You can ensure that the rule config file specifies all\n # of the options. Otherwise, ElastAlert will throw an exception\n # when trying to load the rule.\n required_options = set(['output_file_path'])\n\n # Alert is called\n def alert(self, matches):\n\n # Matches is a list of match dictionaries.\n # It contains more than one match when the alert has\n # the aggregation option set\n for match in matches:\n\n # Config options can be accessed with self.rule\n with open(self.rule['output_file_path'], \"a\") as output_file:\n\n # basic_match_string will transform the match into the default\n # human readable string format\n match_string = str(BasicMatchString(self.rule, match))\n\n output_file.write(match_string)\n\n # get_info is called after an alert is sent to get data that is written back\n # to Elasticsearch in the field \"alert_info\"\n # It should return a dict of information relevant to what the alert does\n def get_info(self):\n return {'type': 'Awesome Alerter',\n 'output_file': self.rule['output_file_path']}\n\n\nIn the rule configuration file, we are going to specify the alert by writing\n\n.. code-block:: yaml\n\n alert: \"elastalert_modules.my_alerts.AwesomeNewAlerter\"\n output_file_path: \"/tmp/alerts.log\"\n\nElastAlert will attempt to import the alert with ``from elastalert_modules.my_alerts import AwesomeNewAlerter``.\nThis means that the folder must be in a location where it can be imported as a python module.\n" }, { "path": "docs/source/recipes/adding_enhancements.rst", "content": ".. _enhancements:\n\nEnhancements\n============\n\nEnhancements are modules which let you modify a match before an alert is sent. They should subclass ``BaseEnhancement``, found in ``elastalert/enhancements.py``.\nThey can be added to rules using the ``match_enhancements`` option::\n\n match_enhancements:\n - module.file.MyEnhancement\n\nwhere module is the name of a Python module, or folder containing ``__init__.py``,\nand file is the name of the Python file containing a ``BaseEnhancement`` subclass named ``MyEnhancement``.\n\nA special exception class ```DropMatchException``` can be used in enhancements to drop matches if custom conditions are met. For example:\n\n.. code-block:: python\n\n class MyEnhancement(BaseEnhancement):\n def process(self, match):\n # Drops a match if \"field_1\" == \"field_2\"\n if match['field_1'] == match['field_2']:\n raise DropMatchException()\n\nExample\n-------\n\nAs an example enhancement, let's add a link to a whois website. The match must contain a field named domain and it will \nadd an entry named domain_whois_link. First, create a modules folder for the enhancement in the ElastAlert directory.\n\n.. code-block:: console\n\n $ mkdir elastalert_modules\n $ cd elastalert_modules\n $ touch __init__.py\n\nNow, in a file named ``my_enhancements.py``, add\n\n\n.. code-block:: python\n\n from elastalert.enhancements import BaseEnhancement\n\n class MyEnhancement(BaseEnhancement):\n\n # The enhancement is run against every match\n # The match is passed to the process function where it can be modified in any way\n # ElastAlert will do this for each enhancement linked to a rule\n def process(self, match):\n if 'domain' in match:\n url = \"http://who.is/whois/%s\" % (match['domain'])\n match['domain_whois_link'] = url\n\nEnhancements will not automatically be run. Inside the rule configuration file, you need to point it to the enhancement(s) that it should run\nby setting the ``match_enhancements`` option::\n\n match_enhancements:\n - \"elastalert_modules.my_enhancements.MyEnhancement\"\n\n" }, { "path": "docs/source/recipes/adding_loaders.rst", "content": ".. _loaders:\n\nRules Loaders\n========================\n\nRulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to\ngather rules for a particular source. Your RulesLoader needs to implement three member functions, and\nwill look something like this:\n\n.. code-block:: python\n\n class AwesomeNewRulesLoader(RulesLoader):\n def get_names(self, conf, use_rule=None):\n ...\n def get_hashes(self, conf, use_rule=None):\n ...\n def get_yaml(self, rule):\n ...\n\nYou can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a\npython module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``.\n\nExample\n-------\n\nAs an example loader, let's retrieve rules from a database rather than from the local file system. First, create a\nmodules folder for the loader in the ElastAlert directory.\n\n.. code-block:: console\n\n $ mkdir elastalert_modules\n $ cd elastalert_modules\n $ touch __init__.py\n\nNow, in a file named ``mongo_loader.py``, add\n\n.. code-block:: python\n\n from pymongo import MongoClient\n from elastalert.loaders import RulesLoader\n import yaml\n\n class MongoRulesLoader(RulesLoader):\n def __init__(self, conf):\n super(MongoRulesLoader, self).__init__(conf)\n self.client = MongoClient(conf['mongo_url'])\n self.db = self.client[conf['mongo_db']]\n self.cache = {}\n\n def get_names(self, conf, use_rule=None):\n if use_rule:\n return [use_rule]\n\n rules = []\n self.cache = {}\n for rule in self.db.rules.find():\n self.cache[rule['name']] = yaml.load(rule['yaml'])\n rules.append(rule['name'])\n\n return rules\n\n def get_hashes(self, conf, use_rule=None):\n if use_rule:\n return [use_rule]\n\n hashes = {}\n self.cache = {}\n for rule in self.db.rules.find():\n self.cache[rule['name']] = rule['yaml']\n hashes[rule['name']] = rule['hash']\n\n return hashes\n\n def get_yaml(self, rule):\n if rule in self.cache:\n return self.cache[rule]\n\n self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml'])\n return self.cache[rule]\n\nFinally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the\ndefault FileRulesLoader, so in your ``elastalert.conf`` file::\n\n rules_loader: \"elastalert_modules.mongo_loader.MongoRulesLoader\"\n\n" }, { "path": "docs/source/recipes/adding_rules.rst", "content": ".. _writingrules:\n\nAdding a New Rule Type\n======================\n\nThis document describes how to create a new rule type. Built in rule types live in ``elastalert/ruletypes.py``\nand are subclasses of ``RuleType``. At the minimum, your rule needs to implement ``add_data``.\n\nYour class may implement several functions from ``RuleType``:\n\n.. code-block:: python\n\n class AwesomeNewRule(RuleType):\n # ...\n def add_data(self, data):\n # ...\n def get_match_str(self, match):\n # ...\n def garbage_collect(self, timestamp):\n # ...\n\nYou can import new rule types by specifying the type as ``module.file.RuleName``, where module is the name of a Python module, or folder\ncontaining ``__init__.py``, and file is the name of the Python file containing a ``RuleType`` subclass named ``RuleName``.\n\nBasics\n------\n\nThe ``RuleType`` instance remains in memory while ElastAlert is running, receives data, keeps track of its state,\nand generates matches. Several important member properties are created in the ``__init__`` method of ``RuleType``:\n\n``self.rules``: This dictionary is loaded from the rule configuration file. If there is a ``timeframe`` configuration\noption, this will be automatically converted to a ``datetime.timedelta`` object when the rules are loaded.\n\n``self.matches``: This is where ElastAlert checks for matches from the rule. Whatever information is relevant to the match\n(generally coming from the fields in Elasticsearch) should be put into a dictionary object and\nadded to ``self.matches``. ElastAlert will pop items out periodically and send alerts based on these objects. It is\nrecommended that you use ``self.add_match(match)`` to add matches. In addition to appending to ``self.matches``,\n``self.add_match`` will convert the datetime ``@timestamp`` back into an ISO8601 timestamp.\n\n``self.required_options``: This is a set of options that must exist in the configuration file. ElastAlert will\nensure that all of these fields exist before trying to instantiate a ``RuleType`` instance.\n\nadd_data(self, data):\n---------------------\n\nWhen ElastAlert queries Elasticsearch, it will pass all of the hits to the rule type by calling ``add_data``.\n``data`` is a list of dictionary objects which contain all of the fields in ``include``, ``query_key`` and ``compare_key``\nif they exist, and ``@timestamp`` as a datetime object. They will always come in chronological order sorted by '@timestamp'.\n\nget_match_str(self, match):\n---------------------------\n\nAlerts will call this function to get a human readable string about a match for an alert. Match will be the same\nobject that was added to ``self.matches``, and ``rules`` the same as ``self.rules``. The ``RuleType`` base implementation\nwill return an empty string. Note that by default, the alert text will already contain the key-value pairs from the match. This\nshould return a string that gives some information about the match in the context of this specific RuleType.\n\ngarbage_collect(self, timestamp):\n---------------------------------\n\nThis will be called after ElastAlert has run over a time period ending in ``timestamp`` and should be used\nto clear any state that may be obsolete as of ``timestamp``. ``timestamp`` is a datetime object.\n\n\nTutorial\n--------\n\nAs an example, we are going to create a rule type for detecting suspicious logins. Let's imagine the data we are querying is login\nevents that contains IP address, username and a timestamp. Our configuration will take a list of usernames and a time range\nand alert if a login occurs in the time range. First, let's create a modules folder in the base ElastAlert folder:\n\n.. code-block:: console\n\n $ mkdir elastalert_modules\n $ cd elastalert_modules\n $ touch __init__.py\n\nNow, in a file named ``my_rules.py``, add\n\n.. code-block:: python\n\n import dateutil.parser\n\n from elastalert.ruletypes import RuleType\n\n # elastalert.util includes useful utility functions\n # such as converting from timestamp to datetime obj\n from elastalert.util import ts_to_dt\n\n class AwesomeRule(RuleType):\n\n # By setting required_options to a set of strings\n # You can ensure that the rule config file specifies all\n # of the options. Otherwise, ElastAlert will throw an exception\n # when trying to load the rule.\n required_options = set(['time_start', 'time_end', 'usernames'])\n\n # add_data will be called each time Elasticsearch is queried.\n # data is a list of documents from Elasticsearch, sorted by timestamp,\n # including all the fields that the config specifies with \"include\"\n def add_data(self, data):\n for document in data:\n\n # To access config options, use self.rules\n if document['username'] in self.rules['usernames']:\n\n # Convert the timestamp to a time object\n login_time = document['@timestamp'].time()\n\n # Convert time_start and time_end to time objects\n time_start = dateutil.parser.parse(self.rules['time_start']).time()\n time_end = dateutil.parser.parse(self.rules['time_end']).time()\n\n # If the time falls between start and end\n if login_time > time_start and login_time < time_end:\n\n # To add a match, use self.add_match\n self.add_match(document)\n\n # The results of get_match_str will appear in the alert text\n def get_match_str(self, match):\n return \"%s logged in between %s and %s\" % (match['username'],\n self.rules['time_start'],\n self.rules['time_end'])\n\n # garbage_collect is called indicating that ElastAlert has already been run up to timestamp\n # It is useful for knowing that there were no query results from Elasticsearch because\n # add_data will not be called with an empty list\n def garbage_collect(self, timestamp):\n pass\n\n\nIn the rule configuration file, ``example_rules/example_login_rule.yaml``, we are going to specify this rule by writing\n\n.. code-block:: yaml\n\n name: \"Example login rule\"\n es_host: elasticsearch.example.com\n es_port: 14900\n type: \"elastalert_modules.my_rules.AwesomeRule\"\n # Alert if admin, userXYZ or foobaz log in between 8 PM and midnight\n time_start: \"20:00\"\n time_end: \"24:00\"\n usernames:\n - \"admin\"\n - \"userXYZ\"\n - \"foobaz\"\n # We require the username field from documents\n include:\n - \"username\"\n alert:\n - debug\n\nElastAlert will attempt to import the rule with ``from elastalert_modules.my_rules import AwesomeRule``.\nThis means that the folder must be in a location where it can be imported as a Python module.\n\nAn alert from this rule will look something like::\n\n Example login rule\n\n userXYZ logged in between 20:00 and 24:00\n\n @timestamp: 2015-03-02T22:23:24Z\n username: userXYZ\n" }, { "path": "docs/source/recipes/signing_requests.rst", "content": ".. _signingrequests:\n\nSigning requests to Amazon Elasticsearch service\n================================================\n\nWhen using Amazon Elasticsearch service, you need to secure your Elasticsearch\nfrom the outside. Currently, there is no way to secure your Elasticsearch using\nnetwork firewall rules, so the only way is to signing the requests using the\naccess key and secret key for a role or user with permissions on the\nElasticsearch service.\n\nYou can sign requests to AWS using any of the standard AWS methods of providing\ncredentials.\n- Environment Variables, ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY``\n- AWS Config or Credential Files, ``~/.aws/config`` and ``~/.aws/credentials``\n- AWS Instance Profiles, uses the EC2 Metadata service\n\nUsing an Instance Profile\n-------------------------\n\nTypically, you'll deploy ElastAlert on a running EC2 instance on AWS. You can\nassign a role to this instance that gives it permissions to read from and write\nto the Elasticsearch service. When using an Instance Profile, you will need to\nspecify the ``aws_region`` in the configuration file or set the\n``AWS_DEFAULT_REGION`` environment variable.\n\nUsing AWS profiles\n------------------\n\nYou can also create a user with permissions on the Elasticsearch service and\ntell ElastAlert to authenticate itself using that user. First, create an AWS\nprofile in the machine where you'd like to run ElastAlert for the user with\npermissions.\n\nYou can use the environment variables ``AWS_DEFAULT_PROFILE`` and\n``AWS_DEFAULT_REGION`` or add two options to the configuration file:\n- ``aws_region``: The AWS region where you want to operate.\n- ``profile``: The name of the AWS profile to use to sign the requests.\n" }, { "path": "docs/source/recipes/writing_filters.rst", "content": ".. _writingfilters:\n\nWriting Filters For Rules\n=========================\n\nThis document describes how to create a filter section for your rule config file.\n\nThe filters used in rules are part of the Elasticsearch query DSL, further documentation for which can be found at\nhttps://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html\nThis document contains a small subset of particularly useful filters.\n\nThe filter section is passed to Elasticsearch exactly as follows::\n\n filter:\n and:\n filters:\n - [filters from rule.yaml]\n\nEvery result that matches these filters will be passed to the rule for processing.\n\nCommon Filter Types:\n--------------------\n\nquery_string\n************\n\nThe query_string type follows the Lucene query format and can be used for partial or full matches to multiple fields.\nSee http://lucene.apache.org/core/2_9_4/queryparsersyntax.html for more information::\n\n filter:\n - query:\n query_string:\n query: \"username: bob\"\n - query:\n query_string:\n query: \"_type: login_logs\"\n - query:\n query_string:\n query: \"field: value OR otherfield: othervalue\"\n - query:\n query_string:\n query: \"this: that AND these: those\"\n\nterm\n****\n\nThe term type allows for exact field matches::\n\n filter:\n - term:\n name_field: \"bob\"\n - term:\n _type: \"login_logs\"\n\nNote that a term query may not behave as expected if a field is analyzed. By default, many string fields will be tokenized by whitespace, and a term query for \"foo bar\" may not match\na field that appears to have the value \"foo bar\", unless it is not analyzed. Conversely, a term query for \"foo\" will match analyzed strings \"foo bar\" and \"foo baz\". For full text\nmatching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html\n\n`terms `_\n*****************************************************************************************************\n\n\n\nTerms allows for easy combination of multiple term filters::\n\n filter:\n - terms:\n field: [\"value1\", \"value2\"] # value1 OR value2\n\nYou can also match on multiple fields::\n\n - terms:\n fieldX: [\"value1\", \"value2\"]\n fieldY: [\"something\", \"something_else\"]\n fieldZ: [\"foo\", \"bar\", \"baz\"]\n\nwildcard\n********\n\nFor wildcard matches::\n\n filter:\n - query:\n wildcard:\n field: \"foo*bar\"\n\nrange\n*****\n\nFor ranges on fields::\n\n filter:\n - range:\n status_code:\n from: 500\n to: 599\n\nNegation, and, or\n*****************\n\nFor Elasticsearch 2.X, any of the filters can be embedded in ``not``, ``and``, and ``or``::\n\n filter:\n - or:\n - term:\n field: \"value\"\n - wildcard:\n field: \"foo*bar\"\n - and:\n - not:\n term:\n field: \"value\"\n - not:\n term:\n _type: \"something\"\n\nFor Elasticsearch 5.x, this will not work and to implement boolean logic use query strings::\n\n filter:\n - query:\n query_string:\n query: \"somefield: somevalue OR foo: bar\"\n \n\nLoading Filters Directly From Kibana 3\n--------------------------------------\n\nThere are two ways to load filters directly from a Kibana 3 dashboard. You can set your filter to::\n\n filter:\n download_dashboard: \"My Dashboard Name\"\n\nand when ElastAlert starts, it will download the dashboard schema from Elasticsearch and use the filters from that.\nHowever, if the dashboard name changes or if there is connectivity problems when ElastAlert starts, the rule will not load and\nElastAlert will exit with an error like \"Could not download filters for ..\"\n\nThe second way is to generate a config file once using the Kibana dashboard. To do this, run ``elastalert-rule-from-kibana``.\n\n.. code-block:: console\n\n $ elastalert-rule-from-kibana\n Elasticsearch host: elasticsearch.example.com\n Elasticsearch port: 14900\n Dashboard name: My Dashboard\n\n Partial Config file\n -----------\n\n name: My Dashboard\n es_host: elasticsearch.example.com\n es_port: 14900\n filter:\n - query:\n query_string: {query: '_exists_:log.message'}\n - query:\n query_string: {query: 'some_field:12345'}\n" }, { "path": "docs/source/ruletypes.rst", "content": "Rule Types and Configuration Options\n************************************\n\nExamples of several types of rule configuration can be found in the example_rules folder.\n\n.. _commonconfig:\n\n.. note:: All \"time\" formats are of the form ``unit: X`` where unit is one of weeks, days, hours, minutes or seconds.\n Such as ``minutes: 15`` or ``hours: 1``.\n\n\nRule Configuration Cheat Sheet\n==============================\n\n\n+--------------------------------------------------------------------------+\n| FOR ALL RULES |\n+==============================================================+===========+\n| ``es_host`` (string) | Required |\n+--------------------------------------------------------------+ |\n| ``es_port`` (number) | |\n+--------------------------------------------------------------+ |\n| ``index`` (string) | |\n+--------------------------------------------------------------+ |\n| ``type`` (string) | |\n+--------------------------------------------------------------+ |\n| ``alert`` (string or list) | |\n+--------------------------------------------------------------+-----------+\n| ``name`` (string, defaults to the filename) | |\n+--------------------------------------------------------------+ |\n| ``use_strftime_index`` (boolean, default False) | Optional |\n+--------------------------------------------------------------+ |\n| ``use_ssl`` (boolean, default False) | |\n+--------------------------------------------------------------+ |\n| ``verify_certs`` (boolean, default True) | |\n+--------------------------------------------------------------+ |\n| ``es_username`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``es_password`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``es_url_prefix`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``es_send_get_body_as`` (string, default \"GET\") | |\n+--------------------------------------------------------------+ |\n| ``aggregation`` (time, no default) | |\n+--------------------------------------------------------------+ |\n| ``description`` (string, default empty string) | |\n+--------------------------------------------------------------+ |\n| ``generate_kibana_link`` (boolean, default False) | |\n+--------------------------------------------------------------+ |\n| ``use_kibana_dashboard`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``kibana_url`` (string, default from es_host) | |\n+--------------------------------------------------------------+ |\n| ``use_kibana4_dashboard`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``kibana4_start_timedelta`` (time, default: 10 min) | |\n+--------------------------------------------------------------+ |\n| ``kibana4_end_timedelta`` (time, default: 10 min) | |\n+--------------------------------------------------------------+ |\n| ``generate_kibana_discover_url`` (boolean, default False) | |\n+--------------------------------------------------------------+ |\n| ``kibana_discover_app_url`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``kibana_discover_version`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``kibana_discover_index_pattern_id`` (string, no default) | |\n+--------------------------------------------------------------+ |\n| ``kibana_discover_columns`` (list of strs, default _source) | |\n+--------------------------------------------------------------+ |\n| ``kibana_discover_from_timedelta`` (time, default: 10 min) | |\n+--------------------------------------------------------------+ |\n| ``kibana_discover_to_timedelta`` (time, default: 10 min) | |\n+--------------------------------------------------------------+ |\n| ``use_local_time`` (boolean, default True) | |\n+--------------------------------------------------------------+ |\n| ``realert`` (time, default: 1 min) | |\n+--------------------------------------------------------------+ |\n| ``exponential_realert`` (time, no default) | |\n+--------------------------------------------------------------+ |\n| ``match_enhancements`` (list of strs, no default) | |\n+--------------------------------------------------------------+ |\n| ``top_count_number`` (int, default 5) | |\n+--------------------------------------------------------------+ |\n| ``top_count_keys`` (list of strs) | |\n+--------------------------------------------------------------+ |\n| ``raw_count_keys`` (boolean, default True) | |\n+--------------------------------------------------------------+ |\n| ``include`` (list of strs, default [\"*\"]) | |\n+--------------------------------------------------------------+ |\n| ``filter`` (ES filter DSL, no default) | |\n+--------------------------------------------------------------+ |\n| ``max_query_size`` (int, default global max_query_size) | |\n+--------------------------------------------------------------+ |\n| ``query_delay`` (time, default 0 min) | |\n+--------------------------------------------------------------+ |\n| ``owner`` (string, default empty string) | |\n+--------------------------------------------------------------+ |\n| ``priority`` (int, default 2) | |\n+--------------------------------------------------------------+ |\n| ``category`` (string, default empty string) | |\n+--------------------------------------------------------------+ |\n| ``scan_entire_timeframe`` (bool, default False) | |\n+--------------------------------------------------------------+ |\n| ``import`` (string) | |\n| | |\n| IGNORED IF ``use_count_query`` or ``use_terms_query`` is true| |\n+--------------------------------------------------------------+ +\n| ``buffer_time`` (time, default from config.yaml) | |\n+--------------------------------------------------------------+ |\n| ``timestamp_type`` (string, default iso) | |\n+--------------------------------------------------------------+ |\n| ``timestamp_format`` (string, default \"%Y-%m-%dT%H:%M:%SZ\") | |\n+--------------------------------------------------------------+ |\n| ``timestamp_format_expr`` (string, no default ) | |\n+--------------------------------------------------------------+ |\n| ``_source_enabled`` (boolean, default True) | |\n+--------------------------------------------------------------+ |\n| ``alert_text_args`` (array of strs) | |\n+--------------------------------------------------------------+ |\n| ``alert_text_kw`` (object) | |\n+--------------------------------------------------------------+ |\n| ``alert_missing_value`` (string, default \"\") | |\n+--------------------------------------------------------------+ |\n| ``is_enabled`` (boolean, default True) | |\n+--------------------------------------------------------------+-----------+\n| ``search_extra_index`` (boolean, default False) | |\n+--------------------------------------------------------------+-----------+\n\n|\n\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| RULE TYPE | Any | Blacklist | Whitelist | Change | Frequency | Spike | Flatline |New_term|Cardinality|\n+====================================================+========+===========+===========+========+===========+=======+==========+========+===========+\n| ``compare_key`` (list of strs, no default) | | Req | Req | Req | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``blacklist`` (list of strs, no default) | | Req | | | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``whitelist`` (list of strs, no default) | | | Req | | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``ignore_null`` (boolean, no default) | | | Req | Req | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``query_key`` (string, no default) | Opt | | | Req | Opt | Opt | Opt | Req | Opt |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``aggregation_key`` (string, no default) | Opt | | | | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``summary_table_fields`` (list, no default) | Opt | | | | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``timeframe`` (time, no default) | | | | Opt | Req | Req | Req | | Req |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``num_events`` (int, no default) | | | | | Req | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``attach_related`` (boolean, no default) | | | | | Opt | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``use_count_query`` (boolean, no default) | | | | | Opt | Opt | Opt | | |\n| | | | | | | | | | |\n|``doc_type`` (string, no default) | | | | | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``use_terms_query`` (boolean, no default) | | | | | Opt | Opt | | Opt | |\n| | | | | | | | | | |\n|``doc_type`` (string, no default) | | | | | | | | | |\n| | | | | | | | | | |\n|``query_key`` (string, no default) | | | | | | | | | |\n| | | | | | | | | | |\n|``terms_size`` (int, default 50) | | | | | | | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n| ``spike_height`` (int, no default) | | | | | | Req | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``spike_type`` ([up|down|both], no default) | | | | | | Req | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``alert_on_new_data`` (boolean, default False) | | | | | | Opt | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``threshold_ref`` (int, no default) | | | | | | Opt | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``threshold_cur`` (int, no default) | | | | | | Opt | | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``threshold`` (int, no default) | | | | | | | Req | | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``fields`` (string or list, no default) | | | | | | | | Req | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``terms_window_size`` (time, default 30 days) | | | | | | | | Opt | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``window_step_size`` (time, default 1 day) | | | | | | | | Opt | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``alert_on_missing_fields`` (boolean, default False)| | | | | | | | Opt | |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``cardinality_field`` (string, no default) | | | | | | | | | Req |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``max_cardinality`` (boolean, no default) | | | | | | | | | Opt |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n|``min_cardinality`` (boolean, no default) | | | | | | | | | Opt |\n+----------------------------------------------------+--------+-----------+-----------+--------+-----------+-------+----------+--------+-----------+\n\nCommon Configuration Options\n============================\n\nEvery file that ends in ``.yaml`` in the ``rules_folder`` will be run by default.\nThe following configuration settings are common to all types of rules.\n\nRequired Settings\n~~~~~~~~~~~~~~~~~\n\nes_host\n^^^^^^^\n\n``es_host``: The hostname of the Elasticsearch cluster the rule will use to query. (Required, string, no default)\nThe environment variable ``ES_HOST`` will override this field.\n\nes_port\n^^^^^^^\n\n``es_port``: The port of the Elasticsearch cluster. (Required, number, no default)\nThe environment variable ``ES_PORT`` will override this field.\n\nindex\n^^^^^\n\n``index``: The name of the index that will be searched. Wildcards can be used here, such as:\n``index: my-index-*`` which will match ``my-index-2014-10-05``. You can also use a format string containing\n``%Y`` for year, ``%m`` for month, and ``%d`` for day. To use this, you must also set ``use_strftime_index`` to true. (Required, string, no default)\n\nname\n^^^^\n\n``name``: The name of the rule. This must be unique across all rules. The name will be used in\nalerts and used as a key when writing and reading search metadata back from Elasticsearch. (Required, string, no default)\n\ntype\n^^^^\n\n``type``: The ``RuleType`` to use. This may either be one of the built in rule types, see :ref:`Rule Types ` section below for more information,\nor loaded from a module. For loading from a module, the type should be specified as ``module.file.RuleName``. (Required, string, no default)\n\nalert\n^^^^^\n\n``alert``: The ``Alerter`` type to use. This may be one or more of the built in alerts, see :ref:`Alert Types ` section below for more information,\nor loaded from a module. For loading from a module, the alert should be specified as ``module.file.AlertName``. (Required, string or list, no default)\n\nOptional Settings\n~~~~~~~~~~~~~~~~~\n\nimport\n^^^^^^\n\n``import``: If specified includes all the settings from this yaml file. This allows common config options to be shared. Note that imported files that aren't\ncomplete rules should not have a ``.yml`` or ``.yaml`` suffix so that ElastAlert doesn't treat them as rules. Filters in imported files are merged (ANDed)\nwith any filters in the rule. You can only have one import per rule, though the imported file can import another file, recursively. The filename\ncan be an absolute path or relative to the rules directory. (Optional, string, no default)\n\nuse_ssl\n^^^^^^^\n\n``use_ssl``: Whether or not to connect to ``es_host`` using TLS. (Optional, boolean, default False)\nThe environment variable ``ES_USE_SSL`` will override this field.\n\nverify_certs\n^^^^^^^^^^^^\n\n``verify_certs``: Whether or not to verify TLS certificates. (Optional, boolean, default True)\n\nclient_cert\n^^^^^^^^^^^\n\n``client_cert``: Path to a PEM certificate to use as the client certificate (Optional, string, no default)\n\nclient_key\n^^^^^^^^^^^\n\n``client_key``: Path to a private key file to use as the client key (Optional, string, no default)\n\nca_certs\n^^^^^^^^\n\n``ca_certs``: Path to a CA cert bundle to use to verify SSL connections (Optional, string, no default)\n\nes_username\n^^^^^^^^^^^\n\n``es_username``: basic-auth username for connecting to ``es_host``. (Optional, string, no default) The environment variable ``ES_USERNAME`` will override this field.\n\nes_password\n^^^^^^^^^^^\n\n``es_password``: basic-auth password for connecting to ``es_host``. (Optional, string, no default) The environment variable ``ES_PASSWORD`` will override this field.\n\nes_url_prefix\n^^^^^^^^^^^^^\n\n``es_url_prefix``: URL prefix for the Elasticsearch endpoint. (Optional, string, no default)\n\nes_send_get_body_as\n^^^^^^^^^^^^^^^^^^^\n\n``es_send_get_body_as``: Method for querying Elasticsearch. (Optional, string, default \"GET\")\n\nuse_strftime_index\n^^^^^^^^^^^^^^^^^^\n\n``use_strftime_index``: If this is true, ElastAlert will format the index using datetime.strftime for each query.\nSee https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior for more details.\nIf a query spans multiple days, the formatted indexes will be concatenated with commas. This is useful\nas narrowing the number of indexes searched, compared to using a wildcard, may be significantly faster. For example, if ``index`` is\n``logstash-%Y.%m.%d``, the query url will be similar to ``elasticsearch.example.com/logstash-2015.02.03/...`` or\n``elasticsearch.example.com/logstash-2015.02.03,logstash-2015.02.04/...``.\n\nsearch_extra_index\n^^^^^^^^^^^^^^^^^^\n\n``search_extra_index``: If this is true, ElastAlert will add an extra index on the early side onto each search. For example, if it's querying\ncompletely within 2018-06-28, it will actually use 2018-06-27,2018-06-28. This can be useful if your timestamp_field is not what's being used\nto generate the index names. If that's the case, sometimes a query would not have been using the right index.\n\naggregation\n^^^^^^^^^^^\n\n``aggregation``: This option allows you to aggregate multiple matches together into one alert. Every time a match is found,\nElastAlert will wait for the ``aggregation`` period, and send all of the matches that have occurred in that time for a particular\nrule together.\n\nFor example::\n\n aggregation:\n hours: 2\n\nmeans that if one match occurred at 12:00, another at 1:00, and a third at 2:30, one\nalert would be sent at 2:00, containing the first two matches, and another at 4:30, containing the third match plus any additional matches\noccurring before 4:30. This can be very useful if you expect a large number of matches and only want a periodic report. (Optional, time, default none)\n\nIf you wish to aggregate all your alerts and send them on a recurring interval, you can do that using the ``schedule`` field.\n\nFor example, if you wish to receive alerts every Monday and Friday::\n\n aggregation:\n schedule: '2 4 * * mon,fri'\n\nThis uses Cron syntax, which you can read more about `here `_. Make sure to `only` include either a schedule field or standard datetime fields (such as ``hours``, ``minutes``, ``days``), not both.\n\nBy default, all events that occur during an aggregation window are grouped together. However, if your rule has the ``aggregation_key`` field set, then each event sharing a common key value will be grouped together. A separate aggregation window will be made for each newly encountered key value.\n\nFor example, if you wish to receive alerts that are grouped by the user who triggered the event, you can set::\n\n aggregation_key: 'my_data.username'\n\nThen, assuming an aggregation window of 10 minutes, if you receive the following data points::\n\n {'my_data': {'username': 'alice', 'event_type': 'login'}, '@timestamp': '2016-09-20T00:00:00'}\n {'my_data': {'username': 'bob', 'event_type': 'something'}, '@timestamp': '2016-09-20T00:05:00'}\n {'my_data': {'username': 'alice', 'event_type': 'something else'}, '@timestamp': '2016-09-20T00:06:00'}\n\nThis should result in 2 alerts: One containing alice's two events, sent at ``2016-09-20T00:10:00`` and one containing bob's one event sent at ``2016-09-20T00:16:00``\n\nFor aggregations, there can sometimes be a large number of documents present in the viewing medium (email, jira ticket, etc..). If you set the ``summary_table_fields`` field, Elastalert will provide a summary of the specified fields from all the results.\n\nFor example, if you wish to summarize the usernames and event_types that appear in the documents so that you can see the most relevant fields at a quick glance, you can set::\n\n summary_table_fields:\n - my_data.username\n - my_data.event_type\n\nThen, for the same sample data shown above listing alice and bob's events, Elastalert will provide the following summary table in the alert medium::\n\n +------------------+--------------------+\n | my_data.username | my_data.event_type |\n +------------------+--------------------+\n | alice | login |\n | bob | something |\n | alice | something else |\n +------------------+--------------------+\n\n\n.. note::\n By default, aggregation time is relative to the current system time, not the time of the match. This means that running elastalert over\n past events will result in different alerts than if elastalert had been running while those events occured. This behavior can be changed\n by setting ``aggregate_by_match_time``.\n\naggregate_by_match_time\n^^^^^^^^^^^^^^^^^^^^^^^\n\nSetting this to true will cause aggregations to be created relative to the timestamp of the first event, rather than the current time. This\nis useful for querying over historic data or if using a very large buffer_time and you want multiple aggregations to occur from a single query.\n\nrealert\n^^^^^^^\n\n``realert``: This option allows you to ignore repeating alerts for a period of time. If the rule uses a ``query_key``, this option\nwill be applied on a per key basis. All matches for a given rule, or for matches with the same ``query_key``, will be ignored for\nthe given time. All matches with a missing ``query_key`` will be grouped together using a value of ``_missing``.\nThis is applied to the time the alert is sent, not to the time of the event. It defaults to one minute, which means\nthat if ElastAlert is run over a large time period which triggers many matches, only the first alert will be sent by default. If you want\nevery alert, set realert to 0 minutes. (Optional, time, default 1 minute)\n\nexponential_realert\n^^^^^^^^^^^^^^^^^^^\n\n``exponential_realert``: This option causes the value of ``realert`` to exponentially increase while alerts continue to fire. If set,\nthe value of ``exponential_realert`` is the maximum ``realert`` will increase to. If the time between alerts is less than twice ``realert``,\n``realert`` will double. For example, if ``realert: minutes: 10`` and ``exponential_realert: hours: 1``, an alerts fires at 1:00 and another\nat 1:15, the next alert will not be until at least 1:35. If another alert fires between 1:35 and 2:15, ``realert`` will increase to the\n1 hour maximum. If more than 2 hours elapse before the next alert, ``realert`` will go back down. Note that alerts that are ignored (e.g.\none that occurred at 1:05) would not change ``realert``. (Optional, time, no default)\n\nbuffer_time\n^^^^^^^^^^^\n\n``buffer_time``: This options allows the rule to override the ``buffer_time`` global setting defined in config.yaml. This value is ignored if\n``use_count_query`` or ``use_terms_query`` is true. (Optional, time)\n\nquery_delay\n^^^^^^^^^^^\n\n``query_delay``: This option will cause ElastAlert to subtract a time delta from every query, causing the rule to run with a delay.\nThis is useful if the data is Elasticsearch doesn't get indexed immediately. (Optional, time)\n\nowner\n^^^^^\n\n``owner``: This value will be used to identify the stakeholder of the alert. Optionally, this field can be included in any alert type. (Optional, string)\n\npriority\n^^^^^^^^\n\n``priority``: This value will be used to identify the relative priority of the alert. Optionally, this field can be included in any alert type (e.g. for use in email subject/body text). (Optional, int, default 2)\n\ncategory\n^^^^^^^^\n\n``category``: This value will be used to identify the category of the alert. Optionally, this field can be included in any alert type (e.g. for use in email subject/body text). (Optional, string, default empty string)\n\nmax_query_size\n^^^^^^^^^^^^^^\n\n``max_query_size``: The maximum number of documents that will be downloaded from Elasticsearch in a single query. If you\nexpect a large number of results, consider using ``use_count_query`` for the rule. If this\nlimit is reached, a warning will be logged but ElastAlert will continue without downloading more results. This setting will\noverride a global ``max_query_size``. (Optional, int, default value of global ``max_query_size``)\n\nfilter\n^^^^^^\n\n``filter``: A list of Elasticsearch query DSL filters that is used to query Elasticsearch. ElastAlert will query Elasticsearch using the format\n``{'filter': {'bool': {'must': [config.filter]}}}`` with an additional timestamp range filter.\nAll of the results of querying with these filters are passed to the ``RuleType`` for analysis.\nFor more information writing filters, see :ref:`Writing Filters `. (Required, Elasticsearch query DSL, no default)\n\ninclude\n^^^^^^^\n\n``include``: A list of terms that should be included in query results and passed to rule types and alerts. When set, only those\nfields, along with '@timestamp', ``query_key``, ``compare_key``, and ``top_count_keys`` are included, if present.\n(Optional, list of strings, default all fields)\n\ntop_count_keys\n^^^^^^^^^^^^^^\n\n``top_count_keys``: A list of fields. ElastAlert will perform a terms query for the top X most common values for each of the fields,\nwhere X is 5 by default, or ``top_count_number`` if it exists.\nFor example, if ``num_events`` is 100, and ``top_count_keys`` is ``- \"username\"``, the alert will say how many of the 100 events\nhave each username, for the top 5 usernames. When this is computed, the time range used is from ``timeframe`` before the most recent event\nto 10 minutes past the most recent event. Because ElastAlert uses an aggregation query to compute this, it will attempt to use the\nfield name plus \".raw\" to count unanalyzed terms. To turn this off, set ``raw_count_keys`` to false.\n\ntop_count_number\n^^^^^^^^^^^^^^^^\n\n``top_count_number``: The number of terms to list if ``top_count_keys`` is set. (Optional, integer, default 5)\n\nraw_count_keys\n^^^^^^^^^^^^^^\n\n``raw_count_keys``: If true, all fields in ``top_count_keys`` will have ``.raw`` appended to them. (Optional, boolean, default true)\n\ndescription\n^^^^^^^^^^^\n\n``description``: text describing the purpose of rule. (Optional, string, default empty string)\nCan be referenced in custom alerters to provide context as to why a rule might trigger.\n\ngenerate_kibana_link\n^^^^^^^^^^^^^^^^^^^^\n\n``generate_kibana_link``: This option is for Kibana 3 only.\nIf true, ElastAlert will generate a temporary Kibana dashboard and include a link to it in alerts. The dashboard\nconsists of an events over time graph and a table with ``include`` fields selected in the table. If the rule uses ``query_key``, the\ndashboard will also contain a filter for the ``query_key`` of the alert. The dashboard schema will\nbe uploaded to the kibana-int index as a temporary dashboard. (Optional, boolean, default False)\n\nkibana_url\n^^^^^^^^^^\n\n``kibana_url``: The url to access Kibana. This will be used if ``generate_kibana_link`` or\n``use_kibana_dashboard`` is true. If not specified, a URL will be constructed using ``es_host`` and ``es_port``.\n(Optional, string, default ``http://:/_plugin/kibana/``)\n\nuse_kibana_dashboard\n^^^^^^^^^^^^^^^^^^^^\n\n``use_kibana_dashboard``: The name of a Kibana 3 dashboard to link to. Instead of generating a dashboard from a template,\nElastAlert can use an existing dashboard. It will set the time range on the dashboard to around the match time,\nupload it as a temporary dashboard, add a filter to the ``query_key`` of the alert if applicable,\nand put the url to the dashboard in the alert. (Optional, string, no default)\n\nuse_kibana4_dashboard\n^^^^^^^^^^^^^^^^^^^^^\n\n``use_kibana4_dashboard``: A link to a Kibana 4 dashboard. For example, \"https://kibana.example.com/#/dashboard/My-Dashboard\".\nThis will set the time setting on the dashboard from the match time minus the timeframe, to 10 minutes after the match time.\nNote that this does not support filtering by ``query_key`` like Kibana 3. This value can use `$VAR` and `${VAR}` references\nto expand environment variables.\n\nkibana4_start_timedelta\n^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana4_start_timedelta``: Defaults to 10 minutes. This option allows you to specify the start time for the generated kibana4 dashboard.\nThis value is added in front of the event. For example,\n\n``kibana4_start_timedelta: minutes: 2``\n\nkibana4_end_timedelta\n^^^^^^^^^^^^^^^^^^^^^\n\n``kibana4_end_timedelta``: Defaults to 10 minutes. This option allows you to specify the end time for the generated kibana4 dashboard.\nThis value is added in back of the event. For example,\n\n``kibana4_end_timedelta: minutes: 2``\n\ngenerate_kibana_discover_url\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``generate_kibana_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Kibana Discover application.\nThis setting requires the following settings are also configured:\n\n- ``kibana_discover_app_url``\n- ``kibana_discover_version``\n- ``kibana_discover_index_pattern_id``\n\n``generate_kibana_discover_url: true``\n\nkibana_discover_app_url\n^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana_discover_app_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_url`` variable.\nThis value can use `$VAR` and `${VAR}` references to expand environment variables.\n\n``kibana_discover_app_url: http://kibana:5601/#/discover``\n\nkibana_discover_version\n^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana_discover_version``: Specifies the version of the Kibana Discover application.\n\nThe currently supported versions of Kibana Discover are:\n\n- `5.6`\n- `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8`\n- `7.0`, `7.1`, `7.2`, `7.3`\n\n``kibana_discover_version: '7.3'``\n\nkibana_discover_index_pattern_id\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application.\nThese ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object.\n\nExample export of an index pattern's saved object:\n\n.. code-block:: text\n\n [\n {\n \"_id\": \"4e97d188-8a45-4418-8a37-07ed69b4d34c\",\n \"_type\": \"index-pattern\",\n \"_source\": { ... }\n }\n ]\n\nYou can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing.\n\n``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c``\n\nkibana_discover_columns\n^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link.\nDefaults to the ``_source`` column.\n\n``kibana_discover_columns: [ timestamp, message ]``\n\nkibana_discover_from_timedelta\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range.\nThe `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes.\n\n``kibana_discover_from_timedelta: minutes: 2``\n\nkibana_discover_to_timedelta\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range.\nThe `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes.\n\n``kibana_discover_to_timedelta: minutes: 2``\n\nuse_local_time\n^^^^^^^^^^^^^^\n\n``use_local_time``: Whether to convert timestamps to the local time zone in alerts. If false, timestamps will\nbe converted to UTC, which is what ElastAlert uses internally. (Optional, boolean, default true)\n\nmatch_enhancements\n^^^^^^^^^^^^^^^^^^\n\n``match_enhancements``: A list of enhancement modules to use with this rule. An enhancement module is a subclass of enhancements.BaseEnhancement\nthat will be given the match dictionary and can modify it before it is passed to the alerter. The enhancements will be run after silence and realert\nis calculated and in the case of aggregated alerts, right before the alert is sent. This can be changed by setting ``run_enhancements_first``.\nThe enhancements should be specified as\n``module.file.EnhancementName``. See :ref:`Enhancements` for more information. (Optional, list of strings, no default)\n\nrun_enhancements_first\n^^^^^^^^^^^^^^^^^^^^^^\n\n``run_enhancements_first``: If set to true, enhancements will be run as soon as a match is found. This means that they can be changed\nor dropped before affecting realert or being added to an aggregation. Silence stashes will still be created before the\nenhancement runs, meaning even if a ``DropMatchException`` is raised, the rule will still be silenced. (Optional, boolean, default false)\n\nquery_key\n^^^^^^^^^\n\n``query_key``: Having a query key means that realert time will be counted separately for each unique value of ``query_key``. For rule types which\ncount documents, such as spike, frequency and flatline, it also means that these counts will be independent for each unique value of ``query_key``.\nFor example, if ``query_key`` is set to ``username`` and ``realert`` is set, and an alert triggers on a document with ``{'username': 'bob'}``,\nadditional alerts for ``{'username': 'bob'}`` will be ignored while other usernames will trigger alerts. Documents which are missing the\n``query_key`` will be grouped together. A list of fields may also be used, which will create a compound query key. This compound key is\ntreated as if it were a single field whose value is the component values, or \"None\", joined by commas. A new field with the key\n\"field1,field2,etc\" will be created in each document and may conflict with existing fields of the same name.\n\naggregation_key\n^^^^^^^^^^^^^^^\n\n``aggregation_key``: Having an aggregation key in conjunction with an aggregation will make it so that each new value encountered for the aggregation_key field will result in a new, separate aggregation window.\n\nsummary_table_fields\n^^^^^^^^^^^^^^^^^^^^\n\n``summary_table_fields``: Specifying the summmary_table_fields in conjunction with an aggregation will make it so that each aggregated alert will contain a table summarizing the values for the specified fields in all the matches that were aggregated together.\n\ntimestamp_type\n^^^^^^^^^^^^^^\n\n``timestamp_type``: One of ``iso``, ``unix``, ``unix_ms``, ``custom``. This option will set the type of ``@timestamp`` (or ``timestamp_field``)\nused to query Elasticsearch. ``iso`` will use ISO8601 timestamps, which will work with most Elasticsearch date type field. ``unix`` will\nquery using an integer unix (seconds since 1/1/1970) timestamp. ``unix_ms`` will use milliseconds unix timestamp. ``custom`` allows you to define\nyour own ``timestamp_format``. The default is ``iso``.\n(Optional, string enum, default iso).\n\ntimestamp_format\n^^^^^^^^^^^^^^^^\n\n``timestamp_format``: In case Elasticsearch used custom date format for date type field, this option provides a way to define custom timestamp\nformat to match the type used for Elastisearch date type field. This option is only valid if ``timestamp_type`` set to ``custom``.\n(Optional, string, default '%Y-%m-%dT%H:%M:%SZ').\n\ntimestamp_format_expr\n^^^^^^^^^^^^^^^^^^^^^\n\n``timestamp_format_expr``: In case Elasticsearch used custom date format for date type field, this option provides a way to adapt the\nvalue obtained converting a datetime through ``timestamp_format``, when the format cannot match perfectly what defined in Elastisearch.\nWhen set, this option is evaluated as a Python expression along with a *globals* dictionary containing the original datetime instance\nnamed ``dt`` and the timestamp to be refined, named ``ts``. The returned value becomes the timestamp obtained from the datetime.\nFor example, when the date type field in Elasticsearch uses milliseconds (``yyyy-MM-dd'T'HH:mm:ss.SSS'Z'``) and ``timestamp_format``\noption is ``'%Y-%m-%dT%H:%M:%S.%fZ'``, Elasticsearch would fail to parse query terms as they contain microsecond values - that is\nit gets 6 digits instead of 3 - since the ``%f`` placeholder stands for microseconds for Python *strftime* method calls.\nSetting ``timestamp_format_expr: 'ts[:23] + ts[26:]'`` will truncate the value to milliseconds granting Elasticsearch compatibility.\nThis option is only valid if ``timestamp_type`` set to ``custom``.\n(Optional, string, no default).\n\n_source_enabled\n^^^^^^^^^^^^^^^\n\n``_source_enabled``: If true, ElastAlert will use _source to retrieve fields from documents in Elasticsearch. If false,\nElastAlert will use ``fields`` to retrieve stored fields. Both of these are represented internally as if they came from ``_source``.\nSee https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html for more details. The fields used come from ``include``,\nsee above for more details. (Optional, boolean, default True)\n\nscan_entire_timeframe\n^^^^^^^^^^^^^^^^^^^^^\n\n``scan_entire_timeframe``: If true, when ElastAlert starts, it will always start querying at the current time minus the timeframe.\n``timeframe`` must exist in the rule. This may be useful, for example, if you are using a flatline rule type with a large timeframe,\nand you want to be sure that if ElastAlert restarts, you can still get alerts. This may cause duplicate alerts for some rule types,\nfor example, Frequency can alert multiple times in a single timeframe, and if ElastAlert were to restart with this setting, it may\nscan the same range again, triggering duplicate alerts.\n\nSome rules and alerts require additional options, which also go in the top level of the rule configuration file.\n\n\n.. _testing :\n\nTesting Your Rule\n=================\n\nOnce you've written a rule configuration, you will want to validate it. To do so, you can either run ElastAlert in debug mode,\nor use ``elastalert-test-rule``, which is a script that makes various aspects of testing easier.\n\nIt can:\n\n- Check that the configuration file loaded successfully.\n\n- Check that the Elasticsearch filter parses.\n\n- Run against the last X day(s) and the show the number of hits that match your filter.\n\n- Show the available terms in one of the results.\n\n- Save documents returned to a JSON file.\n\n- Run ElastAlert using either a JSON file or actual results from Elasticsearch.\n\n- Print out debug alerts or trigger real alerts.\n\n- Check that, if they exist, the primary_key, compare_key and include terms are in the results.\n\n- Show what metadata documents would be written to ``elastalert_status``.\n\nWithout any optional arguments, it will run ElastAlert over the last 24 hours and print out any alerts that would have occurred.\nHere is an example test run which triggered an alert:\n\n.. code-block:: console\n\n $ elastalert-test-rule my_rules/rule1.yaml\n Successfully Loaded Example rule1\n\n Got 105 hits from the last 1 day\n\n Available terms in first hit:\n @timestamp\n field1\n field2\n ...\n Included term this_field_doesnt_exist may be missing or null\n\n INFO:root:Queried rule Example rule1 from 6-16 15:21 PDT to 6-17 15:21 PDT: 105 hits\n INFO:root:Alert for Example rule1 at 2015-06-16T23:53:12Z:\n INFO:root:Example rule1\n\n At least 50 events occurred between 6-16 18:30 PDT and 6-16 20:30 PDT\n\n field1:\n value1: 25\n value2: 25\n\n @timestamp: 2015-06-16T20:30:04-07:00\n field1: value1\n field2: something\n\n\n Would have written the following documents to elastalert_status:\n\n silence - {'rule_name': 'Example rule1', '@timestamp': datetime.datetime( ... ), 'exponent': 0, 'until':\n datetime.datetime( ... )}\n\n elastalert_status - {'hits': 105, 'matches': 1, '@timestamp': datetime.datetime( ... ), 'rule_name': 'Example rule1',\n 'starttime': datetime.datetime( ... ), 'endtime': datetime.datetime( ... ), 'time_taken': 3.1415926}\n\nNote that everything between \"Alert for Example rule1 at ...\" and \"Would have written the following ...\" is the exact text body that an alert would have.\nSee the section below on alert content for more details.\nAlso note that datetime objects are converted to ISO8601 timestamps when uploaded to Elasticsearch. See :ref:`the section on metadata ` for more details.\n\nOther options include:\n\n``--schema-only``: Only perform schema validation on the file. It will not load modules or query Elasticsearch. This may catch invalid YAML\nand missing or misconfigured fields.\n\n``--count-only``: Only find the number of matching documents and list available fields. ElastAlert will not be run and documents will not be downloaded.\n\n``--days N``: Instead of the default 1 day, query N days. For selecting more specific time ranges, you must run ElastAlert itself and use ``--start``\nand ``--end``.\n\n``--save-json FILE``: Save all documents downloaded to a file as JSON. This is useful if you wish to modify data while testing or do offline\ntesting in conjunction with ``--data FILE``. A maximum of 10,000 documents will be downloaded.\n\n``--data FILE``: Use a JSON file as a data source instead of Elasticsearch. The file should be a single list containing objects,\nrather than objects on separate lines. Note than this uses mock functions which mimic some Elasticsearch query methods and is not\nguaranteed to have the exact same results as with Elasticsearch. For example, analyzed string fields may behave differently.\n\n``--alert``: Trigger real alerts instead of the debug (logging text) alert.\n\n``--formatted-output``: Output results in formatted JSON.\n\n.. note::\n Results from running this script may not always be the same as if an actual ElastAlert instance was running. Some rule types, such as spike\n and flatline require a minimum elapsed time before they begin alerting, based on their timeframe. In addition, use_count_query and\n use_terms_query rely on run_every to determine their resolution. This script uses a fixed 5 minute window, which is the same as the default.\n\n\n.. _ruletypes:\n\nRule Types\n==========\n\nThe various ``RuleType`` classes, defined in ``elastalert/ruletypes.py``, form the main logic behind ElastAlert. An instance\nis held in memory for each rule, passed all of the data returned by querying Elasticsearch with a given filter, and generates\nmatches based on that data.\n\nTo select a rule type, set the ``type`` option to the name of the rule type in the rule configuration file:\n\n``type: ``\n\nAny\n~~~\n\n``any``: The any rule will match everything. Every hit that the query returns will generate an alert.\n\nBlacklist\n~~~~~~~~~\n\n``blacklist``: The blacklist rule will check a certain field against a blacklist, and match if it is in the blacklist.\n\nThis rule requires two additional options:\n\n``compare_key``: The name of the field to use to compare to the blacklist. If the field is null, those events will be ignored.\n\n``blacklist``: A list of blacklisted values, and/or a list of paths to flat files which contain the blacklisted values using ``- \"!file /path/to/file\"``; for example::\n\n blacklist:\n - value1\n - value2\n - \"!file /tmp/blacklist1.txt\"\n - \"!file /tmp/blacklist2.txt\"\n\nIt is possible to mix between blacklist value definitions, or use either one. The ``compare_key`` term must be equal to one of these values for it to match.\n\nWhitelist\n~~~~~~~~~\n\n``whitelist``: Similar to ``blacklist``, this rule will compare a certain field to a whitelist, and match if the list does not contain\nthe term.\n\nThis rule requires three additional options:\n\n``compare_key``: The name of the field to use to compare to the whitelist.\n\n``ignore_null``: If true, events without a ``compare_key`` field will not match.\n\n``whitelist``: A list of whitelisted values, and/or a list of paths to flat files which contain the whitelisted values using ``- \"!file /path/to/file\"``; for example::\n\n whitelist:\n - value1\n - value2\n - \"!file /tmp/whitelist1.txt\"\n - \"!file /tmp/whitelist2.txt\"\n\nIt is possible to mix between whitelisted value definitions, or use either one. The ``compare_key`` term must be in this list or else it will match.\n\nChange\n~~~~~~\n\nFor an example configuration file using this rule type, look at ``example_rules/example_change.yaml``.\n\n``change``: This rule will monitor a certain field and match if that field changes. The field\nmust change with respect to the last event with the same ``query_key``.\n\nThis rule requires three additional options:\n\n``compare_key``: The names of the field to monitor for changes. Since this is a list of strings, we can\nhave multiple keys. An alert will trigger if any of the fields change.\n\n``ignore_null``: If true, events without a ``compare_key`` field will not count as changed. Currently this checks for all the fields in ``compare_key``\n\n``query_key``: This rule is applied on a per-``query_key`` basis. This field must be present in all of\nthe events that are checked.\n\nThere is also an optional field:\n\n``timeframe``: The maximum time between changes. After this time period, ElastAlert will forget the old value\nof the ``compare_key`` field.\n\nFrequency\n~~~~~~~~~\n\nFor an example configuration file using this rule type, look at ``example_rules/example_frequency.yaml``.\n\n``frequency``: This rule matches when there are at least a certain number of events in a given time frame. This\nmay be counted on a per-``query_key`` basis.\n\nThis rule requires two additional options:\n\n``num_events``: The number of events which will trigger an alert, inclusive.\n\n``timeframe``: The time that ``num_events`` must occur within.\n\nOptional:\n\n``use_count_query``: If true, ElastAlert will poll Elasticsearch using the count api, and not download all of the matching documents. This is\nuseful is you care only about numbers and not the actual data. It should also be used if you expect a large number of query hits, in the order\nof tens of thousands or more. ``doc_type`` must be set to use this.\n\n``doc_type``: Specify the ``_type`` of document to search for. This must be present if ``use_count_query`` or ``use_terms_query`` is set.\n\n``use_terms_query``: If true, ElastAlert will make an aggregation query against Elasticsearch to get counts of documents matching\neach unique value of ``query_key``. This must be used with ``query_key`` and ``doc_type``. This will only return a maximum of ``terms_size``,\ndefault 50, unique terms.\n\n``terms_size``: When used with ``use_terms_query``, this is the maximum number of terms returned per query. Default is 50.\n\n``query_key``: Counts of documents will be stored independently for each value of ``query_key``. Only ``num_events`` documents,\nall with the same value of ``query_key``, will trigger an alert.\n\n\n``attach_related``: Will attach all the related events to the event that triggered the frequency alert. For example in an alert triggered with ``num_events``: 3,\nthe 3rd event will trigger the alert on itself and add the other 2 events in a key named ``related_events`` that can be accessed in the alerter.\n\nSpike\n~~~~~\n\n``spike``: This rule matches when the volume of events during a given time period is ``spike_height`` times larger or smaller\nthan during the previous time period. It uses two sliding windows to compare the current and reference frequency\nof events. We will call this two windows \"reference\" and \"current\".\n\nThis rule requires three additional options:\n\n``spike_height``: The ratio of number of events in the last ``timeframe`` to the previous ``timeframe`` that when hit\nwill trigger an alert.\n\n``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the number of events is ``spike_height`` times\nhigher. 'Down' meaning the reference number is ``spike_height`` higher than the current number. 'Both' will match either.\n\n``timeframe``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current'\nwindow will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule\nwill not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered\nbefore a baseline rate has been established. This can be overridden using ``alert_on_new_data``.\n\n\nOptional:\n\n``field_value``: When set, uses the value of the field in the document and not the number of matching documents.\nThis is useful to monitor for example a temperature sensor and raise an alarm if the temperature grows too fast.\nNote that the means of the field on the reference and current windows are used to determine if the ``spike_height`` value is reached.\nNote also that the threshold parameters are ignored in this smode.\n\n\n``threshold_ref``: The minimum number of events that must exist in the reference window for an alert to trigger. For example, if\n``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must contain at least 10 events and the 'current' window at\nleast three times that for an alert to be triggered.\n\n``threshold_cur``: The minimum number of events that must exist in the current window for an alert to trigger. For example, if\n``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has more than 60 events and\nthe reference window has less than a third as many.\n\nTo illustrate the use of ``threshold_ref``, ``threshold_cur``, ``alert_on_new_data``, ``timeframe`` and ``spike_height`` together,\nconsider the following examples::\n\n \" Alert if at least 15 events occur within two hours and less than a quarter of that number occurred within the previous two hours. \"\n timeframe: hours: 2\n spike_height: 4\n spike_type: up\n threshold_cur: 15\n\n hour1: 5 events (ref: 0, cur: 5) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour2: 5 events (ref: 0, cur: 10) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour3: 10 events (ref: 5, cur: 15) - No alert because (a) spike_height not met, (b) ref window not filled\n hour4: 35 events (ref: 10, cur: 45) - Alert because (a) spike_height met, (b) threshold_cur met, (c) ref window filled\n\n hour1: 20 events (ref: 0, cur: 20) - No alert because ref window not filled\n hour2: 21 events (ref: 0, cur: 41) - No alert because ref window not filled\n hour3: 19 events (ref: 20, cur: 40) - No alert because (a) spike_height not met, (b) ref window not filled\n hour4: 23 events (ref: 41, cur: 42) - No alert because spike_height not met\n\n hour1: 10 events (ref: 0, cur: 10) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour2: 0 events (ref: 0, cur: 10) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour3: 0 events (ref: 10, cur: 0) - No alert because (a) threshold_cur not met, (b) ref window not filled, (c) spike_height not met\n hour4: 30 events (ref: 10, cur: 30) - No alert because spike_height not met\n hour5: 5 events (ref: 0, cur: 35) - Alert because (a) spike_height met, (b) threshold_cur met, (c) ref window filled\n\n \" Alert if at least 5 events occur within two hours, and twice as many events occur within the next two hours. \"\n timeframe: hours: 2\n spike_height: 2\n spike_type: up\n threshold_ref: 5\n\n hour1: 20 events (ref: 0, cur: 20) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour2: 100 events (ref: 0, cur: 120) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour3: 100 events (ref: 20, cur: 200) - No alert because ref window not filled\n hour4: 100 events (ref: 120, cur: 200) - No alert because spike_height not met\n\n hour1: 0 events (ref: 0, cur: 0) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour2: 20 events (ref: 0, cur: 20) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour3: 100 events (ref: 0, cur: 120) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour4: 100 events (ref: 20, cur: 200) - Alert because (a) spike_height met, (b) threshold_ref met, (c) ref window filled\n\n hour1: 1 events (ref: 0, cur: 1) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour2: 2 events (ref: 0, cur: 3) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour3: 2 events (ref: 1, cur: 4) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour4: 1000 events (ref: 3, cur: 1002) - No alert because threshold_ref not met\n hour5: 2 events (ref: 4, cur: 1002) - No alert because threshold_ref not met\n hour6: 4 events: (ref: 1002, cur: 6) - No alert because spike_height not met\n\n hour1: 1000 events (ref: 0, cur: 1000) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour2: 0 events (ref: 0, cur: 1000) - No alert because (a) threshold_ref not met, (b) ref window not filled\n hour3: 0 events (ref: 1000, cur: 0) - No alert because (a) spike_height not met, (b) ref window not filled\n hour4: 0 events (ref: 1000, cur: 0) - No alert because spike_height not met\n hour5: 1000 events (ref: 0, cur: 1000) - No alert because threshold_ref not met\n hour6: 1050 events (ref: 0, cur: 2050)- No alert because threshold_ref not met\n hour7: 1075 events (ref: 1000, cur: 2125) Alert because (a) spike_height met, (b) threshold_ref met, (c) ref window filled\n\n \" Alert if at least 100 events occur within two hours and less than a fifth of that number occurred in the previous two hours. \"\n timeframe: hours: 2\n spike_height: 5\n spike_type: up\n threshold_cur: 100\n\n hour1: 1000 events (ref: 0, cur: 1000) - No alert because ref window not filled\n\n hour1: 2 events (ref: 0, cur: 2) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour2: 1 events (ref: 0, cur: 3) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour3: 20 events (ref: 2, cur: 21) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour4: 81 events (ref: 3, cur: 101) - Alert because (a) spike_height met, (b) threshold_cur met, (c) ref window filled\n\n hour1: 10 events (ref: 0, cur: 10) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour2: 20 events (ref: 0, cur: 30) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour3: 40 events (ref: 10, cur: 60) - No alert because (a) threshold_cur not met, (b) ref window not filled\n hour4: 80 events (ref: 30, cur: 120) - No alert because spike_height not met\n hour5: 200 events (ref: 60, cur: 280) - No alert because spike_height not met\n\n``alert_on_new_data``: This option is only used if ``query_key`` is set. When this is set to true, any new ``query_key`` encountered may\ntrigger an immediate alert. When set to false, baseline must be established for each new ``query_key`` value, and then subsequent spikes may\ncause alerts. Baseline is established after ``timeframe`` has elapsed twice since first occurrence.\n\n``use_count_query``: If true, ElastAlert will poll Elasticsearch using the count api, and not download all of the matching documents. This is\nuseful is you care only about numbers and not the actual data. It should also be used if you expect a large number of query hits, in the order\nof tens of thousands or more. ``doc_type`` must be set to use this.\n\n``doc_type``: Specify the ``_type`` of document to search for. This must be present if ``use_count_query`` or ``use_terms_query`` is set.\n\n``use_terms_query``: If true, ElastAlert will make an aggregation query against Elasticsearch to get counts of documents matching\neach unique value of ``query_key``. This must be used with ``query_key`` and ``doc_type``. This will only return a maximum of ``terms_size``,\ndefault 50, unique terms.\n\n``terms_size``: When used with ``use_terms_query``, this is the maximum number of terms returned per query. Default is 50.\n\n``query_key``: Counts of documents will be stored independently for each value of ``query_key``.\n\nFlatline\n~~~~~~~~\n\n``flatline``: This rule matches when the total number of events is under a given ``threshold`` for a time period.\n\nThis rule requires two additional options:\n\n``threshold``: The minimum number of events for an alert not to be triggered.\n\n``timeframe``: The time period that must contain less than ``threshold`` events.\n\nOptional:\n\n``use_count_query``: If true, ElastAlert will poll Elasticsearch using the count api, and not download all of the matching documents. This is\nuseful is you care only about numbers and not the actual data. It should also be used if you expect a large number of query hits, in the order\nof tens of thousands or more. ``doc_type`` must be set to use this.\n\n``doc_type``: Specify the ``_type`` of document to search for. This must be present if ``use_count_query`` or ``use_terms_query`` is set.\n\n``use_terms_query``: If true, ElastAlert will make an aggregation query against Elasticsearch to get counts of documents matching\neach unique value of ``query_key``. This must be used with ``query_key`` and ``doc_type``. This will only return a maximum of ``terms_size``,\ndefault 50, unique terms.\n\n``terms_size``: When used with ``use_terms_query``, this is the maximum number of terms returned per query. Default is 50.\n\n``query_key``: With flatline rule, ``query_key`` means that an alert will be triggered if any value of ``query_key`` has been seen at least once\nand then falls below the threshold.\n\n``forget_keys``: Only valid when used with ``query_key``. If this is set to true, ElastAlert will \"forget\" about the ``query_key`` value that\ntriggers an alert, therefore preventing any more alerts for it until it's seen again.\n\nNew Term\n~~~~~~~~\n\n``new_term``: This rule matches when a new value appears in a field that has never been seen before. When ElastAlert starts, it will\nuse an aggregation query to gather all known terms for a list of fields.\n\nThis rule requires one additional option:\n\n``fields``: A list of fields to monitor for new terms. ``query_key`` will be used if ``fields`` is not set. Each entry in the\nlist of fields can itself be a list. If a field entry is provided as a list, it will be interpreted as a set of fields\nthat compose a composite key used for the ElasticSearch query.\n\n.. note::\n\n The composite fields may only refer to primitive types, otherwise the initial ElasticSearch query will not properly return\n the aggregation results, thus causing alerts to fire every time the ElastAlert service initially launches with the rule.\n A warning will be logged to the console if this scenario is encountered. However, future alerts will actually work as\n expected after the initial flurry.\n\nOptional:\n\n``terms_window_size``: The amount of time used for the initial query to find existing terms. No term that has occurred within this time frame\nwill trigger an alert. The default is 30 days.\n\n``window_step_size``: When querying for existing terms, split up the time range into steps of this size. For example, using the default\n30 day window size, and the default 1 day step size, 30 invidivdual queries will be made. This helps to avoid timeouts for very\nexpensive aggregation queries. The default is 1 day.\n\n``alert_on_missing_field``: Whether or not to alert when a field is missing from a document. The default is false.\n\n``use_terms_query``: If true, ElastAlert will use aggregation queries to get terms instead of regular search queries. This is faster\nthan regular searching if there is a large number of documents. If this is used, you may only specify a single field, and must also set\n``query_key`` to that field. Also, note that ``terms_size`` (the number of buckets returned per query) defaults to 50. This means\nthat if a new term appears but there are at least 50 terms which appear more frequently, it will not be found.\n\n.. note::\n\n When using use_terms_query, make sure that the field you are using is not analyzed. If it is, the results of each terms\n query may return tokens rather than full values. ElastAlert will by default turn on use_keyword_postfix, which attempts\n to use the non-analyzed version (.keyword or .raw) to gather initial terms. These will not match the partial values and\n result in false positives.\n\n``use_keyword_postfix``: If true, ElastAlert will automatically try to add .keyword (ES5+) or .raw to the fields when making an\ninitial query. These are non-analyzed fields added by Logstash. If the field used is analyzed, the initial query will return\nonly the tokenized values, potentially causing false positives. Defaults to true.\n\nCardinality\n~~~~~~~~~~~\n\n``cardinality``: This rule matches when a the total number of unique values for a certain field within a time frame is higher or lower\nthan a threshold.\n\nThis rule requires:\n\n``timeframe``: The time period in which the number of unique values will be counted.\n\n``cardinality_field``: Which field to count the cardinality for.\n\nThis rule requires one of the two following options:\n\n``max_cardinality``: If the cardinality of the data is greater than this number, an alert will be triggered. Each new event that\nraises the cardinality will trigger an alert.\n\n``min_cardinality``: If the cardinality of the data is lower than this number, an alert will be triggered. The ``timeframe`` must\nhave elapsed since the first event before any alerts will be sent. When a match occurs, the ``timeframe`` will be reset and must elapse\nagain before additional alerts.\n\nOptional:\n\n``query_key``: Group cardinality counts by this field. For each unique value of the ``query_key`` field, cardinality will be counted separately.\n\nMetric Aggregation\n~~~~~~~~~~~~~~~~~~\n\n``metric_aggregation``: This rule matches when the value of a metric within the calculation window is higher or lower than a threshold. By\ndefault this is ``buffer_time``.\n\nThis rule requires:\n\n``metric_agg_key``: This is the name of the field over which the metric value will be calculated. The underlying type of this field must be\nsupported by the specified aggregation type.\n\n``metric_agg_type``: The type of metric aggregation to perform on the ``metric_agg_key`` field. This must be one of 'min', 'max', 'avg',\n'sum', 'cardinality', 'value_count'.\n\n``doc_type``: Specify the ``_type`` of document to search for.\n\nThis rule also requires at least one of the two following options:\n\n``max_threshold``: If the calculated metric value is greater than this number, an alert will be triggered. This threshold is exclusive.\n\n``min_threshold``: If the calculated metric value is less than this number, an alert will be triggered. This threshold is exclusive.\n\nOptional:\n\n``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and\nevaluated separately against the threshold(s).\n\n``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``,\nthis will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1.\n\n``use_run_every_query_size``: By default the metric value is calculated over a ``buffer_time`` sized window. If this parameter is true\nthe rule will use ``run_every`` as the calculation window.\n\n``allow_buffer_time_overlap``: This setting will only have an effect if ``use_run_every_query_size`` is false and ``buffer_time`` is greater\nthan ``run_every``. If true will allow the start of the metric calculation window to overlap the end time of a previous run. By default the\nstart and end times will not overlap, so if the time elapsed since the last run is less than the metric calculation window size, rule execution\nwill be skipped (to avoid calculations on partial data).\n\n``bucket_interval``: If present this will divide the metric calculation window into ``bucket_interval`` sized segments. The metric value will\nbe calculated and evaluated against the threshold(s) for each segment. If ``bucket_interval`` is specified then ``buffer_time`` must be a\nmultiple of ``bucket_interval``. (Or ``run_every`` if ``use_run_every_query_size`` is true).\n\n``sync_bucket_interval``: This only has an effect if ``bucket_interval`` is present. If true it will sync the start and end times of the metric\ncalculation window to the keys (timestamps) of the underlying date_histogram buckets. Because of the way elasticsearch calculates date_histogram\nbucket keys these usually round evenly to nearest minute, hour, day etc (depending on the bucket size). By default the bucket keys are offset to\nallign with the time elastalert runs, (This both avoid calculations on partial data, and ensures the very latest documents are included).\nSee: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html#_offset for a\nmore comprehensive explaination.\n\nSpike Aggregation\n~~~~~~~~~~~~~~~~~~\n\n``spike_aggregation``: This rule matches when the value of a metric within the calculation window is ``spike_height`` times larger or smaller\nthan during the previous time period. It uses two sliding windows to compare the current and reference metric values.\nWe will call these two windows \"reference\" and \"current\".\n\nThis rule requires:\n\n``metric_agg_key``: This is the name of the field over which the metric value will be calculated. The underlying type of this field must be\nsupported by the specified aggregation type. If using a scripted field via ``metric_agg_script``, this is the name for your scripted field\n\n``metric_agg_type``: The type of metric aggregation to perform on the ``metric_agg_key`` field. This must be one of 'min', 'max', 'avg',\n'sum', 'cardinality', 'value_count'.\n\n``spike_height``: The ratio of the metric value in the last ``timeframe`` to the previous ``timeframe`` that when hit\nwill trigger an alert.\n\n``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the metric value is ``spike_height`` times\nhigher. 'Down' meaning the reference metric value is ``spike_height`` higher than the current metric value. 'Both' will match either.\n\n``buffer_time``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current'\nwindow will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule\nwill not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered\nbefore a baseline rate has been established. This can be overridden using ``alert_on_new_data``.\n\nOptional:\n\n``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and\nevaluated separately against the 'reference'/'current' metric value and ``spike height``.\n\n``metric_agg_script``: A `Painless` formatted script describing how to calculate your metric on-the-fly::\n\n metric_agg_key: myScriptedMetric\n metric_agg_script:\n script: doc['field1'].value * doc['field2'].value\n\n``threshold_ref``: The minimum value of the metric in the reference window for an alert to trigger. For example, if\n``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must have a metric value of 10 and the 'current' window at\nleast three times that for an alert to be triggered.\n\n``threshold_cur``: The minimum value of the metric in the current window for an alert to trigger. For example, if\n``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has a metric value greater than 60 and\nthe reference window is less than a third of that value.\n\n``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``,\nthis will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1.\n\nPercentage Match\n~~~~~~~~~~~~~~~~\n\n``percentage_match``: This rule matches when the percentage of document in the match bucket within a calculation window is higher or lower\nthan a threshold. By default the calculation window is ``buffer_time``.\n\nThis rule requires:\n\n``match_bucket_filter``: ES filter DSL. This defines a filter for the match bucket, which should match a subset of the documents returned by the\nmain query filter.\n\n``doc_type``: Specify the ``_type`` of document to search for.\n\nThis rule also requires at least one of the two following options:\n\n``min_percentage``: If the percentage of matching documents is less than this number, an alert will be triggered.\n\n``max_percentage``: If the percentage of matching documents is greater than this number, an alert will be triggered.\n\nOptional:\n\n``query_key``: Group percentage by this field. For each unique value of the ``query_key`` field, the percentage will be calculated and\nevaluated separately against the threshold(s).\n\n``use_run_every_query_size``: See ``use_run_every_query_size`` in Metric Aggregation rule\n\n``allow_buffer_time_overlap``: See ``allow_buffer_time_overlap`` in Metric Aggregation rule\n\n``bucket_interval``: See ``bucket_interval`` in Metric Aggregation rule\n\n``sync_bucket_interval``: See ``sync_bucket_interval`` in Metric Aggregation rule\n\n``percentage_format_string``: An optional format string to apply to the percentage value in the alert match text. Must be a valid python format string.\nFor example, \"%.2f\" will round it to 2 decimal places.\nSee: https://docs.python.org/3.4/library/string.html#format-specification-mini-language\n\n``min_denominator``: Minimum number of documents on which percentage calculation will apply. Default is 0.\n\n.. _alerts:\n\nAlerts\n======\n\nEach rule may have any number of alerts attached to it. Alerts are subclasses of ``Alerter`` and are passed\na dictionary, or list of dictionaries, from ElastAlert which contain relevant information. They are configured\nin the rule configuration file similarly to rule types.\n\nTo set the alerts for a rule, set the ``alert`` option to the name of the alert, or a list of the names of alerts:\n\n``alert: email``\n\nor\n\n.. code-block:: yaml\n\n alert:\n - email\n - jira\n\nOptions for each alerter can either defined at the top level of the YAML file, or nested within the alert name, allowing for different settings\nfor multiple of the same alerter. For example, consider sending multiple emails, but with different 'To' and 'From' fields:\n\n.. code-block:: yaml\n\n alert:\n - email\n from_addr: \"no-reply@example.com\"\n email: \"customer@example.com\"\n\nversus\n\n.. code-block:: yaml\n\n alert:\n - email:\n from_addr: \"no-reply@example.com\"\n email: \"customer@example.com\"\n - email:\n from_addr: \"elastalert@example.com\"\"\n email: \"devs@example.com\"\n\nIf multiple of the same alerter type are used, top level settings will be used as the default and inline settings will override those\nfor each alerter.\n\nAlert Subject\n~~~~~~~~~~~~~\n\nE-mail subjects, JIRA issue summaries, PagerDuty alerts, or any alerter that has a \"subject\" can be customized by adding an ``alert_subject``\nthat contains a custom summary.\nIt can be further formatted using standard Python formatting syntax::\n\n alert_subject: \"Issue {0} occurred at {1}\"\n\nThe arguments for the formatter will be fed from the matched objects related to the alert.\nThe field names whose values will be used as the arguments can be passed with ``alert_subject_args``::\n\n\n alert_subject_args:\n - issue.name\n - \"@timestamp\"\n\nIt is mandatory to enclose the ``@timestamp`` field in quotes since in YAML format a token cannot begin with the ``@`` character. Not using the quotation marks will trigger a YAML parse error.\n\nIn case the rule matches multiple objects in the index, only the first match is used to populate the arguments for the formatter.\n\nIf the field(s) mentioned in the arguments list are missing, the email alert will have the text ``alert_missing_value`` in place of its expected value. This will also occur if ``use_count_query`` is set to true.\n\nAlert Content\n~~~~~~~~~~~~~\n\nThere are several ways to format the body text of the various types of events. In EBNF::\n\n rule_name = name\n alert_text = alert_text\n ruletype_text = Depends on type\n top_counts_header = top_count_key, \":\"\n top_counts_value = Value, \": \", Count\n top_counts = top_counts_header, LF, top_counts_value\n field_values = Field, \": \", Value\n\nSimilarly to ``alert_subject``, ``alert_text`` can be further formatted using standard Python formatting syntax.\nThe field names whose values will be used as the arguments can be passed with ``alert_text_args`` or ``alert_text_kw``.\nYou may also refer to any top-level rule property in the ``alert_subject_args``, ``alert_text_args``, ``alert_missing_value``, and ``alert_text_kw fields``. However, if the matched document has a key with the same name, that will take preference over the rule property.\n\nBy default::\n\n body = rule_name\n\n [alert_text]\n\n ruletype_text\n\n {top_counts}\n\n {field_values}\n\nWith ``alert_text_type: alert_text_only``::\n\n body = rule_name\n\n alert_text\n\nWith ``alert_text_type: exclude_fields``::\n\n body = rule_name\n\n [alert_text]\n\n ruletype_text\n\n {top_counts}\n\nWith ``alert_text_type: aggregation_summary_only``::\n\n body = rule_name\n\n aggregation_summary\n\nruletype_text is the string returned by RuleType.get_match_str.\n\nfield_values will contain every key value pair included in the results from Elasticsearch. These fields include \"@timestamp\" (or the value of ``timestamp_field``),\nevery key in ``include``, every key in ``top_count_keys``, ``query_key``, and ``compare_key``. If the alert spans multiple events, these values may\ncome from an individual event, usually the one which triggers the alert.\n\nWhen using ``alert_text_args``, you can access nested fields and index into arrays. For example, if your match was ``{\"data\": {\"ips\": [\"127.0.0.1\", \"12.34.56.78\"]}}``, then by using ``\"data.ips[1]\"`` in ``alert_text_args``, it would replace value with ``\"12.34.56.78\"``. This can go arbitrarily deep into fields and will still work on keys that contain dots themselves.\n\nCommand\n~~~~~~~\n\nThe command alert allows you to execute an arbitrary command and pass arguments or stdin from the match. Arguments to the command can use\nPython format string syntax to access parts of the match. The alerter will open a subprocess and optionally pass the match, or matches\nin the case of an aggregated alert, as a JSON array, to the stdin of the process.\n\nThis alert requires one option:\n\n``command``: A list of arguments to execute or a string to execute. If in list format, the first argument is the name of the program to execute. If passed a\nstring, the command is executed through the shell.\n\nStrings can be formatted using the old-style format (``%``) or the new-style format (``.format()``). When the old-style format is used, fields are accessed\nusing ``%(field_name)s``, or ``%(field.subfield)s``. When the new-style format is used, fields are accessed using ``{field_name}``. New-style formatting allows accessing nested\nfields (e.g., ``{field_1[subfield]}``).\n\nIn an aggregated alert, these fields come from the first match.\n\nOptional:\n\n``pipe_match_json``: If true, the match will be converted to JSON and passed to stdin of the command. Note that this will cause ElastAlert to block\nuntil the command exits or sends an EOF to stdout.\n\n``pipe_alert_text``: If true, the standard alert body text will be passed to stdin of the command. Note that this will cause ElastAlert to block\nuntil the command exits or sends an EOF to stdout. It cannot be used at the same time as ``pipe_match_json``.\n\nExample usage using old-style format::\n\n alert:\n - command\n command: [\"/bin/send_alert\", \"--username\", \"%(username)s\"]\n\n.. warning::\n\n Executing commmands with untrusted data can make it vulnerable to shell injection! If you use formatted data in\n your command, it is highly recommended that you use a args list format instead of a shell string.\n\nExample usage using new-style format::\n\n alert:\n - command\n command: [\"/bin/send_alert\", \"--username\", \"{match[username]}\"]\n\n\nEmail\n~~~~~\n\nThis alert will send an email. It connects to an smtp server located at ``smtp_host``, or localhost by default.\nIf available, it will use STARTTLS.\n\nThis alert requires one additional option:\n\n``email``: An address or list of addresses to sent the alert to.\n\nOptional:\n\n``email_from_field``: Use a field from the document that triggered the alert as the recipient. If the field cannot be found,\nthe ``email`` value will be used as a default. Note that this field will not be available in every rule type, for example, if\nyou have ``use_count_query`` or if it's ``type: flatline``. You can optionally add a domain suffix to the field to generate the\naddress using ``email_add_domain``. It can be a single recipient or list of recipients. For example, with the following settings::\n\n email_from_field: \"data.user\"\n email_add_domain: \"@example.com\"\n\nand a match ``{\"@timestamp\": \"2017\", \"data\": {\"foo\": \"bar\", \"user\": \"qlo\"}}``\n\nan email would be sent to ``qlo@example.com``\n\n``smtp_host``: The SMTP host to use, defaults to localhost.\n\n``smtp_port``: The port to use. Default is 25.\n\n``smtp_ssl``: Connect the SMTP host using TLS, defaults to ``false``. If ``smtp_ssl`` is not used, ElastAlert will still attempt\nSTARTTLS.\n\n``smtp_auth_file``: The path to a file which contains SMTP authentication credentials. The path can be either absolute or relative\nto the given rule. It should be YAML formatted and contain two fields, ``user`` and ``password``. If this is not present,\nno authentication will be attempted.\n\n``smtp_cert_file``: Connect the SMTP host using the given path to a TLS certificate file, default to ``None``.\n\n``smtp_key_file``: Connect the SMTP host using the given path to a TLS key file, default to ``None``.\n\n``email_reply_to``: This sets the Reply-To header in the email. By default, the from address is ElastAlert@ and the domain will be set\nby the smtp server.\n\n``from_addr``: This sets the From header in the email. By default, the from address is ElastAlert@ and the domain will be set\nby the smtp server.\n\n``cc``: This adds the CC emails to the list of recipients. By default, this is left empty.\n\n``bcc``: This adds the BCC emails to the list of recipients but does not show up in the email message. By default, this is left empty.\n\n``email_format``: If set to ``html``, the email's MIME type will be set to HTML, and HTML content should correctly render. If you use this,\nyou need to put your own HTML into ``alert_text`` and use ``alert_text_type: alert_text_only``.\n\nJira\n~~~~\n\nThe JIRA alerter will open a ticket on jira whenever an alert is triggered. You must have a service account for ElastAlert to connect with.\nThe credentials of the service account are loaded from a separate file. The ticket number will be written to the alert pipeline, and if it\nis followed by an email alerter, a link will be included in the email.\n\nThis alert requires four additional options:\n\n``jira_server``: The hostname of the JIRA server.\n\n``jira_project``: The project to open the ticket under.\n\n``jira_issuetype``: The type of issue that the ticket will be filed as. Note that this is case sensitive.\n\n``jira_account_file``: The path to the file which contains JIRA account credentials.\n\nFor an example JIRA account file, see ``example_rules/jira_acct.yaml``. The account file is also yaml formatted and must contain two fields:\n\n``user``: The username.\n\n``password``: The password.\n\nOptional:\n\n``jira_component``: The name of the component or components to set the ticket to. This can be a single string or a list of strings. This is provided for backwards compatibility and will eventually be deprecated. It is preferable to use the plural ``jira_components`` instead.\n\n``jira_components``: The name of the component or components to set the ticket to. This can be a single string or a list of strings.\n\n``jira_description``: Similar to ``alert_text``, this text is prepended to the JIRA description.\n\n``jira_label``: The label or labels to add to the JIRA ticket. This can be a single string or a list of strings. This is provided for backwards compatibility and will eventually be deprecated. It is preferable to use the plural ``jira_labels`` instead.\n\n``jira_labels``: The label or labels to add to the JIRA ticket. This can be a single string or a list of strings.\n\n``jira_priority``: The index of the priority to set the issue to. In the JIRA dropdown for priorities, 0 would represent the first priority,\n1 the 2nd, etc.\n\n``jira_watchers``: A list of user names to add as watchers on a JIRA ticket. This can be a single string or a list of strings.\n\n``jira_bump_tickets``: If true, ElastAlert search for existing tickets newer than ``jira_max_age`` and comment on the ticket with\ninformation about the alert instead of opening another ticket. ElastAlert finds the existing ticket by searching by summary. If the\nsummary has changed or contains special characters, it may fail to find the ticket. If you are using a custom ``alert_subject``,\nthe two summaries must be exact matches, except by setting ``jira_ignore_in_title``, you can ignore the value of a field when searching.\nFor example, if the custom subject is \"foo occured at bar\", and \"foo\" is the value field X in the match, you can set ``jira_ignore_in_title``\nto \"X\" and it will only bump tickets with \"bar\" in the subject. Defaults to false.\n\n``jira_ignore_in_title``: ElastAlert will attempt to remove the value for this field from the JIRA subject when searching for tickets to bump.\nSee ``jira_bump_tickets`` description above for an example.\n\n``jira_max_age``: If ``jira_bump_tickets`` is true, the maximum age of a ticket, in days, such that ElastAlert will comment on the ticket\ninstead of opening a new one. Default is 30 days.\n\n``jira_bump_not_in_statuses``: If ``jira_bump_tickets`` is true, a list of statuses the ticket must **not** be in for ElastAlert to comment on\nthe ticket instead of opening a new one. For example, to prevent comments being added to resolved or closed tickets, set this to 'Resolved'\nand 'Closed'. This option should not be set if the ``jira_bump_in_statuses`` option is set.\n\nExample usage::\n\n jira_bump_not_in_statuses:\n - Resolved\n - Closed\n\n``jira_bump_in_statuses``: If ``jira_bump_tickets`` is true, a list of statuses the ticket *must be in* for ElastAlert to comment on\nthe ticket instead of opening a new one. For example, to only comment on 'Open' tickets -- and thus not 'In Progress', 'Analyzing',\n'Resolved', etc. tickets -- set this to 'Open'. This option should not be set if the ``jira_bump_not_in_statuses`` option is set.\n\nExample usage::\n\n jira_bump_in_statuses:\n - Open\n\n``jira_bump_only``: Only update if a ticket is found to bump. This skips ticket creation for rules where you only want to affect existing tickets.\n\nExample usage::\n\n jira_bump_only: true\n\n``jira_transition_to``: If ``jira_bump_tickets`` is true, Transition this ticket to the given Status when bumping. Must match the text of your JIRA implementation's Status field.\n\nExample usage::\n\n jira_transition_to: 'Fixed'\n\n\n\n``jira_bump_after_inactivity``: If this is set, ElastAlert will only comment on tickets that have been inactive for at least this many days.\nIt only applies if ``jira_bump_tickets`` is true. Default is 0 days.\n\nArbitrary Jira fields:\n\nElastAlert supports setting any arbitrary JIRA field that your jira issue supports. For example, if you had a custom field, called \"Affected User\", you can set it by providing that field name in ``snake_case`` prefixed with ``jira_``. These fields can contain primitive strings or arrays of strings. Note that when you create a custom field in your JIRA server, internally, the field is represented as ``customfield_1111``. In elastalert, you may refer to either the public facing name OR the internal representation.\n\nIn addition, if you would like to use a field in the alert as the value for a custom JIRA field, use the field name plus a # symbol in front. For example, if you wanted to set a custom JIRA field called \"user\" to the value of the field \"username\" from the match, you would use the following.\n\nExample::\n\n jira_user: \"#username\"\n\nExample usage::\n\n jira_arbitrary_singular_field: My Name\n jira_arbitrary_multivalue_field:\n - Name 1\n - Name 2\n jira_customfield_12345: My Custom Value\n jira_customfield_9999:\n - My Custom Value 1\n - My Custom Value 2\n\nOpsGenie\n~~~~~~~~\n\nOpsGenie alerter will create an alert which can be used to notify Operations people of issues or log information. An OpsGenie ``API``\nintegration must be created in order to acquire the necessary ``opsgenie_key`` rule variable. Currently the OpsGenieAlerter only creates\nan alert, however it could be extended to update or close existing alerts.\n\nIt is necessary for the user to create an OpsGenie Rest HTTPS API `integration page `_ in order to create alerts.\n\nThe OpsGenie alert requires one option:\n\n``opsgenie_key``: The randomly generated API Integration key created by OpsGenie.\n\nOptional:\n\n``opsgenie_account``: The OpsGenie account to integrate with.\n\n``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert.\n``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients.\n``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful.\n``opsgenie_teams``: A list of OpsGenie teams to notify (useful for schedules with escalation).\n``opsgenie_teams_args``: Map of arguments used to format opsgenie_teams (useful for assigning the alerts to teams based on some data)\n``opsgenie_default_teams``: List of default teams to notify when the formatting of opsgenie_teams is unsuccesful.\n``opsgenie_tags``: A list of tags for this alert.\n\n``opsgenie_message``: Set the OpsGenie message to something other than the rule name. The message can be formatted with fields from the first match e.g. \"Error occurred for {app_name} at {timestamp}.\".\n\n``opsgenie_alias``: Set the OpsGenie alias. The alias can be formatted with fields from the first match e.g \"{app_name} error\".\n\n``opsgenie_subject``: A string used to create the title of the OpsGenie alert. Can use Python string formatting.\n\n``opsgenie_subject_args``: A list of fields to use to format ``opsgenie_subject`` if it contains formaters.\n\n``opsgenie_priority``: Set the OpsGenie priority level. Possible values are P1, P2, P3, P4, P5.\n\n``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value.\n\nExample usage::\n\n opsgenie_details:\n Author: 'Bob Smith' # constant value\n Environment: '$VAR' # environment variable\n Message: { field: message } # field in the first match\n\nSNS\n~~~\n\nThe SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters.\nThe SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or\nvia environment variables. See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html for details.\n\nSNS requires one option:\n\n``sns_topic_arn``: The SNS topic's ARN. For example, ``arn:aws:sns:us-east-1:123456789:somesnstopic``\n\nOptional:\n\n``aws_access_key``: An access key to connect to SNS with.\n\n``aws_secret_key``: The secret key associated with the access key.\n\n``aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1\n\n``profile``: The AWS profile to use. If none specified, the default will be used.\n\nHipChat\n~~~~~~~\n\nHipChat alerter will send a notification to a predefined HipChat room. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following two options:\n\n``hipchat_auth_token``: The randomly generated notification token created by HipChat. Go to https://XXXXX.hipchat.com/account/api and use\n'Create new token' section, choosing 'Send notification' in Scopes list.\n\n``hipchat_room_id``: The id associated with the HipChat room you want to send the alert to. Go to https://XXXXX.hipchat.com/rooms and choose\nthe room you want to post to. The room ID will be the numeric part of the URL.\n\n``hipchat_msg_color``: The color of the message background that is sent to HipChat. May be set to green, yellow or red. Default is red.\n\n``hipchat_domain``: The custom domain in case you have HipChat own server deployment. Default is api.hipchat.com.\n\n``hipchat_ignore_ssl_errors``: Ignore TLS errors (self-signed certificates, etc.). Default is false.\n\n``hipchat_proxy``: By default ElastAlert will not use a network proxy to send notifications to HipChat. Set this option using ``hostname:port`` if you need to use a proxy.\n\n``hipchat_notify``: When set to true, triggers a hipchat bell as if it were a user. Default is true.\n\n``hipchat_from``: When humans report to hipchat, a timestamp appears next to their name. For bots, the name is the name of the token. The from, instead of a timestamp, defaults to empty unless set, which you can do here. This is optional.\n\n``hipchat_message_format``: Determines how the message is treated by HipChat and rendered inside HipChat applications\nhtml - Message is rendered as HTML and receives no special treatment. Must be valid HTML and entities must be escaped (e.g.: '&' instead of '&'). May contain basic tags: a, b, i, strong, em, br, img, pre, code, lists, tables.\ntext - Message is treated just like a message sent by a user. Can include @mentions, emoticons, pastes, and auto-detected URLs (Twitter, YouTube, images, etc).\nValid values: html, text.\nDefaults to 'html'.\n\n``hipchat_mentions``: When using a ``html`` message format, it's not possible to mentions specific users using the ``@user`` syntax.\nIn that case, you can set ``hipchat_mentions`` to a list of users which will be first mentioned using a single text message, then the normal ElastAlert message will be sent to Hipchat.\nIf set, it will mention the users, no matter if the original message format is set to HTML or text.\nValid values: list of strings.\nDefaults to ``[]``.\n\n\nStride\n~~~~~~~\n\nStride alerter will send a notification to a predefined Stride room. The body of the notification is formatted the same as with other alerters.\nSimple HTML such as and tags will be parsed into a format that Stride can consume.\n\nThe alerter requires the following two options:\n\n``stride_access_token``: The randomly generated notification token created by Stride.\n\n``stride_cloud_id``: The site_id associated with the Stride site you want to send the alert to.\n\n``stride_conversation_id``: The conversation_id associated with the Stride conversation you want to send the alert to.\n\n``stride_ignore_ssl_errors``: Ignore TLS errors (self-signed certificates, etc.). Default is false.\n\n``stride_proxy``: By default ElastAlert will not use a network proxy to send notifications to Stride. Set this option using ``hostname:port`` if you need to use a proxy.\n\n\nMS Teams\n~~~~~~~~\n\nMS Teams alerter will send a notification to a predefined Microsoft Teams channel.\n\nThe alerter requires the following options:\n\n``ms_teams_webhook_url``: The webhook URL that includes your auth data and the ID of the channel you want to post to. Go to the Connectors\nmenu in your channel and configure an Incoming Webhook, then copy the resulting URL. You can use a list of URLs to send to multiple channels.\n\n``ms_teams_alert_summary``: Summary should be configured according to `MS documentation `_, although it seems not displayed by Teams currently.\n\nOptional:\n\n``ms_teams_theme_color``: By default the alert will be posted without any color line. To add color, set this attribute to a HTML color value e.g. ``#ff0000`` for red.\n\n``ms_teams_proxy``: By default ElastAlert will not use a network proxy to send notifications to MS Teams. Set this option using ``hostname:port`` if you need to use a proxy.\n\n``ms_teams_alert_fixed_width``: By default this is ``False`` and the notification will be sent to MS Teams as-is. Teams supports a partial Markdown implementation, which means asterisk, underscore and other characters may be interpreted as Markdown. Currenlty, Teams does not fully implement code blocks. Setting this attribute to ``True`` will enable line by line code blocks. It is recommended to enable this to get clearer notifications in Teams.\n\nSlack\n~~~~~\n\nSlack alerter will send a notification to a predefined Slack channel. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following option:\n\n``slack_webhook_url``: The webhook URL that includes your auth data and the ID of the channel (room) you want to post to. Go to the Incoming Webhooks\nsection in your Slack account https://XXXXX.slack.com/services/new/incoming-webhook , choose the channel, click 'Add Incoming Webhooks Integration'\nand copy the resulting URL. You can use a list of URLs to send to multiple channels.\n\nOptional:\n\n``slack_username_override``: By default Slack will use your username when posting to the channel. Use this option to change it (free text).\n\n``slack_channel_override``: Incoming webhooks have a default channel, but it can be overridden. A public channel can be specified \"#other-channel\", and a Direct Message with \"@username\".\n\n``slack_emoji_override``: By default ElastAlert will use the :ghost: emoji when posting to the channel. You can use a different emoji per\nElastAlert rule. Any Apple emoji can be used, see http://emojipedia.org/apple/ . If slack_icon_url_override parameter is provided, emoji is ignored.\n\n``slack_icon_url_override``: By default ElastAlert will use the :ghost: emoji when posting to the channel. You can provide icon_url to use custom image.\nProvide absolute address of the pciture, for example: http://some.address.com/image.jpg .\n\n``slack_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good' or 'warning' colors.\n\n``slack_proxy``: By default ElastAlert will not use a network proxy to send notifications to Slack. Set this option using ``hostname:port`` if you need to use a proxy.\n\n``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`.\n\n``slack_title``: Sets a title for the message, this shows up as a blue text at the start of the message\n\n``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set.\n\n``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles.\n\n``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``.\n\n``slack_kibana_discover_color``: The color of the Kibana Discover url attachment. Defaults to ``#ec4b98``.\n\n``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``.\n\nMattermost\n~~~~~~~~~~\n\nMattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following option:\n\n``mattermost_webhook_url``: The webhook URL. Follow the instructions on https://docs.mattermost.com/developer/webhooks-incoming.html to create an incoming webhook on your Mattermost installation.\n\nOptional:\n\n``mattermost_proxy``: By default ElastAlert will not use a network proxy to send notifications to Mattermost. Set this option using ``hostname:port`` if you need to use a proxy.\n\n``mattermost_ignore_ssl_errors``: By default ElastAlert will verify SSL certificate. Set this option to ``False`` if you want to ignore SSL errors.\n\n``mattermost_username_override``: By default Mattermost will use your username when posting to the channel. Use this option to change it (free text).\n\n``mattermost_channel_override``: Incoming webhooks have a default channel, but it can be overridden. A public channel can be specified \"#other-channel\", and a Direct Message with \"@username\".\n\n``mattermost_icon_url_override``: By default ElastAlert will use the default webhook icon when posting to the channel. You can provide icon_url to use custom image.\nProvide absolute address of the picture (for example: http://some.address.com/image.jpg) or Base64 data url.\n\n``mattermost_msg_pretext``: You can set the message attachment pretext using this option.\n\n``mattermost_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good', 'warning', or hex color code.\n\n``mattermost_msg_fields``: You can add fields to your Mattermost alerts using this option. You can specify the title using `title` and the text value using `value`. Additionally you can specify whether this field should be a `short` field using `short: true`. If you set `args` and `value` is a formattable string, ElastAlert will format the incident key based on the provided array of fields from the rule or match.\nSee https://docs.mattermost.com/developer/message-attachments.html#fields for more information.\n\n\nTelegram\n~~~~~~~~\nTelegram alerter will send a notification to a predefined Telegram username or channel. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following two options:\n\n``telegram_bot_token``: The token is a string along the lines of ``110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw`` that will be required to authorize the bot and send requests to the Bot API. You can learn about obtaining tokens and generating new ones in this document https://core.telegram.org/bots#botfather\n\n``telegram_room_id``: Unique identifier for the target chat or username of the target channel using telegram chat_id (in the format \"-xxxxxxxx\")\n\nOptional:\n\n``telegram_api_url``: Custom domain to call Telegram Bot API. Default to api.telegram.org\n\n``telegram_proxy``: By default ElastAlert will not use a network proxy to send notifications to Telegram. Set this option using ``hostname:port`` if you need to use a proxy.\n\nGoogleChat\n~~~~~~~~~~\nGoogleChat alerter will send a notification to a predefined GoogleChat channel. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following options:\n\n``googlechat_webhook_url``: The webhook URL that includes the channel (room) you want to post to. Go to the Google Chat website https://chat.google.com and choose the channel in which you wish to receive the notifications. Select 'Configure Webhooks' to create a new webhook or to copy the URL from an existing one. You can use a list of URLs to send to multiple channels.\n\nOptional:\n\n``googlechat_format``: Formatting for the notification. Can be either 'card' or 'basic' (default).\n\n``googlechat_header_title``: Sets the text for the card header title. (Only used if format=card)\n\n``googlechat_header_subtitle``: Sets the text for the card header subtitle. (Only used if format=card)\n\n``googlechat_header_image``: URL for the card header icon. (Only used if format=card)\n\n``googlechat_footer_kibanalink``: URL to Kibana to include in the card footer. (Only used if format=card)\n\n\nPagerDuty\n~~~~~~~~~\n\nPagerDuty alerter will trigger an incident to a predefined PagerDuty service. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following option:\n\n``pagerduty_service_key``: Integration Key generated after creating a service with the 'Use our API directly' option at Integration Settings\n\n``pagerduty_client_name``: The name of the monitoring client that is triggering this event.\n\n``pagerduty_event_type``: Any of the following: `trigger`, `resolve`, or `acknowledge`. (Optional, defaults to `trigger`)\n\nOptional:\n\n``alert_subject``: If set, this will be used as the Incident description within PagerDuty. If not set, ElastAlert will default to using the rule name of the alert for the incident.\n\n``alert_subject_args``: If set, and ``alert_subject`` is a formattable string, ElastAlert will format the incident key based on the provided array of fields from the rule or match.\n\n``pagerduty_incident_key``: If not set PagerDuty will trigger a new incident for each alert sent. If set to a unique string per rule PagerDuty will identify the incident that this event should be applied.\nIf there's no open (i.e. unresolved) incident with this key, a new one will be created. If there's already an open incident with a matching key, this event will be appended to that incident's log.\n\n``pagerduty_incident_key_args``: If set, and ``pagerduty_incident_key`` is a formattable string, Elastalert will format the incident key based on the provided array of fields from the rule or match.\n\n``pagerduty_proxy``: By default ElastAlert will not use a network proxy to send notifications to PagerDuty. Set this option using ``hostname:port`` if you need to use a proxy.\n\nV2 API Options (Optional):\n\nThese options are specific to the PagerDuty V2 API\n\nSee https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2\n\n``pagerduty_api_version``: Defaults to `v1`. Set to `v2` to enable the PagerDuty V2 Event API.\n\n``pagerduty_v2_payload_class``: Sets the class of the payload. (the event type in PagerDuty)\n\n``pagerduty_v2_payload_class_args``: If set, and ``pagerduty_v2_payload_class`` is a formattable string, Elastalert will format the class based on the provided array of fields from the rule or match.\n\n``pagerduty_v2_payload_component``: Sets the component of the payload. (what program/interface/etc the event came from)\n\n``pagerduty_v2_payload_component_args``: If set, and ``pagerduty_v2_payload_component`` is a formattable string, Elastalert will format the component based on the provided array of fields from the rule or match.\n\n``pagerduty_v2_payload_group``: Sets the logical grouping (e.g. app-stack)\n\n``pagerduty_v2_payload_group_args``: If set, and ``pagerduty_v2_payload_group`` is a formattable string, Elastalert will format the group based on the provided array of fields from the rule or match.\n\n``pagerduty_v2_payload_severity``: Sets the severity of the page. (defaults to `critical`, valid options: `critical`, `error`, `warning`, `info`)\n\n``pagerduty_v2_payload_source``: Sets the source of the event, preferably the hostname or fqdn.\n\n``pagerduty_v2_payload_source_args``: If set, and ``pagerduty_v2_payload_source`` is a formattable string, Elastalert will format the source based on the provided array of fields from the rule or match.\n\nPagerTree\n~~~~~~~~~\n\nPagerTree alerter will trigger an incident to a predefined PagerTree integration url.\n\nThe alerter requires the following options:\n\n``pagertree_integration_url``: URL generated by PagerTree for the integration.\n\nExotel\n~~~~~~\n\nDevelopers in India can use Exotel alerter, it will trigger an incident to a mobile phone as sms from your exophone. Alert name along with the message body will be sent as an sms.\n\nThe alerter requires the following option:\n\n``exotel_account_sid``: This is sid of your Exotel account.\n\n``exotel_auth_token``: Auth token assosiated with your Exotel account.\n\nIf you don't know how to find your accound sid and auth token, refer - http://support.exotel.in/support/solutions/articles/3000023019-how-to-find-my-exotel-token-and-exotel-sid-\n\n``exotel_to_number``: The phone number where you would like send the notification.\n\n``exotel_from_number``: Your exophone number from which message will be sent.\n\nThe alerter has one optional argument:\n\n``exotel_message_body``: Message you want to send in the sms, is you don't specify this argument only the rule name is sent\n\n\nTwilio\n~~~~~~\n\nTwilio alerter will trigger an incident to a mobile phone as sms from your twilio phone number. Alert name will arrive as sms once this option is chosen.\n\nThe alerter requires the following option:\n\n``twilio_account_sid``: This is sid of your twilio account.\n\n``twilio_auth_token``: Auth token assosiated with your twilio account.\n\n``twilio_to_number``: The phone number where you would like send the notification.\n\n``twilio_from_number``: Your twilio phone number from which message will be sent.\n\n\nVictorOps\n~~~~~~~~~\n\nVictorOps alerter will trigger an incident to a predefined VictorOps routing key. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following options:\n\n``victorops_api_key``: API key generated under the 'REST Endpoint' in the Integrations settings.\n\n``victorops_routing_key``: VictorOps routing key to route the alert to.\n\n``victorops_message_type``: VictorOps field to specify severity level. Must be one of the following: INFO, WARNING, ACKNOWLEDGEMENT, CRITICAL, RECOVERY\n\nOptional:\n\n``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents throughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert.\n\n``victorops_entity_display_name``: Human-readable name of alerting entity to summarize incidents without affecting the life-cycle workflow.\n\n``victorops_proxy``: By default ElastAlert will not use a network proxy to send notifications to VictorOps. Set this option using ``hostname:port`` if you need to use a proxy.\n\nGitter\n~~~~~~\n\nGitter alerter will send a notification to a predefined Gitter channel. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following option:\n\n``gitter_webhook_url``: The webhook URL that includes your auth data and the ID of the channel (room) you want to post to. Go to the Integration Settings\nof the channel https://gitter.im/ORGA/CHANNEL#integrations , click 'CUSTOM' and copy the resulting URL.\n\nOptional:\n\n``gitter_msg_level``: By default the alert will be posted with the 'error' level. You can use 'info' if you want the messages to be black instead of red.\n\n``gitter_proxy``: By default ElastAlert will not use a network proxy to send notifications to Gitter. Set this option using ``hostname:port`` if you need to use a proxy.\n\nServiceNow\n~~~~~~~~~~\n\nThe ServiceNow alerter will create a ne Incident in ServiceNow. The body of the notification is formatted the same as with other alerters.\n\nThe alerter requires the following options:\n\n``servicenow_rest_url``: The ServiceNow RestApi url, this will look like https://instancename.service-now.com/api/now/v1/table/incident\n\n``username``: The ServiceNow Username to access the api.\n\n``password``: The ServiceNow password to access the api.\n\n``short_description``: The ServiceNow password to access the api.\n\n``comments``: Comments to be attached to the incident, this is the equivilant of work notes.\n\n``assignment_group``: The group to assign the incident to.\n\n``category``: The category to attach the incident to, use an existing category.\n\n``subcategory``: The subcategory to attach the incident to, use an existing subcategory.\n\n``cmdb_ci``: The configuration item to attach the incident to.\n\n``caller_id``: The caller id (email address) of the user that created the incident (elastalert@somewhere.com).\n\n\nOptional:\n\n``servicenow_proxy``: By default ElastAlert will not use a network proxy to send notifications to ServiceNow. Set this option using ``hostname:port`` if you need to use a proxy.\n\n\nDebug\n~~~~~\n\nThe debug alerter will log the alert information using the Python logger at the info level. It is logged into a Python Logger object with the name ``elastalert`` that can be easily accessed using the ``getLogger`` command.\n\nStomp\n~~~~~\n\nThis alert type will use the STOMP protocol in order to push a message to a broker like ActiveMQ or RabbitMQ. The message body is a JSON string containing the alert details.\nThe default values will work with a pristine ActiveMQ installation.\n\nOptional:\n\n``stomp_hostname``: The STOMP host to use, defaults to localhost.\n``stomp_hostport``: The STOMP port to use, defaults to 61613.\n``stomp_login``: The STOMP login to use, defaults to admin.\n``stomp_password``: The STOMP password to use, defaults to admin.\n``stomp_destination``: The STOMP destination to use, defaults to /queue/ALERT\n\nThe stomp_destination field depends on the broker, the /queue/ALERT example is the nomenclature used by ActiveMQ. Each broker has its own logic.\n\nAlerta\n~~~~~~\n\nAlerta alerter will post an alert in the Alerta server instance through the alert API endpoint.\nSee http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta JSON format.\n\nFor Alerta 5.0\n\nRequired:\n\n``alerta_api_url``: API server URL.\n\nOptional:\n\n``alerta_api_key``: This is the api key for alerta server, sent in an ``Authorization`` HTTP header. If not defined, no Authorization header is sent.\n\n``alerta_use_qk_as_resource``: If true and query_key is present, this will override ``alerta_resource`` field with the ``query_key value`` (Can be useful if ``query_key`` is a hostname).\n\n``alerta_use_match_timestamp``: If true, it will use the timestamp of the first match as the ``createTime`` of the alert. otherwise, the current server time is used.\n\n``alert_missing_value``: Text to replace any match field not found when formating strings. Defaults to ````.\n\nThe following options dictate the values of the API JSON payload:\n\n``alerta_severity``: Defaults to \"warning\".\n\n``alerta_timeout``: Defaults 84600 (1 Day).\n\n``alerta_type``: Defaults to \"elastalert\".\n\nThe following options use Python-like string syntax ``{}`` or ``%()s`` to access parts of the match, similar to the CommandAlerter. Ie: \"Alert for {clientip}\".\nIf the referenced key is not found in the match, it is replaced by the text indicated by the option ``alert_missing_value``.\n\n``alerta_resource``: Defaults to \"elastalert\".\n\n``alerta_service``: Defaults to \"elastalert\".\n\n``alerta_origin``: Defaults to \"elastalert\".\n\n``alerta_environment``: Defaults to \"Production\".\n\n``alerta_group``: Defaults to \"\".\n\n``alerta_correlate``: Defaults to an empty list.\n\n``alerta_tags``: Defaults to an empty list.\n\n``alerta_event``: Defaults to the rule's name.\n\n``alerta_text``: Defaults to the rule's text according to its type.\n\n``alerta_value``: Defaults to \"\".\n\nThe ``attributes`` dictionary is built by joining the lists from ``alerta_attributes_keys`` and ``alerta_attributes_values``, considered in order.\n\n\nExample usage using old-style format::\n\n alert:\n - alerta\n alerta_api_url: \"http://youralertahost/api/alert\"\n alerta_attributes_keys: [\"hostname\", \"TimestampEvent\", \"senderIP\" ]\n alerta_attributes_values: [\"%(key)s\", \"%(logdate)s\", \"%(sender_ip)s\" ]\n alerta_correlate: [\"ProbeUP\",\"ProbeDOWN\"]\n alerta_event: \"ProbeUP\"\n alerta_text: \"Probe %(hostname)s is UP at %(logdate)s GMT\"\n alerta_value: \"UP\"\n\nExample usage using new-style format::\n\n alert:\n - alerta\n alerta_attributes_values: [\"{key}\", \"{logdate}\", \"{sender_ip}\" ]\n alerta_text: \"Probe {hostname} is UP at {logdate} GMT\"\n\n\n\nHTTP POST\n~~~~~~~~~\n\nThis alert type will send results to a JSON endpoint using HTTP POST. The key names are configurable so this is compatible with almost any endpoint. By default, the JSON will contain all the items from the match, unless you specify http_post_payload, in which case it will only contain those items.\n\nRequired:\n\n``http_post_url``: The URL to POST.\n\nOptional:\n\n``http_post_payload``: List of keys:values to use as the content of the POST. Example - ip:clientip will map the value from the clientip index of Elasticsearch to JSON key named ip. If not defined, all the Elasticsearch keys will be sent.\n\n``http_post_static_payload``: Key:value pairs of static parameters to be sent, along with the Elasticsearch results. Put your authentication or other information here.\n\n``http_post_headers``: Key:value pairs of headers to be sent as part of the request.\n\n``http_post_proxy``: URL of proxy, if required.\n\n``http_post_all_values``: Boolean of whether or not to include every key value pair from the match in addition to those in http_post_payload and http_post_static_payload. Defaults to True if http_post_payload is not specified, otherwise False.\n\n``http_post_timeout``: The timeout value, in seconds, for making the post. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles.\n\nExample usage::\n\n alert: post\n http_post_url: \"http://example.com/api\"\n http_post_payload:\n ip: clientip\n http_post_static_payload:\n apikey: abc123\n http_post_headers:\n authorization: Basic 123dr3234\n\n\nAlerter\n~~~~~~~\n\nFor all Alerter subclasses, you may reference values from a top-level rule property in your Alerter fields by referring to the property name surrounded by dollar signs. This can be useful when you have rule-level properties that you would like to reference many times in your alert. For example:\n\nExample usage::\n\n jira_priority: $priority$\n jira_alert_owner: $owner$\n\n\n\nLine Notify\n~~~~~~~~~~~\n\nLine Notify will send notification to a Line application. The body of the notification is formatted the same as with other alerters.\n\nRequired:\n\n``linenotify_access_token``: The access token that you got from https://notify-bot.line.me/my/\n\ntheHive\n~~~~~~~\n\ntheHive alert type will send JSON request to theHive (Security Incident Response Platform) with TheHive4py API. Sent request will be stored like Hive Alert with description and observables.\n\nRequired:\n\n``hive_connection``: The connection details as key:values. Required keys are ``hive_host``, ``hive_port`` and ``hive_apikey``.\n\n``hive_alert_config``: Configuration options for the alert.\n\nOptional:\n\n``hive_proxies``: Proxy configuration.\n\n``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting.\n\nExample usage::\n\n alert: hivealerter\n\n hive_connection:\n hive_host: http://localhost\n hive_port: \n hive_apikey: \n hive_proxies:\n http: ''\n https: ''\n\n hive_alert_config:\n title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided\n type: 'external'\n source: 'elastalert'\n description: '{match[field1]} {rule[name]} Sample description'\n severity: 2\n tags: ['tag1', 'tag2 {rule[name]}']\n tlp: 3\n status: 'New'\n follow: True\n\n hive_observable_data_mapping:\n - domain: \"{match[field1]}_{rule[name]}\"\n - domain: \"{match[field]}\"\n - ip: \"{match[ip_field]}\"\n\n\nZabbix\n~~~~~~~~~~~\n\nZabbix will send notification to a Zabbix server. The item in the host specified receive a 1 value for each hit. For example, if the elastic query produce 3 hits in the last execution of elastalert, three '1' (integer) values will be send from elastalert to Zabbix Server. If the query have 0 hits, any value will be sent.\n\nRequired:\n\n``zbx_sender_host``: The address where zabbix server is running.\n``zbx_sender_port``: The port where zabbix server is listenning.\n``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert.\n``zbx_item``: This field setup the item in the host that receives the value sent by Elastalert.\n" }, { "path": "docs/source/running_elastalert.rst", "content": ".. _tutorial:\n\nRunning ElastAlert for the First Time\n=====================================\n\nRequirements\n------------\n\n- Elasticsearch\n- ISO8601 or Unix timestamped data\n- Python 3.6\n- pip, see requirements.txt\n- Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev\n\nDownloading and Configuring\n---------------------------\n\nYou can either install the latest released version of ElastAlert using pip::\n\n $ pip install elastalert\n\nor you can clone the ElastAlert repository for the most recent changes::\n\n $ git clone https://github.com/Yelp/elastalert.git\n\nInstall the module::\n\n $ pip install \"setuptools>=11.3\"\n $ python setup.py install\n\nDepending on the version of Elasticsearch, you may need to manually install the correct version of elasticsearch-py.\n\nElasticsearch 5.0+::\n\n $ pip install \"elasticsearch>=5.0.0\"\n\nElasticsearch 2.X::\n\n $ pip install \"elasticsearch<3.0.0\"\n\nNext, open up config.yaml.example. In it, you will find several configuration options. ElastAlert may be run without changing any of these settings.\n\n``rules_folder`` is where ElastAlert will load rule configuration files from. It will attempt to load every .yaml file in the folder. Without any valid rules, ElastAlert will not start. ElastAlert will also load new rules, stop running missing rules, and restart modified rules as the files in this folder change. For this tutorial, we will use the example_rules folder.\n\n``run_every`` is how often ElastAlert will query Elasticsearch.\n\n``buffer_time`` is the size of the query window, stretching backwards from the time each query is run. This value is ignored for rules where ``use_count_query`` or ``use_terms_query`` is set to true.\n\n``es_host`` is the address of an Elasticsearch cluster where ElastAlert will store data about its state, queries run, alerts, and errors. Each rule may also use a different Elasticsearch host to query against.\n\n``es_port`` is the port corresponding to ``es_host``.\n\n``use_ssl``: Optional; whether or not to connect to ``es_host`` using TLS; set to ``True`` or ``False``.\n\n``verify_certs``: Optional; whether or not to verify TLS certificates; set to ``True`` or ``False``. The default is ``True``\n\n``client_cert``: Optional; path to a PEM certificate to use as the client certificate\n\n``client_key``: Optional; path to a private key file to use as the client key\n\n``ca_certs``: Optional; path to a CA cert bundle to use to verify SSL connections\n\n``es_username``: Optional; basic-auth username for connecting to ``es_host``.\n\n``es_password``: Optional; basic-auth password for connecting to ``es_host``.\n\n``es_url_prefix``: Optional; URL prefix for the Elasticsearch endpoint.\n\n``es_send_get_body_as``: Optional; Method for querying Elasticsearch - ``GET``, ``POST`` or ``source``. The default is ``GET``\n\n``writeback_index`` is the name of the index in which ElastAlert will store data. We will create this index later.\n\n``alert_time_limit`` is the retry window for failed alerts.\n\nSave the file as ``config.yaml``\n\nSetting Up Elasticsearch\n------------------------\n\nElastAlert saves information and metadata about its queries and its alerts back to Elasticsearch. This is useful for auditing, debugging, and it allows ElastAlert to restart and resume exactly where it left off. This is not required for ElastAlert to run, but highly recommended.\n\nFirst, we need to create an index for ElastAlert to write to by running ``elastalert-create-index`` and following the instructions::\n\n $ elastalert-create-index\n New index name (Default elastalert_status)\n Name of existing index to copy (Default None)\n New index elastalert_status created\n Done!\n\nFor information about what data will go here, see :ref:`ElastAlert Metadata Index `.\n\nCreating a Rule\n---------------\n\nEach rule defines a query to perform, parameters on what triggers a match, and a list of alerts to fire for each match. We are going to use ``example_rules/example_frequency.yaml`` as a template::\n\n # From example_rules/example_frequency.yaml\n es_host: elasticsearch.example.com\n es_port: 14900\n name: Example rule\n type: frequency\n index: logstash-*\n num_events: 50\n timeframe:\n hours: 4\n filter:\n - term:\n some_field: \"some_value\"\n alert:\n - \"email\"\n email:\n - \"elastalert@example.com\"\n\n``es_host`` and ``es_port`` should point to the Elasticsearch cluster we want to query.\n\n``name`` is the unique name for this rule. ElastAlert will not start if two rules share the same name.\n\n``type``: Each rule has a different type which may take different parameters. The ``frequency`` type means \"Alert when more than ``num_events`` occur within ``timeframe``.\" For information other types, see :ref:`Rule types `.\n\n``index``: The name of the index(es) to query. If you are using Logstash, by default the indexes will match ``\"logstash-*\"``.\n\n``num_events``: This parameter is specific to ``frequency`` type and is the threshold for when an alert is triggered.\n\n``timeframe`` is the time period in which ``num_events`` must occur.\n\n``filter`` is a list of Elasticsearch filters that are used to filter results. Here we have a single term filter for documents with ``some_field`` matching ``some_value``. See :ref:`Writing Filters For Rules ` for more information. If no filters are desired, it should be specified as an empty list: ``filter: []``\n\n``alert`` is a list of alerts to run on each match. For more information on alert types, see :ref:`Alerts `. The email alert requires an SMTP server for sending mail. By default, it will attempt to use localhost. This can be changed with the ``smtp_host`` option.\n\n``email`` is a list of addresses to which alerts will be sent.\n\nThere are many other optional configuration options, see :ref:`Common configuration options `.\n\nAll documents must have a timestamp field. ElastAlert will try to use ``@timestamp`` by default, but this can be changed with the ``timestamp_field`` option. By default, ElastAlert uses ISO8601 timestamps, though unix timestamps are supported by setting ``timestamp_type``.\n\nAs is, this rule means \"Send an email to elastalert@example.com when there are more than 50 documents with ``some_field == some_value`` within a 4 hour period.\"\n\nTesting Your Rule\n-----------------\n\nRunning the ``elastalert-test-rule`` tool will test that your config file successfully loads and run it in debug mode over the last 24 hours::\n\n $ elastalert-test-rule example_rules/example_frequency.yaml\n\nIf you want to specify a configuration file to use, you can run it with the config flag::\n\n $ elastalert-test-rule --config example_rules/example_frequency.yaml\n\nThe configuration preferences will be loaded as follows:\n 1. Configurations specified in the yaml file.\n 2. Configurations specified in the config file, if specified.\n 3. Default configurations, for the tool to run.\n\nSee :ref:`the testing section for more details `\n\nRunning ElastAlert\n------------------\n\nThere are two ways of invoking ElastAlert. As a daemon, through Supervisor (http://supervisord.org/), or directly with Python. For easier debugging purposes in this tutorial, we will invoke it directly::\n\n $ python -m elastalert.elastalert --verbose --rule example_frequency.yaml # or use the entry point: elastalert --verbose --rule ...\n No handlers could be found for logger \"Elasticsearch\"\n INFO:root:Queried rule Example rule from 1-15 14:22 PST to 1-15 15:07 PST: 5 hits\n INFO:Elasticsearch:POST http://elasticsearch.example.com:14900/elastalert_status/elastalert_status?op_type=create [status:201 request:0.025s]\n INFO:root:Ran Example rule from 1-15 14:22 PST to 1-15 15:07 PST: 5 query hits (0 already seen), 0 matches, 0 alerts sent\n INFO:root:Sleeping for 297 seconds\n\nElastAlert uses the python logging system and ``--verbose`` sets it to display INFO level messages. ``--rule example_frequency.yaml`` specifies the rule to run, otherwise ElastAlert will attempt to load the other rules in the example_rules folder.\n\nLet's break down the response to see what's happening.\n\n``Queried rule Example rule from 1-15 14:22 PST to 1-15 15:07 PST: 5 hits``\n\nElastAlert periodically queries the most recent ``buffer_time`` (default 45 minutes) for data matching the filters. Here we see that it matched 5 hits.\n\n``POST http://elasticsearch.example.com:14900/elastalert_status/elastalert_status?op_type=create [status:201 request:0.025s]``\n\nThis line showing that ElastAlert uploaded a document to the elastalert_status index with information about the query it just made.\n\n``Ran Example rule from 1-15 14:22 PST to 1-15 15:07 PST: 5 query hits (0 already seen), 0 matches, 0 alerts sent``\n\nThe line means ElastAlert has finished processing the rule. For large time periods, sometimes multiple queries may be run, but their data will be processed together. ``query hits`` is the number of documents that are downloaded from Elasticsearch, ``already seen`` refers to documents that were already counted in a previous overlapping query and will be ignored, ``matches`` is the number of matches the rule type outputted, and ``alerts sent`` is the number of alerts actually sent. This may differ from ``matches`` because of options like ``realert`` and ``aggregation`` or because of an error.\n\n``Sleeping for 297 seconds``\n\nThe default ``run_every`` is 5 minutes, meaning ElastAlert will sleep until 5 minutes have elapsed from the last cycle before running queries for each rule again with time ranges shifted forward 5 minutes.\n\nSay, over the next 297 seconds, 46 more matching documents were added to Elasticsearch::\n\n\n INFO:root:Queried rule Example rule from 1-15 14:27 PST to 1-15 15:12 PST: 51 hits\n ...\n INFO:root:Sent email to ['elastalert@example.com']\n ...\n INFO:root:Ran Example rule from 1-15 14:27 PST to 1-15 15:12 PST: 51 query hits, 1 matches, 1 alerts sent\n\nThe body of the email will contain something like::\n\n Example rule\n\n At least 50 events occurred between 1-15 11:12 PST and 1-15 15:12 PST\n\n @timestamp: 2015-01-15T15:12:00-08:00\n\nIf an error occurred, such as an unreachable SMTP server, you may see:\n\n\n``ERROR:root:Error while running alert email: Error connecting to SMTP host: [Errno 61] Connection refused``\n\n\nNote that if you stop ElastAlert and then run it again later, it will look up ``elastalert_status`` and begin querying\nat the end time of the last query. This is to prevent duplication or skipping of alerts if ElastAlert is restarted.\n\nBy using the ``--debug`` flag instead of ``--verbose``, the body of email will instead be logged and the email will not be sent. In addition, the queries will not be saved to ``elastalert_status``.\n" }, { "path": "elastalert/__init__.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport time\n\nfrom elasticsearch import Elasticsearch\nfrom elasticsearch import RequestsHttpConnection\nfrom elasticsearch.client import _make_path\nfrom elasticsearch.client import query_params\nfrom elasticsearch.exceptions import TransportError\n\n\nclass ElasticSearchClient(Elasticsearch):\n \"\"\" Extension of low level :class:`Elasticsearch` client with additional version resolving features \"\"\"\n\n def __init__(self, conf):\n \"\"\"\n :arg conf: es_conn_config dictionary. Ref. :func:`~util.build_es_conn_config`\n \"\"\"\n super(ElasticSearchClient, self).__init__(host=conf['es_host'],\n port=conf['es_port'],\n url_prefix=conf['es_url_prefix'],\n use_ssl=conf['use_ssl'],\n verify_certs=conf['verify_certs'],\n ca_certs=conf['ca_certs'],\n connection_class=RequestsHttpConnection,\n http_auth=conf['http_auth'],\n timeout=conf['es_conn_timeout'],\n send_get_body_as=conf['send_get_body_as'],\n client_cert=conf['client_cert'],\n client_key=conf['client_key'])\n self._conf = copy.copy(conf)\n self._es_version = None\n\n @property\n def conf(self):\n \"\"\"\n Returns the provided es_conn_config used when initializing the class instance.\n \"\"\"\n return self._conf\n\n @property\n def es_version(self):\n \"\"\"\n Returns the reported version from the Elasticsearch server.\n \"\"\"\n if self._es_version is None:\n for retry in range(3):\n try:\n self._es_version = self.info()['version']['number']\n break\n except TransportError:\n if retry == 2:\n raise\n time.sleep(3)\n return self._es_version\n\n def is_atleastfive(self):\n \"\"\"\n Returns True when the Elasticsearch server version >= 5\n \"\"\"\n return int(self.es_version.split(\".\")[0]) >= 5\n\n def is_atleastsix(self):\n \"\"\"\n Returns True when the Elasticsearch server version >= 6\n \"\"\"\n return int(self.es_version.split(\".\")[0]) >= 6\n\n def is_atleastsixtwo(self):\n \"\"\"\n Returns True when the Elasticsearch server version >= 6.2\n \"\"\"\n major, minor = list(map(int, self.es_version.split(\".\")[:2]))\n return major > 6 or (major == 6 and minor >= 2)\n\n def is_atleastsixsix(self):\n \"\"\"\n Returns True when the Elasticsearch server version >= 6.6\n \"\"\"\n major, minor = list(map(int, self.es_version.split(\".\")[:2]))\n return major > 6 or (major == 6 and minor >= 6)\n\n def is_atleastseven(self):\n \"\"\"\n Returns True when the Elasticsearch server version >= 7\n \"\"\"\n return int(self.es_version.split(\".\")[0]) >= 7\n\n def resolve_writeback_index(self, writeback_index, doc_type):\n \"\"\" In ES6, you cannot have multiple _types per index,\n therefore we use self.writeback_index as the prefix for the actual\n index name, based on doc_type. \"\"\"\n if not self.is_atleastsix():\n return writeback_index\n elif doc_type == 'silence':\n return writeback_index + '_silence'\n elif doc_type == 'past_elastalert':\n return writeback_index + '_past'\n elif doc_type == 'elastalert_status':\n return writeback_index + '_status'\n elif doc_type == 'elastalert_error':\n return writeback_index + '_error'\n return writeback_index\n\n @query_params(\n \"_source\",\n \"_source_exclude\",\n \"_source_excludes\",\n \"_source_include\",\n \"_source_includes\",\n \"allow_no_indices\",\n \"allow_partial_search_results\",\n \"analyze_wildcard\",\n \"analyzer\",\n \"batched_reduce_size\",\n \"default_operator\",\n \"df\",\n \"docvalue_fields\",\n \"expand_wildcards\",\n \"explain\",\n \"from_\",\n \"ignore_unavailable\",\n \"lenient\",\n \"max_concurrent_shard_requests\",\n \"pre_filter_shard_size\",\n \"preference\",\n \"q\",\n \"rest_total_hits_as_int\",\n \"request_cache\",\n \"routing\",\n \"scroll\",\n \"search_type\",\n \"seq_no_primary_term\",\n \"size\",\n \"sort\",\n \"stats\",\n \"stored_fields\",\n \"suggest_field\",\n \"suggest_mode\",\n \"suggest_size\",\n \"suggest_text\",\n \"terminate_after\",\n \"timeout\",\n \"track_scores\",\n \"track_total_hits\",\n \"typed_keys\",\n \"version\",\n )\n def deprecated_search(self, index=None, doc_type=None, body=None, params=None):\n \"\"\"\n Execute a search query and get back search hits that match the query.\n ``_\n :arg index: A list of index names to search, or a string containing a\n comma-separated list of index names to search; use `_all`\n or empty string to perform the operation on all indices\n :arg doc_type: A comma-separated list of document types to search; leave\n empty to perform the operation on all types\n :arg body: The search definition using the Query DSL\n :arg _source: True or false to return the _source field or not, or a\n list of fields to return\n :arg _source_exclude: A list of fields to exclude from the returned\n _source field\n :arg _source_include: A list of fields to extract and return from the\n _source field\n :arg allow_no_indices: Whether to ignore if a wildcard indices\n expression resolves into no concrete indices. (This includes `_all`\n string or when no indices have been specified)\n :arg allow_partial_search_results: Set to false to return an overall\n failure if the request would produce partial results. Defaults to\n True, which will allow partial results in the case of timeouts or\n partial failures\n :arg analyze_wildcard: Specify whether wildcard and prefix queries\n should be analyzed (default: false)\n :arg analyzer: The analyzer to use for the query string\n :arg batched_reduce_size: The number of shard results that should be\n reduced at once on the coordinating node. This value should be used\n as a protection mechanism to reduce the memory overhead per search\n request if the potential number of shards in the request can be\n large., default 512\n :arg default_operator: The default operator for query string query (AND\n or OR), default 'OR', valid choices are: 'AND', 'OR'\n :arg df: The field to use as default where no field prefix is given in\n the query string\n :arg docvalue_fields: A comma-separated list of fields to return as the\n docvalue representation of a field for each hit\n :arg expand_wildcards: Whether to expand wildcard expression to concrete\n indices that are open, closed or both., default 'open', valid\n choices are: 'open', 'closed', 'none', 'all'\n :arg explain: Specify whether to return detailed information about score\n computation as part of a hit\n :arg from\\\\_: Starting offset (default: 0)\n :arg ignore_unavailable: Whether specified concrete indices should be\n ignored when unavailable (missing or closed)\n :arg lenient: Specify whether format-based query failures (such as\n providing text to a numeric field) should be ignored\n :arg max_concurrent_shard_requests: The number of concurrent shard\n requests this search executes concurrently. This value should be\n used to limit the impact of the search on the cluster in order to\n limit the number of concurrent shard requests, default 'The default\n grows with the number of nodes in the cluster but is at most 256.'\n :arg pre_filter_shard_size: A threshold that enforces a pre-filter\n roundtrip to prefilter search shards based on query rewriting if\n the number of shards the search request expands to exceeds the\n threshold. This filter roundtrip can limit the number of shards\n significantly if for instance a shard can not match any documents\n based on it's rewrite method ie. if date filters are mandatory to\n match but the shard bounds and the query are disjoint., default 128\n :arg preference: Specify the node or shard the operation should be\n performed on (default: random)\n :arg q: Query in the Lucene query string syntax\n :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number\n in the response. This param is added version 6.x to handle mixed cluster queries where nodes\n are in multiple versions (7.0 and 6.latest)\n :arg request_cache: Specify if request cache should be used for this\n request or not, defaults to index level setting\n :arg routing: A comma-separated list of specific routing values\n :arg scroll: Specify how long a consistent view of the index should be\n maintained for scrolled search\n :arg search_type: Search operation type, valid choices are:\n 'query_then_fetch', 'dfs_query_then_fetch'\n :arg size: Number of hits to return (default: 10)\n :arg sort: A comma-separated list of : pairs\n :arg stats: Specific 'tag' of the request for logging and statistical\n purposes\n :arg stored_fields: A comma-separated list of stored fields to return as\n part of a hit\n :arg suggest_field: Specify which field to use for suggestions\n :arg suggest_mode: Specify suggest mode, default 'missing', valid\n choices are: 'missing', 'popular', 'always'\n :arg suggest_size: How many suggestions to return in response\n :arg suggest_text: The source text for which the suggestions should be\n returned\n :arg terminate_after: The maximum number of documents to collect for\n each shard, upon reaching which the query execution will terminate\n early.\n :arg timeout: Explicit operation timeout\n :arg track_scores: Whether to calculate and return scores even if they\n are not used for sorting\n :arg track_total_hits: Indicate if the number of documents that match\n the query should be tracked\n :arg typed_keys: Specify whether aggregation and suggester names should\n be prefixed by their respective types in the response\n :arg version: Specify whether to return document version as part of a\n hit\n \"\"\"\n # from is a reserved word so it cannot be used, use from_ instead\n if \"from_\" in params:\n params[\"from\"] = params.pop(\"from_\")\n\n if not index:\n index = \"_all\"\n res = self.transport.perform_request(\n \"GET\", _make_path(index, doc_type, \"_search\"), params=params, body=body\n )\n if type(res) == list or type(res) == tuple:\n return res[1]\n return res\n" }, { "path": "elastalert/alerts.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport datetime\nimport json\nimport logging\nimport os\nimport re\nimport subprocess\nimport sys\nimport time\nimport uuid\nimport warnings\nfrom email.mime.text import MIMEText\nfrom email.utils import formatdate\nfrom html.parser import HTMLParser\nfrom smtplib import SMTP\nfrom smtplib import SMTP_SSL\nfrom smtplib import SMTPAuthenticationError\nfrom smtplib import SMTPException\nfrom socket import error\n\nimport boto3\nimport requests\nimport stomp\nfrom exotel import Exotel\nfrom jira.client import JIRA\nfrom jira.exceptions import JIRAError\nfrom requests.auth import HTTPProxyAuth\nfrom requests.exceptions import RequestException\nfrom staticconf.loader import yaml_loader\nfrom texttable import Texttable\nfrom twilio.base.exceptions import TwilioRestException\nfrom twilio.rest import Client as TwilioClient\n\nfrom .util import EAException\nfrom .util import elastalert_logger\nfrom .util import lookup_es_key\nfrom .util import pretty_ts\nfrom .util import resolve_string\nfrom .util import ts_now\nfrom .util import ts_to_dt\n\n\nclass DateTimeEncoder(json.JSONEncoder):\n def default(self, obj):\n if hasattr(obj, 'isoformat'):\n return obj.isoformat()\n else:\n return json.JSONEncoder.default(self, obj)\n\n\nclass BasicMatchString(object):\n \"\"\" Creates a string containing fields in match for the given rule. \"\"\"\n\n def __init__(self, rule, match):\n self.rule = rule\n self.match = match\n\n def _ensure_new_line(self):\n while self.text[-2:] != '\\n\\n':\n self.text += '\\n'\n\n def _add_custom_alert_text(self):\n missing = self.rule.get('alert_missing_value', '')\n alert_text = str(self.rule.get('alert_text', ''))\n if 'alert_text_args' in self.rule:\n alert_text_args = self.rule.get('alert_text_args')\n alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args]\n\n # Support referencing other top-level rule properties\n # This technically may not work if there is a top-level rule property with the same name\n # as an es result key, since it would have been matched in the lookup_es_key call above\n for i, text_value in enumerate(alert_text_values):\n if text_value is None:\n alert_value = self.rule.get(alert_text_args[i])\n if alert_value:\n alert_text_values[i] = alert_value\n\n alert_text_values = [missing if val is None else val for val in alert_text_values]\n alert_text = alert_text.format(*alert_text_values)\n elif 'alert_text_kw' in self.rule:\n kw = {}\n for name, kw_name in list(self.rule.get('alert_text_kw').items()):\n val = lookup_es_key(self.match, name)\n\n # Support referencing other top-level rule properties\n # This technically may not work if there is a top-level rule property with the same name\n # as an es result key, since it would have been matched in the lookup_es_key call above\n if val is None:\n val = self.rule.get(name)\n\n kw[kw_name] = missing if val is None else val\n alert_text = alert_text.format(**kw)\n\n self.text += alert_text\n\n def _add_rule_text(self):\n self.text += self.rule['type'].get_match_str(self.match)\n\n def _add_top_counts(self):\n for key, counts in list(self.match.items()):\n if key.startswith('top_events_'):\n self.text += '%s:\\n' % (key[11:])\n top_events = list(counts.items())\n\n if not top_events:\n self.text += 'No events found.\\n'\n else:\n top_events.sort(key=lambda x: x[1], reverse=True)\n for term, count in top_events:\n self.text += '%s: %s\\n' % (term, count)\n\n self.text += '\\n'\n\n def _add_match_items(self):\n match_items = list(self.match.items())\n match_items.sort(key=lambda x: x[0])\n for key, value in match_items:\n if key.startswith('top_events_'):\n continue\n value_str = str(value)\n value_str.replace('\\\\n', '\\n')\n if type(value) in [list, dict]:\n try:\n value_str = self._pretty_print_as_json(value)\n except TypeError:\n # Non serializable object, fallback to str\n pass\n self.text += '%s: %s\\n' % (key, value_str)\n\n def _pretty_print_as_json(self, blob):\n try:\n return json.dumps(blob, cls=DateTimeEncoder, sort_keys=True, indent=4, ensure_ascii=False)\n except UnicodeDecodeError:\n # This blob contains non-unicode, so lets pretend it's Latin-1 to show something\n return json.dumps(blob, cls=DateTimeEncoder, sort_keys=True, indent=4, encoding='Latin-1', ensure_ascii=False)\n\n def __str__(self):\n self.text = ''\n if 'alert_text' not in self.rule:\n self.text += self.rule['name'] + '\\n\\n'\n\n self._add_custom_alert_text()\n self._ensure_new_line()\n if self.rule.get('alert_text_type') != 'alert_text_only':\n self._add_rule_text()\n self._ensure_new_line()\n if self.rule.get('top_count_keys'):\n self._add_top_counts()\n if self.rule.get('alert_text_type') != 'exclude_fields':\n self._add_match_items()\n return self.text\n\n\nclass JiraFormattedMatchString(BasicMatchString):\n def _add_match_items(self):\n match_items = dict([(x, y) for x, y in list(self.match.items()) if not x.startswith('top_events_')])\n json_blob = self._pretty_print_as_json(match_items)\n preformatted_text = '{{code}}{0}{{code}}'.format(json_blob)\n self.text += preformatted_text\n\n\nclass Alerter(object):\n \"\"\" Base class for types of alerts.\n\n :param rule: The rule configuration.\n \"\"\"\n required_options = frozenset([])\n\n def __init__(self, rule):\n self.rule = rule\n # pipeline object is created by ElastAlerter.send_alert()\n # and attached to each alerters used by a rule before calling alert()\n self.pipeline = None\n self.resolve_rule_references(self.rule)\n\n def resolve_rule_references(self, root):\n # Support referencing other top-level rule properties to avoid redundant copy/paste\n if type(root) == list:\n # Make a copy since we may be modifying the contents of the structure we're walking\n for i, item in enumerate(copy.copy(root)):\n if type(item) == dict or type(item) == list:\n self.resolve_rule_references(root[i])\n else:\n root[i] = self.resolve_rule_reference(item)\n elif type(root) == dict:\n # Make a copy since we may be modifying the contents of the structure we're walking\n for key, value in root.copy().items():\n if type(value) == dict or type(value) == list:\n self.resolve_rule_references(root[key])\n else:\n root[key] = self.resolve_rule_reference(value)\n\n def resolve_rule_reference(self, value):\n strValue = str(value)\n if strValue.startswith('$') and strValue.endswith('$') and strValue[1:-1] in self.rule:\n if type(value) == int:\n return int(self.rule[strValue[1:-1]])\n else:\n return self.rule[strValue[1:-1]]\n else:\n return value\n\n def alert(self, match):\n \"\"\" Send an alert. Match is a dictionary of information about the alert.\n\n :param match: A dictionary of relevant information to the alert.\n \"\"\"\n raise NotImplementedError()\n\n def get_info(self):\n \"\"\" Returns a dictionary of data related to this alert. At minimum, this should contain\n a field type corresponding to the type of Alerter. \"\"\"\n return {'type': 'Unknown'}\n\n def create_title(self, matches):\n \"\"\" Creates custom alert title to be used, e.g. as an e-mail subject or JIRA issue summary.\n\n :param matches: A list of dictionaries of relevant information to the alert.\n \"\"\"\n if 'alert_subject' in self.rule:\n return self.create_custom_title(matches)\n\n return self.create_default_title(matches)\n\n def create_custom_title(self, matches):\n alert_subject = str(self.rule['alert_subject'])\n alert_subject_max_len = int(self.rule.get('alert_subject_max_len', 2048))\n\n if 'alert_subject_args' in self.rule:\n alert_subject_args = self.rule['alert_subject_args']\n alert_subject_values = [lookup_es_key(matches[0], arg) for arg in alert_subject_args]\n\n # Support referencing other top-level rule properties\n # This technically may not work if there is a top-level rule property with the same name\n # as an es result key, since it would have been matched in the lookup_es_key call above\n for i, subject_value in enumerate(alert_subject_values):\n if subject_value is None:\n alert_value = self.rule.get(alert_subject_args[i])\n if alert_value:\n alert_subject_values[i] = alert_value\n\n missing = self.rule.get('alert_missing_value', '')\n alert_subject_values = [missing if val is None else val for val in alert_subject_values]\n alert_subject = alert_subject.format(*alert_subject_values)\n\n if len(alert_subject) > alert_subject_max_len:\n alert_subject = alert_subject[:alert_subject_max_len]\n\n return alert_subject\n\n def create_alert_body(self, matches):\n body = self.get_aggregation_summary_text(matches)\n if self.rule.get('alert_text_type') != 'aggregation_summary_only':\n for match in matches:\n body += str(BasicMatchString(self.rule, match))\n # Separate text of aggregated alerts with dashes\n if len(matches) > 1:\n body += '\\n----------------------------------------\\n'\n return body\n\n def get_aggregation_summary_text__maximum_width(self):\n \"\"\"Get maximum width allowed for summary text.\"\"\"\n return 80\n\n def get_aggregation_summary_text(self, matches):\n text = ''\n if 'aggregation' in self.rule and 'summary_table_fields' in self.rule:\n text = self.rule.get('summary_prefix', '')\n summary_table_fields = self.rule['summary_table_fields']\n if not isinstance(summary_table_fields, list):\n summary_table_fields = [summary_table_fields]\n # Include a count aggregation so that we can see at a glance how many of each aggregation_key were encountered\n summary_table_fields_with_count = summary_table_fields + ['count']\n text += \"Aggregation resulted in the following data for summary_table_fields ==> {0}:\\n\\n\".format(\n summary_table_fields_with_count\n )\n text_table = Texttable(max_width=self.get_aggregation_summary_text__maximum_width())\n text_table.header(summary_table_fields_with_count)\n # Format all fields as 'text' to avoid long numbers being shown as scientific notation\n text_table.set_cols_dtype(['t' for i in summary_table_fields_with_count])\n match_aggregation = {}\n\n # Maintain an aggregate count for each unique key encountered in the aggregation period\n for match in matches:\n key_tuple = tuple([str(lookup_es_key(match, key)) for key in summary_table_fields])\n if key_tuple not in match_aggregation:\n match_aggregation[key_tuple] = 1\n else:\n match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1\n for keys, count in match_aggregation.items():\n text_table.add_row([key for key in keys] + [count])\n text += text_table.draw() + '\\n\\n'\n text += self.rule.get('summary_prefix', '')\n return str(text)\n\n def create_default_title(self, matches):\n return self.rule['name']\n\n def get_account(self, account_file):\n \"\"\" Gets the username and password from an account file.\n\n :param account_file: Path to the file which contains user and password information.\n It can be either an absolute file path or one that is relative to the given rule.\n \"\"\"\n if os.path.isabs(account_file):\n account_file_path = account_file\n else:\n account_file_path = os.path.join(os.path.dirname(self.rule['rule_file']), account_file)\n account_conf = yaml_loader(account_file_path)\n if 'user' not in account_conf or 'password' not in account_conf:\n raise EAException('Account file must have user and password fields')\n self.user = account_conf['user']\n self.password = account_conf['password']\n\n\nclass StompAlerter(Alerter):\n \"\"\" The stomp alerter publishes alerts via stomp to a broker. \"\"\"\n required_options = frozenset(\n ['stomp_hostname', 'stomp_hostport', 'stomp_login', 'stomp_password'])\n\n def alert(self, matches):\n alerts = []\n\n qk = self.rule.get('query_key', None)\n\n fullmessage = {}\n for match in matches:\n if qk is not None:\n resmatch = lookup_es_key(match, qk)\n else:\n resmatch = None\n\n if resmatch is not None:\n elastalert_logger.info(\n 'Alert for %s, %s at %s:' % (self.rule['name'], resmatch, lookup_es_key(match, self.rule['timestamp_field'])))\n alerts.append(\n 'Alert for %s, %s at %s:' % (self.rule['name'], resmatch, lookup_es_key(\n match, self.rule['timestamp_field']))\n )\n fullmessage['match'] = resmatch\n else:\n elastalert_logger.info('Rule %s generated an alert at %s:' % (\n self.rule['name'], lookup_es_key(match, self.rule['timestamp_field'])))\n alerts.append(\n 'Rule %s generated an alert at %s:' % (self.rule['name'], lookup_es_key(\n match, self.rule['timestamp_field']))\n )\n fullmessage['match'] = lookup_es_key(\n match, self.rule['timestamp_field'])\n elastalert_logger.info(str(BasicMatchString(self.rule, match)))\n\n fullmessage['alerts'] = alerts\n fullmessage['rule'] = self.rule['name']\n fullmessage['rule_file'] = self.rule['rule_file']\n\n fullmessage['matching'] = str(BasicMatchString(self.rule, match))\n fullmessage['alertDate'] = datetime.datetime.now(\n ).strftime(\"%Y-%m-%d %H:%M:%S\")\n fullmessage['body'] = self.create_alert_body(matches)\n\n fullmessage['matches'] = matches\n\n self.stomp_hostname = self.rule.get('stomp_hostname', 'localhost')\n self.stomp_hostport = self.rule.get('stomp_hostport', '61613')\n self.stomp_login = self.rule.get('stomp_login', 'admin')\n self.stomp_password = self.rule.get('stomp_password', 'admin')\n self.stomp_destination = self.rule.get(\n 'stomp_destination', '/queue/ALERT')\n self.stomp_ssl = self.rule.get('stomp_ssl', False)\n\n conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)], use_ssl=self.stomp_ssl)\n\n conn.start()\n conn.connect(self.stomp_login, self.stomp_password)\n # Ensures that the CONNECTED frame is received otherwise, the disconnect call will fail.\n time.sleep(1)\n conn.send(self.stomp_destination, json.dumps(fullmessage))\n conn.disconnect()\n\n def get_info(self):\n return {'type': 'stomp'}\n\n\nclass DebugAlerter(Alerter):\n \"\"\" The debug alerter uses a Python logger (by default, alerting to terminal). \"\"\"\n\n def alert(self, matches):\n qk = self.rule.get('query_key', None)\n for match in matches:\n if qk in match:\n elastalert_logger.info(\n 'Alert for %s, %s at %s:' % (self.rule['name'], match[qk], lookup_es_key(match, self.rule['timestamp_field'])))\n else:\n elastalert_logger.info('Alert for %s at %s:' % (self.rule['name'], lookup_es_key(match, self.rule['timestamp_field'])))\n elastalert_logger.info(str(BasicMatchString(self.rule, match)))\n\n def get_info(self):\n return {'type': 'debug'}\n\n\nclass EmailAlerter(Alerter):\n \"\"\" Sends an email alert \"\"\"\n required_options = frozenset(['email'])\n\n def __init__(self, *args):\n super(EmailAlerter, self).__init__(*args)\n\n self.smtp_host = self.rule.get('smtp_host', 'localhost')\n self.smtp_ssl = self.rule.get('smtp_ssl', False)\n self.from_addr = self.rule.get('from_addr', 'ElastAlert')\n self.smtp_port = self.rule.get('smtp_port')\n if self.rule.get('smtp_auth_file'):\n self.get_account(self.rule['smtp_auth_file'])\n self.smtp_key_file = self.rule.get('smtp_key_file')\n self.smtp_cert_file = self.rule.get('smtp_cert_file')\n # Convert email to a list if it isn't already\n if isinstance(self.rule['email'], str):\n self.rule['email'] = [self.rule['email']]\n # If there is a cc then also convert it a list if it isn't\n cc = self.rule.get('cc')\n if cc and isinstance(cc, str):\n self.rule['cc'] = [self.rule['cc']]\n # If there is a bcc then also convert it to a list if it isn't\n bcc = self.rule.get('bcc')\n if bcc and isinstance(bcc, str):\n self.rule['bcc'] = [self.rule['bcc']]\n add_suffix = self.rule.get('email_add_domain')\n if add_suffix and not add_suffix.startswith('@'):\n self.rule['email_add_domain'] = '@' + add_suffix\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n # Add JIRA ticket if it exists\n if self.pipeline is not None and 'jira_ticket' in self.pipeline:\n url = '%s/browse/%s' % (self.pipeline['jira_server'], self.pipeline['jira_ticket'])\n body += '\\nJIRA ticket: %s' % (url)\n\n to_addr = self.rule['email']\n if 'email_from_field' in self.rule:\n recipient = lookup_es_key(matches[0], self.rule['email_from_field'])\n if isinstance(recipient, str):\n if '@' in recipient:\n to_addr = [recipient]\n elif 'email_add_domain' in self.rule:\n to_addr = [recipient + self.rule['email_add_domain']]\n elif isinstance(recipient, list):\n to_addr = recipient\n if 'email_add_domain' in self.rule:\n to_addr = [name + self.rule['email_add_domain'] for name in to_addr]\n if self.rule.get('email_format') == 'html':\n email_msg = MIMEText(body, 'html', _charset='UTF-8')\n else:\n email_msg = MIMEText(body, _charset='UTF-8')\n email_msg['Subject'] = self.create_title(matches)\n email_msg['To'] = ', '.join(to_addr)\n email_msg['From'] = self.from_addr\n email_msg['Reply-To'] = self.rule.get('email_reply_to', email_msg['To'])\n email_msg['Date'] = formatdate()\n if self.rule.get('cc'):\n email_msg['CC'] = ','.join(self.rule['cc'])\n to_addr = to_addr + self.rule['cc']\n if self.rule.get('bcc'):\n to_addr = to_addr + self.rule['bcc']\n\n try:\n if self.smtp_ssl:\n if self.smtp_port:\n self.smtp = SMTP_SSL(self.smtp_host, self.smtp_port, keyfile=self.smtp_key_file, certfile=self.smtp_cert_file)\n else:\n self.smtp = SMTP_SSL(self.smtp_host, keyfile=self.smtp_key_file, certfile=self.smtp_cert_file)\n else:\n if self.smtp_port:\n self.smtp = SMTP(self.smtp_host, self.smtp_port)\n else:\n self.smtp = SMTP(self.smtp_host)\n self.smtp.ehlo()\n if self.smtp.has_extn('STARTTLS'):\n self.smtp.starttls(keyfile=self.smtp_key_file, certfile=self.smtp_cert_file)\n if 'smtp_auth_file' in self.rule:\n self.smtp.login(self.user, self.password)\n except (SMTPException, error) as e:\n raise EAException(\"Error connecting to SMTP host: %s\" % (e))\n except SMTPAuthenticationError as e:\n raise EAException(\"SMTP username/password rejected: %s\" % (e))\n self.smtp.sendmail(self.from_addr, to_addr, email_msg.as_string())\n self.smtp.quit()\n\n elastalert_logger.info(\"Sent email to %s\" % (to_addr))\n\n def create_default_title(self, matches):\n subject = 'ElastAlert: %s' % (self.rule['name'])\n\n # If the rule has a query_key, add that value plus timestamp to subject\n if 'query_key' in self.rule:\n qk = matches[0].get(self.rule['query_key'])\n if qk:\n subject += ' - %s' % (qk)\n\n return subject\n\n def get_info(self):\n return {'type': 'email',\n 'recipients': self.rule['email']}\n\n\nclass JiraAlerter(Alerter):\n \"\"\" Creates a Jira ticket for each alert \"\"\"\n required_options = frozenset(['jira_server', 'jira_account_file', 'jira_project', 'jira_issuetype'])\n\n # Maintain a static set of built-in fields that we explicitly know how to set\n # For anything else, we will do best-effort and try to set a string value\n known_field_list = [\n 'jira_account_file',\n 'jira_assignee',\n 'jira_bump_after_inactivity',\n 'jira_bump_in_statuses',\n 'jira_bump_not_in_statuses',\n 'jira_bump_only',\n 'jira_bump_tickets',\n 'jira_component',\n 'jira_components',\n 'jira_description',\n 'jira_ignore_in_title',\n 'jira_issuetype',\n 'jira_label',\n 'jira_labels',\n 'jira_max_age',\n 'jira_priority',\n 'jira_project',\n 'jira_server',\n 'jira_transition_to',\n 'jira_watchers',\n ]\n\n # Some built-in jira types that can be used as custom fields require special handling\n # Here is a sample of one of them:\n # {\"id\":\"customfield_12807\",\"name\":\"My Custom Field\",\"custom\":true,\"orderable\":true,\"navigable\":true,\"searchable\":true,\n # \"clauseNames\":[\"cf[12807]\",\"My Custom Field\"],\"schema\":{\"type\":\"array\",\"items\":\"string\",\n # \"custom\":\"com.atlassian.jira.plugin.system.customfieldtypes:multiselect\",\"customId\":12807}}\n # There are likely others that will need to be updated on a case-by-case basis\n custom_string_types_with_special_handling = [\n 'com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes',\n 'com.atlassian.jira.plugin.system.customfieldtypes:multiselect',\n 'com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons',\n ]\n\n def __init__(self, rule):\n super(JiraAlerter, self).__init__(rule)\n self.server = self.rule['jira_server']\n self.get_account(self.rule['jira_account_file'])\n self.project = self.rule['jira_project']\n self.issue_type = self.rule['jira_issuetype']\n\n # Deferred settings refer to values that can only be resolved when a match\n # is found and as such loading them will be delayed until we find a match\n self.deferred_settings = []\n\n # We used to support only a single component. This allows us to maintain backwards compatibility\n # while also giving the user-facing API a more representative name\n self.components = self.rule.get('jira_components', self.rule.get('jira_component'))\n\n # We used to support only a single label. This allows us to maintain backwards compatibility\n # while also giving the user-facing API a more representative name\n self.labels = self.rule.get('jira_labels', self.rule.get('jira_label'))\n\n self.description = self.rule.get('jira_description', '')\n self.assignee = self.rule.get('jira_assignee')\n self.max_age = self.rule.get('jira_max_age', 30)\n self.priority = self.rule.get('jira_priority')\n self.bump_tickets = self.rule.get('jira_bump_tickets', False)\n self.bump_not_in_statuses = self.rule.get('jira_bump_not_in_statuses')\n self.bump_in_statuses = self.rule.get('jira_bump_in_statuses')\n self.bump_after_inactivity = self.rule.get('jira_bump_after_inactivity', 0)\n self.bump_only = self.rule.get('jira_bump_only', False)\n self.transition = self.rule.get('jira_transition_to', False)\n self.watchers = self.rule.get('jira_watchers')\n self.client = None\n\n if self.bump_in_statuses and self.bump_not_in_statuses:\n msg = 'Both jira_bump_in_statuses (%s) and jira_bump_not_in_statuses (%s) are set.' % \\\n (','.join(self.bump_in_statuses), ','.join(self.bump_not_in_statuses))\n intersection = list(set(self.bump_in_statuses) & set(self.bump_in_statuses))\n if intersection:\n msg = '%s Both have common statuses of (%s). As such, no tickets will ever be found.' % (\n msg, ','.join(intersection))\n msg += ' This should be simplified to use only one or the other.'\n logging.warning(msg)\n\n self.reset_jira_args()\n\n try:\n self.client = JIRA(self.server, basic_auth=(self.user, self.password))\n self.get_priorities()\n self.jira_fields = self.client.fields()\n self.get_arbitrary_fields()\n except JIRAError as e:\n # JIRAError may contain HTML, pass along only first 1024 chars\n raise EAException(\"Error connecting to JIRA: %s\" % (str(e)[:1024])).with_traceback(sys.exc_info()[2])\n\n self.set_priority()\n\n def set_priority(self):\n try:\n if self.priority is not None and self.client is not None:\n self.jira_args['priority'] = {'id': self.priority_ids[self.priority]}\n except KeyError:\n logging.error(\"Priority %s not found. Valid priorities are %s\" % (self.priority, list(self.priority_ids.keys())))\n\n def reset_jira_args(self):\n self.jira_args = {'project': {'key': self.project},\n 'issuetype': {'name': self.issue_type}}\n\n if self.components:\n # Support single component or list\n if type(self.components) != list:\n self.jira_args['components'] = [{'name': self.components}]\n else:\n self.jira_args['components'] = [{'name': component} for component in self.components]\n if self.labels:\n # Support single label or list\n if type(self.labels) != list:\n self.labels = [self.labels]\n self.jira_args['labels'] = self.labels\n if self.watchers:\n # Support single watcher or list\n if type(self.watchers) != list:\n self.watchers = [self.watchers]\n if self.assignee:\n self.jira_args['assignee'] = {'name': self.assignee}\n\n self.set_priority()\n\n def set_jira_arg(self, jira_field, value, fields):\n # Remove the jira_ part. Convert underscores to spaces\n normalized_jira_field = jira_field[5:].replace('_', ' ').lower()\n # All jira fields should be found in the 'id' or the 'name' field. Therefore, try both just in case\n for identifier in ['name', 'id']:\n field = next((f for f in fields if normalized_jira_field == f[identifier].replace('_', ' ').lower()), None)\n if field:\n break\n if not field:\n # Log a warning to ElastAlert saying that we couldn't find that type?\n # OR raise and fail to load the alert entirely? Probably the latter...\n raise Exception(\"Could not find a definition for the jira field '{0}'\".format(normalized_jira_field))\n arg_name = field['id']\n # Check the schema information to decide how to set the value correctly\n # If the schema information is not available, raise an exception since we don't know how to set it\n # Note this is only the case for two built-in types, id: issuekey and id: thumbnail\n if not ('schema' in field or 'type' in field['schema']):\n raise Exception(\"Could not determine schema information for the jira field '{0}'\".format(normalized_jira_field))\n arg_type = field['schema']['type']\n\n # Handle arrays of simple types like strings or numbers\n if arg_type == 'array':\n # As a convenience, support the scenario wherein the user only provides\n # a single value for a multi-value field e.g. jira_labels: Only_One_Label\n if type(value) != list:\n value = [value]\n array_items = field['schema']['items']\n # Simple string types\n if array_items in ['string', 'date', 'datetime']:\n # Special case for multi-select custom types (the JIRA metadata says that these are strings, but\n # in reality, they are required to be provided as an object.\n if 'custom' in field['schema'] and field['schema']['custom'] in self.custom_string_types_with_special_handling:\n self.jira_args[arg_name] = [{'value': v} for v in value]\n else:\n self.jira_args[arg_name] = value\n elif array_items == 'number':\n self.jira_args[arg_name] = [int(v) for v in value]\n # Also attempt to handle arrays of complex types that have to be passed as objects with an identifier 'key'\n elif array_items == 'option':\n self.jira_args[arg_name] = [{'value': v} for v in value]\n else:\n # Try setting it as an object, using 'name' as the key\n # This may not work, as the key might actually be 'key', 'id', 'value', or something else\n # If it works, great! If not, it will manifest itself as an API error that will bubble up\n self.jira_args[arg_name] = [{'name': v} for v in value]\n # Handle non-array types\n else:\n # Simple string types\n if arg_type in ['string', 'date', 'datetime']:\n # Special case for custom types (the JIRA metadata says that these are strings, but\n # in reality, they are required to be provided as an object.\n if 'custom' in field['schema'] and field['schema']['custom'] in self.custom_string_types_with_special_handling:\n self.jira_args[arg_name] = {'value': value}\n else:\n self.jira_args[arg_name] = value\n # Number type\n elif arg_type == 'number':\n self.jira_args[arg_name] = int(value)\n elif arg_type == 'option':\n self.jira_args[arg_name] = {'value': value}\n # Complex type\n else:\n self.jira_args[arg_name] = {'name': value}\n\n def get_arbitrary_fields(self):\n # Clear jira_args\n self.reset_jira_args()\n\n for jira_field, value in self.rule.items():\n # If we find a field that is not covered by the set that we are aware of, it means it is either:\n # 1. A built-in supported field in JIRA that we don't have on our radar\n # 2. A custom field that a JIRA admin has configured\n if jira_field.startswith('jira_') and jira_field not in self.known_field_list and str(value)[:1] != '#':\n self.set_jira_arg(jira_field, value, self.jira_fields)\n if jira_field.startswith('jira_') and jira_field not in self.known_field_list and str(value)[:1] == '#':\n self.deferred_settings.append(jira_field)\n\n def get_priorities(self):\n \"\"\" Creates a mapping of priority index to id. \"\"\"\n priorities = self.client.priorities()\n self.priority_ids = {}\n for x in range(len(priorities)):\n self.priority_ids[x] = priorities[x].id\n\n def set_assignee(self, assignee):\n self.assignee = assignee\n if assignee:\n self.jira_args['assignee'] = {'name': assignee}\n elif 'assignee' in self.jira_args:\n self.jira_args.pop('assignee')\n\n def find_existing_ticket(self, matches):\n # Default title, get stripped search version\n if 'alert_subject' not in self.rule:\n title = self.create_default_title(matches, True)\n else:\n title = self.create_title(matches)\n\n if 'jira_ignore_in_title' in self.rule:\n title = title.replace(matches[0].get(self.rule['jira_ignore_in_title'], ''), '')\n\n # This is necessary for search to work. Other special characters and dashes\n # directly adjacent to words appear to be ok\n title = title.replace(' - ', ' ')\n title = title.replace('\\\\', '\\\\\\\\')\n\n date = (datetime.datetime.now() - datetime.timedelta(days=self.max_age)).strftime('%Y-%m-%d')\n jql = 'project=%s AND summary~\"%s\" and created >= \"%s\"' % (self.project, title, date)\n if self.bump_in_statuses:\n jql = '%s and status in (%s)' % (jql, ','.join([\"\\\"%s\\\"\" % status if ' ' in status else status for status\n in self.bump_in_statuses]))\n if self.bump_not_in_statuses:\n jql = '%s and status not in (%s)' % (jql, ','.join([\"\\\"%s\\\"\" % status if ' ' in status else status\n for status in self.bump_not_in_statuses]))\n try:\n issues = self.client.search_issues(jql)\n except JIRAError as e:\n logging.exception(\"Error while searching for JIRA ticket using jql '%s': %s\" % (jql, e))\n return None\n\n if len(issues):\n return issues[0]\n\n def comment_on_ticket(self, ticket, match):\n text = str(JiraFormattedMatchString(self.rule, match))\n timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field']))\n comment = \"This alert was triggered again at %s\\n%s\" % (timestamp, text)\n self.client.add_comment(ticket, comment)\n\n def transition_ticket(self, ticket):\n transitions = self.client.transitions(ticket)\n for t in transitions:\n if t['name'] == self.transition:\n self.client.transition_issue(ticket, t['id'])\n\n def alert(self, matches):\n # Reset arbitrary fields to pick up changes\n self.get_arbitrary_fields()\n if len(self.deferred_settings) > 0:\n fields = self.client.fields()\n for jira_field in self.deferred_settings:\n value = lookup_es_key(matches[0], self.rule[jira_field][1:])\n self.set_jira_arg(jira_field, value, fields)\n\n title = self.create_title(matches)\n\n if self.bump_tickets:\n ticket = self.find_existing_ticket(matches)\n if ticket:\n inactivity_datetime = ts_now() - datetime.timedelta(days=self.bump_after_inactivity)\n if ts_to_dt(ticket.fields.updated) >= inactivity_datetime:\n if self.pipeline is not None:\n self.pipeline['jira_ticket'] = None\n self.pipeline['jira_server'] = self.server\n return None\n elastalert_logger.info('Commenting on existing ticket %s' % (ticket.key))\n for match in matches:\n try:\n self.comment_on_ticket(ticket, match)\n except JIRAError as e:\n logging.exception(\"Error while commenting on ticket %s: %s\" % (ticket, e))\n if self.labels:\n for label in self.labels:\n try:\n ticket.fields.labels.append(label)\n except JIRAError as e:\n logging.exception(\"Error while appending labels to ticket %s: %s\" % (ticket, e))\n if self.transition:\n elastalert_logger.info('Transitioning existing ticket %s' % (ticket.key))\n try:\n self.transition_ticket(ticket)\n except JIRAError as e:\n logging.exception(\"Error while transitioning ticket %s: %s\" % (ticket, e))\n\n if self.pipeline is not None:\n self.pipeline['jira_ticket'] = ticket\n self.pipeline['jira_server'] = self.server\n return None\n if self.bump_only:\n return None\n\n self.jira_args['summary'] = title\n self.jira_args['description'] = self.create_alert_body(matches)\n\n try:\n self.issue = self.client.create_issue(**self.jira_args)\n\n # You can not add watchers on initial creation. Only as a follow-up action\n if self.watchers:\n for watcher in self.watchers:\n try:\n self.client.add_watcher(self.issue.key, watcher)\n except Exception as ex:\n # Re-raise the exception, preserve the stack-trace, and give some\n # context as to which watcher failed to be added\n raise Exception(\n \"Exception encountered when trying to add '{0}' as a watcher. Does the user exist?\\n{1}\" .format(\n watcher,\n ex\n )).with_traceback(sys.exc_info()[2])\n\n except JIRAError as e:\n raise EAException(\"Error creating JIRA ticket using jira_args (%s): %s\" % (self.jira_args, e))\n elastalert_logger.info(\"Opened Jira ticket: %s\" % (self.issue))\n\n if self.pipeline is not None:\n self.pipeline['jira_ticket'] = self.issue\n self.pipeline['jira_server'] = self.server\n\n def create_alert_body(self, matches):\n body = self.description + '\\n'\n body += self.get_aggregation_summary_text(matches)\n if self.rule.get('alert_text_type') != 'aggregation_summary_only':\n for match in matches:\n body += str(JiraFormattedMatchString(self.rule, match))\n if len(matches) > 1:\n body += '\\n----------------------------------------\\n'\n return body\n\n def get_aggregation_summary_text(self, matches):\n text = super(JiraAlerter, self).get_aggregation_summary_text(matches)\n if text:\n text = '{{noformat}}{0}{{noformat}}'.format(text)\n return text\n\n def create_default_title(self, matches, for_search=False):\n # If there is a query_key, use that in the title\n\n if 'query_key' in self.rule and lookup_es_key(matches[0], self.rule['query_key']):\n title = 'ElastAlert: %s matched %s' % (lookup_es_key(matches[0], self.rule['query_key']), self.rule['name'])\n else:\n title = 'ElastAlert: %s' % (self.rule['name'])\n\n if for_search:\n return title\n\n timestamp = matches[0].get(self.rule['timestamp_field'])\n if timestamp:\n title += ' - %s' % (pretty_ts(timestamp, self.rule.get('use_local_time')))\n\n # Add count for spikes\n count = matches[0].get('spike_count')\n if count:\n title += ' - %s+ events' % (count)\n\n return title\n\n def get_info(self):\n return {'type': 'jira'}\n\n\nclass CommandAlerter(Alerter):\n required_options = set(['command'])\n\n def __init__(self, *args):\n super(CommandAlerter, self).__init__(*args)\n\n self.last_command = []\n\n self.shell = False\n if isinstance(self.rule['command'], str):\n self.shell = True\n if '%' in self.rule['command']:\n logging.warning('Warning! You could be vulnerable to shell injection!')\n self.rule['command'] = [self.rule['command']]\n\n self.new_style_string_format = False\n if 'new_style_string_format' in self.rule and self.rule['new_style_string_format']:\n self.new_style_string_format = True\n\n def alert(self, matches):\n # Format the command and arguments\n try:\n command = [resolve_string(command_arg, matches[0]) for command_arg in self.rule['command']]\n self.last_command = command\n except KeyError as e:\n raise EAException(\"Error formatting command: %s\" % (e))\n\n # Run command and pipe data\n try:\n subp = subprocess.Popen(command, stdin=subprocess.PIPE, shell=self.shell)\n\n if self.rule.get('pipe_match_json'):\n match_json = json.dumps(matches, cls=DateTimeEncoder) + '\\n'\n stdout, stderr = subp.communicate(input=match_json.encode())\n elif self.rule.get('pipe_alert_text'):\n alert_text = self.create_alert_body(matches)\n stdout, stderr = subp.communicate(input=alert_text.encode())\n if self.rule.get(\"fail_on_non_zero_exit\", False) and subp.wait():\n raise EAException(\"Non-zero exit code while running command %s\" % (' '.join(command)))\n except OSError as e:\n raise EAException(\"Error while running command %s: %s\" % (' '.join(command), e))\n\n def get_info(self):\n return {'type': 'command',\n 'command': ' '.join(self.last_command)}\n\n\nclass SnsAlerter(Alerter):\n \"\"\" Send alert using AWS SNS service \"\"\"\n required_options = frozenset(['sns_topic_arn'])\n\n def __init__(self, *args):\n super(SnsAlerter, self).__init__(*args)\n self.sns_topic_arn = self.rule.get('sns_topic_arn', '')\n self.aws_access_key_id = self.rule.get('aws_access_key_id')\n self.aws_secret_access_key = self.rule.get('aws_secret_access_key')\n self.aws_region = self.rule.get('aws_region', 'us-east-1')\n self.profile = self.rule.get('boto_profile', None) # Deprecated\n self.profile = self.rule.get('aws_profile', None)\n\n def create_default_title(self, matches):\n subject = 'ElastAlert: %s' % (self.rule['name'])\n return subject\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n session = boto3.Session(\n aws_access_key_id=self.aws_access_key_id,\n aws_secret_access_key=self.aws_secret_access_key,\n region_name=self.aws_region,\n profile_name=self.profile\n )\n sns_client = session.client('sns')\n sns_client.publish(\n TopicArn=self.sns_topic_arn,\n Message=body,\n Subject=self.create_title(matches)\n )\n elastalert_logger.info(\"Sent sns notification to %s\" % (self.sns_topic_arn))\n\n\nclass HipChatAlerter(Alerter):\n \"\"\" Creates a HipChat room notification for each alert \"\"\"\n required_options = frozenset(['hipchat_auth_token', 'hipchat_room_id'])\n\n def __init__(self, rule):\n super(HipChatAlerter, self).__init__(rule)\n self.hipchat_msg_color = self.rule.get('hipchat_msg_color', 'red')\n self.hipchat_message_format = self.rule.get('hipchat_message_format', 'html')\n self.hipchat_auth_token = self.rule['hipchat_auth_token']\n self.hipchat_room_id = self.rule['hipchat_room_id']\n self.hipchat_domain = self.rule.get('hipchat_domain', 'api.hipchat.com')\n self.hipchat_ignore_ssl_errors = self.rule.get('hipchat_ignore_ssl_errors', False)\n self.hipchat_notify = self.rule.get('hipchat_notify', True)\n self.hipchat_from = self.rule.get('hipchat_from', '')\n self.url = 'https://%s/v2/room/%s/notification?auth_token=%s' % (\n self.hipchat_domain, self.hipchat_room_id, self.hipchat_auth_token)\n self.hipchat_proxy = self.rule.get('hipchat_proxy', None)\n\n def create_alert_body(self, matches):\n body = super(HipChatAlerter, self).create_alert_body(matches)\n\n # HipChat sends 400 bad request on messages longer than 10000 characters\n if self.hipchat_message_format == 'html':\n # Use appropriate line ending for text/html\n br = '
'\n body = body.replace('\\n', br)\n\n truncated_message = '
...(truncated)'\n truncate_to = 10000 - len(truncated_message)\n else:\n truncated_message = '..(truncated)'\n truncate_to = 10000 - len(truncated_message)\n\n if (len(body) > 9999):\n body = body[:truncate_to] + truncated_message\n\n return body\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n # Post to HipChat\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.hipchat_proxy} if self.hipchat_proxy else None\n payload = {\n 'color': self.hipchat_msg_color,\n 'message': body,\n 'message_format': self.hipchat_message_format,\n 'notify': self.hipchat_notify,\n 'from': self.hipchat_from\n }\n\n try:\n if self.hipchat_ignore_ssl_errors:\n requests.packages.urllib3.disable_warnings()\n\n if self.rule.get('hipchat_mentions', []):\n ping_users = self.rule.get('hipchat_mentions', [])\n ping_msg = payload.copy()\n ping_msg['message'] = \"ping {}\".format(\n \", \".join(\"@{}\".format(user) for user in ping_users)\n )\n ping_msg['message_format'] = \"text\"\n\n response = requests.post(\n self.url,\n data=json.dumps(ping_msg, cls=DateTimeEncoder),\n headers=headers,\n verify=not self.hipchat_ignore_ssl_errors,\n proxies=proxies)\n\n response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers,\n verify=not self.hipchat_ignore_ssl_errors,\n proxies=proxies)\n warnings.resetwarnings()\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to HipChat: %s\" % e)\n elastalert_logger.info(\"Alert sent to HipChat room %s\" % self.hipchat_room_id)\n\n def get_info(self):\n return {'type': 'hipchat',\n 'hipchat_room_id': self.hipchat_room_id}\n\n\nclass MsTeamsAlerter(Alerter):\n \"\"\" Creates a Microsoft Teams Conversation Message for each alert \"\"\"\n required_options = frozenset(['ms_teams_webhook_url', 'ms_teams_alert_summary'])\n\n def __init__(self, rule):\n super(MsTeamsAlerter, self).__init__(rule)\n self.ms_teams_webhook_url = self.rule['ms_teams_webhook_url']\n if isinstance(self.ms_teams_webhook_url, str):\n self.ms_teams_webhook_url = [self.ms_teams_webhook_url]\n self.ms_teams_proxy = self.rule.get('ms_teams_proxy', None)\n self.ms_teams_alert_summary = self.rule.get('ms_teams_alert_summary', 'ElastAlert Message')\n self.ms_teams_alert_fixed_width = self.rule.get('ms_teams_alert_fixed_width', False)\n self.ms_teams_theme_color = self.rule.get('ms_teams_theme_color', '')\n\n def format_body(self, body):\n if self.ms_teams_alert_fixed_width:\n body = body.replace('`', \"'\")\n body = \"```{0}```\".format('```\\n\\n```'.join(x for x in body.split('\\n'))).replace('\\n``````', '')\n return body\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n body = self.format_body(body)\n # post to Teams\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.ms_teams_proxy} if self.ms_teams_proxy else None\n payload = {\n '@type': 'MessageCard',\n '@context': 'http://schema.org/extensions',\n 'summary': self.ms_teams_alert_summary,\n 'title': self.create_title(matches),\n 'text': body\n }\n if self.ms_teams_theme_color != '':\n payload['themeColor'] = self.ms_teams_theme_color\n\n for url in self.ms_teams_webhook_url:\n try:\n response = requests.post(url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to ms teams: %s\" % e)\n elastalert_logger.info(\"Alert sent to MS Teams\")\n\n def get_info(self):\n return {'type': 'ms_teams',\n 'ms_teams_webhook_url': self.ms_teams_webhook_url}\n\n\nclass SlackAlerter(Alerter):\n \"\"\" Creates a Slack room message for each alert \"\"\"\n required_options = frozenset(['slack_webhook_url'])\n\n def __init__(self, rule):\n super(SlackAlerter, self).__init__(rule)\n self.slack_webhook_url = self.rule['slack_webhook_url']\n if isinstance(self.slack_webhook_url, str):\n self.slack_webhook_url = [self.slack_webhook_url]\n self.slack_proxy = self.rule.get('slack_proxy', None)\n self.slack_username_override = self.rule.get('slack_username_override', 'elastalert')\n self.slack_channel_override = self.rule.get('slack_channel_override', '')\n if isinstance(self.slack_channel_override, str):\n self.slack_channel_override = [self.slack_channel_override]\n self.slack_title_link = self.rule.get('slack_title_link', '')\n self.slack_title = self.rule.get('slack_title', '')\n self.slack_emoji_override = self.rule.get('slack_emoji_override', ':ghost:')\n self.slack_icon_url_override = self.rule.get('slack_icon_url_override', '')\n self.slack_msg_color = self.rule.get('slack_msg_color', 'danger')\n self.slack_parse_override = self.rule.get('slack_parse_override', 'none')\n self.slack_text_string = self.rule.get('slack_text_string', '')\n self.slack_alert_fields = self.rule.get('slack_alert_fields', '')\n self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False)\n self.slack_timeout = self.rule.get('slack_timeout', 10)\n self.slack_ca_certs = self.rule.get('slack_ca_certs')\n self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False)\n self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98')\n self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana')\n\n def format_body(self, body):\n # https://api.slack.com/docs/formatting\n return body\n\n def get_aggregation_summary_text__maximum_width(self):\n width = super(SlackAlerter, self).get_aggregation_summary_text__maximum_width()\n # Reduced maximum width for prettier Slack display.\n return min(width, 75)\n\n def get_aggregation_summary_text(self, matches):\n text = super(SlackAlerter, self).get_aggregation_summary_text(matches)\n if text:\n text = '```\\n{0}```\\n'.format(text)\n return text\n\n def populate_fields(self, matches):\n alert_fields = []\n for arg in self.slack_alert_fields:\n arg = copy.copy(arg)\n arg['value'] = lookup_es_key(matches[0], arg['value'])\n alert_fields.append(arg)\n return alert_fields\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n body = self.format_body(body)\n # post to slack\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.slack_proxy} if self.slack_proxy else None\n payload = {\n 'username': self.slack_username_override,\n 'parse': self.slack_parse_override,\n 'text': self.slack_text_string,\n 'attachments': [\n {\n 'color': self.slack_msg_color,\n 'title': self.create_title(matches),\n 'text': body,\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ]\n }\n\n # if we have defined fields, populate noteable fields for the alert\n if self.slack_alert_fields != '':\n payload['attachments'][0]['fields'] = self.populate_fields(matches)\n\n if self.slack_icon_url_override != '':\n payload['icon_url'] = self.slack_icon_url_override\n else:\n payload['icon_emoji'] = self.slack_emoji_override\n\n if self.slack_title != '':\n payload['attachments'][0]['title'] = self.slack_title\n\n if self.slack_title_link != '':\n payload['attachments'][0]['title_link'] = self.slack_title_link\n\n if self.slack_attach_kibana_discover_url:\n kibana_discover_url = lookup_es_key(matches[0], 'kibana_discover_url')\n if kibana_discover_url:\n payload['attachments'].append({\n 'color': self.slack_kibana_discover_color,\n 'title': self.slack_kibana_discover_title,\n 'title_link': kibana_discover_url\n })\n\n for url in self.slack_webhook_url:\n for channel_override in self.slack_channel_override:\n try:\n if self.slack_ca_certs:\n verify = self.slack_ca_certs\n else:\n verify = self.slack_ignore_ssl_errors\n if self.slack_ignore_ssl_errors:\n requests.packages.urllib3.disable_warnings()\n payload['channel'] = channel_override\n response = requests.post(\n url, data=json.dumps(payload, cls=DateTimeEncoder),\n headers=headers, verify=verify,\n proxies=proxies,\n timeout=self.slack_timeout)\n warnings.resetwarnings()\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to slack: %s\" % e)\n elastalert_logger.info(\"Alert '%s' sent to Slack\" % self.rule['name'])\n\n def get_info(self):\n return {'type': 'slack',\n 'slack_username_override': self.slack_username_override}\n\n\nclass MattermostAlerter(Alerter):\n \"\"\" Creates a Mattermsot post for each alert \"\"\"\n required_options = frozenset(['mattermost_webhook_url'])\n\n def __init__(self, rule):\n super(MattermostAlerter, self).__init__(rule)\n\n # HTTP config\n self.mattermost_webhook_url = self.rule['mattermost_webhook_url']\n if isinstance(self.mattermost_webhook_url, str):\n self.mattermost_webhook_url = [self.mattermost_webhook_url]\n self.mattermost_proxy = self.rule.get('mattermost_proxy', None)\n self.mattermost_ignore_ssl_errors = self.rule.get('mattermost_ignore_ssl_errors', False)\n\n # Override webhook config\n self.mattermost_username_override = self.rule.get('mattermost_username_override', 'elastalert')\n self.mattermost_channel_override = self.rule.get('mattermost_channel_override', '')\n self.mattermost_icon_url_override = self.rule.get('mattermost_icon_url_override', '')\n\n # Message properties\n self.mattermost_msg_pretext = self.rule.get('mattermost_msg_pretext', '')\n self.mattermost_msg_color = self.rule.get('mattermost_msg_color', 'danger')\n self.mattermost_msg_fields = self.rule.get('mattermost_msg_fields', '')\n\n def get_aggregation_summary_text__maximum_width(self):\n width = super(MattermostAlerter, self).get_aggregation_summary_text__maximum_width()\n # Reduced maximum width for prettier Mattermost display.\n return min(width, 75)\n\n def get_aggregation_summary_text(self, matches):\n text = super(MattermostAlerter, self).get_aggregation_summary_text(matches)\n if text:\n text = '```\\n{0}```\\n'.format(text)\n return text\n\n def populate_fields(self, matches):\n alert_fields = []\n missing = self.rule.get('alert_missing_value', '')\n for field in self.mattermost_msg_fields:\n field = copy.copy(field)\n if 'args' in field:\n args_values = [lookup_es_key(matches[0], arg) or missing for arg in field['args']]\n if 'value' in field:\n field['value'] = field['value'].format(*args_values)\n else:\n field['value'] = \"\\n\".join(str(arg) for arg in args_values)\n del(field['args'])\n alert_fields.append(field)\n return alert_fields\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n title = self.create_title(matches)\n\n # post to mattermost\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.mattermost_proxy} if self.mattermost_proxy else None\n payload = {\n 'attachments': [\n {\n 'fallback': \"{0}: {1}\".format(title, self.mattermost_msg_pretext),\n 'color': self.mattermost_msg_color,\n 'title': title,\n 'pretext': self.mattermost_msg_pretext,\n 'fields': []\n }\n ]\n }\n\n if self.rule.get('alert_text_type') == 'alert_text_only':\n payload['attachments'][0]['text'] = body\n else:\n payload['text'] = body\n\n if self.mattermost_msg_fields != '':\n payload['attachments'][0]['fields'] = self.populate_fields(matches)\n\n if self.mattermost_icon_url_override != '':\n payload['icon_url'] = self.mattermost_icon_url_override\n\n if self.mattermost_username_override != '':\n payload['username'] = self.mattermost_username_override\n\n if self.mattermost_channel_override != '':\n payload['channel'] = self.mattermost_channel_override\n\n for url in self.mattermost_webhook_url:\n try:\n if self.mattermost_ignore_ssl_errors:\n requests.urllib3.disable_warnings()\n\n response = requests.post(\n url, data=json.dumps(payload, cls=DateTimeEncoder),\n headers=headers, verify=not self.mattermost_ignore_ssl_errors,\n proxies=proxies)\n\n warnings.resetwarnings()\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to Mattermost: %s\" % e)\n elastalert_logger.info(\"Alert sent to Mattermost\")\n\n def get_info(self):\n return {'type': 'mattermost',\n 'mattermost_username_override': self.mattermost_username_override,\n 'mattermost_webhook_url': self.mattermost_webhook_url}\n\n\nclass PagerDutyAlerter(Alerter):\n \"\"\" Create an incident on PagerDuty for each alert \"\"\"\n required_options = frozenset(['pagerduty_service_key', 'pagerduty_client_name'])\n\n def __init__(self, rule):\n super(PagerDutyAlerter, self).__init__(rule)\n self.pagerduty_service_key = self.rule['pagerduty_service_key']\n self.pagerduty_client_name = self.rule['pagerduty_client_name']\n self.pagerduty_incident_key = self.rule.get('pagerduty_incident_key', '')\n self.pagerduty_incident_key_args = self.rule.get('pagerduty_incident_key_args', None)\n self.pagerduty_event_type = self.rule.get('pagerduty_event_type', 'trigger')\n self.pagerduty_proxy = self.rule.get('pagerduty_proxy', None)\n\n self.pagerduty_api_version = self.rule.get('pagerduty_api_version', 'v1')\n self.pagerduty_v2_payload_class = self.rule.get('pagerduty_v2_payload_class', '')\n self.pagerduty_v2_payload_class_args = self.rule.get('pagerduty_v2_payload_class_args', None)\n self.pagerduty_v2_payload_component = self.rule.get('pagerduty_v2_payload_component', '')\n self.pagerduty_v2_payload_component_args = self.rule.get('pagerduty_v2_payload_component_args', None)\n self.pagerduty_v2_payload_group = self.rule.get('pagerduty_v2_payload_group', '')\n self.pagerduty_v2_payload_group_args = self.rule.get('pagerduty_v2_payload_group_args', None)\n self.pagerduty_v2_payload_severity = self.rule.get('pagerduty_v2_payload_severity', 'critical')\n self.pagerduty_v2_payload_source = self.rule.get('pagerduty_v2_payload_source', 'ElastAlert')\n self.pagerduty_v2_payload_source_args = self.rule.get('pagerduty_v2_payload_source_args', None)\n\n if self.pagerduty_api_version == 'v2':\n self.url = 'https://events.pagerduty.com/v2/enqueue'\n else:\n self.url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n # post to pagerduty\n headers = {'content-type': 'application/json'}\n if self.pagerduty_api_version == 'v2':\n payload = {\n 'routing_key': self.pagerduty_service_key,\n 'event_action': self.pagerduty_event_type,\n 'dedup_key': self.get_incident_key(matches),\n 'client': self.pagerduty_client_name,\n 'payload': {\n 'class': self.resolve_formatted_key(self.pagerduty_v2_payload_class,\n self.pagerduty_v2_payload_class_args,\n matches),\n 'component': self.resolve_formatted_key(self.pagerduty_v2_payload_component,\n self.pagerduty_v2_payload_component_args,\n matches),\n 'group': self.resolve_formatted_key(self.pagerduty_v2_payload_group,\n self.pagerduty_v2_payload_group_args,\n matches),\n 'severity': self.pagerduty_v2_payload_severity,\n 'source': self.resolve_formatted_key(self.pagerduty_v2_payload_source,\n self.pagerduty_v2_payload_source_args,\n matches),\n 'summary': self.create_title(matches),\n 'custom_details': {\n 'information': body,\n },\n },\n }\n match_timestamp = lookup_es_key(matches[0], self.rule.get('timestamp_field', '@timestamp'))\n if match_timestamp:\n payload['payload']['timestamp'] = match_timestamp\n else:\n payload = {\n 'service_key': self.pagerduty_service_key,\n 'description': self.create_title(matches),\n 'event_type': self.pagerduty_event_type,\n 'incident_key': self.get_incident_key(matches),\n 'client': self.pagerduty_client_name,\n 'details': {\n \"information\": body,\n },\n }\n\n # set https proxy, if it was provided\n proxies = {'https': self.pagerduty_proxy} if self.pagerduty_proxy else None\n try:\n response = requests.post(\n self.url,\n data=json.dumps(payload, cls=DateTimeEncoder, ensure_ascii=False),\n headers=headers,\n proxies=proxies\n )\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to pagerduty: %s\" % e)\n\n if self.pagerduty_event_type == 'trigger':\n elastalert_logger.info(\"Trigger sent to PagerDuty\")\n elif self.pagerduty_event_type == 'resolve':\n elastalert_logger.info(\"Resolve sent to PagerDuty\")\n elif self.pagerduty_event_type == 'acknowledge':\n elastalert_logger.info(\"acknowledge sent to PagerDuty\")\n\n def resolve_formatted_key(self, key, args, matches):\n if args:\n key_values = [lookup_es_key(matches[0], arg) for arg in args]\n\n # Populate values with rule level properties too\n for i in range(len(key_values)):\n if key_values[i] is None:\n key_value = self.rule.get(args[i])\n if key_value:\n key_values[i] = key_value\n\n missing = self.rule.get('alert_missing_value', '')\n key_values = [missing if val is None else val for val in key_values]\n return key.format(*key_values)\n else:\n return key\n\n def get_incident_key(self, matches):\n if self.pagerduty_incident_key_args:\n incident_key_values = [lookup_es_key(matches[0], arg) for arg in self.pagerduty_incident_key_args]\n\n # Populate values with rule level properties too\n for i in range(len(incident_key_values)):\n if incident_key_values[i] is None:\n key_value = self.rule.get(self.pagerduty_incident_key_args[i])\n if key_value:\n incident_key_values[i] = key_value\n\n missing = self.rule.get('alert_missing_value', '')\n incident_key_values = [missing if val is None else val for val in incident_key_values]\n return self.pagerduty_incident_key.format(*incident_key_values)\n else:\n return self.pagerduty_incident_key\n\n def get_info(self):\n return {'type': 'pagerduty',\n 'pagerduty_client_name': self.pagerduty_client_name}\n\n\nclass PagerTreeAlerter(Alerter):\n \"\"\" Creates a PagerTree Incident for each alert \"\"\"\n required_options = frozenset(['pagertree_integration_url'])\n\n def __init__(self, rule):\n super(PagerTreeAlerter, self).__init__(rule)\n self.url = self.rule['pagertree_integration_url']\n self.pagertree_proxy = self.rule.get('pagertree_proxy', None)\n\n def alert(self, matches):\n # post to pagertree\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.pagertree_proxy} if self.pagertree_proxy else None\n payload = {\n \"event_type\": \"create\",\n \"Id\": str(uuid.uuid4()),\n \"Title\": self.create_title(matches),\n \"Description\": self.create_alert_body(matches)\n }\n\n try:\n response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to PagerTree: %s\" % e)\n elastalert_logger.info(\"Trigger sent to PagerTree\")\n\n def get_info(self):\n return {'type': 'pagertree',\n 'pagertree_integration_url': self.url}\n\n\nclass ExotelAlerter(Alerter):\n required_options = frozenset(['exotel_account_sid', 'exotel_auth_token', 'exotel_to_number', 'exotel_from_number'])\n\n def __init__(self, rule):\n super(ExotelAlerter, self).__init__(rule)\n self.exotel_account_sid = self.rule['exotel_account_sid']\n self.exotel_auth_token = self.rule['exotel_auth_token']\n self.exotel_to_number = self.rule['exotel_to_number']\n self.exotel_from_number = self.rule['exotel_from_number']\n self.sms_body = self.rule.get('exotel_message_body', '')\n\n def alert(self, matches):\n client = Exotel(self.exotel_account_sid, self.exotel_auth_token)\n\n try:\n message_body = self.rule['name'] + self.sms_body\n response = client.sms(self.rule['exotel_from_number'], self.rule['exotel_to_number'], message_body)\n if response != 200:\n raise EAException(\"Error posting to Exotel, response code is %s\" % response)\n except RequestException:\n raise EAException(\"Error posting to Exotel\").with_traceback(sys.exc_info()[2])\n elastalert_logger.info(\"Trigger sent to Exotel\")\n\n def get_info(self):\n return {'type': 'exotel', 'exotel_account': self.exotel_account_sid}\n\n\nclass TwilioAlerter(Alerter):\n required_options = frozenset(['twilio_account_sid', 'twilio_auth_token', 'twilio_to_number', 'twilio_from_number'])\n\n def __init__(self, rule):\n super(TwilioAlerter, self).__init__(rule)\n self.twilio_account_sid = self.rule['twilio_account_sid']\n self.twilio_auth_token = self.rule['twilio_auth_token']\n self.twilio_to_number = self.rule['twilio_to_number']\n self.twilio_from_number = self.rule['twilio_from_number']\n\n def alert(self, matches):\n client = TwilioClient(self.twilio_account_sid, self.twilio_auth_token)\n\n try:\n client.messages.create(body=self.rule['name'],\n to=self.twilio_to_number,\n from_=self.twilio_from_number)\n\n except TwilioRestException as e:\n raise EAException(\"Error posting to twilio: %s\" % e)\n\n elastalert_logger.info(\"Trigger sent to Twilio\")\n\n def get_info(self):\n return {'type': 'twilio',\n 'twilio_client_name': self.twilio_from_number}\n\n\nclass VictorOpsAlerter(Alerter):\n \"\"\" Creates a VictorOps Incident for each alert \"\"\"\n required_options = frozenset(['victorops_api_key', 'victorops_routing_key', 'victorops_message_type'])\n\n def __init__(self, rule):\n super(VictorOpsAlerter, self).__init__(rule)\n self.victorops_api_key = self.rule['victorops_api_key']\n self.victorops_routing_key = self.rule['victorops_routing_key']\n self.victorops_message_type = self.rule['victorops_message_type']\n self.victorops_entity_id = self.rule.get('victorops_entity_id', None)\n self.victorops_entity_display_name = self.rule.get('victorops_entity_display_name', 'no entity display name')\n self.url = 'https://alert.victorops.com/integrations/generic/20131114/alert/%s/%s' % (\n self.victorops_api_key, self.victorops_routing_key)\n self.victorops_proxy = self.rule.get('victorops_proxy', None)\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n # post to victorops\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.victorops_proxy} if self.victorops_proxy else None\n payload = {\n \"message_type\": self.victorops_message_type,\n \"entity_display_name\": self.victorops_entity_display_name,\n \"monitoring_tool\": \"ElastAlert\",\n \"state_message\": body\n }\n if self.victorops_entity_id:\n payload[\"entity_id\"] = self.victorops_entity_id\n\n try:\n response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to VictorOps: %s\" % e)\n elastalert_logger.info(\"Trigger sent to VictorOps\")\n\n def get_info(self):\n return {'type': 'victorops',\n 'victorops_routing_key': self.victorops_routing_key}\n\n\nclass TelegramAlerter(Alerter):\n \"\"\" Send a Telegram message via bot api for each alert \"\"\"\n required_options = frozenset(['telegram_bot_token', 'telegram_room_id'])\n\n def __init__(self, rule):\n super(TelegramAlerter, self).__init__(rule)\n self.telegram_bot_token = self.rule['telegram_bot_token']\n self.telegram_room_id = self.rule['telegram_room_id']\n self.telegram_api_url = self.rule.get('telegram_api_url', 'api.telegram.org')\n self.url = 'https://%s/bot%s/%s' % (self.telegram_api_url, self.telegram_bot_token, \"sendMessage\")\n self.telegram_proxy = self.rule.get('telegram_proxy', None)\n self.telegram_proxy_login = self.rule.get('telegram_proxy_login', None)\n self.telegram_proxy_password = self.rule.get('telegram_proxy_pass', None)\n\n def alert(self, matches):\n body = 'āš  *%s* āš  ```\\n' % (self.create_title(matches))\n for match in matches:\n body += str(BasicMatchString(self.rule, match))\n # Separate text of aggregated alerts with dashes\n if len(matches) > 1:\n body += '\\n----------------------------------------\\n'\n if len(body) > 4095:\n body = body[0:4000] + \"\\nāš  *message was cropped according to telegram limits!* āš \"\n body += ' ```'\n\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.telegram_proxy} if self.telegram_proxy else None\n auth = HTTPProxyAuth(self.telegram_proxy_login, self.telegram_proxy_password) if self.telegram_proxy_login else None\n payload = {\n 'chat_id': self.telegram_room_id,\n 'text': body,\n 'parse_mode': 'markdown',\n 'disable_web_page_preview': True\n }\n\n try:\n response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies, auth=auth)\n warnings.resetwarnings()\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to Telegram: %s. Details: %s\" % (e, \"\" if e.response is None else e.response.text))\n\n elastalert_logger.info(\n \"Alert sent to Telegram room %s\" % self.telegram_room_id)\n\n def get_info(self):\n return {'type': 'telegram',\n 'telegram_room_id': self.telegram_room_id}\n\n\nclass GoogleChatAlerter(Alerter):\n \"\"\" Send a notification via Google Chat webhooks \"\"\"\n required_options = frozenset(['googlechat_webhook_url'])\n\n def __init__(self, rule):\n super(GoogleChatAlerter, self).__init__(rule)\n self.googlechat_webhook_url = self.rule['googlechat_webhook_url']\n if isinstance(self.googlechat_webhook_url, str):\n self.googlechat_webhook_url = [self.googlechat_webhook_url]\n self.googlechat_format = self.rule.get('googlechat_format', 'basic')\n self.googlechat_header_title = self.rule.get('googlechat_header_title', None)\n self.googlechat_header_subtitle = self.rule.get('googlechat_header_subtitle', None)\n self.googlechat_header_image = self.rule.get('googlechat_header_image', None)\n self.googlechat_footer_kibanalink = self.rule.get('googlechat_footer_kibanalink', None)\n\n def create_header(self):\n header = None\n if self.googlechat_header_title:\n header = {\n \"title\": self.googlechat_header_title,\n \"subtitle\": self.googlechat_header_subtitle,\n \"imageUrl\": self.googlechat_header_image\n }\n return header\n\n def create_footer(self):\n footer = None\n if self.googlechat_footer_kibanalink:\n footer = {\"widgets\": [{\n \"buttons\": [{\n \"textButton\": {\n \"text\": \"VISIT KIBANA\",\n \"onClick\": {\n \"openLink\": {\n \"url\": self.googlechat_footer_kibanalink\n }\n }\n }\n }]\n }]\n }\n return footer\n\n def create_card(self, matches):\n card = {\"cards\": [{\n \"sections\": [{\n \"widgets\": [\n {\"textParagraph\": {\"text\": self.create_alert_body(matches)}}\n ]}\n ]}\n ]}\n\n # Add the optional header\n header = self.create_header()\n if header:\n card['cards'][0]['header'] = header\n\n # Add the optional footer\n footer = self.create_footer()\n if footer:\n card['cards'][0]['sections'].append(footer)\n return card\n\n def create_basic(self, matches):\n body = self.create_alert_body(matches)\n return {'text': body}\n\n def alert(self, matches):\n # Format message\n if self.googlechat_format == 'card':\n message = self.create_card(matches)\n else:\n message = self.create_basic(matches)\n\n # Post to webhook\n headers = {'content-type': 'application/json'}\n for url in self.googlechat_webhook_url:\n try:\n response = requests.post(url, data=json.dumps(message), headers=headers)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to google chat: {}\".format(e))\n elastalert_logger.info(\"Alert sent to Google Chat!\")\n\n def get_info(self):\n return {'type': 'googlechat',\n 'googlechat_webhook_url': self.googlechat_webhook_url}\n\n\nclass GitterAlerter(Alerter):\n \"\"\" Creates a Gitter activity message for each alert \"\"\"\n required_options = frozenset(['gitter_webhook_url'])\n\n def __init__(self, rule):\n super(GitterAlerter, self).__init__(rule)\n self.gitter_webhook_url = self.rule['gitter_webhook_url']\n self.gitter_proxy = self.rule.get('gitter_proxy', None)\n self.gitter_msg_level = self.rule.get('gitter_msg_level', 'error')\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n\n # post to Gitter\n headers = {'content-type': 'application/json'}\n # set https proxy, if it was provided\n proxies = {'https': self.gitter_proxy} if self.gitter_proxy else None\n payload = {\n 'message': body,\n 'level': self.gitter_msg_level\n }\n\n try:\n response = requests.post(self.gitter_webhook_url, json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to Gitter: %s\" % e)\n elastalert_logger.info(\"Alert sent to Gitter\")\n\n def get_info(self):\n return {'type': 'gitter',\n 'gitter_webhook_url': self.gitter_webhook_url}\n\n\nclass ServiceNowAlerter(Alerter):\n \"\"\" Creates a ServiceNow alert \"\"\"\n required_options = set([\n 'username',\n 'password',\n 'servicenow_rest_url',\n 'short_description',\n 'comments',\n 'assignment_group',\n 'category',\n 'subcategory',\n 'cmdb_ci',\n 'caller_id'\n ])\n\n def __init__(self, rule):\n super(ServiceNowAlerter, self).__init__(rule)\n self.servicenow_rest_url = self.rule['servicenow_rest_url']\n self.servicenow_proxy = self.rule.get('servicenow_proxy', None)\n\n def alert(self, matches):\n for match in matches:\n # Parse everything into description.\n description = str(BasicMatchString(self.rule, match))\n\n # Set proper headers\n headers = {\n \"Content-Type\": \"application/json\",\n \"Accept\": \"application/json;charset=utf-8\"\n }\n proxies = {'https': self.servicenow_proxy} if self.servicenow_proxy else None\n payload = {\n \"description\": description,\n \"short_description\": self.rule['short_description'],\n \"comments\": self.rule['comments'],\n \"assignment_group\": self.rule['assignment_group'],\n \"category\": self.rule['category'],\n \"subcategory\": self.rule['subcategory'],\n \"cmdb_ci\": self.rule['cmdb_ci'],\n \"caller_id\": self.rule[\"caller_id\"]\n }\n try:\n response = requests.post(\n self.servicenow_rest_url,\n auth=(self.rule['username'], self.rule['password']),\n headers=headers,\n data=json.dumps(payload, cls=DateTimeEncoder),\n proxies=proxies\n )\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to ServiceNow: %s\" % e)\n elastalert_logger.info(\"Alert sent to ServiceNow\")\n\n def get_info(self):\n return {'type': 'ServiceNow',\n 'self.servicenow_rest_url': self.servicenow_rest_url}\n\n\nclass AlertaAlerter(Alerter):\n \"\"\" Creates an Alerta event for each alert \"\"\"\n required_options = frozenset(['alerta_api_url'])\n\n def __init__(self, rule):\n super(AlertaAlerter, self).__init__(rule)\n\n # Setup defaul parameters\n self.url = self.rule.get('alerta_api_url', None)\n self.api_key = self.rule.get('alerta_api_key', None)\n self.timeout = self.rule.get('alerta_timeout', 86400)\n self.use_match_timestamp = self.rule.get('alerta_use_match_timestamp', False)\n self.use_qk_as_resource = self.rule.get('alerta_use_qk_as_resource', False)\n self.verify_ssl = not self.rule.get('alerta_api_skip_ssl', False)\n self.missing_text = self.rule.get('alert_missing_value', '')\n\n # Fill up default values of the API JSON payload\n self.severity = self.rule.get('alerta_severity', 'warning')\n self.resource = self.rule.get('alerta_resource', 'elastalert')\n self.environment = self.rule.get('alerta_environment', 'Production')\n self.origin = self.rule.get('alerta_origin', 'elastalert')\n self.service = self.rule.get('alerta_service', ['elastalert'])\n self.text = self.rule.get('alerta_text', 'elastalert')\n self.type = self.rule.get('alerta_type', 'elastalert')\n self.event = self.rule.get('alerta_event', 'elastalert')\n self.correlate = self.rule.get('alerta_correlate', [])\n self.tags = self.rule.get('alerta_tags', [])\n self.group = self.rule.get('alerta_group', '')\n self.attributes_keys = self.rule.get('alerta_attributes_keys', [])\n self.attributes_values = self.rule.get('alerta_attributes_values', [])\n self.value = self.rule.get('alerta_value', '')\n\n def alert(self, matches):\n # Override the resource if requested\n if self.use_qk_as_resource and 'query_key' in self.rule and lookup_es_key(matches[0], self.rule['query_key']):\n self.resource = lookup_es_key(matches[0], self.rule['query_key'])\n\n headers = {'content-type': 'application/json'}\n if self.api_key is not None:\n headers['Authorization'] = 'Key %s' % (self.rule['alerta_api_key'])\n alerta_payload = self.get_json_payload(matches[0])\n\n try:\n response = requests.post(self.url, data=alerta_payload, headers=headers, verify=self.verify_ssl)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to Alerta: %s\" % e)\n elastalert_logger.info(\"Alert sent to Alerta\")\n\n def create_default_title(self, matches):\n title = '%s' % (self.rule['name'])\n # If the rule has a query_key, add that value\n if 'query_key' in self.rule:\n qk = matches[0].get(self.rule['query_key'])\n if qk:\n title += '.%s' % (qk)\n return title\n\n def get_info(self):\n return {'type': 'alerta',\n 'alerta_url': self.url}\n\n def get_json_payload(self, match):\n \"\"\"\n Builds the API Create Alert body, as in\n http://alerta.readthedocs.io/en/latest/api/reference.html#create-an-alert\n\n For the values that could have references to fields on the match, resolve those references.\n\n \"\"\"\n\n # Using default text and event title if not defined in rule\n alerta_text = self.rule['type'].get_match_str([match]) if self.text == '' else resolve_string(self.text, match, self.missing_text)\n alerta_event = self.create_default_title([match]) if self.event == '' else resolve_string(self.event, match, self.missing_text)\n\n match_timestamp = lookup_es_key(match, self.rule.get('timestamp_field', '@timestamp'))\n if match_timestamp is None:\n match_timestamp = datetime.datetime.now().strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")\n if self.use_match_timestamp:\n createTime = ts_to_dt(match_timestamp).strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")\n else:\n createTime = datetime.datetime.now().strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")\n\n alerta_payload_dict = {\n 'resource': resolve_string(self.resource, match, self.missing_text),\n 'severity': self.severity,\n 'timeout': self.timeout,\n 'createTime': createTime,\n 'type': self.type,\n 'environment': resolve_string(self.environment, match, self.missing_text),\n 'origin': resolve_string(self.origin, match, self.missing_text),\n 'group': resolve_string(self.group, match, self.missing_text),\n 'event': alerta_event,\n 'text': alerta_text,\n 'value': resolve_string(self.value, match, self.missing_text),\n 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service],\n 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags],\n 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate],\n 'attributes': dict(list(zip(self.attributes_keys,\n [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values]))),\n 'rawData': self.create_alert_body([match]),\n }\n\n try:\n payload = json.dumps(alerta_payload_dict, cls=DateTimeEncoder)\n except Exception as e:\n raise Exception(\"Error building Alerta request: %s\" % e)\n return payload\n\n\nclass HTTPPostAlerter(Alerter):\n \"\"\" Requested elasticsearch indices are sent by HTTP POST. Encoded with JSON. \"\"\"\n\n def __init__(self, rule):\n super(HTTPPostAlerter, self).__init__(rule)\n post_url = self.rule.get('http_post_url')\n if isinstance(post_url, str):\n post_url = [post_url]\n self.post_url = post_url\n self.post_proxy = self.rule.get('http_post_proxy')\n self.post_payload = self.rule.get('http_post_payload', {})\n self.post_static_payload = self.rule.get('http_post_static_payload', {})\n self.post_all_values = self.rule.get('http_post_all_values', not self.post_payload)\n self.post_http_headers = self.rule.get('http_post_headers', {})\n self.timeout = self.rule.get('http_post_timeout', 10)\n\n def alert(self, matches):\n \"\"\" Each match will trigger a POST to the specified endpoint(s). \"\"\"\n for match in matches:\n payload = match if self.post_all_values else {}\n payload.update(self.post_static_payload)\n for post_key, es_key in list(self.post_payload.items()):\n payload[post_key] = lookup_es_key(match, es_key)\n headers = {\n \"Content-Type\": \"application/json\",\n \"Accept\": \"application/json;charset=utf-8\"\n }\n headers.update(self.post_http_headers)\n proxies = {'https': self.post_proxy} if self.post_proxy else None\n for url in self.post_url:\n try:\n response = requests.post(url, data=json.dumps(payload, cls=DateTimeEncoder),\n headers=headers, proxies=proxies, timeout=self.timeout)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting HTTP Post alert: %s\" % e)\n elastalert_logger.info(\"HTTP Post alert sent.\")\n\n def get_info(self):\n return {'type': 'http_post',\n 'http_post_webhook_url': self.post_url}\n\n\nclass StrideHTMLParser(HTMLParser):\n \"\"\"Parse html into stride's fabric structure\"\"\"\n\n def __init__(self):\n \"\"\"\n Define a couple markup place holders.\n \"\"\"\n self.content = []\n self.mark = None\n HTMLParser.__init__(self)\n\n def handle_starttag(self, tag, attrs):\n \"\"\"Identify and verify starting tag is fabric compatible.\"\"\"\n if tag == 'b' or tag == 'strong':\n self.mark = dict(type='strong')\n if tag == 'u':\n self.mark = dict(type='underline')\n if tag == 'a':\n self.mark = dict(type='link', attrs=dict(attrs))\n\n def handle_endtag(self, tag):\n \"\"\"Clear mark on endtag.\"\"\"\n self.mark = None\n\n def handle_data(self, data):\n \"\"\"Construct data node for our data.\"\"\"\n node = dict(type='text', text=data)\n if self.mark:\n node['marks'] = [self.mark]\n self.content.append(node)\n\n\nclass StrideAlerter(Alerter):\n \"\"\" Creates a Stride conversation message for each alert \"\"\"\n\n required_options = frozenset(\n ['stride_access_token', 'stride_cloud_id', 'stride_conversation_id'])\n\n def __init__(self, rule):\n super(StrideAlerter, self).__init__(rule)\n\n self.stride_access_token = self.rule['stride_access_token']\n self.stride_cloud_id = self.rule['stride_cloud_id']\n self.stride_conversation_id = self.rule['stride_conversation_id']\n self.stride_ignore_ssl_errors = self.rule.get('stride_ignore_ssl_errors', False)\n self.stride_proxy = self.rule.get('stride_proxy', None)\n self.url = 'https://api.atlassian.com/site/%s/conversation/%s/message' % (\n self.stride_cloud_id, self.stride_conversation_id)\n\n def alert(self, matches):\n body = self.create_alert_body(matches).strip()\n\n # parse body with StrideHTMLParser\n parser = StrideHTMLParser()\n parser.feed(body)\n\n # Post to Stride\n headers = {\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(self.stride_access_token)\n }\n\n # set https proxy, if it was provided\n proxies = {'https': self.stride_proxy} if self.stride_proxy else None\n\n # build stride json payload\n # https://developer.atlassian.com/cloud/stride/apis/document/structure/\n payload = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': parser.content}\n ]}\n ]}}\n\n try:\n if self.stride_ignore_ssl_errors:\n requests.packages.urllib3.disable_warnings()\n response = requests.post(\n self.url, data=json.dumps(payload, cls=DateTimeEncoder),\n headers=headers, verify=not self.stride_ignore_ssl_errors,\n proxies=proxies)\n warnings.resetwarnings()\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to Stride: %s\" % e)\n elastalert_logger.info(\n \"Alert sent to Stride conversation %s\" % self.stride_conversation_id)\n\n def get_info(self):\n return {'type': 'stride',\n 'stride_cloud_id': self.stride_cloud_id,\n 'stride_converstation_id': self.stride_converstation_id}\n\n\nclass LineNotifyAlerter(Alerter):\n \"\"\" Created a Line Notify for each alert \"\"\"\n required_option = frozenset([\"linenotify_access_token\"])\n\n def __init__(self, rule):\n super(LineNotifyAlerter, self).__init__(rule)\n self.linenotify_access_token = self.rule[\"linenotify_access_token\"]\n\n def alert(self, matches):\n body = self.create_alert_body(matches)\n # post to Line Notify\n headers = {\n \"Content-Type\": \"application/x-www-form-urlencoded\",\n \"Authorization\": \"Bearer {}\".format(self.linenotify_access_token)\n }\n payload = {\n \"message\": body\n }\n try:\n response = requests.post(\"https://notify-api.line.me/api/notify\", data=payload, headers=headers)\n response.raise_for_status()\n except RequestException as e:\n raise EAException(\"Error posting to Line Notify: %s\" % e)\n elastalert_logger.info(\"Alert sent to Line Notify\")\n\n def get_info(self):\n return {\"type\": \"linenotify\", \"linenotify_access_token\": self.linenotify_access_token}\n\n\nclass HiveAlerter(Alerter):\n \"\"\"\n Use matched data to create alerts containing observables in an instance of TheHive\n \"\"\"\n\n required_options = set(['hive_connection', 'hive_alert_config'])\n\n def alert(self, matches):\n\n connection_details = self.rule['hive_connection']\n\n for match in matches:\n context = {'rule': self.rule, 'match': match}\n\n artifacts = []\n for mapping in self.rule.get('hive_observable_data_mapping', []):\n for observable_type, match_data_key in mapping.items():\n try:\n match_data_keys = re.findall(r'\\{match\\[([^\\]]*)\\]', match_data_key)\n rule_data_keys = re.findall(r'\\{rule\\[([^\\]]*)\\]', match_data_key)\n data_keys = match_data_keys + rule_data_keys\n context_keys = list(context['match'].keys()) + list(context['rule'].keys())\n if all([True if k in context_keys else False for k in data_keys]):\n artifact = {'tlp': 2, 'tags': [], 'message': None, 'dataType': observable_type,\n 'data': match_data_key.format(**context)}\n artifacts.append(artifact)\n except KeyError:\n raise KeyError('\\nformat string\\n{}\\nmatch data\\n{}'.format(match_data_key, context))\n\n alert_config = {\n 'artifacts': artifacts,\n 'sourceRef': str(uuid.uuid4())[0:6],\n 'customFields': {},\n 'caseTemplate': None,\n 'title': '{rule[index]}_{rule[name]}'.format(**context),\n 'date': int(time.time()) * 1000\n }\n alert_config.update(self.rule.get('hive_alert_config', {}))\n custom_fields = {}\n for alert_config_field, alert_config_value in alert_config.items():\n if alert_config_field == 'customFields':\n n = 0\n for cf_key, cf_value in alert_config_value.items():\n cf = {'order': n, cf_value['type']: cf_value['value'].format(**context)}\n n += 1\n custom_fields[cf_key] = cf\n elif isinstance(alert_config_value, str):\n alert_config[alert_config_field] = alert_config_value.format(**context)\n elif isinstance(alert_config_value, (list, tuple)):\n formatted_list = []\n for element in alert_config_value:\n try:\n formatted_list.append(element.format(**context))\n except (AttributeError, KeyError, IndexError):\n formatted_list.append(element)\n alert_config[alert_config_field] = formatted_list\n if custom_fields:\n alert_config['customFields'] = custom_fields\n\n alert_body = json.dumps(alert_config, indent=4, sort_keys=True)\n req = '{}:{}/api/alert'.format(connection_details['hive_host'], connection_details['hive_port'])\n headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer {}'.format(connection_details.get('hive_apikey', ''))}\n proxies = connection_details.get('hive_proxies', {'http': '', 'https': ''})\n verify = connection_details.get('hive_verify', False)\n response = requests.post(req, headers=headers, data=alert_body, proxies=proxies, verify=verify)\n\n if response.status_code != 201:\n raise Exception('alert not successfully created in TheHive\\n{}'.format(response.text))\n\n def get_info(self):\n\n return {\n 'type': 'hivealerter',\n 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '')\n }\n" }, { "path": "elastalert/auth.py", "content": "# -*- coding: utf-8 -*-\nimport os\nimport boto3\nfrom aws_requests_auth.aws_auth import AWSRequestsAuth\n\n\nclass RefeshableAWSRequestsAuth(AWSRequestsAuth):\n \"\"\"\n A class ensuring that AWS request signing uses a refreshed credential\n \"\"\"\n\n def __init__(self,\n refreshable_credential,\n aws_host,\n aws_region,\n aws_service):\n \"\"\"\n :param refreshable_credential: A credential class that refreshes STS or IAM Instance Profile credentials\n :type refreshable_credential: :class:`botocore.credentials.RefreshableCredentials`\n \"\"\"\n self.refreshable_credential = refreshable_credential\n self.aws_host = aws_host\n self.aws_region = aws_region\n self.service = aws_service\n\n @property\n def aws_access_key(self):\n return self.refreshable_credential.access_key\n\n @property\n def aws_secret_access_key(self):\n return self.refreshable_credential.secret_key\n\n @property\n def aws_token(self):\n return self.refreshable_credential.token\n\n\nclass Auth(object):\n\n def __call__(self, host, username, password, aws_region, profile_name):\n \"\"\" Return the authorization header.\n\n :param host: Elasticsearch host.\n :param username: Username used for authenticating the requests to Elasticsearch.\n :param password: Password used for authenticating the requests to Elasticsearch.\n :param aws_region: AWS Region to use. Only required when signing requests.\n :param profile_name: AWS profile to use for connecting. Only required when signing requests.\n \"\"\"\n if username and password:\n return username + ':' + password\n\n if not aws_region and not os.environ.get('AWS_DEFAULT_REGION'):\n return None\n\n session = boto3.session.Session(profile_name=profile_name, region_name=aws_region)\n\n return RefeshableAWSRequestsAuth(\n refreshable_credential=session.get_credentials(),\n aws_host=host,\n aws_region=session.region_name,\n aws_service='es')\n" }, { "path": "elastalert/config.py", "content": "# -*- coding: utf-8 -*-\nimport datetime\nimport logging\nimport logging.config\n\nfrom envparse import Env\nfrom staticconf.loader import yaml_loader\n\nfrom . import loaders\nfrom .util import EAException\nfrom .util import elastalert_logger\nfrom .util import get_module\n\n# Required global (config.yaml) configuration options\nrequired_globals = frozenset(['run_every', 'es_host', 'es_port', 'writeback_index', 'buffer_time'])\n\n# Settings that can be derived from ENV variables\nenv_settings = {'ES_USE_SSL': 'use_ssl',\n 'ES_PASSWORD': 'es_password',\n 'ES_USERNAME': 'es_username',\n 'ES_HOST': 'es_host',\n 'ES_PORT': 'es_port',\n 'ES_URL_PREFIX': 'es_url_prefix'}\n\nenv = Env(ES_USE_SSL=bool)\n\n\n# Used to map the names of rule loaders to their classes\nloader_mapping = {\n 'file': loaders.FileRulesLoader,\n}\n\n\ndef load_conf(args, defaults=None, overwrites=None):\n \"\"\" Creates a conf dictionary for ElastAlerter. Loads the global\n config file and then each rule found in rules_folder.\n\n :param args: The parsed arguments to ElastAlert\n :param defaults: Dictionary of default conf values\n :param overwrites: Dictionary of conf values to override\n :return: The global configuration, a dictionary.\n \"\"\"\n filename = args.config\n if filename:\n conf = yaml_loader(filename)\n else:\n try:\n conf = yaml_loader('config.yaml')\n except FileNotFoundError:\n raise EAException('No --config or config.yaml found')\n\n # init logging from config and set log levels according to command line options\n configure_logging(args, conf)\n\n for env_var, conf_var in list(env_settings.items()):\n val = env(env_var, None)\n if val is not None:\n conf[conf_var] = val\n\n for key, value in (iter(defaults.items()) if defaults is not None else []):\n if key not in conf:\n conf[key] = value\n\n for key, value in (iter(overwrites.items()) if overwrites is not None else []):\n conf[key] = value\n\n # Make sure we have all required globals\n if required_globals - frozenset(list(conf.keys())):\n raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(list(conf.keys())))))\n\n conf.setdefault('writeback_alias', 'elastalert_alerts')\n conf.setdefault('max_query_size', 10000)\n conf.setdefault('scroll_keepalive', '30s')\n conf.setdefault('max_scrolling_count', 0)\n conf.setdefault('disable_rules_on_error', True)\n conf.setdefault('scan_subdirectories', True)\n conf.setdefault('rules_loader', 'file')\n\n # Convert run_every, buffer_time into a timedelta object\n try:\n conf['run_every'] = datetime.timedelta(**conf['run_every'])\n conf['buffer_time'] = datetime.timedelta(**conf['buffer_time'])\n if 'alert_time_limit' in conf:\n conf['alert_time_limit'] = datetime.timedelta(**conf['alert_time_limit'])\n else:\n conf['alert_time_limit'] = datetime.timedelta(days=2)\n if 'old_query_limit' in conf:\n conf['old_query_limit'] = datetime.timedelta(**conf['old_query_limit'])\n else:\n conf['old_query_limit'] = datetime.timedelta(weeks=1)\n except (KeyError, TypeError) as e:\n raise EAException('Invalid time format used: %s' % e)\n\n # Initialise the rule loader and load each rule configuration\n rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader'])\n rules_loader = rules_loader_class(conf)\n conf['rules_loader'] = rules_loader\n # Make sure we have all the required globals for the loader\n # Make sure we have all required globals\n if rules_loader.required_globals - frozenset(list(conf.keys())):\n raise EAException(\n '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(list(conf.keys())))))\n\n return conf\n\n\ndef configure_logging(args, conf):\n # configure logging from config file if provided\n if 'logging' in conf:\n # load new logging config\n logging.config.dictConfig(conf['logging'])\n\n if args.verbose and args.debug:\n elastalert_logger.info(\n \"Note: --debug and --verbose flags are set. --debug takes precedent.\"\n )\n\n # re-enable INFO log level on elastalert_logger in verbose/debug mode\n # (but don't touch it if it is already set to INFO or below by config)\n if args.verbose or args.debug:\n if elastalert_logger.level > logging.INFO or elastalert_logger.level == logging.NOTSET:\n elastalert_logger.setLevel(logging.INFO)\n\n if args.debug:\n elastalert_logger.info(\n \"\"\"Note: In debug mode, alerts will be logged to console but NOT actually sent.\n To send them but remain verbose, use --verbose instead.\"\"\"\n )\n\n if not args.es_debug and 'logging' not in conf:\n logging.getLogger('elasticsearch').setLevel(logging.WARNING)\n\n if args.es_debug_trace:\n tracer = logging.getLogger('elasticsearch.trace')\n tracer.setLevel(logging.INFO)\n tracer.addHandler(logging.FileHandler(args.es_debug_trace))\n" }, { "path": "elastalert/create_index.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport argparse\nimport getpass\nimport json\nimport os\nimport time\n\nimport elasticsearch.helpers\nimport yaml\nfrom elasticsearch import RequestsHttpConnection\nfrom elasticsearch.client import Elasticsearch\nfrom elasticsearch.client import IndicesClient\nfrom elasticsearch.exceptions import NotFoundError\nfrom envparse import Env\n\nfrom .auth import Auth\n\nenv = Env(ES_USE_SSL=bool)\n\n\ndef create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None):\n esversion = es_client.info()[\"version\"][\"number\"]\n print(\"Elastic Version: \" + esversion)\n\n es_index_mappings = read_es_index_mappings() if is_atleastsix(esversion) else read_es_index_mappings(5)\n\n es_index = IndicesClient(es_client)\n if not recreate:\n if es_index.exists(ea_index):\n print('Index ' + ea_index + ' already exists. Skipping index creation.')\n return None\n\n # (Re-)Create indices.\n if is_atleastsix(esversion):\n index_names = (\n ea_index,\n ea_index + '_status',\n ea_index + '_silence',\n ea_index + '_error',\n ea_index + '_past',\n )\n else:\n index_names = (\n ea_index,\n )\n for index_name in index_names:\n if es_index.exists(index_name):\n print('Deleting index ' + index_name + '.')\n try:\n es_index.delete(index_name)\n except NotFoundError:\n # Why does this ever occur?? It shouldn't. But it does.\n pass\n es_index.create(index_name)\n\n # To avoid a race condition. TODO: replace this with a real check\n time.sleep(2)\n\n if is_atleastseven(esversion):\n # TODO remove doc_type completely when elasicsearch client allows doc_type=None\n # doc_type is a deprecated feature and will be completely removed in Elasicsearch 8\n es_client.indices.put_mapping(index=ea_index, doc_type='_doc',\n body=es_index_mappings['elastalert'], include_type_name=True)\n es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',\n body=es_index_mappings['elastalert_status'], include_type_name=True)\n es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc',\n body=es_index_mappings['silence'], include_type_name=True)\n es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc',\n body=es_index_mappings['elastalert_error'], include_type_name=True)\n es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',\n body=es_index_mappings['past_elastalert'], include_type_name=True)\n elif is_atleastsixtwo(esversion):\n es_client.indices.put_mapping(index=ea_index, doc_type='_doc',\n body=es_index_mappings['elastalert'])\n es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',\n body=es_index_mappings['elastalert_status'])\n es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc',\n body=es_index_mappings['silence'])\n es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc',\n body=es_index_mappings['elastalert_error'])\n es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',\n body=es_index_mappings['past_elastalert'])\n elif is_atleastsix(esversion):\n es_client.indices.put_mapping(index=ea_index, doc_type='elastalert',\n body=es_index_mappings['elastalert'])\n es_client.indices.put_mapping(index=ea_index + '_status', doc_type='elastalert_status',\n body=es_index_mappings['elastalert_status'])\n es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='silence',\n body=es_index_mappings['silence'])\n es_client.indices.put_mapping(index=ea_index + '_error', doc_type='elastalert_error',\n body=es_index_mappings['elastalert_error'])\n es_client.indices.put_mapping(index=ea_index + '_past', doc_type='past_elastalert',\n body=es_index_mappings['past_elastalert'])\n else:\n es_client.indices.put_mapping(index=ea_index, doc_type='elastalert',\n body=es_index_mappings['elastalert'])\n es_client.indices.put_mapping(index=ea_index, doc_type='elastalert_status',\n body=es_index_mappings['elastalert_status'])\n es_client.indices.put_mapping(index=ea_index, doc_type='silence',\n body=es_index_mappings['silence'])\n es_client.indices.put_mapping(index=ea_index, doc_type='elastalert_error',\n body=es_index_mappings['elastalert_error'])\n es_client.indices.put_mapping(index=ea_index, doc_type='past_elastalert',\n body=es_index_mappings['past_elastalert'])\n\n print('New index %s created' % ea_index)\n if old_ea_index:\n print(\"Copying all data from old index '{0}' to new index '{1}'\".format(old_ea_index, ea_index))\n # Use the defaults for chunk_size, scroll, scan_kwargs, and bulk_kwargs\n elasticsearch.helpers.reindex(es_client, old_ea_index, ea_index)\n\n print('Done!')\n\n\ndef read_es_index_mappings(es_version=6):\n print('Reading Elastic {0} index mappings:'.format(es_version))\n return {\n 'silence': read_es_index_mapping('silence', es_version),\n 'elastalert_status': read_es_index_mapping('elastalert_status', es_version),\n 'elastalert': read_es_index_mapping('elastalert', es_version),\n 'past_elastalert': read_es_index_mapping('past_elastalert', es_version),\n 'elastalert_error': read_es_index_mapping('elastalert_error', es_version)\n }\n\n\ndef read_es_index_mapping(mapping, es_version=6):\n base_path = os.path.abspath(os.path.dirname(__file__))\n mapping_path = 'es_mappings/{0}/{1}.json'.format(es_version, mapping)\n path = os.path.join(base_path, mapping_path)\n with open(path, 'r') as f:\n print(\"Reading index mapping '{0}'\".format(mapping_path))\n return json.load(f)\n\n\ndef is_atleastsix(es_version):\n return int(es_version.split(\".\")[0]) >= 6\n\n\ndef is_atleastsixtwo(es_version):\n major, minor = list(map(int, es_version.split(\".\")[:2]))\n return major > 6 or (major == 6 and minor >= 2)\n\n\ndef is_atleastseven(es_version):\n return int(es_version.split(\".\")[0]) >= 7\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('--host', default=os.environ.get('ES_HOST', None), help='Elasticsearch host')\n parser.add_argument('--port', default=os.environ.get('ES_PORT', None), type=int, help='Elasticsearch port')\n parser.add_argument('--username', default=os.environ.get('ES_USERNAME', None), help='Elasticsearch username')\n parser.add_argument('--password', default=os.environ.get('ES_PASSWORD', None), help='Elasticsearch password')\n parser.add_argument('--url-prefix', help='Elasticsearch URL prefix')\n parser.add_argument('--no-auth', action='store_const', const=True, help='Suppress prompt for basic auth')\n parser.add_argument('--ssl', action='store_true', default=env('ES_USE_SSL', None), help='Use TLS')\n parser.add_argument('--no-ssl', dest='ssl', action='store_false', help='Do not use TLS')\n parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates')\n parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false',\n help='Do not verify TLS certificates')\n parser.add_argument('--index', help='Index name to create')\n parser.add_argument('--alias', help='Alias name to create')\n parser.add_argument('--old-index', help='Old index name to copy')\n parser.add_argument('--send_get_body_as', default='GET',\n help='Method for querying Elasticsearch - POST, GET or source')\n parser.add_argument(\n '--boto-profile',\n default=None,\n dest='profile',\n help='DEPRECATED: (use --profile) Boto profile to use for signing requests')\n parser.add_argument(\n '--profile',\n default=None,\n help='AWS profile to use for signing requests. Optionally use the AWS_DEFAULT_PROFILE environment variable')\n parser.add_argument(\n '--aws-region',\n default=None,\n help='AWS Region to use for signing requests. Optionally use the AWS_DEFAULT_REGION environment variable')\n parser.add_argument('--timeout', default=60, type=int, help='Elasticsearch request timeout')\n parser.add_argument('--config', default='config.yaml', help='Global config file (default: config.yaml)')\n parser.add_argument('--recreate', type=bool, default=False,\n help='Force re-creation of the index (this will cause data loss).')\n args = parser.parse_args()\n\n if os.path.isfile(args.config):\n filename = args.config\n elif os.path.isfile('../config.yaml'):\n filename = '../config.yaml'\n else:\n filename = ''\n\n if filename:\n with open(filename) as config_file:\n data = yaml.load(config_file, Loader=yaml.FullLoader)\n host = args.host if args.host else data.get('es_host')\n port = args.port if args.port else data.get('es_port')\n username = args.username if args.username else data.get('es_username')\n password = args.password if args.password else data.get('es_password')\n url_prefix = args.url_prefix if args.url_prefix is not None else data.get('es_url_prefix', '')\n use_ssl = args.ssl if args.ssl is not None else data.get('use_ssl')\n verify_certs = args.verify_certs if args.verify_certs is not None else data.get('verify_certs') is not False\n aws_region = data.get('aws_region', None)\n send_get_body_as = data.get('send_get_body_as', 'GET')\n ca_certs = data.get('ca_certs')\n client_cert = data.get('client_cert')\n client_key = data.get('client_key')\n index = args.index if args.index is not None else data.get('writeback_index')\n alias = args.alias if args.alias is not None else data.get('writeback_alias')\n old_index = args.old_index if args.old_index is not None else None\n else:\n username = args.username if args.username else None\n password = args.password if args.password else None\n aws_region = args.aws_region\n host = args.host if args.host else input('Enter Elasticsearch host: ')\n port = args.port if args.port else int(input('Enter Elasticsearch port: '))\n use_ssl = (args.ssl if args.ssl is not None\n else input('Use SSL? t/f: ').lower() in ('t', 'true'))\n if use_ssl:\n verify_certs = (args.verify_certs if args.verify_certs is not None\n else input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false'))\n else:\n verify_certs = True\n if args.no_auth is None and username is None:\n username = input('Enter optional basic-auth username (or leave blank): ')\n password = getpass.getpass('Enter optional basic-auth password (or leave blank): ')\n url_prefix = (args.url_prefix if args.url_prefix is not None\n else input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): '))\n send_get_body_as = args.send_get_body_as\n ca_certs = None\n client_cert = None\n client_key = None\n index = args.index if args.index is not None else input('New index name? (Default elastalert_status) ')\n if not index:\n index = 'elastalert_status'\n alias = args.alias if args.alias is not None else input('New alias name? (Default elastalert_alerts) ')\n if not alias:\n alias = 'elastalert_alias'\n old_index = (args.old_index if args.old_index is not None\n else input('Name of existing index to copy? (Default None) '))\n\n timeout = args.timeout\n\n auth = Auth()\n http_auth = auth(host=host,\n username=username,\n password=password,\n aws_region=aws_region,\n profile_name=args.profile)\n es = Elasticsearch(\n host=host,\n port=port,\n timeout=timeout,\n use_ssl=use_ssl,\n verify_certs=verify_certs,\n connection_class=RequestsHttpConnection,\n http_auth=http_auth,\n url_prefix=url_prefix,\n send_get_body_as=send_get_body_as,\n client_cert=client_cert,\n ca_certs=ca_certs,\n client_key=client_key)\n\n create_index_mappings(es_client=es, ea_index=index, recreate=args.recreate, old_ea_index=old_index)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "elastalert/elastalert.py", "content": "# -*- coding: utf-8 -*-\nimport argparse\nimport copy\nimport datetime\nimport json\nimport logging\nimport os\nimport random\nimport signal\nimport sys\nimport threading\nimport time\nimport timeit\nimport traceback\nfrom email.mime.text import MIMEText\nfrom smtplib import SMTP\nfrom smtplib import SMTPException\nfrom socket import error\n\nimport dateutil.tz\nimport pytz\nfrom apscheduler.schedulers.background import BackgroundScheduler\nfrom croniter import croniter\nfrom elasticsearch.exceptions import ConnectionError\nfrom elasticsearch.exceptions import ElasticsearchException\nfrom elasticsearch.exceptions import NotFoundError\nfrom elasticsearch.exceptions import TransportError\n\nfrom . import kibana\nfrom .alerts import DebugAlerter\nfrom .config import load_conf\nfrom .enhancements import DropMatchException\nfrom .kibana_discover import generate_kibana_discover_url\nfrom .ruletypes import FlatlineRule\nfrom .util import add_raw_postfix\nfrom .util import cronite_datetime_to_timestamp\nfrom .util import dt_to_ts\nfrom .util import dt_to_unix\nfrom .util import EAException\nfrom .util import elastalert_logger\nfrom .util import elasticsearch_client\nfrom .util import format_index\nfrom .util import lookup_es_key\nfrom .util import parse_deadline\nfrom .util import parse_duration\nfrom .util import pretty_ts\nfrom .util import replace_dots_in_field_names\nfrom .util import seconds\nfrom .util import set_es_key\nfrom .util import should_scrolling_continue\nfrom .util import total_seconds\nfrom .util import ts_add\nfrom .util import ts_now\nfrom .util import ts_to_dt\nfrom .util import unix_to_dt\n\n\nclass ElastAlerter(object):\n \"\"\" The main ElastAlert runner. This class holds all state about active rules,\n controls when queries are run, and passes information between rules and alerts.\n\n :param args: An argparse arguments instance. Should contain debug and start\n\n :param conf: The configuration dictionary. At the top level, this\n contains global options, and under 'rules', contains all state relating\n to rules and alerts. In each rule in conf['rules'], the RuleType and Alerter\n instances live under 'type' and 'alerts', respectively. The conf dictionary\n should not be passed directly from a configuration file, but must be populated\n by config.py:load_rules instead. \"\"\"\n\n thread_data = threading.local()\n\n def parse_args(self, args):\n parser = argparse.ArgumentParser()\n parser.add_argument(\n '--config',\n action='store',\n dest='config',\n default=\"config.yaml\",\n help='Global config file (default: config.yaml)')\n parser.add_argument('--debug', action='store_true', dest='debug', help='Suppresses alerts and prints information instead. '\n 'Not compatible with `--verbose`')\n parser.add_argument('--rule', dest='rule', help='Run only a specific rule (by filename, must still be in rules folder)')\n parser.add_argument('--silence', dest='silence', help='Silence rule for a time period. Must be used with --rule. Usage: '\n '--silence =, eg. --silence hours=2')\n parser.add_argument('--start', dest='start', help='YYYY-MM-DDTHH:MM:SS Start querying from this timestamp. '\n 'Use \"NOW\" to start from current time. (Default: present)')\n parser.add_argument('--end', dest='end', help='YYYY-MM-DDTHH:MM:SS Query to this timestamp. (Default: present)')\n parser.add_argument('--verbose', action='store_true', dest='verbose', help='Increase verbosity without suppressing alerts. '\n 'Not compatible with `--debug`')\n parser.add_argument('--patience', action='store', dest='timeout',\n type=parse_duration,\n default=datetime.timedelta(),\n help='Maximum time to wait for ElasticSearch to become responsive. Usage: '\n '--patience =. e.g. --patience minutes=5')\n parser.add_argument(\n '--pin_rules',\n action='store_true',\n dest='pin_rules',\n help='Stop ElastAlert from monitoring config file changes')\n parser.add_argument('--es_debug', action='store_true', dest='es_debug', help='Enable verbose logging from Elasticsearch queries')\n parser.add_argument(\n '--es_debug_trace',\n action='store',\n dest='es_debug_trace',\n help='Enable logging from Elasticsearch queries as curl command. Queries will be logged to file. Note that '\n 'this will incorrectly display localhost:9200 as the host/port')\n self.args = parser.parse_args(args)\n\n def __init__(self, args):\n self.es_clients = {}\n self.parse_args(args)\n self.debug = self.args.debug\n self.verbose = self.args.verbose\n\n if self.verbose and self.debug:\n elastalert_logger.info(\n \"Note: --debug and --verbose flags are set. --debug takes precedent.\"\n )\n\n if self.verbose or self.debug:\n elastalert_logger.setLevel(logging.INFO)\n\n if self.debug:\n elastalert_logger.info(\n \"\"\"Note: In debug mode, alerts will be logged to console but NOT actually sent.\n To send them but remain verbose, use --verbose instead.\"\"\"\n )\n\n if not self.args.es_debug:\n logging.getLogger('elasticsearch').setLevel(logging.WARNING)\n\n if self.args.es_debug_trace:\n tracer = logging.getLogger('elasticsearch.trace')\n tracer.setLevel(logging.INFO)\n tracer.addHandler(logging.FileHandler(self.args.es_debug_trace))\n\n self.conf = load_conf(self.args)\n self.rules_loader = self.conf['rules_loader']\n self.rules = self.rules_loader.load(self.conf, self.args)\n\n print(len(self.rules), 'rules loaded')\n\n self.max_query_size = self.conf['max_query_size']\n self.scroll_keepalive = self.conf['scroll_keepalive']\n self.writeback_index = self.conf['writeback_index']\n self.writeback_alias = self.conf['writeback_alias']\n self.run_every = self.conf['run_every']\n self.alert_time_limit = self.conf['alert_time_limit']\n self.old_query_limit = self.conf['old_query_limit']\n self.disable_rules_on_error = self.conf['disable_rules_on_error']\n self.notify_email = self.conf.get('notify_email', [])\n self.from_addr = self.conf.get('from_addr', 'ElastAlert')\n self.smtp_host = self.conf.get('smtp_host', 'localhost')\n self.max_aggregation = self.conf.get('max_aggregation', 10000)\n self.buffer_time = self.conf['buffer_time']\n self.silence_cache = {}\n self.rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule)\n self.starttime = self.args.start\n self.disabled_rules = []\n self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False)\n self.thread_data.num_hits = 0\n self.thread_data.num_dupes = 0\n self.scheduler = BackgroundScheduler()\n self.string_multi_field_name = self.conf.get('string_multi_field_name', False)\n self.add_metadata_alert = self.conf.get('add_metadata_alert', False)\n self.show_disabled_rules = self.conf.get('show_disabled_rules', True)\n\n self.writeback_es = elasticsearch_client(self.conf)\n\n remove = []\n for rule in self.rules:\n if not self.init_rule(rule):\n remove.append(rule)\n list(map(self.rules.remove, remove))\n\n if self.args.silence:\n self.silence()\n\n @staticmethod\n def get_index(rule, starttime=None, endtime=None):\n \"\"\" Gets the index for a rule. If strftime is set and starttime and endtime\n are provided, it will return a comma seperated list of indices. If strftime\n is set but starttime and endtime are not provided, it will replace all format\n tokens with a wildcard. \"\"\"\n index = rule['index']\n add_extra = rule.get('search_extra_index', False)\n if rule.get('use_strftime_index'):\n if starttime and endtime:\n return format_index(index, starttime, endtime, add_extra)\n else:\n # Replace the substring containing format characters with a *\n format_start = index.find('%')\n format_end = index.rfind('%') + 2\n return index[:format_start] + '*' + index[format_end:]\n else:\n return index\n\n @staticmethod\n def get_query(filters, starttime=None, endtime=None, sort=True, timestamp_field='@timestamp', to_ts_func=dt_to_ts, desc=False,\n five=False):\n \"\"\" Returns a query dict that will apply a list of filters, filter by\n start and end time, and sort results by timestamp.\n\n :param filters: A list of Elasticsearch filters to use.\n :param starttime: A timestamp to use as the start time of the query.\n :param endtime: A timestamp to use as the end time of the query.\n :param sort: If true, sort results by timestamp. (Default True)\n :return: A query dictionary to pass to Elasticsearch.\n \"\"\"\n starttime = to_ts_func(starttime)\n endtime = to_ts_func(endtime)\n filters = copy.copy(filters)\n es_filters = {'filter': {'bool': {'must': filters}}}\n if starttime and endtime:\n es_filters['filter']['bool']['must'].insert(0, {'range': {timestamp_field: {'gt': starttime,\n 'lte': endtime}}})\n if five:\n query = {'query': {'bool': es_filters}}\n else:\n query = {'query': {'filtered': es_filters}}\n if sort:\n query['sort'] = [{timestamp_field: {'order': 'desc' if desc else 'asc'}}]\n return query\n\n def get_terms_query(self, query, rule, size, field, five=False):\n \"\"\" Takes a query generated by get_query and outputs a aggregation query \"\"\"\n query_element = query['query']\n if 'sort' in query_element:\n query_element.pop('sort')\n if not five:\n query_element['filtered'].update({'aggs': {'counts': {'terms': {'field': field,\n 'size': size,\n 'min_doc_count': rule.get('min_doc_count', 1)}}}})\n aggs_query = {'aggs': query_element}\n else:\n aggs_query = query\n aggs_query['aggs'] = {'counts': {'terms': {'field': field,\n 'size': size,\n 'min_doc_count': rule.get('min_doc_count', 1)}}}\n return aggs_query\n\n def get_aggregation_query(self, query, rule, query_key, terms_size, timestamp_field='@timestamp'):\n \"\"\" Takes a query generated by get_query and outputs a aggregation query \"\"\"\n query_element = query['query']\n if 'sort' in query_element:\n query_element.pop('sort')\n metric_agg_element = rule['aggregation_query_element']\n\n bucket_interval_period = rule.get('bucket_interval_period')\n if bucket_interval_period is not None:\n aggs_element = {\n 'interval_aggs': {\n 'date_histogram': {\n 'field': timestamp_field,\n 'interval': bucket_interval_period},\n 'aggs': metric_agg_element\n }\n }\n if rule.get('bucket_offset_delta'):\n aggs_element['interval_aggs']['date_histogram']['offset'] = '+%ss' % (rule['bucket_offset_delta'])\n else:\n aggs_element = metric_agg_element\n\n if query_key is not None:\n for idx, key in reversed(list(enumerate(query_key.split(',')))):\n aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size,\n 'min_doc_count': rule.get('min_doc_count', 1)},\n 'aggs': aggs_element}}\n\n if not rule['five']:\n query_element['filtered'].update({'aggs': aggs_element})\n aggs_query = {'aggs': query_element}\n else:\n aggs_query = query\n aggs_query['aggs'] = aggs_element\n return aggs_query\n\n def get_index_start(self, index, timestamp_field='@timestamp'):\n \"\"\" Query for one result sorted by timestamp to find the beginning of the index.\n\n :param index: The index of which to find the earliest event.\n :return: Timestamp of the earliest event.\n \"\"\"\n query = {'sort': {timestamp_field: {'order': 'asc'}}}\n try:\n if self.thread_data.current_es.is_atleastsixsix():\n res = self.thread_data.current_es.search(index=index, size=1, body=query,\n _source_includes=[timestamp_field], ignore_unavailable=True)\n else:\n res = self.thread_data.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field],\n ignore_unavailable=True)\n except ElasticsearchException as e:\n self.handle_error(\"Elasticsearch query error: %s\" % (e), {'index': index, 'query': query})\n return '1969-12-30T00:00:00Z'\n if len(res['hits']['hits']) == 0:\n # Index is completely empty, return a date before the epoch\n return '1969-12-30T00:00:00Z'\n return res['hits']['hits'][0][timestamp_field]\n\n @staticmethod\n def process_hits(rule, hits):\n \"\"\" Update the _source field for each hit received from ES based on the rule configuration.\n\n This replaces timestamps with datetime objects,\n folds important fields into _source and creates compound query_keys.\n\n :return: A list of processed _source dictionaries.\n \"\"\"\n\n processed_hits = []\n for hit in hits:\n # Merge fields and _source\n hit.setdefault('_source', {})\n for key, value in list(hit.get('fields', {}).items()):\n # Fields are returned as lists, assume any with length 1 are not arrays in _source\n # Except sometimes they aren't lists. This is dependent on ES version\n hit['_source'].setdefault(key, value[0] if type(value) is list and len(value) == 1 else value)\n\n # Convert the timestamp to a datetime\n ts = lookup_es_key(hit['_source'], rule['timestamp_field'])\n if not ts and not rule[\"_source_enabled\"]:\n raise EAException(\n \"Error: No timestamp was found for hit. '_source_enabled' is set to false, check your mappings for stored fields\"\n )\n\n set_es_key(hit['_source'], rule['timestamp_field'], rule['ts_to_dt'](ts))\n set_es_key(hit, rule['timestamp_field'], lookup_es_key(hit['_source'], rule['timestamp_field']))\n\n # Tack metadata fields into _source\n for field in ['_id', '_index', '_type']:\n if field in hit:\n hit['_source'][field] = hit[field]\n\n if rule.get('compound_query_key'):\n values = [lookup_es_key(hit['_source'], key) for key in rule['compound_query_key']]\n hit['_source'][rule['query_key']] = ', '.join([str(value) for value in values])\n\n if rule.get('compound_aggregation_key'):\n values = [lookup_es_key(hit['_source'], key) for key in rule['compound_aggregation_key']]\n hit['_source'][rule['aggregation_key']] = ', '.join([str(value) for value in values])\n\n processed_hits.append(hit['_source'])\n\n return processed_hits\n\n def get_hits(self, rule, starttime, endtime, index, scroll=False):\n \"\"\" Query Elasticsearch for the given rule and return the results.\n :param rule: The rule configuration.\n :param starttime: The earliest time to query.\n :param endtime: The latest time to query.\n :return: A list of hits, bounded by rule['max_query_size'] (or self.max_query_size).\n \"\"\"\n\n query = self.get_query(\n rule['filter'],\n starttime,\n endtime,\n timestamp_field=rule['timestamp_field'],\n to_ts_func=rule['dt_to_ts'],\n five=rule['five'],\n )\n if self.thread_data.current_es.is_atleastsixsix():\n extra_args = {'_source_includes': rule['include']}\n else:\n extra_args = {'_source_include': rule['include']}\n scroll_keepalive = rule.get('scroll_keepalive', self.scroll_keepalive)\n if not rule.get('_source_enabled'):\n if rule['five']:\n query['stored_fields'] = rule['include']\n else:\n query['fields'] = rule['include']\n extra_args = {}\n\n try:\n if scroll:\n res = self.thread_data.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive)\n else:\n res = self.thread_data.current_es.search(\n scroll=scroll_keepalive,\n index=index,\n size=rule.get('max_query_size', self.max_query_size),\n body=query,\n ignore_unavailable=True,\n **extra_args\n )\n if '_scroll_id' in res:\n rule['scroll_id'] = res['_scroll_id']\n\n if self.thread_data.current_es.is_atleastseven():\n self.thread_data.total_hits = int(res['hits']['total']['value'])\n else:\n self.thread_data.total_hits = int(res['hits']['total'])\n\n if len(res.get('_shards', {}).get('failures', [])) > 0:\n try:\n errs = [e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']]\n if len(errs):\n raise ElasticsearchException(errs)\n except (TypeError, KeyError):\n # Different versions of ES have this formatted in different ways. Fallback to str-ing the whole thing\n raise ElasticsearchException(str(res['_shards']['failures']))\n\n logging.debug(str(res))\n except ElasticsearchException as e:\n # Elasticsearch sometimes gives us GIGANTIC error messages\n # (so big that they will fill the entire terminal buffer)\n if len(str(e)) > 1024:\n e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)\n self.handle_error('Error running query: %s' % (e), {'rule': rule['name'], 'query': query})\n return None\n hits = res['hits']['hits']\n self.thread_data.num_hits += len(hits)\n lt = rule.get('use_local_time')\n status_log = \"Queried rule %s from %s to %s: %s / %s hits\" % (\n rule['name'],\n pretty_ts(starttime, lt),\n pretty_ts(endtime, lt),\n self.thread_data.num_hits,\n len(hits)\n )\n if self.thread_data.total_hits > rule.get('max_query_size', self.max_query_size):\n elastalert_logger.info(\"%s (scrolling..)\" % status_log)\n else:\n elastalert_logger.info(status_log)\n\n hits = self.process_hits(rule, hits)\n\n # Record doc_type for use in get_top_counts\n if 'doc_type' not in rule and len(hits):\n rule['doc_type'] = hits[0]['_type']\n return hits\n\n def get_hits_count(self, rule, starttime, endtime, index):\n \"\"\" Query Elasticsearch for the count of results and returns a list of timestamps\n equal to the endtime. This allows the results to be passed to rules which expect\n an object for each hit.\n\n :param rule: The rule configuration dictionary.\n :param starttime: The earliest time to query.\n :param endtime: The latest time to query.\n :return: A dictionary mapping timestamps to number of hits for that time period.\n \"\"\"\n query = self.get_query(\n rule['filter'],\n starttime,\n endtime,\n timestamp_field=rule['timestamp_field'],\n sort=False,\n to_ts_func=rule['dt_to_ts'],\n five=rule['five']\n )\n\n try:\n res = self.thread_data.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True)\n except ElasticsearchException as e:\n # Elasticsearch sometimes gives us GIGANTIC error messages\n # (so big that they will fill the entire terminal buffer)\n if len(str(e)) > 1024:\n e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)\n self.handle_error('Error running count query: %s' % (e), {'rule': rule['name'], 'query': query})\n return None\n\n self.thread_data.num_hits += res['count']\n lt = rule.get('use_local_time')\n elastalert_logger.info(\n \"Queried rule %s from %s to %s: %s hits\" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count'])\n )\n return {endtime: res['count']}\n\n def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=None):\n rule_filter = copy.copy(rule['filter'])\n if qk:\n qk_list = qk.split(\",\")\n end = None\n if rule['five']:\n end = '.keyword'\n else:\n end = '.raw'\n\n if len(qk_list) == 1:\n qk = qk_list[0]\n filter_key = rule['query_key']\n if rule.get('raw_count_keys', True) and not rule['query_key'].endswith(end):\n filter_key = add_raw_postfix(filter_key, rule['five'])\n rule_filter.extend([{'term': {filter_key: qk}}])\n else:\n filter_keys = rule['compound_query_key']\n for i in range(len(filter_keys)):\n key_with_postfix = filter_keys[i]\n if rule.get('raw_count_keys', True) and not key.endswith(end):\n key_with_postfix = add_raw_postfix(key_with_postfix, rule['five'])\n rule_filter.extend([{'term': {key_with_postfix: qk_list[i]}}])\n\n base_query = self.get_query(\n rule_filter,\n starttime,\n endtime,\n timestamp_field=rule['timestamp_field'],\n sort=False,\n to_ts_func=rule['dt_to_ts'],\n five=rule['five']\n )\n if size is None:\n size = rule.get('terms_size', 50)\n query = self.get_terms_query(base_query, rule, size, key, rule['five'])\n\n try:\n if not rule['five']:\n res = self.thread_data.current_es.deprecated_search(\n index=index,\n doc_type=rule['doc_type'],\n body=query,\n search_type='count',\n ignore_unavailable=True\n )\n else:\n res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'],\n body=query, size=0, ignore_unavailable=True)\n except ElasticsearchException as e:\n # Elasticsearch sometimes gives us GIGANTIC error messages\n # (so big that they will fill the entire terminal buffer)\n if len(str(e)) > 1024:\n e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)\n self.handle_error('Error running terms query: %s' % (e), {'rule': rule['name'], 'query': query})\n return None\n\n if 'aggregations' not in res:\n return {}\n if not rule['five']:\n buckets = res['aggregations']['filtered']['counts']['buckets']\n else:\n buckets = res['aggregations']['counts']['buckets']\n self.thread_data.num_hits += len(buckets)\n lt = rule.get('use_local_time')\n elastalert_logger.info(\n 'Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets))\n )\n return {endtime: buckets}\n\n def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_size=None):\n rule_filter = copy.copy(rule['filter'])\n base_query = self.get_query(\n rule_filter,\n starttime,\n endtime,\n timestamp_field=rule['timestamp_field'],\n sort=False,\n to_ts_func=rule['dt_to_ts'],\n five=rule['five']\n )\n if term_size is None:\n term_size = rule.get('terms_size', 50)\n query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field'])\n try:\n if not rule['five']:\n res = self.thread_data.current_es.deprecated_search(\n index=index,\n doc_type=rule.get('doc_type'),\n body=query,\n search_type='count',\n ignore_unavailable=True\n )\n else:\n res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'),\n body=query, size=0, ignore_unavailable=True)\n except ElasticsearchException as e:\n if len(str(e)) > 1024:\n e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024)\n self.handle_error('Error running query: %s' % (e), {'rule': rule['name']})\n return None\n if 'aggregations' not in res:\n return {}\n if not rule['five']:\n payload = res['aggregations']['filtered']\n else:\n payload = res['aggregations']\n\n if self.thread_data.current_es.is_atleastseven():\n self.thread_data.num_hits += res['hits']['total']['value']\n else:\n self.thread_data.num_hits += res['hits']['total']\n\n return {endtime: payload}\n\n def remove_duplicate_events(self, data, rule):\n new_events = []\n for event in data:\n if event['_id'] in rule['processed_hits']:\n continue\n\n # Remember the new data's IDs\n rule['processed_hits'][event['_id']] = lookup_es_key(event, rule['timestamp_field'])\n new_events.append(event)\n\n return new_events\n\n def remove_old_events(self, rule):\n # Anything older than the buffer time we can forget\n now = ts_now()\n remove = []\n buffer_time = rule.get('buffer_time', self.buffer_time)\n if rule.get('query_delay'):\n buffer_time += rule['query_delay']\n for _id, timestamp in rule['processed_hits'].items():\n if now - timestamp > buffer_time:\n remove.append(_id)\n list(map(rule['processed_hits'].pop, remove))\n\n def run_query(self, rule, start=None, end=None, scroll=False):\n \"\"\" Query for the rule and pass all of the results to the RuleType instance.\n\n :param rule: The rule configuration.\n :param start: The earliest time to query.\n :param end: The latest time to query.\n Returns True on success and False on failure.\n \"\"\"\n if start is None:\n start = self.get_index_start(rule['index'])\n if end is None:\n end = ts_now()\n\n # Reset hit counter and query\n rule_inst = rule['type']\n rule['scrolling_cycle'] = rule.get('scrolling_cycle', 0) + 1\n index = self.get_index(rule, start, end)\n if rule.get('use_count_query'):\n data = self.get_hits_count(rule, start, end, index)\n elif rule.get('use_terms_query'):\n data = self.get_hits_terms(rule, start, end, index, rule['query_key'])\n elif rule.get('aggregation_query_element'):\n data = self.get_hits_aggregation(rule, start, end, index, rule.get('query_key', None))\n else:\n data = self.get_hits(rule, start, end, index, scroll)\n if data:\n old_len = len(data)\n data = self.remove_duplicate_events(data, rule)\n self.thread_data.num_dupes += old_len - len(data)\n\n # There was an exception while querying\n if data is None:\n return False\n elif data:\n if rule.get('use_count_query'):\n rule_inst.add_count_data(data)\n elif rule.get('use_terms_query'):\n rule_inst.add_terms_data(data)\n elif rule.get('aggregation_query_element'):\n rule_inst.add_aggregation_data(data)\n else:\n rule_inst.add_data(data)\n\n try:\n if rule.get('scroll_id') and self.thread_data.num_hits < self.thread_data.total_hits and should_scrolling_continue(rule):\n if not self.run_query(rule, start, end, scroll=True):\n return False\n except RuntimeError:\n # It's possible to scroll far enough to hit max recursive depth\n pass\n\n if 'scroll_id' in rule:\n scroll_id = rule.pop('scroll_id')\n try:\n self.thread_data.current_es.clear_scroll(scroll_id=scroll_id)\n except NotFoundError:\n pass\n\n return True\n\n def get_starttime(self, rule):\n \"\"\" Query ES for the last time we ran this rule.\n\n :param rule: The rule configuration.\n :return: A timestamp or None.\n \"\"\"\n sort = {'sort': {'@timestamp': {'order': 'desc'}}}\n query = {'filter': {'term': {'rule_name': '%s' % (rule['name'])}}}\n if self.writeback_es.is_atleastfive():\n query = {'query': {'bool': query}}\n query.update(sort)\n\n try:\n doc_type = 'elastalert_status'\n index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type)\n if self.writeback_es.is_atleastsixtwo():\n if self.writeback_es.is_atleastsixsix():\n res = self.writeback_es.search(index=index, size=1, body=query,\n _source_includes=['endtime', 'rule_name'])\n else:\n res = self.writeback_es.search(index=index, size=1, body=query,\n _source_include=['endtime', 'rule_name'])\n else:\n res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type,\n size=1, body=query, _source_include=['endtime', 'rule_name'])\n if res['hits']['hits']:\n endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime'])\n\n if ts_now() - endtime < self.old_query_limit:\n return endtime\n else:\n elastalert_logger.info(\"Found expired previous run for %s at %s\" % (rule['name'], endtime))\n return None\n except (ElasticsearchException, KeyError) as e:\n self.handle_error('Error querying for last run: %s' % (e), {'rule': rule['name']})\n\n def set_starttime(self, rule, endtime):\n \"\"\" Given a rule and an endtime, sets the appropriate starttime for it. \"\"\"\n # This means we are starting fresh\n if 'starttime' not in rule:\n if not rule.get('scan_entire_timeframe'):\n # Try to get the last run from Elasticsearch\n last_run_end = self.get_starttime(rule)\n if last_run_end:\n rule['starttime'] = last_run_end\n self.adjust_start_time_for_overlapping_agg_query(rule)\n self.adjust_start_time_for_interval_sync(rule, endtime)\n rule['minimum_starttime'] = rule['starttime']\n return None\n\n # Use buffer for normal queries, or run_every increments otherwise\n # or, if scan_entire_timeframe, use timeframe\n\n if not rule.get('use_count_query') and not rule.get('use_terms_query'):\n if not rule.get('scan_entire_timeframe'):\n buffer_time = rule.get('buffer_time', self.buffer_time)\n buffer_delta = endtime - buffer_time\n else:\n buffer_delta = endtime - rule['timeframe']\n # If we started using a previous run, don't go past that\n if 'minimum_starttime' in rule and rule['minimum_starttime'] > buffer_delta:\n rule['starttime'] = rule['minimum_starttime']\n # If buffer_time doesn't bring us past the previous endtime, use that instead\n elif 'previous_endtime' in rule and rule['previous_endtime'] < buffer_delta:\n rule['starttime'] = rule['previous_endtime']\n self.adjust_start_time_for_overlapping_agg_query(rule)\n else:\n rule['starttime'] = buffer_delta\n\n self.adjust_start_time_for_interval_sync(rule, endtime)\n\n else:\n if not rule.get('scan_entire_timeframe'):\n # Query from the end of the last run, if it exists, otherwise a run_every sized window\n rule['starttime'] = rule.get('previous_endtime', endtime - self.run_every)\n else:\n rule['starttime'] = rule.get('previous_endtime', endtime - rule['timeframe'])\n\n def adjust_start_time_for_overlapping_agg_query(self, rule):\n if rule.get('aggregation_query_element'):\n if rule.get('allow_buffer_time_overlap') and not rule.get('use_run_every_query_size') and (\n rule['buffer_time'] > rule['run_every']):\n rule['starttime'] = rule['starttime'] - (rule['buffer_time'] - rule['run_every'])\n rule['original_starttime'] = rule['starttime']\n\n def adjust_start_time_for_interval_sync(self, rule, endtime):\n # If aggregation query adjust bucket offset\n if rule.get('aggregation_query_element'):\n\n if rule.get('bucket_interval'):\n es_interval_delta = rule.get('bucket_interval_timedelta')\n unix_starttime = dt_to_unix(rule['starttime'])\n es_interval_delta_in_sec = total_seconds(es_interval_delta)\n offset = int(unix_starttime % es_interval_delta_in_sec)\n\n if rule.get('sync_bucket_interval'):\n rule['starttime'] = unix_to_dt(unix_starttime - offset)\n endtime = unix_to_dt(dt_to_unix(endtime) - offset)\n else:\n rule['bucket_offset_delta'] = offset\n\n def get_segment_size(self, rule):\n \"\"\" The segment size is either buffer_size for queries which can overlap or run_every for queries\n which must be strictly separate. This mimicks the query size for when ElastAlert is running continuously. \"\"\"\n if not rule.get('use_count_query') and not rule.get('use_terms_query') and not rule.get('aggregation_query_element'):\n return rule.get('buffer_time', self.buffer_time)\n elif rule.get('aggregation_query_element'):\n if rule.get('use_run_every_query_size'):\n return self.run_every\n else:\n return rule.get('buffer_time', self.buffer_time)\n else:\n return self.run_every\n\n def get_query_key_value(self, rule, match):\n # get the value for the match's query_key (or none) to form the key used for the silence_cache.\n # Flatline ruletype sets \"key\" instead of the actual query_key\n if isinstance(rule['type'], FlatlineRule) and 'key' in match:\n return str(match['key'])\n return self.get_named_key_value(rule, match, 'query_key')\n\n def get_aggregation_key_value(self, rule, match):\n # get the value for the match's aggregation_key (or none) to form the key used for grouped aggregates.\n return self.get_named_key_value(rule, match, 'aggregation_key')\n\n def get_named_key_value(self, rule, match, key_name):\n # search the match for the key specified in the rule to get the value\n if key_name in rule:\n try:\n key_value = lookup_es_key(match, rule[key_name])\n if key_value is not None:\n # Only do the unicode conversion if we actually found something)\n # otherwise we might transform None --> 'None'\n key_value = str(key_value)\n except KeyError:\n # Some matches may not have the specified key\n # use a special token for these\n key_value = '_missing'\n else:\n key_value = None\n\n return key_value\n\n def enhance_filter(self, rule):\n \"\"\" If there is a blacklist or whitelist in rule then we add it to the filter.\n It adds it as a query_string. If there is already an query string its is appended\n with blacklist or whitelist.\n\n :param rule:\n :return:\n \"\"\"\n if not rule.get('filter_by_list', True):\n return\n if 'blacklist' in rule:\n listname = 'blacklist'\n elif 'whitelist' in rule:\n listname = 'whitelist'\n else:\n return\n\n filters = rule['filter']\n additional_terms = []\n for term in rule[listname]:\n if not term.startswith('/') or not term.endswith('/'):\n additional_terms.append(rule['compare_key'] + ':\"' + term + '\"')\n else:\n # These are regular expressions and won't work if they are quoted\n additional_terms.append(rule['compare_key'] + ':' + term)\n if listname == 'whitelist':\n query = \"NOT \" + \" AND NOT \".join(additional_terms)\n else:\n query = \" OR \".join(additional_terms)\n query_str_filter = {'query_string': {'query': query}}\n if self.writeback_es.is_atleastfive():\n filters.append(query_str_filter)\n else:\n filters.append({'query': query_str_filter})\n logging.debug(\"Enhanced filter with {} terms: {}\".format(listname, str(query_str_filter)))\n\n def run_rule(self, rule, endtime, starttime=None):\n \"\"\" Run a rule for a given time period, including querying and alerting on results.\n\n :param rule: The rule configuration.\n :param starttime: The earliest timestamp to query.\n :param endtime: The latest timestamp to query.\n :return: The number of matches that the rule produced.\n \"\"\"\n run_start = time.time()\n self.thread_data.current_es = self.es_clients.setdefault(rule['name'], elasticsearch_client(rule))\n\n # If there are pending aggregate matches, try processing them\n for x in range(len(rule['agg_matches'])):\n match = rule['agg_matches'].pop()\n self.add_aggregated_alert(match, rule)\n\n # Start from provided time if it's given\n if starttime:\n rule['starttime'] = starttime\n else:\n self.set_starttime(rule, endtime)\n\n rule['original_starttime'] = rule['starttime']\n rule['scrolling_cycle'] = 0\n\n # Don't run if starttime was set to the future\n if ts_now() <= rule['starttime']:\n logging.warning(\"Attempted to use query start time in the future (%s), sleeping instead\" % (starttime))\n return 0\n\n # Run the rule. If querying over a large time period, split it up into segments\n self.thread_data.num_hits = 0\n self.thread_data.num_dupes = 0\n self.thread_data.cumulative_hits = 0\n segment_size = self.get_segment_size(rule)\n\n tmp_endtime = rule['starttime']\n\n while endtime - rule['starttime'] > segment_size:\n tmp_endtime = tmp_endtime + segment_size\n if not self.run_query(rule, rule['starttime'], tmp_endtime):\n return 0\n self.thread_data.cumulative_hits += self.thread_data.num_hits\n self.thread_data.num_hits = 0\n rule['starttime'] = tmp_endtime\n rule['type'].garbage_collect(tmp_endtime)\n\n if rule.get('aggregation_query_element'):\n if endtime - tmp_endtime == segment_size:\n if not self.run_query(rule, tmp_endtime, endtime):\n return 0\n self.thread_data.cumulative_hits += self.thread_data.num_hits\n elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0:\n rule['starttime'] = rule['original_starttime']\n return 0\n else:\n endtime = tmp_endtime\n else:\n if not self.run_query(rule, rule['starttime'], endtime):\n return 0\n self.thread_data.cumulative_hits += self.thread_data.num_hits\n rule['type'].garbage_collect(endtime)\n\n # Process any new matches\n num_matches = len(rule['type'].matches)\n while rule['type'].matches:\n match = rule['type'].matches.pop(0)\n match['num_hits'] = self.thread_data.cumulative_hits\n match['num_matches'] = num_matches\n\n # If realert is set, silence the rule for that duration\n # Silence is cached by query_key, if it exists\n # Default realert time is 0 seconds\n silence_cache_key = rule['name']\n query_key_value = self.get_query_key_value(rule, match)\n if query_key_value is not None:\n silence_cache_key += '.' + query_key_value\n\n if self.is_silenced(rule['name'] + \"._silence\") or self.is_silenced(silence_cache_key):\n elastalert_logger.info('Ignoring match for silenced rule %s' % (silence_cache_key,))\n continue\n\n if rule['realert']:\n next_alert, exponent = self.next_alert_time(rule, silence_cache_key, ts_now())\n self.set_realert(silence_cache_key, next_alert, exponent)\n\n if rule.get('run_enhancements_first'):\n try:\n for enhancement in rule['match_enhancements']:\n try:\n enhancement.process(match)\n except EAException as e:\n self.handle_error(\"Error running match enhancement: %s\" % (e), {'rule': rule['name']})\n except DropMatchException:\n continue\n\n # If no aggregation, alert immediately\n if not rule['aggregation']:\n self.alert([match], rule)\n continue\n\n # Add it as an aggregated match\n self.add_aggregated_alert(match, rule)\n\n # Mark this endtime for next run's start\n rule['previous_endtime'] = endtime\n\n time_taken = time.time() - run_start\n # Write to ES that we've run this rule against this time period\n body = {'rule_name': rule['name'],\n 'endtime': endtime,\n 'starttime': rule['original_starttime'],\n 'matches': num_matches,\n 'hits': max(self.thread_data.num_hits, self.thread_data.cumulative_hits),\n '@timestamp': ts_now(),\n 'time_taken': time_taken}\n self.writeback('elastalert_status', body)\n\n return num_matches\n\n def init_rule(self, new_rule, new=True):\n ''' Copies some necessary non-config state from an exiting rule to a new rule. '''\n if not new:\n self.scheduler.remove_job(job_id=new_rule['name'])\n\n try:\n self.modify_rule_for_ES5(new_rule)\n except TransportError as e:\n elastalert_logger.warning('Error connecting to Elasticsearch for rule {}. '\n 'The rule has been disabled.'.format(new_rule['name']))\n self.send_notification_email(exception=e, rule=new_rule)\n return False\n\n self.enhance_filter(new_rule)\n\n # Change top_count_keys to .raw\n if 'top_count_keys' in new_rule and new_rule.get('raw_count_keys', True):\n if self.string_multi_field_name:\n string_multi_field_name = self.string_multi_field_name\n elif self.writeback_es.is_atleastfive():\n string_multi_field_name = '.keyword'\n else:\n string_multi_field_name = '.raw'\n\n for i, key in enumerate(new_rule['top_count_keys']):\n if not key.endswith(string_multi_field_name):\n new_rule['top_count_keys'][i] += string_multi_field_name\n\n if 'download_dashboard' in new_rule['filter']:\n # Download filters from Kibana and set the rules filters to them\n db_filters = self.filters_from_kibana(new_rule, new_rule['filter']['download_dashboard'])\n if db_filters is not None:\n new_rule['filter'] = db_filters\n else:\n raise EAException(\"Could not download filters from %s\" % (new_rule['filter']['download_dashboard']))\n\n blank_rule = {'agg_matches': [],\n 'aggregate_alert_time': {},\n 'current_aggregate_id': {},\n 'processed_hits': {},\n 'run_every': self.run_every,\n 'has_run_once': False}\n rule = blank_rule\n\n # Set rule to either a blank template or existing rule with same name\n if not new:\n for rule in self.rules:\n if rule['name'] == new_rule['name']:\n break\n else:\n rule = blank_rule\n\n copy_properties = ['agg_matches',\n 'current_aggregate_id',\n 'aggregate_alert_time',\n 'processed_hits',\n 'starttime',\n 'minimum_starttime',\n 'has_run_once']\n for prop in copy_properties:\n if prop not in rule:\n continue\n new_rule[prop] = rule[prop]\n\n job = self.scheduler.add_job(self.handle_rule_execution, 'interval',\n args=[new_rule],\n seconds=new_rule['run_every'].total_seconds(),\n id=new_rule['name'],\n max_instances=1,\n jitter=5)\n job.modify(next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=random.randint(0, 15)))\n\n return new_rule\n\n @staticmethod\n def modify_rule_for_ES5(new_rule):\n # Get ES version per rule\n rule_es = elasticsearch_client(new_rule)\n if rule_es.is_atleastfive():\n new_rule['five'] = True\n else:\n new_rule['five'] = False\n return\n\n # In ES5, filters starting with 'query' should have the top wrapper removed\n new_filters = []\n for es_filter in new_rule.get('filter', []):\n if es_filter.get('query'):\n new_filters.append(es_filter['query'])\n else:\n new_filters.append(es_filter)\n new_rule['filter'] = new_filters\n\n def load_rule_changes(self):\n \"\"\" Using the modification times of rule config files, syncs the running rules\n to match the files in rules_folder by removing, adding or reloading rules. \"\"\"\n new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule)\n\n # Check each current rule for changes\n for rule_file, hash_value in self.rule_hashes.items():\n if rule_file not in new_rule_hashes:\n # Rule file was deleted\n elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file))\n for rule in self.rules:\n if rule['rule_file'] == rule_file:\n break\n else:\n continue\n self.scheduler.remove_job(job_id=rule['name'])\n self.rules.remove(rule)\n continue\n if hash_value != new_rule_hashes[rule_file]:\n # Rule file was changed, reload rule\n try:\n new_rule = self.rules_loader.load_configuration(rule_file, self.conf)\n if not new_rule:\n logging.error('Invalid rule file skipped: %s' % rule_file)\n continue\n if 'is_enabled' in new_rule and not new_rule['is_enabled']:\n elastalert_logger.info('Rule file %s is now disabled.' % (rule_file))\n # Remove this rule if it's been disabled\n self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file]\n continue\n except EAException as e:\n message = 'Could not load rule %s: %s' % (rule_file, e)\n self.handle_error(message)\n # Want to send email to address specified in the rule. Try and load the YAML to find it.\n try:\n rule_yaml = self.rules_loader.load_yaml(rule_file)\n except EAException:\n self.send_notification_email(exception=e)\n continue\n\n self.send_notification_email(exception=e, rule=rule_yaml)\n continue\n elastalert_logger.info(\"Reloading configuration for rule %s\" % (rule_file))\n\n # Re-enable if rule had been disabled\n for disabled_rule in self.disabled_rules:\n if disabled_rule['name'] == new_rule['name']:\n self.rules.append(disabled_rule)\n self.disabled_rules.remove(disabled_rule)\n break\n\n # Initialize the rule that matches rule_file\n new_rule = self.init_rule(new_rule, False)\n self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file]\n if new_rule:\n self.rules.append(new_rule)\n\n # Load new rules\n if not self.args.rule:\n for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()):\n try:\n new_rule = self.rules_loader.load_configuration(rule_file, self.conf)\n if not new_rule:\n logging.error('Invalid rule file skipped: %s' % rule_file)\n continue\n if 'is_enabled' in new_rule and not new_rule['is_enabled']:\n continue\n if new_rule['name'] in [rule['name'] for rule in self.rules]:\n raise EAException(\"A rule with the name %s already exists\" % (new_rule['name']))\n except EAException as e:\n self.handle_error('Could not load rule %s: %s' % (rule_file, e))\n self.send_notification_email(exception=e, rule_file=rule_file)\n continue\n if self.init_rule(new_rule):\n elastalert_logger.info('Loaded new rule %s' % (rule_file))\n if new_rule['name'] in self.es_clients:\n self.es_clients.pop(new_rule['name'])\n self.rules.append(new_rule)\n\n self.rule_hashes = new_rule_hashes\n\n def start(self):\n \"\"\" Periodically go through each rule and run it \"\"\"\n if self.starttime:\n if self.starttime == 'NOW':\n self.starttime = ts_now()\n else:\n try:\n self.starttime = ts_to_dt(self.starttime)\n except (TypeError, ValueError):\n self.handle_error(\"%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)\" % (self.starttime))\n exit(1)\n\n for rule in self.rules:\n rule['initial_starttime'] = self.starttime\n self.wait_until_responsive(timeout=self.args.timeout)\n self.running = True\n elastalert_logger.info(\"Starting up\")\n self.scheduler.add_job(self.handle_pending_alerts, 'interval',\n seconds=self.run_every.total_seconds(), id='_internal_handle_pending_alerts')\n self.scheduler.add_job(self.handle_config_change, 'interval',\n seconds=self.run_every.total_seconds(), id='_internal_handle_config_change')\n self.scheduler.start()\n while self.running:\n next_run = datetime.datetime.utcnow() + self.run_every\n\n # Quit after end_time has been reached\n if self.args.end:\n endtime = ts_to_dt(self.args.end)\n\n if next_run.replace(tzinfo=dateutil.tz.tzutc()) > endtime:\n exit(0)\n\n if next_run < datetime.datetime.utcnow():\n continue\n\n # Show disabled rules\n if self.show_disabled_rules:\n elastalert_logger.info(\"Disabled rules are: %s\" % (str(self.get_disabled_rules())))\n\n # Wait before querying again\n sleep_duration = total_seconds(next_run - datetime.datetime.utcnow())\n self.sleep_for(sleep_duration)\n\n def wait_until_responsive(self, timeout, clock=timeit.default_timer):\n \"\"\"Wait until ElasticSearch becomes responsive (or too much time passes).\"\"\"\n\n # Elapsed time is a floating point number of seconds.\n timeout = timeout.total_seconds()\n\n # Don't poll unless we're asked to.\n if timeout <= 0.0:\n return\n\n # Periodically poll ElasticSearch. Keep going until ElasticSearch is\n # responsive *and* the writeback index exists.\n ref = clock()\n while (clock() - ref) < timeout:\n try:\n if self.writeback_es.indices.exists(self.writeback_alias):\n return\n except ConnectionError:\n pass\n time.sleep(1.0)\n\n if self.writeback_es.ping():\n logging.error(\n 'Writeback alias \"%s\" does not exist, did you run `elastalert-create-index`?',\n self.writeback_alias,\n )\n else:\n logging.error(\n 'Could not reach ElasticSearch at \"%s:%d\".',\n self.conf['es_host'],\n self.conf['es_port'],\n )\n exit(1)\n\n def run_all_rules(self):\n \"\"\" Run each rule one time \"\"\"\n self.handle_pending_alerts()\n\n for rule in self.rules:\n self.handle_rule_execution(rule)\n\n self.handle_config_change()\n\n def handle_pending_alerts(self):\n self.thread_data.alerts_sent = 0\n self.send_pending_alerts()\n elastalert_logger.info(\"Background alerts thread %s pending alerts sent at %s\" % (self.thread_data.alerts_sent,\n pretty_ts(ts_now())))\n\n def handle_config_change(self):\n if not self.args.pin_rules:\n self.load_rule_changes()\n elastalert_logger.info(\"Background configuration change check run at %s\" % (pretty_ts(ts_now())))\n\n def handle_rule_execution(self, rule):\n self.thread_data.alerts_sent = 0\n next_run = datetime.datetime.utcnow() + rule['run_every']\n # Set endtime based on the rule's delay\n delay = rule.get('query_delay')\n if hasattr(self.args, 'end') and self.args.end:\n endtime = ts_to_dt(self.args.end)\n elif delay:\n endtime = ts_now() - delay\n else:\n endtime = ts_now()\n\n # Apply rules based on execution time limits\n if rule.get('limit_execution'):\n rule['next_starttime'] = None\n rule['next_min_starttime'] = None\n exec_next = next(croniter(rule['limit_execution']))\n endtime_epoch = dt_to_unix(endtime)\n # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time\n # That means that we need to pause execution after this run\n if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59:\n # apscheduler requires pytz tzinfos, so don't use unix_to_dt here!\n rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc)\n if rule.get('limit_execution_coverage'):\n rule['next_min_starttime'] = rule['next_starttime']\n if not rule['has_run_once']:\n self.reset_rule_schedule(rule)\n return\n\n rule['has_run_once'] = True\n try:\n num_matches = self.run_rule(rule, endtime, rule.get('initial_starttime'))\n except EAException as e:\n self.handle_error(\"Error running rule %s: %s\" % (rule['name'], e), {'rule': rule['name']})\n except Exception as e:\n self.handle_uncaught_exception(e, rule)\n else:\n old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time'))\n elastalert_logger.info(\"Ran %s from %s to %s: %s query hits (%s already seen), %s matches,\"\n \" %s alerts sent\" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')),\n self.thread_data.num_hits, self.thread_data.num_dupes, num_matches,\n self.thread_data.alerts_sent))\n self.thread_data.alerts_sent = 0\n\n if next_run < datetime.datetime.utcnow():\n # We were processing for longer than our refresh interval\n # This can happen if --start was specified with a large time period\n # or if we are running too slow to process events in real time.\n logging.warning(\n \"Querying from %s to %s took longer than %s!\" % (\n old_starttime,\n pretty_ts(endtime, rule.get('use_local_time')),\n self.run_every\n )\n )\n\n rule['initial_starttime'] = None\n\n self.remove_old_events(rule)\n\n self.reset_rule_schedule(rule)\n\n def reset_rule_schedule(self, rule):\n # We hit the end of a execution schedule, pause ourselves until next run\n if rule.get('limit_execution') and rule['next_starttime']:\n self.scheduler.modify_job(job_id=rule['name'], next_run_time=rule['next_starttime'])\n # If we are preventing covering non-scheduled time periods, reset min_starttime and previous_endtime\n if rule['next_min_starttime']:\n rule['minimum_starttime'] = rule['next_min_starttime']\n rule['previous_endtime'] = rule['next_min_starttime']\n elastalert_logger.info('Pausing %s until next run at %s' % (rule['name'], pretty_ts(rule['next_starttime'])))\n\n def stop(self):\n \"\"\" Stop an ElastAlert runner that's been started \"\"\"\n self.running = False\n\n def get_disabled_rules(self):\n \"\"\" Return disabled rules \"\"\"\n return [rule['name'] for rule in self.disabled_rules]\n\n def sleep_for(self, duration):\n \"\"\" Sleep for a set duration \"\"\"\n elastalert_logger.info(\"Sleeping for %s seconds\" % (duration))\n time.sleep(duration)\n\n def generate_kibana4_db(self, rule, match):\n ''' Creates a link for a kibana4 dashboard which has time set to the match. '''\n db_name = rule.get('use_kibana4_dashboard')\n start = ts_add(\n lookup_es_key(match, rule['timestamp_field']),\n -rule.get('kibana4_start_timedelta', rule.get('timeframe', datetime.timedelta(minutes=10)))\n )\n end = ts_add(\n lookup_es_key(match, rule['timestamp_field']),\n rule.get('kibana4_end_timedelta', rule.get('timeframe', datetime.timedelta(minutes=10)))\n )\n return kibana.kibana4_dashboard_link(db_name, start, end)\n\n def generate_kibana_db(self, rule, match):\n ''' Uses a template dashboard to upload a temp dashboard showing the match.\n Returns the url to the dashboard. '''\n db = copy.deepcopy(kibana.dashboard_temp)\n\n # Set timestamp fields to match our rule especially if\n # we have configured something other than @timestamp\n kibana.set_timestamp_field(db, rule['timestamp_field'])\n\n # Set filters\n for filter in rule['filter']:\n if filter:\n kibana.add_filter(db, filter)\n kibana.set_included_fields(db, rule['include'])\n\n # Set index\n index = self.get_index(rule)\n kibana.set_index_name(db, index)\n\n return self.upload_dashboard(db, rule, match)\n\n def upload_dashboard(self, db, rule, match):\n ''' Uploads a dashboard schema to the kibana-int Elasticsearch index associated with rule.\n Returns the url to the dashboard. '''\n # Set time range\n start = ts_add(lookup_es_key(match, rule['timestamp_field']), -rule.get('timeframe', datetime.timedelta(minutes=10)))\n end = ts_add(lookup_es_key(match, rule['timestamp_field']), datetime.timedelta(minutes=10))\n kibana.set_time(db, start, end)\n\n # Set dashboard name\n db_name = 'ElastAlert - %s - %s' % (rule['name'], end)\n kibana.set_name(db, db_name)\n\n # Add filter for query_key value\n if 'query_key' in rule:\n for qk in rule.get('compound_query_key', [rule['query_key']]):\n if qk in match:\n term = {'term': {qk: match[qk]}}\n kibana.add_filter(db, term)\n\n # Add filter for aggregation_key value\n if 'aggregation_key' in rule:\n for qk in rule.get('compound_aggregation_key', [rule['aggregation_key']]):\n if qk in match:\n term = {'term': {qk: match[qk]}}\n kibana.add_filter(db, term)\n\n # Convert to json\n db_js = json.dumps(db)\n db_body = {'user': 'guest',\n 'group': 'guest',\n 'title': db_name,\n 'dashboard': db_js}\n\n # Upload\n es = elasticsearch_client(rule)\n # TODO: doc_type = _doc for elastic >= 6\n res = es.index(index='kibana-int',\n doc_type='temp',\n body=db_body)\n\n # Return dashboard URL\n kibana_url = rule.get('kibana_url')\n if not kibana_url:\n kibana_url = 'http://%s:%s/_plugin/kibana/' % (rule['es_host'],\n rule['es_port'])\n return kibana_url + '#/dashboard/temp/%s' % (res['_id'])\n\n def get_dashboard(self, rule, db_name):\n \"\"\" Download dashboard which matches use_kibana_dashboard from Elasticsearch. \"\"\"\n es = elasticsearch_client(rule)\n if not db_name:\n raise EAException(\"use_kibana_dashboard undefined\")\n query = {'query': {'term': {'_id': db_name}}}\n try:\n # TODO use doc_type = _doc\n res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard'])\n except ElasticsearchException as e:\n raise EAException(\"Error querying for dashboard: %s\" % (e)).with_traceback(sys.exc_info()[2])\n\n if res['hits']['hits']:\n return json.loads(res['hits']['hits'][0]['_source']['dashboard'])\n else:\n raise EAException(\"Could not find dashboard named %s\" % (db_name))\n\n def use_kibana_link(self, rule, match):\n \"\"\" Uploads an existing dashboard as a temp dashboard modified for match time.\n Returns the url to the dashboard. \"\"\"\n # Download or get cached dashboard\n dashboard = rule.get('dashboard_schema')\n if not dashboard:\n db_name = rule.get('use_kibana_dashboard')\n dashboard = self.get_dashboard(rule, db_name)\n if dashboard:\n rule['dashboard_schema'] = dashboard\n else:\n return None\n dashboard = copy.deepcopy(dashboard)\n return self.upload_dashboard(dashboard, rule, match)\n\n def filters_from_kibana(self, rule, db_name):\n \"\"\" Downloads a dashboard from Kibana and returns corresponding filters, None on error. \"\"\"\n try:\n db = rule.get('dashboard_schema')\n if not db:\n db = self.get_dashboard(rule, db_name)\n filters = kibana.filters_from_dashboard(db)\n except EAException:\n return None\n return filters\n\n def alert(self, matches, rule, alert_time=None, retried=False):\n \"\"\" Wraps alerting, Kibana linking and enhancements in an exception handler \"\"\"\n try:\n return self.send_alert(matches, rule, alert_time=alert_time, retried=retried)\n except Exception as e:\n self.handle_uncaught_exception(e, rule)\n\n def send_alert(self, matches, rule, alert_time=None, retried=False):\n \"\"\" Send out an alert.\n\n :param matches: A list of matches.\n :param rule: A rule configuration.\n \"\"\"\n if not matches:\n return\n\n if alert_time is None:\n alert_time = ts_now()\n\n # Compute top count keys\n if rule.get('top_count_keys'):\n for match in matches:\n if 'query_key' in rule:\n qk = lookup_es_key(match, rule['query_key'])\n else:\n qk = None\n\n if isinstance(rule['type'], FlatlineRule):\n # flatline rule triggers when there have been no events from now()-timeframe to now(),\n # so using now()-timeframe will return no results. for now we can just mutliple the timeframe\n # by 2, but this could probably be timeframe+run_every to prevent too large of a lookup?\n timeframe = datetime.timedelta(seconds=2 * rule.get('timeframe').total_seconds())\n else:\n timeframe = rule.get('timeframe', datetime.timedelta(minutes=10))\n\n start = ts_to_dt(lookup_es_key(match, rule['timestamp_field'])) - timeframe\n end = ts_to_dt(lookup_es_key(match, rule['timestamp_field'])) + datetime.timedelta(minutes=10)\n keys = rule.get('top_count_keys')\n counts = self.get_top_counts(rule, start, end, keys, qk=qk)\n match.update(counts)\n\n # Generate a kibana3 dashboard for the first match\n if rule.get('generate_kibana_link') or rule.get('use_kibana_dashboard'):\n try:\n if rule.get('generate_kibana_link'):\n kb_link = self.generate_kibana_db(rule, matches[0])\n else:\n kb_link = self.use_kibana_link(rule, matches[0])\n except EAException as e:\n self.handle_error(\"Could not generate Kibana dash for %s match: %s\" % (rule['name'], e))\n else:\n if kb_link:\n matches[0]['kibana_link'] = kb_link\n\n if rule.get('use_kibana4_dashboard'):\n kb_link = self.generate_kibana4_db(rule, matches[0])\n if kb_link:\n matches[0]['kibana_link'] = kb_link\n\n if rule.get('generate_kibana_discover_url'):\n kb_link = generate_kibana_discover_url(rule, matches[0])\n if kb_link:\n matches[0]['kibana_discover_url'] = kb_link\n\n # Enhancements were already run at match time if\n # run_enhancements_first is set or\n # retried==True, which means this is a retry of a failed alert\n if not rule.get('run_enhancements_first') and not retried:\n for enhancement in rule['match_enhancements']:\n valid_matches = []\n for match in matches:\n try:\n enhancement.process(match)\n valid_matches.append(match)\n except DropMatchException:\n pass\n except EAException as e:\n self.handle_error(\"Error running match enhancement: %s\" % (e), {'rule': rule['name']})\n matches = valid_matches\n if not matches:\n return None\n\n # Don't send real alerts in debug mode\n if self.debug:\n alerter = DebugAlerter(rule)\n alerter.alert(matches)\n return None\n\n # Run the alerts\n alert_sent = False\n alert_exception = None\n # Alert.pipeline is a single object shared between every alerter\n # This allows alerters to pass objects and data between themselves\n alert_pipeline = {\"alert_time\": alert_time}\n for alert in rule['alert']:\n alert.pipeline = alert_pipeline\n try:\n alert.alert(matches)\n except EAException as e:\n self.handle_error('Error while running alert %s: %s' % (alert.get_info()['type'], e), {'rule': rule['name']})\n alert_exception = str(e)\n else:\n self.thread_data.alerts_sent += 1\n alert_sent = True\n\n # Write the alert(s) to ES\n agg_id = None\n for match in matches:\n alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception)\n # Set all matches to aggregate together\n if agg_id:\n alert_body['aggregate_id'] = agg_id\n res = self.writeback('elastalert', alert_body, rule)\n if res and not agg_id:\n agg_id = res['_id']\n\n def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=None):\n body = {\n 'match_body': match,\n 'rule_name': rule['name'],\n 'alert_info': rule['alert'][0].get_info() if not self.debug else {},\n 'alert_sent': alert_sent,\n 'alert_time': alert_time\n }\n\n if rule.get('include_match_in_root'):\n body.update({k: v for k, v in match.items() if not k.startswith('_')})\n\n if self.add_metadata_alert:\n body['category'] = rule['category']\n body['description'] = rule['description']\n body['owner'] = rule['owner']\n body['priority'] = rule['priority']\n\n match_time = lookup_es_key(match, rule['timestamp_field'])\n if match_time is not None:\n body['match_time'] = match_time\n\n # TODO record info about multiple alerts\n\n # If the alert failed to send, record the exception\n if not alert_sent:\n body['alert_exception'] = alert_exception\n return body\n\n def writeback(self, doc_type, body, rule=None, match_body=None):\n # ES 2.0 - 2.3 does not support dots in field names.\n if self.replace_dots_in_field_names:\n writeback_body = replace_dots_in_field_names(body)\n else:\n writeback_body = body\n\n for key in list(writeback_body.keys()):\n # Convert any datetime objects to timestamps\n if isinstance(writeback_body[key], datetime.datetime):\n writeback_body[key] = dt_to_ts(writeback_body[key])\n\n if self.debug:\n elastalert_logger.info(\"Skipping writing to ES: %s\" % (writeback_body))\n return None\n\n if '@timestamp' not in writeback_body:\n writeback_body['@timestamp'] = dt_to_ts(ts_now())\n\n try:\n index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type)\n if self.writeback_es.is_atleastsixtwo():\n res = self.writeback_es.index(index=index, body=body)\n else:\n res = self.writeback_es.index(index=index, doc_type=doc_type, body=body)\n return res\n except ElasticsearchException as e:\n logging.exception(\"Error writing alert info to Elasticsearch: %s\" % (e))\n\n def find_recent_pending_alerts(self, time_limit):\n \"\"\" Queries writeback_es to find alerts that did not send\n and are newer than time_limit \"\"\"\n\n # XXX only fetches 1000 results. If limit is reached, next loop will catch them\n # unless there is constantly more than 1000 alerts to send.\n\n # Fetch recent, unsent alerts that aren't part of an aggregate, earlier alerts first.\n inner_query = {'query_string': {'query': '!_exists_:aggregate_id AND alert_sent:false'}}\n time_filter = {'range': {'alert_time': {'from': dt_to_ts(ts_now() - time_limit),\n 'to': dt_to_ts(ts_now())}}}\n sort = {'sort': {'alert_time': {'order': 'asc'}}}\n if self.writeback_es.is_atleastfive():\n query = {'query': {'bool': {'must': inner_query, 'filter': time_filter}}}\n else:\n query = {'query': inner_query, 'filter': time_filter}\n query.update(sort)\n try:\n if self.writeback_es.is_atleastsixtwo():\n res = self.writeback_es.search(index=self.writeback_index, body=query, size=1000)\n else:\n res = self.writeback_es.deprecated_search(index=self.writeback_index,\n doc_type='elastalert', body=query, size=1000)\n if res['hits']['hits']:\n return res['hits']['hits']\n except ElasticsearchException as e:\n logging.exception(\"Error finding recent pending alerts: %s %s\" % (e, query))\n return []\n\n def send_pending_alerts(self):\n pending_alerts = self.find_recent_pending_alerts(self.alert_time_limit)\n for alert in pending_alerts:\n _id = alert['_id']\n alert = alert['_source']\n try:\n rule_name = alert.pop('rule_name')\n alert_time = alert.pop('alert_time')\n match_body = alert.pop('match_body')\n except KeyError:\n # Malformed alert, drop it\n continue\n\n # Find original rule\n for rule in self.rules:\n if rule['name'] == rule_name:\n break\n else:\n # Original rule is missing, keep alert for later if rule reappears\n continue\n\n # Set current_es for top_count_keys query\n self.thread_data.current_es = elasticsearch_client(rule)\n\n # Send the alert unless it's a future alert\n if ts_now() > ts_to_dt(alert_time):\n aggregated_matches = self.get_aggregated_matches(_id)\n if aggregated_matches:\n matches = [match_body] + [agg_match['match_body'] for agg_match in aggregated_matches]\n self.alert(matches, rule, alert_time=alert_time)\n else:\n # If this rule isn't using aggregation, this must be a retry of a failed alert\n retried = False\n if not rule.get('aggregation'):\n retried = True\n self.alert([match_body], rule, alert_time=alert_time, retried=retried)\n\n if rule['current_aggregate_id']:\n for qk, agg_id in rule['current_aggregate_id'].items():\n if agg_id == _id:\n rule['current_aggregate_id'].pop(qk)\n break\n\n # Delete it from the index\n try:\n if self.writeback_es.is_atleastsixtwo():\n self.writeback_es.delete(index=self.writeback_index, id=_id)\n else:\n self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=_id)\n except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil.\n self.handle_error(\"Failed to delete alert %s at %s\" % (_id, alert_time))\n\n # Send in memory aggregated alerts\n for rule in self.rules:\n if rule['agg_matches']:\n for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].items():\n if ts_now() > aggregate_alert_time:\n alertable_matches = [\n agg_match\n for agg_match\n in rule['agg_matches']\n if self.get_aggregation_key_value(rule, agg_match) == aggregation_key_value\n ]\n self.alert(alertable_matches, rule)\n rule['agg_matches'] = [\n agg_match\n for agg_match\n in rule['agg_matches']\n if self.get_aggregation_key_value(rule, agg_match) != aggregation_key_value\n ]\n\n def get_aggregated_matches(self, _id):\n \"\"\" Removes and returns all matches from writeback_es that have aggregate_id == _id \"\"\"\n\n # XXX if there are more than self.max_aggregation matches, you have big alerts and we will leave entries in ES.\n query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}}\n matches = []\n try:\n if self.writeback_es.is_atleastsixtwo():\n res = self.writeback_es.search(index=self.writeback_index, body=query,\n size=self.max_aggregation)\n else:\n res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert',\n body=query, size=self.max_aggregation)\n for match in res['hits']['hits']:\n matches.append(match['_source'])\n if self.writeback_es.is_atleastsixtwo():\n self.writeback_es.delete(index=self.writeback_index, id=match['_id'])\n else:\n self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=match['_id'])\n except (KeyError, ElasticsearchException) as e:\n self.handle_error(\"Error fetching aggregated matches: %s\" % (e), {'id': _id})\n return matches\n\n def find_pending_aggregate_alert(self, rule, aggregation_key_value=None):\n query = {'filter': {'bool': {'must': [{'term': {'rule_name': rule['name']}},\n {'range': {'alert_time': {'gt': ts_now()}}},\n {'term': {'alert_sent': 'false'}}],\n 'must_not': [{'exists': {'field': 'aggregate_id'}}]}}}\n if aggregation_key_value:\n query['filter']['bool']['must'].append({'term': {'aggregation_key': aggregation_key_value}})\n if self.writeback_es.is_atleastfive():\n query = {'query': {'bool': query}}\n query['sort'] = {'alert_time': {'order': 'desc'}}\n try:\n if self.writeback_es.is_atleastsixtwo():\n res = self.writeback_es.search(index=self.writeback_index, body=query, size=1)\n else:\n res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', body=query, size=1)\n if len(res['hits']['hits']) == 0:\n return None\n except (KeyError, ElasticsearchException) as e:\n self.handle_error(\"Error searching for pending aggregated matches: %s\" % (e), {'rule_name': rule['name']})\n return None\n\n return res['hits']['hits'][0]\n\n def add_aggregated_alert(self, match, rule):\n \"\"\" Save a match as a pending aggregate alert to Elasticsearch. \"\"\"\n\n # Optionally include the 'aggregation_key' as a dimension for aggregations\n aggregation_key_value = self.get_aggregation_key_value(rule, match)\n\n if (not rule['current_aggregate_id'].get(aggregation_key_value) or\n ('aggregate_alert_time' in rule and aggregation_key_value in rule['aggregate_alert_time'] and rule[\n 'aggregate_alert_time'].get(aggregation_key_value) < ts_to_dt(lookup_es_key(match, rule['timestamp_field'])))):\n\n # ElastAlert may have restarted while pending alerts exist\n pending_alert = self.find_pending_aggregate_alert(rule, aggregation_key_value)\n if pending_alert:\n alert_time = ts_to_dt(pending_alert['_source']['alert_time'])\n rule['aggregate_alert_time'][aggregation_key_value] = alert_time\n agg_id = pending_alert['_id']\n rule['current_aggregate_id'] = {aggregation_key_value: agg_id}\n elastalert_logger.info(\n 'Adding alert for %s to aggregation(id: %s, aggregation_key: %s), next alert at %s' % (\n rule['name'],\n agg_id,\n aggregation_key_value,\n alert_time\n )\n )\n else:\n # First match, set alert_time\n alert_time = ''\n if isinstance(rule['aggregation'], dict) and rule['aggregation'].get('schedule'):\n croniter._datetime_to_timestamp = cronite_datetime_to_timestamp # For Python 2.6 compatibility\n try:\n iter = croniter(rule['aggregation']['schedule'], ts_now())\n alert_time = unix_to_dt(iter.get_next())\n except Exception as e:\n self.handle_error(\"Error parsing aggregate send time Cron format %s\" % (e), rule['aggregation']['schedule'])\n else:\n if rule.get('aggregate_by_match_time', False):\n match_time = ts_to_dt(lookup_es_key(match, rule['timestamp_field']))\n alert_time = match_time + rule['aggregation']\n else:\n alert_time = ts_now() + rule['aggregation']\n\n rule['aggregate_alert_time'][aggregation_key_value] = alert_time\n agg_id = None\n elastalert_logger.info(\n 'New aggregation for %s, aggregation_key: %s. next alert at %s.' % (rule['name'], aggregation_key_value, alert_time)\n )\n else:\n # Already pending aggregation, use existing alert_time\n alert_time = rule['aggregate_alert_time'].get(aggregation_key_value)\n agg_id = rule['current_aggregate_id'].get(aggregation_key_value)\n elastalert_logger.info(\n 'Adding alert for %s to aggregation(id: %s, aggregation_key: %s), next alert at %s' % (\n rule['name'],\n agg_id,\n aggregation_key_value,\n alert_time\n )\n )\n\n alert_body = self.get_alert_body(match, rule, False, alert_time)\n if agg_id:\n alert_body['aggregate_id'] = agg_id\n if aggregation_key_value:\n alert_body['aggregation_key'] = aggregation_key_value\n res = self.writeback('elastalert', alert_body, rule)\n\n # If new aggregation, save _id\n if res and not agg_id:\n rule['current_aggregate_id'][aggregation_key_value] = res['_id']\n\n # Couldn't write the match to ES, save it in memory for now\n if not res:\n rule['agg_matches'].append(match)\n\n return res\n\n def silence(self, silence_cache_key=None):\n \"\"\" Silence an alert for a period of time. --silence and --rule must be passed as args. \"\"\"\n if self.debug:\n logging.error('--silence not compatible with --debug')\n exit(1)\n\n if not self.args.rule:\n logging.error('--silence must be used with --rule')\n exit(1)\n\n # With --rule, self.rules will only contain that specific rule\n if not silence_cache_key:\n silence_cache_key = self.rules[0]['name'] + \"._silence\"\n\n try:\n silence_ts = parse_deadline(self.args.silence)\n except (ValueError, TypeError):\n logging.error('%s is not a valid time period' % (self.args.silence))\n exit(1)\n\n if not self.set_realert(silence_cache_key, silence_ts, 0):\n logging.error('Failed to save silence command to Elasticsearch')\n exit(1)\n\n elastalert_logger.info('Success. %s will be silenced until %s' % (silence_cache_key, silence_ts))\n\n def set_realert(self, silence_cache_key, timestamp, exponent):\n \"\"\" Write a silence to Elasticsearch for silence_cache_key until timestamp. \"\"\"\n body = {'exponent': exponent,\n 'rule_name': silence_cache_key,\n '@timestamp': ts_now(),\n 'until': timestamp}\n\n self.silence_cache[silence_cache_key] = (timestamp, exponent)\n return self.writeback('silence', body)\n\n def is_silenced(self, rule_name):\n \"\"\" Checks if rule_name is currently silenced. Returns false on exception. \"\"\"\n if rule_name in self.silence_cache:\n if ts_now() < self.silence_cache[rule_name][0]:\n return True\n\n if self.debug:\n return False\n query = {'term': {'rule_name': rule_name}}\n sort = {'sort': {'until': {'order': 'desc'}}}\n if self.writeback_es.is_atleastfive():\n query = {'query': query}\n else:\n query = {'filter': query}\n query.update(sort)\n\n try:\n doc_type = 'silence'\n index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type)\n if self.writeback_es.is_atleastsixtwo():\n if self.writeback_es.is_atleastsixsix():\n res = self.writeback_es.search(index=index, size=1, body=query,\n _source_includes=['until', 'exponent'])\n else:\n res = self.writeback_es.search(index=index, size=1, body=query,\n _source_include=['until', 'exponent'])\n else:\n res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type,\n size=1, body=query, _source_include=['until', 'exponent'])\n except ElasticsearchException as e:\n self.handle_error(\"Error while querying for alert silence status: %s\" % (e), {'rule': rule_name})\n\n return False\n if res['hits']['hits']:\n until_ts = res['hits']['hits'][0]['_source']['until']\n exponent = res['hits']['hits'][0]['_source'].get('exponent', 0)\n if rule_name not in list(self.silence_cache.keys()):\n self.silence_cache[rule_name] = (ts_to_dt(until_ts), exponent)\n else:\n self.silence_cache[rule_name] = (ts_to_dt(until_ts), self.silence_cache[rule_name][1])\n if ts_now() < ts_to_dt(until_ts):\n return True\n return False\n\n def handle_error(self, message, data=None):\n ''' Logs message at error level and writes message, data and traceback to Elasticsearch. '''\n logging.error(message)\n body = {'message': message}\n tb = traceback.format_exc()\n body['traceback'] = tb.strip().split('\\n')\n if data:\n body['data'] = data\n self.writeback('elastalert_error', body)\n\n def handle_uncaught_exception(self, exception, rule):\n \"\"\" Disables a rule and sends a notification. \"\"\"\n logging.error(traceback.format_exc())\n self.handle_error('Uncaught exception running rule %s: %s' % (rule['name'], exception), {'rule': rule['name']})\n if self.disable_rules_on_error:\n self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']]\n self.disabled_rules.append(rule)\n self.scheduler.pause_job(job_id=rule['name'])\n elastalert_logger.info('Rule %s disabled', rule['name'])\n if self.notify_email:\n self.send_notification_email(exception=exception, rule=rule)\n\n def send_notification_email(self, text='', exception=None, rule=None, subject=None, rule_file=None):\n email_body = text\n rule_name = None\n if rule:\n rule_name = rule['name']\n elif rule_file:\n rule_name = rule_file\n if exception and rule_name:\n if not subject:\n subject = 'Uncaught exception in ElastAlert - %s' % (rule_name)\n email_body += '\\n\\n'\n email_body += 'The rule %s has raised an uncaught exception.\\n\\n' % (rule_name)\n if self.disable_rules_on_error:\n modified = ' or if the rule config file has been modified' if not self.args.pin_rules else ''\n email_body += 'It has been disabled and will be re-enabled when ElastAlert restarts%s.\\n\\n' % (modified)\n tb = traceback.format_exc()\n email_body += tb\n\n if isinstance(self.notify_email, str):\n self.notify_email = [self.notify_email]\n email = MIMEText(email_body)\n email['Subject'] = subject if subject else 'ElastAlert notification'\n recipients = self.notify_email\n if rule and rule.get('notify_email'):\n if isinstance(rule['notify_email'], str):\n rule['notify_email'] = [rule['notify_email']]\n recipients = recipients + rule['notify_email']\n recipients = list(set(recipients))\n email['To'] = ', '.join(recipients)\n email['From'] = self.from_addr\n email['Reply-To'] = self.conf.get('email_reply_to', email['To'])\n\n try:\n smtp = SMTP(self.smtp_host)\n smtp.sendmail(self.from_addr, recipients, email.as_string())\n except (SMTPException, error) as e:\n self.handle_error('Error connecting to SMTP host: %s' % (e), {'email_body': email_body})\n\n def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None):\n \"\"\" Counts the number of events for each unique value for each key field.\n Returns a dictionary with top_events_ mapped to the top 5 counts for each key. \"\"\"\n all_counts = {}\n if not number:\n number = rule.get('top_count_number', 5)\n for key in keys:\n index = self.get_index(rule, starttime, endtime)\n\n hits_terms = self.get_hits_terms(rule, starttime, endtime, index, key, qk, number)\n if hits_terms is None:\n top_events_count = {}\n else:\n buckets = list(hits_terms.values())[0]\n\n # get_hits_terms adds to num_hits, but we don't want to count these\n self.thread_data.num_hits -= len(buckets)\n terms = {}\n for bucket in buckets:\n terms[bucket['key']] = bucket['doc_count']\n counts = list(terms.items())\n counts.sort(key=lambda x: x[1], reverse=True)\n top_events_count = dict(counts[:number])\n\n # Save a dict with the top 5 events by key\n all_counts['top_events_%s' % (key)] = top_events_count\n\n return all_counts\n\n def next_alert_time(self, rule, name, timestamp):\n \"\"\" Calculate an 'until' time and exponent based on how much past the last 'until' we are. \"\"\"\n if name in self.silence_cache:\n last_until, exponent = self.silence_cache[name]\n else:\n # If this isn't cached, this is the first alert or writeback_es is down, normal realert\n return timestamp + rule['realert'], 0\n\n if not rule.get('exponential_realert'):\n return timestamp + rule['realert'], 0\n diff = seconds(timestamp - last_until)\n # Increase exponent if we've alerted recently\n if diff < seconds(rule['realert']) * 2 ** exponent:\n exponent += 1\n else:\n # Continue decreasing exponent the longer it's been since the last alert\n while diff > seconds(rule['realert']) * 2 ** exponent and exponent > 0:\n diff -= seconds(rule['realert']) * 2 ** exponent\n exponent -= 1\n\n wait = datetime.timedelta(seconds=seconds(rule['realert']) * 2 ** exponent)\n if wait >= rule['exponential_realert']:\n return timestamp + rule['exponential_realert'], exponent - 1\n return timestamp + wait, exponent\n\n\ndef handle_signal(signal, frame):\n elastalert_logger.info('SIGINT received, stopping ElastAlert...')\n # use os._exit to exit immediately and avoid someone catching SystemExit\n os._exit(0)\n\n\ndef main(args=None):\n signal.signal(signal.SIGINT, handle_signal)\n if not args:\n args = sys.argv[1:]\n client = ElastAlerter(args)\n if not client.args.silence:\n client.start()\n\n\nif __name__ == '__main__':\n sys.exit(main(sys.argv[1:]))\n" }, { "path": "elastalert/enhancements.py", "content": "# -*- coding: utf-8 -*-\nfrom .util import pretty_ts\n\n\nclass BaseEnhancement(object):\n \"\"\" Enhancements take a match dictionary object and modify it in some way to\n enhance an alert. These are specified in each rule under the match_enhancements option.\n Generally, the key value pairs in the match module will be contained in the alert body. \"\"\"\n\n def __init__(self, rule):\n self.rule = rule\n\n def process(self, match):\n \"\"\" Modify the contents of match, a dictionary, in some way \"\"\"\n raise NotImplementedError()\n\n\nclass TimeEnhancement(BaseEnhancement):\n def process(self, match):\n match['@timestamp'] = pretty_ts(match['@timestamp'])\n\n\nclass DropMatchException(Exception):\n \"\"\" ElastAlert will drop a match if this exception type is raised by an enhancement \"\"\"\n pass\n" }, { "path": "elastalert/es_mappings/5/elastalert.json", "content": "{\n \"elastalert\": {\n \"properties\": {\n \"rule_name\": {\n \"index\": \"not_analyzed\",\n \"type\": \"string\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"alert_time\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"match_time\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"match_body\": {\n \"type\": \"object\",\n \"enabled\": \"false\"\n },\n \"aggregate_id\": {\n \"index\": \"not_analyzed\",\n \"type\": \"string\"\n }\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/5/elastalert_error.json", "content": "{\n \"elastalert_error\": {\n \"properties\": {\n \"data\": {\n \"type\": \"object\",\n \"enabled\": \"false\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n }\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/5/elastalert_status.json", "content": "{\n \"elastalert_status\": {\n \"properties\": {\n \"rule_name\": {\n \"index\": \"not_analyzed\",\n \"type\": \"string\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n }\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/5/past_elastalert.json", "content": "{\n \"past_elastalert\": {\n \"properties\": {\n \"rule_name\": {\n \"index\": \"not_analyzed\",\n \"type\": \"string\"\n },\n \"match_body\": {\n \"type\": \"object\",\n \"enabled\": \"false\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"aggregate_id\": {\n \"index\": \"not_analyzed\",\n \"type\": \"string\"\n }\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/5/silence.json", "content": "{\n \"silence\": {\n \"properties\": {\n \"rule_name\": {\n \"index\": \"not_analyzed\",\n \"type\": \"string\"\n },\n \"until\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n }\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/6/elastalert.json", "content": "{\n \"numeric_detection\": true,\n \"date_detection\": false,\n \"dynamic_templates\": [\n {\n \"strings_as_keyword\": {\n \"mapping\": {\n \"ignore_above\": 1024,\n \"type\": \"keyword\"\n },\n \"match_mapping_type\": \"string\"\n }\n }\n ],\n \"properties\": {\n \"rule_name\": {\n \"type\": \"keyword\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"alert_time\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"match_time\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"match_body\": {\n \"type\": \"object\"\n },\n \"aggregate_id\": {\n \"type\": \"keyword\"\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/6/elastalert_error.json", "content": "{\n \"properties\": {\n \"data\": {\n \"type\": \"object\",\n \"enabled\": \"false\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/6/elastalert_status.json", "content": "{\n \"properties\": {\n \"rule_name\": {\n \"type\": \"keyword\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/6/past_elastalert.json", "content": "{\n \"properties\": {\n \"rule_name\": {\n \"type\": \"keyword\"\n },\n \"match_body\": {\n \"type\": \"object\",\n \"enabled\": \"false\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"aggregate_id\": {\n \"type\": \"keyword\"\n }\n }\n}\n" }, { "path": "elastalert/es_mappings/6/silence.json", "content": "{\n \"properties\": {\n \"rule_name\": {\n \"type\": \"keyword\"\n },\n \"until\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n },\n \"@timestamp\": {\n \"type\": \"date\",\n \"format\": \"dateOptionalTime\"\n }\n }\n}\n" }, { "path": "elastalert/kibana.py", "content": "# -*- coding: utf-8 -*-\n# flake8: noqa\nimport os.path\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom .util import EAException\n\n\ndashboard_temp = {'editable': True,\n 'failover': False,\n 'index': {'default': 'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED',\n 'interval': 'none',\n 'pattern': '',\n 'warm_fields': True},\n 'loader': {'hide': False,\n 'load_elasticsearch': True,\n 'load_elasticsearch_size': 20,\n 'load_gist': True,\n 'load_local': True,\n 'save_default': True,\n 'save_elasticsearch': True,\n 'save_gist': False,\n 'save_local': True,\n 'save_temp': True,\n 'save_temp_ttl': '30d',\n 'save_temp_ttl_enable': True},\n 'nav': [{'collapse': False,\n 'enable': True,\n 'filter_id': 0,\n 'notice': False,\n 'now': False,\n 'refresh_intervals': ['5s',\n '10s',\n '30s',\n '1m',\n '5m',\n '15m',\n '30m',\n '1h',\n '2h',\n '1d'],\n 'status': 'Stable',\n 'time_options': ['5m',\n '15m',\n '1h',\n '6h',\n '12h',\n '24h',\n '2d',\n '7d',\n '30d'],\n 'timefield': '@timestamp',\n 'type': 'timepicker'}],\n 'panel_hints': True,\n 'pulldowns': [{'collapse': False,\n 'enable': True,\n 'notice': True,\n 'type': 'filtering'}],\n 'refresh': False,\n 'rows': [{'collapsable': True,\n 'collapse': False,\n 'editable': True,\n 'height': '350px',\n 'notice': False,\n 'panels': [{'annotate': {'enable': False,\n 'field': '_type',\n 'query': '*',\n 'size': 20,\n 'sort': ['_score', 'desc']},\n 'auto_int': True,\n 'bars': True,\n 'derivative': False,\n 'editable': True,\n 'fill': 3,\n 'grid': {'max': None, 'min': 0},\n 'group': ['default'],\n 'interactive': True,\n 'interval': '1m',\n 'intervals': ['auto',\n '1s',\n '1m',\n '5m',\n '10m',\n '30m',\n '1h',\n '3h',\n '12h',\n '1d',\n '1w',\n '1M',\n '1y'],\n 'legend': True,\n 'legend_counts': True,\n 'lines': False,\n 'linewidth': 3,\n 'mode': 'count',\n 'options': True,\n 'percentage': False,\n 'pointradius': 5,\n 'points': False,\n 'queries': {'ids': [0], 'mode': 'all'},\n 'resolution': 100,\n 'scale': 1,\n 'show_query': True,\n 'span': 12,\n 'spyable': True,\n 'stack': True,\n 'time_field': '@timestamp',\n 'timezone': 'browser',\n 'title': 'Events over time',\n 'tooltip': {'query_as_alias': True,\n 'value_type': 'cumulative'},\n 'type': 'histogram',\n 'value_field': None,\n 'x-axis': True,\n 'y-axis': True,\n 'y_format': 'none',\n 'zerofill': True,\n 'zoomlinks': True}],\n 'title': 'Graph'},\n {'collapsable': True,\n 'collapse': False,\n 'editable': True,\n 'height': '350px',\n 'notice': False,\n 'panels': [{'all_fields': False,\n 'editable': True,\n 'error': False,\n 'field_list': True,\n 'fields': [],\n 'group': ['default'],\n 'header': True,\n 'highlight': [],\n 'localTime': True,\n 'normTimes': True,\n 'offset': 0,\n 'overflow': 'min-height',\n 'pages': 5,\n 'paging': True,\n 'queries': {'ids': [0], 'mode': 'all'},\n 'size': 100,\n 'sort': ['@timestamp', 'desc'],\n 'sortable': True,\n 'span': 12,\n 'spyable': True,\n 'status': 'Stable',\n 'style': {'font-size': '9pt'},\n 'timeField': '@timestamp',\n 'title': 'All events',\n 'trimFactor': 300,\n 'type': 'table'}],\n 'title': 'Events'}],\n 'services': {'filter': {'ids': [0],\n 'list': {'0': {'active': True,\n 'alias': '',\n 'field': '@timestamp',\n 'from': 'now-24h',\n 'id': 0,\n 'mandate': 'must',\n 'to': 'now',\n 'type': 'time'}}},\n 'query': {'ids': [0],\n 'list': {'0': {'alias': '',\n 'color': '#7EB26D',\n 'enable': True,\n 'id': 0,\n 'pin': False,\n 'query': '',\n 'type': 'lucene'}}}},\n 'style': 'dark',\n 'title': 'ElastAlert Alert Dashboard'}\n\nkibana4_time_temp = \"(refreshInterval:(display:Off,section:0,value:0),time:(from:'%s',mode:absolute,to:'%s'))\"\n\n\ndef set_time(dashboard, start, end):\n dashboard['services']['filter']['list']['0']['from'] = start\n dashboard['services']['filter']['list']['0']['to'] = end\n\n\ndef set_index_name(dashboard, name):\n dashboard['index']['default'] = name\n\n\ndef set_timestamp_field(dashboard, field):\n # set the nav timefield if we don't want @timestamp\n dashboard['nav'][0]['timefield'] = field\n\n # set the time_field for each of our panels\n for row in dashboard.get('rows'):\n for panel in row.get('panels'):\n panel['time_field'] = field\n\n # set our filter's time field\n dashboard['services']['filter']['list']['0']['field'] = field\n\n\ndef add_filter(dashboard, es_filter):\n next_id = max(dashboard['services']['filter']['ids']) + 1\n\n kibana_filter = {'active': True,\n 'alias': '',\n 'id': next_id,\n 'mandate': 'must'}\n\n if 'not' in es_filter:\n es_filter = es_filter['not']\n kibana_filter['mandate'] = 'mustNot'\n\n if 'query' in es_filter:\n es_filter = es_filter['query']\n if 'query_string' in es_filter:\n kibana_filter['type'] = 'querystring'\n kibana_filter['query'] = es_filter['query_string']['query']\n elif 'term' in es_filter:\n kibana_filter['type'] = 'field'\n f_field, f_query = list(es_filter['term'].items())[0]\n # Wrap query in quotes, otherwise certain characters cause Kibana to throw errors\n if isinstance(f_query, str):\n f_query = '\"%s\"' % (f_query.replace('\"', '\\\\\"'))\n if isinstance(f_query, list):\n # Escape quotes\n f_query = [item.replace('\"', '\\\\\"') for item in f_query]\n # Wrap in quotes\n f_query = ['\"%s\"' % (item) for item in f_query]\n # Convert into joined query\n f_query = '(%s)' % (' AND '.join(f_query))\n kibana_filter['field'] = f_field\n kibana_filter['query'] = f_query\n elif 'range' in es_filter:\n kibana_filter['type'] = 'range'\n f_field, f_range = list(es_filter['range'].items())[0]\n kibana_filter['field'] = f_field\n kibana_filter.update(f_range)\n else:\n raise EAException(\"Could not parse filter %s for Kibana\" % (es_filter))\n\n dashboard['services']['filter']['ids'].append(next_id)\n dashboard['services']['filter']['list'][str(next_id)] = kibana_filter\n\n\ndef set_name(dashboard, name):\n dashboard['title'] = name\n\n\ndef set_included_fields(dashboard, fields):\n dashboard['rows'][1]['panels'][0]['fields'] = list(set(fields))\n\n\ndef filters_from_dashboard(db):\n filters = db['services']['filter']['list']\n config_filters = []\n or_filters = []\n for filter in list(filters.values()):\n filter_type = filter['type']\n if filter_type == 'time':\n continue\n\n if filter_type == 'querystring':\n config_filter = {'query': {'query_string': {'query': filter['query']}}}\n\n if filter_type == 'field':\n config_filter = {'term': {filter['field']: filter['query']}}\n\n if filter_type == 'range':\n config_filter = {'range': {filter['field']: {'from': filter['from'], 'to': filter['to']}}}\n\n if filter['mandate'] == 'mustNot':\n config_filter = {'not': config_filter}\n\n if filter['mandate'] == 'either':\n or_filters.append(config_filter)\n else:\n config_filters.append(config_filter)\n\n if or_filters:\n config_filters.append({'or': or_filters})\n\n return config_filters\n\n\ndef kibana4_dashboard_link(dashboard, starttime, endtime):\n dashboard = os.path.expandvars(dashboard)\n time_settings = kibana4_time_temp % (starttime, endtime)\n time_settings = urllib.parse.quote(time_settings)\n return \"%s?_g=%s\" % (dashboard, time_settings)\n" }, { "path": "elastalert/kibana_discover.py", "content": "# -*- coding: utf-8 -*-\n# flake8: noqa\nimport datetime\nimport logging\nimport json\nimport os.path\nimport prison\nimport urllib.parse\n\nfrom .util import EAException\nfrom .util import lookup_es_key\nfrom .util import ts_add\n\nkibana_default_timedelta = datetime.timedelta(minutes=10)\n\nkibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8'])\nkibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3'])\n\ndef generate_kibana_discover_url(rule, match):\n ''' Creates a link for a kibana discover app. '''\n\n discover_app_url = rule.get('kibana_discover_app_url')\n if not discover_app_url:\n logging.warning(\n 'Missing kibana_discover_app_url for rule %s' % (\n rule.get('name', '')\n )\n )\n return None\n\n kibana_version = rule.get('kibana_discover_version')\n if not kibana_version:\n logging.warning(\n 'Missing kibana_discover_version for rule %s' % (\n rule.get('name', '')\n )\n )\n return None\n\n index = rule.get('kibana_discover_index_pattern_id')\n if not index:\n logging.warning(\n 'Missing kibana_discover_index_pattern_id for rule %s' % (\n rule.get('name', '')\n )\n )\n return None\n\n columns = rule.get('kibana_discover_columns', ['_source'])\n filters = rule.get('filter', [])\n\n if 'query_key' in rule:\n query_keys = rule.get('compound_query_key', [rule['query_key']])\n else:\n query_keys = []\n\n timestamp = lookup_es_key(match, rule['timestamp_field'])\n timeframe = rule.get('timeframe', kibana_default_timedelta)\n from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe)\n from_time = ts_add(timestamp, -from_timedelta)\n to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe)\n to_time = ts_add(timestamp, to_timedelta)\n\n if kibana_version in kibana5_kibana6_versions:\n globalState = kibana6_disover_global_state(from_time, to_time)\n appState = kibana_discover_app_state(index, columns, filters, query_keys, match)\n\n elif kibana_version in kibana7_versions:\n globalState = kibana7_disover_global_state(from_time, to_time)\n appState = kibana_discover_app_state(index, columns, filters, query_keys, match)\n\n else:\n logging.warning(\n 'Unknown kibana discover application version %s for rule %s' % (\n kibana_version,\n rule.get('name', '')\n )\n )\n return None\n\n return \"%s?_g=%s&_a=%s\" % (\n os.path.expandvars(discover_app_url),\n urllib.parse.quote(globalState),\n urllib.parse.quote(appState)\n )\n\n\ndef kibana6_disover_global_state(from_time, to_time):\n return prison.dumps( {\n 'refreshInterval': {\n 'pause': True,\n 'value': 0\n },\n 'time': {\n 'from': from_time,\n 'mode': 'absolute',\n 'to': to_time\n }\n } )\n\n\ndef kibana7_disover_global_state(from_time, to_time):\n return prison.dumps( {\n 'filters': [],\n 'refreshInterval': {\n 'pause': True,\n 'value': 0\n },\n 'time': {\n 'from': from_time,\n 'to': to_time\n }\n } )\n\n\ndef kibana_discover_app_state(index, columns, filters, query_keys, match):\n app_filters = []\n\n if filters:\n bool_filter = { 'must': filters }\n app_filters.append( {\n '$state': {\n 'store': 'appState'\n },\n 'bool': bool_filter,\n 'meta': {\n 'alias': 'filter',\n 'disabled': False,\n 'index': index,\n 'key': 'bool',\n 'negate': False,\n 'type': 'custom',\n 'value': json.dumps(bool_filter, separators=(',', ':'))\n },\n } )\n\n for query_key in query_keys:\n query_value = lookup_es_key(match, query_key)\n\n if query_value is None:\n app_filters.append( {\n '$state': {\n 'store': 'appState'\n },\n 'exists': {\n 'field': query_key\n },\n 'meta': {\n 'alias': None,\n 'disabled': False,\n 'index': index,\n 'key': query_key,\n 'negate': True,\n 'type': 'exists',\n 'value': 'exists'\n }\n } )\n\n else:\n app_filters.append( {\n '$state': {\n 'store': 'appState'\n },\n 'meta': {\n 'alias': None,\n 'disabled': False,\n 'index': index,\n 'key': query_key,\n 'negate': False,\n 'params': {\n 'query': query_value,\n 'type': 'phrase'\n },\n 'type': 'phrase',\n 'value': str(query_value)\n },\n 'query': {\n 'match': {\n query_key: {\n 'query': query_value,\n 'type': 'phrase'\n }\n }\n }\n } )\n\n return prison.dumps( {\n 'columns': columns,\n 'filters': app_filters,\n 'index': index,\n 'interval': 'auto'\n } )\n" }, { "path": "elastalert/loaders.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport datetime\nimport hashlib\nimport logging\nimport os\nimport sys\n\nimport jsonschema\nimport yaml\nimport yaml.scanner\nfrom staticconf.loader import yaml_loader\n\nfrom . import alerts\nfrom . import enhancements\nfrom . import ruletypes\nfrom .opsgenie import OpsGenieAlerter\nfrom .util import dt_to_ts\nfrom .util import dt_to_ts_with_format\nfrom .util import dt_to_unix\nfrom .util import dt_to_unixms\nfrom .util import EAException\nfrom .util import get_module\nfrom .util import ts_to_dt\nfrom .util import ts_to_dt_with_format\nfrom .util import unix_to_dt\nfrom .util import unixms_to_dt\n\n\nclass RulesLoader(object):\n # import rule dependency\n import_rules = {}\n\n # Required global (config.yaml) configuration options for the loader\n required_globals = frozenset([])\n\n # Required local (rule.yaml) configuration options\n required_locals = frozenset(['alert', 'type', 'name', 'index'])\n\n # Used to map the names of rules to their classes\n rules_mapping = {\n 'frequency': ruletypes.FrequencyRule,\n 'any': ruletypes.AnyRule,\n 'spike': ruletypes.SpikeRule,\n 'blacklist': ruletypes.BlacklistRule,\n 'whitelist': ruletypes.WhitelistRule,\n 'change': ruletypes.ChangeRule,\n 'flatline': ruletypes.FlatlineRule,\n 'new_term': ruletypes.NewTermsRule,\n 'cardinality': ruletypes.CardinalityRule,\n 'metric_aggregation': ruletypes.MetricAggregationRule,\n 'percentage_match': ruletypes.PercentageMatchRule,\n 'spike_aggregation': ruletypes.SpikeMetricAggregationRule,\n }\n\n # Used to map names of alerts to their classes\n alerts_mapping = {\n 'email': alerts.EmailAlerter,\n 'jira': alerts.JiraAlerter,\n 'opsgenie': OpsGenieAlerter,\n 'stomp': alerts.StompAlerter,\n 'debug': alerts.DebugAlerter,\n 'command': alerts.CommandAlerter,\n 'sns': alerts.SnsAlerter,\n 'hipchat': alerts.HipChatAlerter,\n 'stride': alerts.StrideAlerter,\n 'ms_teams': alerts.MsTeamsAlerter,\n 'slack': alerts.SlackAlerter,\n 'mattermost': alerts.MattermostAlerter,\n 'pagerduty': alerts.PagerDutyAlerter,\n 'exotel': alerts.ExotelAlerter,\n 'twilio': alerts.TwilioAlerter,\n 'victorops': alerts.VictorOpsAlerter,\n 'telegram': alerts.TelegramAlerter,\n 'googlechat': alerts.GoogleChatAlerter,\n 'gitter': alerts.GitterAlerter,\n 'servicenow': alerts.ServiceNowAlerter,\n 'alerta': alerts.AlertaAlerter,\n 'post': alerts.HTTPPostAlerter,\n 'hivealerter': alerts.HiveAlerter\n }\n\n # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list\n # For example, jira goes before email so the ticket # will be added to the resulting email.\n alerts_order = {\n 'jira': 0,\n 'email': 1\n }\n\n base_config = {}\n\n def __init__(self, conf):\n # schema for rule yaml\n self.rule_schema = jsonschema.Draft7Validator(\n yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader))\n\n self.base_config = copy.deepcopy(conf)\n\n def load(self, conf, args=None):\n \"\"\"\n Discover and load all the rules as defined in the conf and args.\n :param dict conf: Configuration dict\n :param dict args: Arguments dict\n :return: List of rules\n :rtype: list\n \"\"\"\n names = []\n use_rule = None if args is None else args.rule\n\n # Load each rule configuration file\n rules = []\n rule_files = self.get_names(conf, use_rule)\n for rule_file in rule_files:\n try:\n rule = self.load_configuration(rule_file, conf, args)\n # A rule failed to load, don't try to process it\n if not rule:\n logging.error('Invalid rule file skipped: %s' % rule_file)\n continue\n # By setting \"is_enabled: False\" in rule file, a rule is easily disabled\n if 'is_enabled' in rule and not rule['is_enabled']:\n continue\n if rule['name'] in names:\n raise EAException('Duplicate rule named %s' % (rule['name']))\n except EAException as e:\n raise EAException('Error loading file %s: %s' % (rule_file, e))\n\n rules.append(rule)\n names.append(rule['name'])\n\n return rules\n\n def get_names(self, conf, use_rule=None):\n \"\"\"\n Return a list of rule names that can be passed to `get_yaml` to retrieve.\n :param dict conf: Configuration dict\n :param str use_rule: Limit to only specified rule\n :return: A list of rule names\n :rtype: list\n \"\"\"\n raise NotImplementedError()\n\n def get_hashes(self, conf, use_rule=None):\n \"\"\"\n Discover and get the hashes of all the rules as defined in the conf.\n :param dict conf: Configuration\n :param str use_rule: Limit to only specified rule\n :return: Dict of rule name to hash\n :rtype: dict\n \"\"\"\n raise NotImplementedError()\n\n def get_yaml(self, filename):\n \"\"\"\n Get and parse the yaml of the specified rule.\n :param str filename: Rule to get the yaml\n :return: Rule YAML dict\n :rtype: dict\n \"\"\"\n raise NotImplementedError()\n\n def get_import_rule(self, rule):\n \"\"\"\n Retrieve the name of the rule to import.\n :param dict rule: Rule dict\n :return: rule name that will all `get_yaml` to retrieve the yaml of the rule\n :rtype: str\n \"\"\"\n return rule['import']\n\n def load_configuration(self, filename, conf, args=None):\n \"\"\" Load a yaml rule file and fill in the relevant fields with objects.\n\n :param str filename: The name of a rule configuration file.\n :param dict conf: The global configuration dictionary, used for populating defaults.\n :param dict args: Arguments\n :return: The rule configuration, a dictionary.\n \"\"\"\n rule = self.load_yaml(filename)\n self.load_options(rule, conf, filename, args)\n self.load_modules(rule, args)\n return rule\n\n def load_yaml(self, filename):\n \"\"\"\n Load the rule including all dependency rules.\n :param str filename: Rule to load\n :return: Loaded rule dict\n :rtype: dict\n \"\"\"\n rule = {\n 'rule_file': filename,\n }\n\n self.import_rules.pop(filename, None) # clear `filename` dependency\n while True:\n loaded = self.get_yaml(filename)\n\n # Special case for merging filters - if both files specify a filter merge (AND) them\n if 'filter' in rule and 'filter' in loaded:\n rule['filter'] = loaded['filter'] + rule['filter']\n\n loaded.update(rule)\n rule = loaded\n if 'import' in rule:\n # Find the path of the next file.\n import_filename = self.get_import_rule(rule)\n # set dependencies\n rules = self.import_rules.get(filename, [])\n rules.append(import_filename)\n self.import_rules[filename] = rules\n filename = import_filename\n del (rule['import']) # or we could go on forever!\n else:\n break\n\n return rule\n\n def load_options(self, rule, conf, filename, args=None):\n \"\"\" Converts time objects, sets defaults, and validates some settings.\n\n :param rule: A dictionary of parsed YAML from a rule config file.\n :param conf: The global configuration dictionary, used for populating defaults.\n :param filename: Name of the rule\n :param args: Arguments\n \"\"\"\n self.adjust_deprecated_values(rule)\n\n try:\n self.rule_schema.validate(rule)\n except jsonschema.ValidationError as e:\n raise EAException(\"Invalid Rule file: %s\\n%s\" % (filename, e))\n\n try:\n # Set all time based parameters\n if 'timeframe' in rule:\n rule['timeframe'] = datetime.timedelta(**rule['timeframe'])\n if 'realert' in rule:\n rule['realert'] = datetime.timedelta(**rule['realert'])\n else:\n if 'aggregation' in rule:\n rule['realert'] = datetime.timedelta(minutes=0)\n else:\n rule['realert'] = datetime.timedelta(minutes=1)\n if 'aggregation' in rule and not rule['aggregation'].get('schedule'):\n rule['aggregation'] = datetime.timedelta(**rule['aggregation'])\n if 'query_delay' in rule:\n rule['query_delay'] = datetime.timedelta(**rule['query_delay'])\n if 'buffer_time' in rule:\n rule['buffer_time'] = datetime.timedelta(**rule['buffer_time'])\n if 'run_every' in rule:\n rule['run_every'] = datetime.timedelta(**rule['run_every'])\n if 'bucket_interval' in rule:\n rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval'])\n if 'exponential_realert' in rule:\n rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert'])\n if 'kibana4_start_timedelta' in rule:\n rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta'])\n if 'kibana4_end_timedelta' in rule:\n rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta'])\n if 'kibana_discover_from_timedelta' in rule:\n rule['kibana_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta'])\n if 'kibana_discover_to_timedelta' in rule:\n rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta'])\n except (KeyError, TypeError) as e:\n raise EAException('Invalid time format used: %s' % e)\n\n # Set defaults, copy defaults from config.yaml\n for key, val in list(self.base_config.items()):\n rule.setdefault(key, val)\n rule.setdefault('name', os.path.splitext(filename)[0])\n rule.setdefault('realert', datetime.timedelta(seconds=0))\n rule.setdefault('aggregation', datetime.timedelta(seconds=0))\n rule.setdefault('query_delay', datetime.timedelta(seconds=0))\n rule.setdefault('timestamp_field', '@timestamp')\n rule.setdefault('filter', [])\n rule.setdefault('timestamp_type', 'iso')\n rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ')\n rule.setdefault('_source_enabled', True)\n rule.setdefault('use_local_time', True)\n rule.setdefault('description', \"\")\n\n # Set timestamp_type conversion function, used when generating queries and processing hits\n rule['timestamp_type'] = rule['timestamp_type'].strip().lower()\n if rule['timestamp_type'] == 'iso':\n rule['ts_to_dt'] = ts_to_dt\n rule['dt_to_ts'] = dt_to_ts\n elif rule['timestamp_type'] == 'unix':\n rule['ts_to_dt'] = unix_to_dt\n rule['dt_to_ts'] = dt_to_unix\n elif rule['timestamp_type'] == 'unix_ms':\n rule['ts_to_dt'] = unixms_to_dt\n rule['dt_to_ts'] = dt_to_unixms\n elif rule['timestamp_type'] == 'custom':\n def _ts_to_dt_with_format(ts):\n return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format'])\n\n def _dt_to_ts_with_format(dt):\n ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format'])\n if 'timestamp_format_expr' in rule:\n # eval expression passing 'ts' and 'dt'\n return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt})\n else:\n return ts\n\n rule['ts_to_dt'] = _ts_to_dt_with_format\n rule['dt_to_ts'] = _dt_to_ts_with_format\n else:\n raise EAException('timestamp_type must be one of iso, unix, or unix_ms')\n\n # Add support for client ssl certificate auth\n if 'verify_certs' in conf:\n rule.setdefault('verify_certs', conf.get('verify_certs'))\n rule.setdefault('ca_certs', conf.get('ca_certs'))\n rule.setdefault('client_cert', conf.get('client_cert'))\n rule.setdefault('client_key', conf.get('client_key'))\n\n # Set HipChat options from global config\n rule.setdefault('hipchat_msg_color', 'red')\n rule.setdefault('hipchat_domain', 'api.hipchat.com')\n rule.setdefault('hipchat_notify', True)\n rule.setdefault('hipchat_from', '')\n rule.setdefault('hipchat_ignore_ssl_errors', False)\n\n # Make sure we have required options\n if self.required_locals - frozenset(list(rule.keys())):\n raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys())))))\n\n if 'include' in rule and type(rule['include']) != list:\n raise EAException('include option must be a list')\n\n raw_query_key = rule.get('query_key')\n if isinstance(raw_query_key, list):\n if len(raw_query_key) > 1:\n rule['compound_query_key'] = raw_query_key\n rule['query_key'] = ','.join(raw_query_key)\n elif len(raw_query_key) == 1:\n rule['query_key'] = raw_query_key[0]\n else:\n del(rule['query_key'])\n\n if isinstance(rule.get('aggregation_key'), list):\n rule['compound_aggregation_key'] = rule['aggregation_key']\n rule['aggregation_key'] = ','.join(rule['aggregation_key'])\n\n if isinstance(rule.get('compare_key'), list):\n rule['compound_compare_key'] = rule['compare_key']\n rule['compare_key'] = ','.join(rule['compare_key'])\n elif 'compare_key' in rule:\n rule['compound_compare_key'] = [rule['compare_key']]\n # Add QK, CK and timestamp to include\n include = rule.get('include', ['*'])\n if 'query_key' in rule:\n include.append(rule['query_key'])\n if 'compound_query_key' in rule:\n include += rule['compound_query_key']\n if 'compound_aggregation_key' in rule:\n include += rule['compound_aggregation_key']\n if 'compare_key' in rule:\n include.append(rule['compare_key'])\n if 'compound_compare_key' in rule:\n include += rule['compound_compare_key']\n if 'top_count_keys' in rule:\n include += rule['top_count_keys']\n include.append(rule['timestamp_field'])\n rule['include'] = list(set(include))\n\n # Check that generate_kibana_url is compatible with the filters\n if rule.get('generate_kibana_link'):\n for es_filter in rule.get('filter'):\n if es_filter:\n if 'not' in es_filter:\n es_filter = es_filter['not']\n if 'query' in es_filter:\n es_filter = es_filter['query']\n if list(es_filter.keys())[0] not in ('term', 'query_string', 'range'):\n raise EAException(\n 'generate_kibana_link is incompatible with filters other than term, query_string and range.'\n 'Consider creating a dashboard and using use_kibana_dashboard instead.')\n\n # Check that doc_type is provided if use_count/terms_query\n if rule.get('use_count_query') or rule.get('use_terms_query'):\n if 'doc_type' not in rule:\n raise EAException('doc_type must be specified.')\n\n # Check that query_key is set if use_terms_query\n if rule.get('use_terms_query'):\n if 'query_key' not in rule:\n raise EAException('query_key must be specified with use_terms_query')\n\n # Warn if use_strf_index is used with %y, %M or %D\n # (%y = short year, %M = minutes, %D = full date)\n if rule.get('use_strftime_index'):\n for token in ['%y', '%M', '%D']:\n if token in rule.get('index'):\n logging.warning('Did you mean to use %s in the index? '\n 'The index will be formatted like %s' % (token,\n datetime.datetime.now().strftime(\n rule.get('index'))))\n\n if rule.get('scan_entire_timeframe') and not rule.get('timeframe'):\n raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified')\n\n def load_modules(self, rule, args=None):\n \"\"\" Loads things that could be modules. Enhancements, alerts and rule type. \"\"\"\n # Set match enhancements\n match_enhancements = []\n for enhancement_name in rule.get('match_enhancements', []):\n if enhancement_name in dir(enhancements):\n enhancement = getattr(enhancements, enhancement_name)\n else:\n enhancement = get_module(enhancement_name)\n if not issubclass(enhancement, enhancements.BaseEnhancement):\n raise EAException(\"Enhancement module %s not a subclass of BaseEnhancement\" % enhancement_name)\n match_enhancements.append(enhancement(rule))\n rule['match_enhancements'] = match_enhancements\n\n # Convert rule type into RuleType object\n if rule['type'] in self.rules_mapping:\n rule['type'] = self.rules_mapping[rule['type']]\n else:\n rule['type'] = get_module(rule['type'])\n if not issubclass(rule['type'], ruletypes.RuleType):\n raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type']))\n\n # Make sure we have required alert and type options\n reqs = rule['type'].required_options\n\n if reqs - frozenset(list(rule.keys())):\n raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(list(rule.keys())))))\n # Instantiate rule\n try:\n rule['type'] = rule['type'](rule, args)\n except (KeyError, EAException) as e:\n raise EAException('Error initializing rule %s: %s' % (rule['name'], e)).with_traceback(sys.exc_info()[2])\n # Instantiate alerts only if we're not in debug mode\n # In debug mode alerts are not actually sent so don't bother instantiating them\n if not args or not args.debug:\n rule['alert'] = self.load_alerts(rule, alert_field=rule['alert'])\n\n def load_alerts(self, rule, alert_field):\n def normalize_config(alert):\n \"\"\"Alert config entries are either \"alertType\" or {\"alertType\": {\"key\": \"data\"}}.\n This function normalizes them both to the latter format. \"\"\"\n if isinstance(alert, str):\n return alert, rule\n elif isinstance(alert, dict):\n name, config = next(iter(list(alert.items())))\n config_copy = copy.copy(rule)\n config_copy.update(config) # warning, this (intentionally) mutates the rule dict\n return name, config_copy\n else:\n raise EAException()\n\n def create_alert(alert, alert_config):\n alert_class = self.alerts_mapping.get(alert) or get_module(alert)\n if not issubclass(alert_class, alerts.Alerter):\n raise EAException('Alert module %s is not a subclass of Alerter' % alert)\n missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset(\n alert_config or [])\n if missing_options:\n raise EAException('Missing required option(s): %s' % (', '.join(missing_options)))\n return alert_class(alert_config)\n\n try:\n if type(alert_field) != list:\n alert_field = [alert_field]\n\n alert_field = [normalize_config(x) for x in alert_field]\n alert_field = sorted(alert_field, key=lambda a_b: self.alerts_order.get(a_b[0], 1))\n # Convert all alerts into Alerter objects\n alert_field = [create_alert(a, b) for a, b in alert_field]\n\n except (KeyError, EAException) as e:\n raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)).with_traceback(sys.exc_info()[2])\n\n return alert_field\n\n @staticmethod\n def adjust_deprecated_values(rule):\n # From rename of simple HTTP alerter\n if rule.get('type') == 'simple':\n rule['type'] = 'post'\n if 'simple_proxy' in rule:\n rule['http_post_proxy'] = rule['simple_proxy']\n if 'simple_webhook_url' in rule:\n rule['http_post_url'] = rule['simple_webhook_url']\n logging.warning(\n '\"simple\" alerter has been renamed \"post\" and comptability may be removed in a future release.')\n\n\nclass FileRulesLoader(RulesLoader):\n\n # Required global (config.yaml) configuration options for the loader\n required_globals = frozenset(['rules_folder'])\n\n def get_names(self, conf, use_rule=None):\n # Passing a filename directly can bypass rules_folder and .yaml checks\n if use_rule and os.path.isfile(use_rule):\n return [use_rule]\n rule_folder = conf['rules_folder']\n rule_files = []\n if 'scan_subdirectories' in conf and conf['scan_subdirectories']:\n for root, folders, files in os.walk(rule_folder):\n for filename in files:\n if use_rule and use_rule != filename:\n continue\n if self.is_yaml(filename):\n rule_files.append(os.path.join(root, filename))\n else:\n for filename in os.listdir(rule_folder):\n fullpath = os.path.join(rule_folder, filename)\n if os.path.isfile(fullpath) and self.is_yaml(filename):\n rule_files.append(fullpath)\n return rule_files\n\n def get_hashes(self, conf, use_rule=None):\n rule_files = self.get_names(conf, use_rule)\n rule_mod_times = {}\n for rule_file in rule_files:\n rule_mod_times[rule_file] = self.get_rule_file_hash(rule_file)\n return rule_mod_times\n\n def get_yaml(self, filename):\n try:\n return yaml_loader(filename)\n except yaml.scanner.ScannerError as e:\n raise EAException('Could not parse file %s: %s' % (filename, e))\n\n def get_import_rule(self, rule):\n \"\"\"\n Allow for relative paths to the import rule.\n :param dict rule:\n :return: Path the import rule\n :rtype: str\n \"\"\"\n if os.path.isabs(rule['import']):\n return rule['import']\n else:\n return os.path.join(os.path.dirname(rule['rule_file']), rule['import'])\n\n def get_rule_file_hash(self, rule_file):\n rule_file_hash = ''\n if os.path.exists(rule_file):\n with open(rule_file, 'rb') as fh:\n rule_file_hash = hashlib.sha1(fh.read()).digest()\n for import_rule_file in self.import_rules.get(rule_file, []):\n rule_file_hash += self.get_rule_file_hash(import_rule_file)\n return rule_file_hash\n\n @staticmethod\n def is_yaml(filename):\n return filename.endswith('.yaml') or filename.endswith('.yml')\n" }, { "path": "elastalert/opsgenie.py", "content": "# -*- coding: utf-8 -*-\nimport json\nimport logging\nimport os.path\nimport requests\n\nfrom .alerts import Alerter\nfrom .alerts import BasicMatchString\nfrom .util import EAException\nfrom .util import elastalert_logger\nfrom .util import lookup_es_key\n\n\nclass OpsGenieAlerter(Alerter):\n '''Sends a http request to the OpsGenie API to signal for an alert'''\n required_options = frozenset(['opsgenie_key'])\n\n def __init__(self, *args):\n super(OpsGenieAlerter, self).__init__(*args)\n self.account = self.rule.get('opsgenie_account')\n self.api_key = self.rule.get('opsgenie_key', 'key')\n self.default_reciepients = self.rule.get('opsgenie_default_receipients', None)\n self.recipients = self.rule.get('opsgenie_recipients')\n self.recipients_args = self.rule.get('opsgenie_recipients_args')\n self.default_teams = self.rule.get('opsgenie_default_teams', None)\n self.teams = self.rule.get('opsgenie_teams')\n self.teams_args = self.rule.get('opsgenie_teams_args')\n self.tags = self.rule.get('opsgenie_tags', []) + ['ElastAlert', self.rule['name']]\n self.to_addr = self.rule.get('opsgenie_addr', 'https://api.opsgenie.com/v2/alerts')\n self.custom_message = self.rule.get('opsgenie_message')\n self.opsgenie_subject = self.rule.get('opsgenie_subject')\n self.opsgenie_subject_args = self.rule.get('opsgenie_subject_args')\n self.alias = self.rule.get('opsgenie_alias')\n self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None)\n self.priority = self.rule.get('opsgenie_priority')\n self.opsgenie_details = self.rule.get('opsgenie_details', {})\n\n def _parse_responders(self, responders, responder_args, matches, default_responders):\n if responder_args:\n formated_responders = list()\n responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.items())\n responders_values = dict((k, v) for k, v in responders_values.items() if v)\n\n for responder in responders:\n responder = str(responder)\n try:\n formated_responders.append(responder.format(**responders_values))\n except KeyError as error:\n logging.warn(\"OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. \" % (error))\n if not formated_responders:\n logging.warn(\"OpsGenieAlerter: no responders can be formed. Trying the default responder \")\n if not default_responders:\n logging.warn(\"OpsGenieAlerter: default responder not set. Falling back\")\n formated_responders = responders\n else:\n formated_responders = default_responders\n responders = formated_responders\n return responders\n\n def _fill_responders(self, responders, type_):\n return [{'id': r, 'type': type_} for r in responders]\n\n def alert(self, matches):\n body = ''\n for match in matches:\n body += str(BasicMatchString(self.rule, match))\n # Separate text of aggregated alerts with dashes\n if len(matches) > 1:\n body += '\\n----------------------------------------\\n'\n\n if self.custom_message is None:\n self.message = self.create_title(matches)\n else:\n self.message = self.custom_message.format(**matches[0])\n self.recipients = self._parse_responders(self.recipients, self.recipients_args, matches, self.default_reciepients)\n self.teams = self._parse_responders(self.teams, self.teams_args, matches, self.default_teams)\n post = {}\n post['message'] = self.message\n if self.account:\n post['user'] = self.account\n if self.recipients:\n post['responders'] = [{'username': r, 'type': 'user'} for r in self.recipients]\n if self.teams:\n post['teams'] = [{'name': r, 'type': 'team'} for r in self.teams]\n post['description'] = body\n post['source'] = 'ElastAlert'\n\n for i, tag in enumerate(self.tags):\n self.tags[i] = tag.format(**matches[0])\n post['tags'] = self.tags\n\n if self.priority and self.priority not in ('P1', 'P2', 'P3', 'P4', 'P5'):\n logging.warn(\"Priority level does not appear to be specified correctly. \\\n Please make sure to set it to a value between P1 and P5\")\n else:\n post['priority'] = self.priority\n\n if self.alias is not None:\n post['alias'] = self.alias.format(**matches[0])\n\n details = self.get_details(matches)\n if details:\n post['details'] = details\n\n logging.debug(json.dumps(post))\n\n headers = {\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey {}'.format(self.api_key),\n }\n # set https proxy, if it was provided\n proxies = {'https': self.opsgenie_proxy} if self.opsgenie_proxy else None\n\n try:\n r = requests.post(self.to_addr, json=post, headers=headers, proxies=proxies)\n\n logging.debug('request response: {0}'.format(r))\n if r.status_code != 202:\n elastalert_logger.info(\"Error response from {0} \\n \"\n \"API Response: {1}\".format(self.to_addr, r))\n r.raise_for_status()\n logging.info(\"Alert sent to OpsGenie\")\n except Exception as err:\n raise EAException(\"Error sending alert: {0}\".format(err))\n\n def create_default_title(self, matches):\n subject = 'ElastAlert: %s' % (self.rule['name'])\n\n # If the rule has a query_key, add that value plus timestamp to subject\n if 'query_key' in self.rule:\n qk = matches[0].get(self.rule['query_key'])\n if qk:\n subject += ' - %s' % (qk)\n\n return subject\n\n def create_title(self, matches):\n \"\"\" Creates custom alert title to be used as subject for opsgenie alert.\"\"\"\n if self.opsgenie_subject:\n return self.create_custom_title(matches)\n\n return self.create_default_title(matches)\n\n def create_custom_title(self, matches):\n opsgenie_subject = str(self.rule['opsgenie_subject'])\n\n if self.opsgenie_subject_args:\n opsgenie_subject_values = [lookup_es_key(matches[0], arg) for arg in self.opsgenie_subject_args]\n\n for i, subject_value in enumerate(opsgenie_subject_values):\n if subject_value is None:\n alert_value = self.rule.get(self.opsgenie_subject_args[i])\n if alert_value:\n opsgenie_subject_values[i] = alert_value\n\n opsgenie_subject_values = ['' if val is None else val for val in opsgenie_subject_values]\n return opsgenie_subject.format(*opsgenie_subject_values)\n\n return opsgenie_subject\n\n def get_info(self):\n ret = {'type': 'opsgenie'}\n if self.recipients:\n ret['recipients'] = self.recipients\n if self.account:\n ret['account'] = self.account\n if self.teams:\n ret['teams'] = self.teams\n return ret\n\n def get_details(self, matches):\n details = {}\n\n for key, value in self.opsgenie_details.items():\n\n if type(value) is dict:\n if 'field' in value:\n field_value = lookup_es_key(matches[0], value['field'])\n if field_value is not None:\n details[key] = str(field_value)\n\n elif type(value) is str:\n details[key] = os.path.expandvars(value)\n\n return details\n" }, { "path": "elastalert/rule_from_kibana.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport json\n\nimport yaml\n\nfrom elastalert.kibana import filters_from_dashboard\nfrom elastalert.util import elasticsearch_client\n\n\ndef main():\n es_host = input(\"Elasticsearch host: \")\n es_port = input(\"Elasticsearch port: \")\n db_name = input(\"Dashboard name: \")\n send_get_body_as = input(\"Method for querying Elasticsearch[GET]: \") or 'GET'\n\n es = elasticsearch_client({'es_host': es_host, 'es_port': es_port, 'send_get_body_as': send_get_body_as})\n\n print(\"Elastic Version:\" + es.es_version)\n\n query = {'query': {'term': {'_id': db_name}}}\n\n if es.is_atleastsixsix():\n # TODO check support for kibana 7\n # TODO use doc_type='_doc' instead\n res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_includes=['dashboard'])\n else:\n res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard'])\n\n if not res['hits']['hits']:\n print(\"No dashboard %s found\" % (db_name))\n exit()\n\n db = json.loads(res['hits']['hits'][0]['_source']['dashboard'])\n config_filters = filters_from_dashboard(db)\n\n print(\"\\nPartial Config file\")\n print(\"-----------\\n\")\n print(\"name: %s\" % (db_name))\n print(\"es_host: %s\" % (es_host))\n print(\"es_port: %s\" % (es_port))\n print(\"filter:\")\n print(yaml.safe_dump(config_filters))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "elastalert/ruletypes.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport datetime\nimport sys\n\nfrom blist import sortedlist\n\nfrom .util import add_raw_postfix\nfrom .util import dt_to_ts\nfrom .util import EAException\nfrom .util import elastalert_logger\nfrom .util import elasticsearch_client\nfrom .util import format_index\nfrom .util import hashable\nfrom .util import lookup_es_key\nfrom .util import new_get_event_ts\nfrom .util import pretty_ts\nfrom .util import total_seconds\nfrom .util import ts_now\nfrom .util import ts_to_dt\n\n\nclass RuleType(object):\n \"\"\" The base class for a rule type.\n The class must implement add_data and add any matches to self.matches.\n\n :param rules: A rule configuration.\n \"\"\"\n required_options = frozenset()\n\n def __init__(self, rules, args=None):\n self.matches = []\n self.rules = rules\n self.occurrences = {}\n self.rules['category'] = self.rules.get('category', '')\n self.rules['description'] = self.rules.get('description', '')\n self.rules['owner'] = self.rules.get('owner', '')\n self.rules['priority'] = self.rules.get('priority', '2')\n\n def add_data(self, data):\n \"\"\" The function that the ElastAlert client calls with results from ES.\n Data is a list of dictionaries, from Elasticsearch.\n\n :param data: A list of events, each of which is a dictionary of terms.\n \"\"\"\n raise NotImplementedError()\n\n def add_match(self, event):\n \"\"\" This function is called on all matching events. Rules use it to add\n extra information about the context of a match. Event is a dictionary\n containing terms directly from Elasticsearch and alerts will report\n all of the information.\n\n :param event: The matching event, a dictionary of terms.\n \"\"\"\n # Convert datetime's back to timestamps\n ts = self.rules.get('timestamp_field')\n if ts in event:\n event[ts] = dt_to_ts(event[ts])\n\n self.matches.append(copy.deepcopy(event))\n\n def get_match_str(self, match):\n \"\"\" Returns a string that gives more context about a match.\n\n :param match: The matching event, a dictionary of terms.\n :return: A user facing string describing the match.\n \"\"\"\n return ''\n\n def garbage_collect(self, timestamp):\n \"\"\" Gets called periodically to remove old data that is useless beyond given timestamp.\n May also be used to compute things in the absence of new data.\n\n :param timestamp: A timestamp indicating the rule has been run up to that point.\n \"\"\"\n pass\n\n def add_count_data(self, counts):\n \"\"\" Gets called when a rule has use_count_query set to True. Called to add data from querying to the rule.\n\n :param counts: A dictionary mapping timestamps to hit counts.\n \"\"\"\n raise NotImplementedError()\n\n def add_terms_data(self, terms):\n \"\"\" Gets called when a rule has use_terms_query set to True.\n\n :param terms: A list of buckets with a key, corresponding to query_key, and the count \"\"\"\n raise NotImplementedError()\n\n def add_aggregation_data(self, payload):\n \"\"\" Gets called when a rule has use_terms_query set to True.\n :param terms: A list of buckets with a key, corresponding to query_key, and the count \"\"\"\n raise NotImplementedError()\n\n\nclass CompareRule(RuleType):\n \"\"\" A base class for matching a specific term by passing it to a compare function \"\"\"\n required_options = frozenset(['compound_compare_key'])\n\n def expand_entries(self, list_type):\n \"\"\" Expand entries specified in files using the '!file' directive, if there are\n any, then add everything to a set.\n \"\"\"\n entries_set = set()\n for entry in self.rules[list_type]:\n if entry.startswith(\"!file\"): # - \"!file /path/to/list\"\n filename = entry.split()[1]\n with open(filename, 'r') as f:\n for line in f:\n entries_set.add(line.rstrip())\n else:\n entries_set.add(entry)\n self.rules[list_type] = entries_set\n\n def compare(self, event):\n \"\"\" An event is a match if this returns true \"\"\"\n raise NotImplementedError()\n\n def add_data(self, data):\n # If compare returns true, add it as a match\n for event in data:\n if self.compare(event):\n self.add_match(event)\n\n\nclass BlacklistRule(CompareRule):\n \"\"\" A CompareRule where the compare function checks a given key against a blacklist \"\"\"\n required_options = frozenset(['compare_key', 'blacklist'])\n\n def __init__(self, rules, args=None):\n super(BlacklistRule, self).__init__(rules, args=None)\n self.expand_entries('blacklist')\n\n def compare(self, event):\n term = lookup_es_key(event, self.rules['compare_key'])\n if term in self.rules['blacklist']:\n return True\n return False\n\n\nclass WhitelistRule(CompareRule):\n \"\"\" A CompareRule where the compare function checks a given term against a whitelist \"\"\"\n required_options = frozenset(['compare_key', 'whitelist', 'ignore_null'])\n\n def __init__(self, rules, args=None):\n super(WhitelistRule, self).__init__(rules, args=None)\n self.expand_entries('whitelist')\n\n def compare(self, event):\n term = lookup_es_key(event, self.rules['compare_key'])\n if term is None:\n return not self.rules['ignore_null']\n if term not in self.rules['whitelist']:\n return True\n return False\n\n\nclass ChangeRule(CompareRule):\n \"\"\" A rule that will store values for a certain term and match if those values change \"\"\"\n required_options = frozenset(['query_key', 'compound_compare_key', 'ignore_null'])\n change_map = {}\n occurrence_time = {}\n\n def compare(self, event):\n key = hashable(lookup_es_key(event, self.rules['query_key']))\n values = []\n elastalert_logger.debug(\" Previous Values of compare keys \" + str(self.occurrences))\n for val in self.rules['compound_compare_key']:\n lookup_value = lookup_es_key(event, val)\n values.append(lookup_value)\n elastalert_logger.debug(\" Current Values of compare keys \" + str(values))\n\n changed = False\n for val in values:\n if not isinstance(val, bool) and not val and self.rules['ignore_null']:\n return False\n # If we have seen this key before, compare it to the new value\n if key in self.occurrences:\n for idx, previous_values in enumerate(self.occurrences[key]):\n elastalert_logger.debug(\" \" + str(previous_values) + \" \" + str(values[idx]))\n changed = previous_values != values[idx]\n if changed:\n break\n if changed:\n self.change_map[key] = (self.occurrences[key], values)\n # If using timeframe, only return true if the time delta is < timeframe\n if key in self.occurrence_time:\n changed = event[self.rules['timestamp_field']] - self.occurrence_time[key] <= self.rules['timeframe']\n\n # Update the current value and time\n elastalert_logger.debug(\" Setting current value of compare keys values \" + str(values))\n self.occurrences[key] = values\n if 'timeframe' in self.rules:\n self.occurrence_time[key] = event[self.rules['timestamp_field']]\n elastalert_logger.debug(\"Final result of comparision between previous and current values \" + str(changed))\n return changed\n\n def add_match(self, match):\n # TODO this is not technically correct\n # if the term changes multiple times before an alert is sent\n # this data will be overwritten with the most recent change\n change = self.change_map.get(hashable(lookup_es_key(match, self.rules['query_key'])))\n extra = {}\n if change:\n extra = {'old_value': change[0],\n 'new_value': change[1]}\n elastalert_logger.debug(\"Description of the changed records \" + str(dict(list(match.items()) + list(extra.items()))))\n super(ChangeRule, self).add_match(dict(list(match.items()) + list(extra.items())))\n\n\nclass FrequencyRule(RuleType):\n \"\"\" A rule that matches if num_events number of events occur within a timeframe \"\"\"\n required_options = frozenset(['num_events', 'timeframe'])\n\n def __init__(self, *args):\n super(FrequencyRule, self).__init__(*args)\n self.ts_field = self.rules.get('timestamp_field', '@timestamp')\n self.get_ts = new_get_event_ts(self.ts_field)\n self.attach_related = self.rules.get('attach_related', False)\n\n def add_count_data(self, data):\n \"\"\" Add count data to the rule. Data should be of the form {ts: count}. \"\"\"\n if len(data) > 1:\n raise EAException('add_count_data can only accept one count at a time')\n\n (ts, count), = list(data.items())\n\n event = ({self.ts_field: ts}, count)\n self.occurrences.setdefault('all', EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append(event)\n self.check_for_match('all')\n\n def add_terms_data(self, terms):\n for timestamp, buckets in terms.items():\n for bucket in buckets:\n event = ({self.ts_field: timestamp,\n self.rules['query_key']: bucket['key']}, bucket['doc_count'])\n self.occurrences.setdefault(bucket['key'], EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append(event)\n self.check_for_match(bucket['key'])\n\n def add_data(self, data):\n if 'query_key' in self.rules:\n qk = self.rules['query_key']\n else:\n qk = None\n\n for event in data:\n if qk:\n key = hashable(lookup_es_key(event, qk))\n else:\n # If no query_key, we use the key 'all' for all events\n key = 'all'\n\n # Store the timestamps of recent occurrences, per key\n self.occurrences.setdefault(key, EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append((event, 1))\n self.check_for_match(key, end=False)\n\n # We call this multiple times with the 'end' parameter because subclasses\n # may or may not want to check while only partial data has been added\n if key in self.occurrences: # could have been emptied by previous check\n self.check_for_match(key, end=True)\n\n def check_for_match(self, key, end=False):\n # Match if, after removing old events, we hit num_events.\n # the 'end' parameter depends on whether this was called from the\n # middle or end of an add_data call and is used in subclasses\n if self.occurrences[key].count() >= self.rules['num_events']:\n event = self.occurrences[key].data[-1][0]\n if self.attach_related:\n event['related_events'] = [data[0] for data in self.occurrences[key].data[:-1]]\n self.add_match(event)\n self.occurrences.pop(key)\n\n def garbage_collect(self, timestamp):\n \"\"\" Remove all occurrence data that is beyond the timeframe away \"\"\"\n stale_keys = []\n for key, window in self.occurrences.items():\n if timestamp - lookup_es_key(window.data[-1][0], self.ts_field) > self.rules['timeframe']:\n stale_keys.append(key)\n list(map(self.occurrences.pop, stale_keys))\n\n def get_match_str(self, match):\n lt = self.rules.get('use_local_time')\n match_ts = lookup_es_key(match, self.ts_field)\n starttime = pretty_ts(dt_to_ts(ts_to_dt(match_ts) - self.rules['timeframe']), lt)\n endtime = pretty_ts(match_ts, lt)\n message = 'At least %d events occurred between %s and %s\\n\\n' % (self.rules['num_events'],\n starttime,\n endtime)\n return message\n\n\nclass AnyRule(RuleType):\n \"\"\" A rule that will match on any input data \"\"\"\n\n def add_data(self, data):\n for datum in data:\n self.add_match(datum)\n\n\nclass EventWindow(object):\n \"\"\" A container for hold event counts for rules which need a chronological ordered event window. \"\"\"\n\n def __init__(self, timeframe, onRemoved=None, getTimestamp=new_get_event_ts('@timestamp')):\n self.timeframe = timeframe\n self.onRemoved = onRemoved\n self.get_ts = getTimestamp\n self.data = sortedlist(key=self.get_ts)\n self.running_count = 0\n\n def clear(self):\n self.data = sortedlist(key=self.get_ts)\n self.running_count = 0\n\n def append(self, event):\n \"\"\" Add an event to the window. Event should be of the form (dict, count).\n This will also pop the oldest events and call onRemoved on them until the\n window size is less than timeframe. \"\"\"\n self.data.add(event)\n self.running_count += event[1]\n\n while self.duration() >= self.timeframe:\n oldest = self.data[0]\n self.data.remove(oldest)\n self.running_count -= oldest[1]\n self.onRemoved and self.onRemoved(oldest)\n\n def duration(self):\n \"\"\" Get the size in timedelta of the window. \"\"\"\n if not self.data:\n return datetime.timedelta(0)\n return self.get_ts(self.data[-1]) - self.get_ts(self.data[0])\n\n def count(self):\n \"\"\" Count the number of events in the window. \"\"\"\n return self.running_count\n\n def mean(self):\n \"\"\" Compute the mean of the value_field in the window. \"\"\"\n if len(self.data) > 0:\n datasum = 0\n datalen = 0\n for dat in self.data:\n if \"placeholder\" not in dat[0]:\n datasum += dat[1]\n datalen += 1\n if datalen > 0:\n return datasum / float(datalen)\n return None\n else:\n return None\n\n def __iter__(self):\n return iter(self.data)\n\n def append_middle(self, event):\n \"\"\" Attempt to place the event in the correct location in our deque.\n Returns True if successful, otherwise False. \"\"\"\n rotation = 0\n ts = self.get_ts(event)\n\n # Append left if ts is earlier than first event\n if self.get_ts(self.data[0]) > ts:\n self.data.appendleft(event)\n self.running_count += event[1]\n return\n\n # Rotate window until we can insert event\n while self.get_ts(self.data[-1]) > ts:\n self.data.rotate(1)\n rotation += 1\n if rotation == len(self.data):\n # This should never happen\n return\n self.data.append(event)\n self.running_count += event[1]\n self.data.rotate(-rotation)\n\n\nclass SpikeRule(RuleType):\n \"\"\" A rule that uses two sliding windows to compare relative event frequency. \"\"\"\n required_options = frozenset(['timeframe', 'spike_height', 'spike_type'])\n\n def __init__(self, *args):\n super(SpikeRule, self).__init__(*args)\n self.timeframe = self.rules['timeframe']\n\n self.ref_windows = {}\n self.cur_windows = {}\n\n self.ts_field = self.rules.get('timestamp_field', '@timestamp')\n self.get_ts = new_get_event_ts(self.ts_field)\n self.first_event = {}\n self.skip_checks = {}\n\n self.field_value = self.rules.get('field_value')\n\n self.ref_window_filled_once = False\n\n def add_count_data(self, data):\n \"\"\" Add count data to the rule. Data should be of the form {ts: count}. \"\"\"\n if len(data) > 1:\n raise EAException('add_count_data can only accept one count at a time')\n for ts, count in data.items():\n self.handle_event({self.ts_field: ts}, count, 'all')\n\n def add_terms_data(self, terms):\n for timestamp, buckets in terms.items():\n for bucket in buckets:\n count = bucket['doc_count']\n event = {self.ts_field: timestamp,\n self.rules['query_key']: bucket['key']}\n key = bucket['key']\n self.handle_event(event, count, key)\n\n def add_data(self, data):\n for event in data:\n qk = self.rules.get('query_key', 'all')\n if qk != 'all':\n qk = hashable(lookup_es_key(event, qk))\n if qk is None:\n qk = 'other'\n if self.field_value is not None:\n count = lookup_es_key(event, self.field_value)\n if count is not None:\n try:\n count = int(count)\n except ValueError:\n elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count))\n else:\n self.handle_event(event, count, qk)\n else:\n self.handle_event(event, 1, qk)\n\n def clear_windows(self, qk, event):\n # Reset the state and prevent alerts until windows filled again\n self.ref_windows[qk].clear()\n self.first_event.pop(qk)\n self.skip_checks[qk] = lookup_es_key(event, self.ts_field) + self.rules['timeframe'] * 2\n\n def handle_event(self, event, count, qk='all'):\n self.first_event.setdefault(qk, event)\n\n self.ref_windows.setdefault(qk, EventWindow(self.timeframe, getTimestamp=self.get_ts))\n self.cur_windows.setdefault(qk, EventWindow(self.timeframe, self.ref_windows[qk].append, self.get_ts))\n\n self.cur_windows[qk].append((event, count))\n\n # Don't alert if ref window has not yet been filled for this key AND\n if lookup_es_key(event, self.ts_field) - self.first_event[qk][self.ts_field] < self.rules['timeframe'] * 2:\n # ElastAlert has not been running long enough for any alerts OR\n if not self.ref_window_filled_once:\n return\n # This rule is not using alert_on_new_data (with query_key) OR\n if not (self.rules.get('query_key') and self.rules.get('alert_on_new_data')):\n return\n # An alert for this qk has recently fired\n if qk in self.skip_checks and lookup_es_key(event, self.ts_field) < self.skip_checks[qk]:\n return\n else:\n self.ref_window_filled_once = True\n\n if self.field_value is not None:\n if self.find_matches(self.ref_windows[qk].mean(), self.cur_windows[qk].mean()):\n # skip over placeholder events\n for match, count in self.cur_windows[qk].data:\n if \"placeholder\" not in match:\n break\n self.add_match(match, qk)\n self.clear_windows(qk, match)\n else:\n if self.find_matches(self.ref_windows[qk].count(), self.cur_windows[qk].count()):\n # skip over placeholder events which have count=0\n for match, count in self.cur_windows[qk].data:\n if count:\n break\n\n self.add_match(match, qk)\n self.clear_windows(qk, match)\n\n def add_match(self, match, qk):\n extra_info = {}\n if self.field_value is None:\n spike_count = self.cur_windows[qk].count()\n reference_count = self.ref_windows[qk].count()\n else:\n spike_count = self.cur_windows[qk].mean()\n reference_count = self.ref_windows[qk].mean()\n extra_info = {'spike_count': spike_count,\n 'reference_count': reference_count}\n\n match = dict(list(match.items()) + list(extra_info.items()))\n\n super(SpikeRule, self).add_match(match)\n\n def find_matches(self, ref, cur):\n \"\"\" Determines if an event spike or dip happening. \"\"\"\n # Apply threshold limits\n if self.field_value is None:\n if (cur < self.rules.get('threshold_cur', 0) or\n ref < self.rules.get('threshold_ref', 0)):\n return False\n elif ref is None or ref == 0 or cur is None or cur == 0:\n return False\n\n spike_up, spike_down = False, False\n if cur <= ref / self.rules['spike_height']:\n spike_down = True\n if cur >= ref * self.rules['spike_height']:\n spike_up = True\n\n if (self.rules['spike_type'] in ['both', 'up'] and spike_up) or \\\n (self.rules['spike_type'] in ['both', 'down'] and spike_down):\n return True\n return False\n\n def get_match_str(self, match):\n if self.field_value is None:\n message = 'An abnormal number (%d) of events occurred around %s.\\n' % (\n match['spike_count'],\n pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time'))\n )\n message += 'Preceding that time, there were only %d events within %s\\n\\n' % (match['reference_count'], self.rules['timeframe'])\n else:\n message = 'An abnormal average value (%.2f) of field \\'%s\\' occurred around %s.\\n' % (\n match['spike_count'],\n self.field_value,\n pretty_ts(match[self.rules['timestamp_field']],\n self.rules.get('use_local_time'))\n )\n message += 'Preceding that time, the field had an average value of (%.2f) within %s\\n\\n' % (\n match['reference_count'], self.rules['timeframe'])\n return message\n\n def garbage_collect(self, ts):\n # Windows are sized according to their newest event\n # This is a placeholder to accurately size windows in the absence of events\n for qk in list(self.cur_windows.keys()):\n # If we havn't seen this key in a long time, forget it\n if qk != 'all' and self.ref_windows[qk].count() == 0 and self.cur_windows[qk].count() == 0:\n self.cur_windows.pop(qk)\n self.ref_windows.pop(qk)\n continue\n placeholder = {self.ts_field: ts, \"placeholder\": True}\n # The placeholder may trigger an alert, in which case, qk will be expected\n if qk != 'all':\n placeholder.update({self.rules['query_key']: qk})\n self.handle_event(placeholder, 0, qk)\n\n\nclass FlatlineRule(FrequencyRule):\n \"\"\" A rule that matches when there is a low number of events given a timeframe. \"\"\"\n required_options = frozenset(['timeframe', 'threshold'])\n\n def __init__(self, *args):\n super(FlatlineRule, self).__init__(*args)\n self.threshold = self.rules['threshold']\n\n # Dictionary mapping query keys to the first events\n self.first_event = {}\n\n def check_for_match(self, key, end=True):\n # This function gets called between every added document with end=True after the last\n # We ignore the calls before the end because it may trigger false positives\n if not end:\n return\n\n most_recent_ts = self.get_ts(self.occurrences[key].data[-1])\n if self.first_event.get(key) is None:\n self.first_event[key] = most_recent_ts\n\n # Don't check for matches until timeframe has elapsed\n if most_recent_ts - self.first_event[key] < self.rules['timeframe']:\n return\n\n # Match if, after removing old events, we hit num_events\n count = self.occurrences[key].count()\n if count < self.rules['threshold']:\n # Do a deep-copy, otherwise we lose the datetime type in the timestamp field of the last event\n event = copy.deepcopy(self.occurrences[key].data[-1][0])\n event.update(key=key, count=count)\n self.add_match(event)\n\n if not self.rules.get('forget_keys'):\n # After adding this match, leave the occurrences windows alone since it will\n # be pruned in the next add_data or garbage_collect, but reset the first_event\n # so that alerts continue to fire until the threshold is passed again.\n least_recent_ts = self.get_ts(self.occurrences[key].data[0])\n timeframe_ago = most_recent_ts - self.rules['timeframe']\n self.first_event[key] = min(least_recent_ts, timeframe_ago)\n else:\n # Forget about this key until we see it again\n self.first_event.pop(key)\n self.occurrences.pop(key)\n\n def get_match_str(self, match):\n ts = match[self.rules['timestamp_field']]\n lt = self.rules.get('use_local_time')\n message = 'An abnormally low number of events occurred around %s.\\n' % (pretty_ts(ts, lt))\n message += 'Between %s and %s, there were less than %s events.\\n\\n' % (\n pretty_ts(dt_to_ts(ts_to_dt(ts) - self.rules['timeframe']), lt),\n pretty_ts(ts, lt),\n self.rules['threshold']\n )\n return message\n\n def garbage_collect(self, ts):\n # We add an event with a count of zero to the EventWindow for each key. This will cause the EventWindow\n # to remove events that occurred more than one `timeframe` ago, and call onRemoved on them.\n default = ['all'] if 'query_key' not in self.rules else []\n for key in list(self.occurrences.keys()) or default:\n self.occurrences.setdefault(\n key,\n EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)\n ).append(\n ({self.ts_field: ts}, 0)\n )\n self.first_event.setdefault(key, ts)\n self.check_for_match(key)\n\n\nclass NewTermsRule(RuleType):\n \"\"\" Alerts on a new value in a list of fields. \"\"\"\n\n def __init__(self, rule, args=None):\n super(NewTermsRule, self).__init__(rule, args)\n self.seen_values = {}\n # Allow the use of query_key or fields\n if 'fields' not in self.rules:\n if 'query_key' not in self.rules:\n raise EAException(\"fields or query_key must be specified\")\n self.fields = self.rules['query_key']\n else:\n self.fields = self.rules['fields']\n if not self.fields:\n raise EAException(\"fields must not be an empty list\")\n if type(self.fields) != list:\n self.fields = [self.fields]\n if self.rules.get('use_terms_query') and \\\n (len(self.fields) != 1 or (len(self.fields) == 1 and type(self.fields[0]) == list)):\n raise EAException(\"use_terms_query can only be used with a single non-composite field\")\n if self.rules.get('use_terms_query'):\n if [self.rules['query_key']] != self.fields:\n raise EAException('If use_terms_query is specified, you cannot specify different query_key and fields')\n if not self.rules.get('query_key').endswith('.keyword') and not self.rules.get('query_key').endswith('.raw'):\n if self.rules.get('use_keyword_postfix', True):\n elastalert_logger.warn('Warning: If query_key is a non-keyword field, you must set '\n 'use_keyword_postfix to false, or add .keyword/.raw to your query_key.')\n try:\n self.get_all_terms(args)\n except Exception as e:\n # Refuse to start if we cannot get existing terms\n raise EAException('Error searching for existing terms: %s' % (repr(e))).with_traceback(sys.exc_info()[2])\n\n def get_all_terms(self, args):\n \"\"\" Performs a terms aggregation for each field to get every existing term. \"\"\"\n self.es = elasticsearch_client(self.rules)\n window_size = datetime.timedelta(**self.rules.get('terms_window_size', {'days': 30}))\n field_name = {\"field\": \"\", \"size\": 2147483647} # Integer.MAX_VALUE\n query_template = {\"aggs\": {\"values\": {\"terms\": field_name}}}\n if args and hasattr(args, 'start') and args.start:\n end = ts_to_dt(args.start)\n elif 'start_date' in self.rules:\n end = ts_to_dt(self.rules['start_date'])\n else:\n end = ts_now()\n start = end - window_size\n step = datetime.timedelta(**self.rules.get('window_step_size', {'days': 1}))\n\n for field in self.fields:\n tmp_start = start\n tmp_end = min(start + step, end)\n\n time_filter = {self.rules['timestamp_field']: {'lt': self.rules['dt_to_ts'](tmp_end), 'gte': self.rules['dt_to_ts'](tmp_start)}}\n query_template['filter'] = {'bool': {'must': [{'range': time_filter}]}}\n query = {'aggs': {'filtered': query_template}}\n\n if 'filter' in self.rules:\n for item in self.rules['filter']:\n query_template['filter']['bool']['must'].append(item)\n\n # For composite keys, we will need to perform sub-aggregations\n if type(field) == list:\n self.seen_values.setdefault(tuple(field), [])\n level = query_template['aggs']\n # Iterate on each part of the composite key and add a sub aggs clause to the elastic search query\n for i, sub_field in enumerate(field):\n if self.rules.get('use_keyword_postfix', True):\n level['values']['terms']['field'] = add_raw_postfix(sub_field, self.is_five_or_above())\n else:\n level['values']['terms']['field'] = sub_field\n if i < len(field) - 1:\n # If we have more fields after the current one, then set up the next nested structure\n level['values']['aggs'] = {'values': {'terms': copy.deepcopy(field_name)}}\n level = level['values']['aggs']\n else:\n self.seen_values.setdefault(field, [])\n # For non-composite keys, only a single agg is needed\n if self.rules.get('use_keyword_postfix', True):\n field_name['field'] = add_raw_postfix(field, self.is_five_or_above())\n else:\n field_name['field'] = field\n\n # Query the entire time range in small chunks\n while tmp_start < end:\n if self.rules.get('use_strftime_index'):\n index = format_index(self.rules['index'], tmp_start, tmp_end)\n else:\n index = self.rules['index']\n res = self.es.search(body=query, index=index, ignore_unavailable=True, timeout='50s')\n if 'aggregations' in res:\n buckets = res['aggregations']['filtered']['values']['buckets']\n if type(field) == list:\n # For composite keys, make the lookup based on all fields\n # Make it a tuple since it can be hashed and used in dictionary lookups\n for bucket in buckets:\n # We need to walk down the hierarchy and obtain the value at each level\n self.seen_values[tuple(field)] += self.flatten_aggregation_hierarchy(bucket)\n else:\n keys = [bucket['key'] for bucket in buckets]\n self.seen_values[field] += keys\n else:\n if type(field) == list:\n self.seen_values.setdefault(tuple(field), [])\n else:\n self.seen_values.setdefault(field, [])\n if tmp_start == tmp_end:\n break\n tmp_start = tmp_end\n tmp_end = min(tmp_start + step, end)\n time_filter[self.rules['timestamp_field']] = {'lt': self.rules['dt_to_ts'](tmp_end),\n 'gte': self.rules['dt_to_ts'](tmp_start)}\n\n for key, values in self.seen_values.items():\n if not values:\n if type(key) == tuple:\n # If we don't have any results, it could either be because of the absence of any baseline data\n # OR it may be because the composite key contained a non-primitive type. Either way, give the\n # end-users a heads up to help them debug what might be going on.\n elastalert_logger.warning((\n 'No results were found from all sub-aggregations. This can either indicate that there is '\n 'no baseline data OR that a non-primitive field was used in a composite key.'\n ))\n else:\n elastalert_logger.info('Found no values for %s' % (field))\n continue\n self.seen_values[key] = list(set(values))\n elastalert_logger.info('Found %s unique values for %s' % (len(set(values)), key))\n\n def flatten_aggregation_hierarchy(self, root, hierarchy_tuple=()):\n \"\"\" For nested aggregations, the results come back in the following format:\n {\n \"aggregations\" : {\n \"filtered\" : {\n \"doc_count\" : 37,\n \"values\" : {\n \"doc_count_error_upper_bound\" : 0,\n \"sum_other_doc_count\" : 0,\n \"buckets\" : [ {\n \"key\" : \"1.1.1.1\", # IP address (root)\n \"doc_count\" : 13,\n \"values\" : {\n \"doc_count_error_upper_bound\" : 0,\n \"sum_other_doc_count\" : 0,\n \"buckets\" : [ {\n \"key\" : \"80\", # Port (sub-aggregation)\n \"doc_count\" : 3,\n \"values\" : {\n \"doc_count_error_upper_bound\" : 0,\n \"sum_other_doc_count\" : 0,\n \"buckets\" : [ {\n \"key\" : \"ack\", # Reason (sub-aggregation, leaf-node)\n \"doc_count\" : 3\n }, {\n \"key\" : \"syn\", # Reason (sub-aggregation, leaf-node)\n \"doc_count\" : 1\n } ]\n }\n }, {\n \"key\" : \"82\", # Port (sub-aggregation)\n \"doc_count\" : 3,\n \"values\" : {\n \"doc_count_error_upper_bound\" : 0,\n \"sum_other_doc_count\" : 0,\n \"buckets\" : [ {\n \"key\" : \"ack\", # Reason (sub-aggregation, leaf-node)\n \"doc_count\" : 3\n }, {\n \"key\" : \"syn\", # Reason (sub-aggregation, leaf-node)\n \"doc_count\" : 3\n } ]\n }\n } ]\n }\n }, {\n \"key\" : \"2.2.2.2\", # IP address (root)\n \"doc_count\" : 4,\n \"values\" : {\n \"doc_count_error_upper_bound\" : 0,\n \"sum_other_doc_count\" : 0,\n \"buckets\" : [ {\n \"key\" : \"443\", # Port (sub-aggregation)\n \"doc_count\" : 3,\n \"values\" : {\n \"doc_count_error_upper_bound\" : 0,\n \"sum_other_doc_count\" : 0,\n \"buckets\" : [ {\n \"key\" : \"ack\", # Reason (sub-aggregation, leaf-node)\n \"doc_count\" : 3\n }, {\n \"key\" : \"syn\", # Reason (sub-aggregation, leaf-node)\n \"doc_count\" : 3\n } ]\n }\n } ]\n }\n } ]\n }\n }\n }\n }\n\n Each level will either have more values and buckets, or it will be a leaf node\n We'll ultimately return a flattened list with the hierarchies appended as strings,\n e.g the above snippet would yield a list with:\n\n [\n ('1.1.1.1', '80', 'ack'),\n ('1.1.1.1', '80', 'syn'),\n ('1.1.1.1', '82', 'ack'),\n ('1.1.1.1', '82', 'syn'),\n ('2.2.2.2', '443', 'ack'),\n ('2.2.2.2', '443', 'syn')\n ]\n\n A similar formatting will be performed in the add_data method and used as the basis for comparison\n\n \"\"\"\n results = []\n # There are more aggregation hierarchies left. Traverse them.\n if 'values' in root:\n results += self.flatten_aggregation_hierarchy(root['values']['buckets'], hierarchy_tuple + (root['key'],))\n else:\n # We've gotten to a sub-aggregation, which may have further sub-aggregations\n # See if we need to traverse further\n for node in root:\n if 'values' in node:\n results += self.flatten_aggregation_hierarchy(node, hierarchy_tuple)\n else:\n results.append(hierarchy_tuple + (node['key'],))\n return results\n\n def add_data(self, data):\n for document in data:\n for field in self.fields:\n value = ()\n lookup_field = field\n if type(field) == list:\n # For composite keys, make the lookup based on all fields\n # Make it a tuple since it can be hashed and used in dictionary lookups\n lookup_field = tuple(field)\n for sub_field in field:\n lookup_result = lookup_es_key(document, sub_field)\n if not lookup_result:\n value = None\n break\n value += (lookup_result,)\n else:\n value = lookup_es_key(document, field)\n if not value and self.rules.get('alert_on_missing_field'):\n document['missing_field'] = lookup_field\n self.add_match(copy.deepcopy(document))\n elif value:\n if value not in self.seen_values[lookup_field]:\n document['new_field'] = lookup_field\n self.add_match(copy.deepcopy(document))\n self.seen_values[lookup_field].append(value)\n\n def add_terms_data(self, terms):\n # With terms query, len(self.fields) is always 1 and the 0'th entry is always a string\n field = self.fields[0]\n for timestamp, buckets in terms.items():\n for bucket in buckets:\n if bucket['doc_count']:\n if bucket['key'] not in self.seen_values[field]:\n match = {field: bucket['key'],\n self.rules['timestamp_field']: timestamp,\n 'new_field': field}\n self.add_match(match)\n self.seen_values[field].append(bucket['key'])\n\n def is_five_or_above(self):\n version = self.es.info()['version']['number']\n return int(version[0]) >= 5\n\n\nclass CardinalityRule(RuleType):\n \"\"\" A rule that matches if cardinality of a field is above or below a threshold within a timeframe \"\"\"\n required_options = frozenset(['timeframe', 'cardinality_field'])\n\n def __init__(self, *args):\n super(CardinalityRule, self).__init__(*args)\n if 'max_cardinality' not in self.rules and 'min_cardinality' not in self.rules:\n raise EAException(\"CardinalityRule must have one of either max_cardinality or min_cardinality\")\n self.ts_field = self.rules.get('timestamp_field', '@timestamp')\n self.cardinality_field = self.rules['cardinality_field']\n self.cardinality_cache = {}\n self.first_event = {}\n self.timeframe = self.rules['timeframe']\n\n def add_data(self, data):\n qk = self.rules.get('query_key')\n for event in data:\n if qk:\n key = hashable(lookup_es_key(event, qk))\n else:\n # If no query_key, we use the key 'all' for all events\n key = 'all'\n self.cardinality_cache.setdefault(key, {})\n self.first_event.setdefault(key, lookup_es_key(event, self.ts_field))\n value = hashable(lookup_es_key(event, self.cardinality_field))\n if value is not None:\n # Store this timestamp as most recent occurence of the term\n self.cardinality_cache[key][value] = lookup_es_key(event, self.ts_field)\n self.check_for_match(key, event)\n\n def check_for_match(self, key, event, gc=True):\n # Check to see if we are past max/min_cardinality for a given key\n time_elapsed = lookup_es_key(event, self.ts_field) - self.first_event.get(key, lookup_es_key(event, self.ts_field))\n timeframe_elapsed = time_elapsed > self.timeframe\n if (len(self.cardinality_cache[key]) > self.rules.get('max_cardinality', float('inf')) or\n (len(self.cardinality_cache[key]) < self.rules.get('min_cardinality', float('-inf')) and timeframe_elapsed)):\n # If there might be a match, run garbage collect first, as outdated terms are only removed in GC\n # Only run it if there might be a match so it doesn't impact performance\n if gc:\n self.garbage_collect(lookup_es_key(event, self.ts_field))\n self.check_for_match(key, event, False)\n else:\n self.first_event.pop(key, None)\n self.add_match(event)\n\n def garbage_collect(self, timestamp):\n \"\"\" Remove all occurrence data that is beyond the timeframe away \"\"\"\n for qk, terms in list(self.cardinality_cache.items()):\n for term, last_occurence in list(terms.items()):\n if timestamp - last_occurence > self.rules['timeframe']:\n self.cardinality_cache[qk].pop(term)\n\n # Create a placeholder event for if a min_cardinality match occured\n if 'min_cardinality' in self.rules:\n event = {self.ts_field: timestamp}\n if 'query_key' in self.rules:\n event.update({self.rules['query_key']: qk})\n self.check_for_match(qk, event, False)\n\n def get_match_str(self, match):\n lt = self.rules.get('use_local_time')\n starttime = pretty_ts(dt_to_ts(ts_to_dt(lookup_es_key(match, self.ts_field)) - self.rules['timeframe']), lt)\n endtime = pretty_ts(lookup_es_key(match, self.ts_field), lt)\n if 'max_cardinality' in self.rules:\n message = ('A maximum of %d unique %s(s) occurred since last alert or between %s and %s\\n\\n' % (self.rules['max_cardinality'],\n self.rules['cardinality_field'],\n starttime, endtime))\n else:\n message = ('Less than %d unique %s(s) occurred since last alert or between %s and %s\\n\\n' % (self.rules['min_cardinality'],\n self.rules['cardinality_field'],\n starttime, endtime))\n return message\n\n\nclass BaseAggregationRule(RuleType):\n def __init__(self, *args):\n super(BaseAggregationRule, self).__init__(*args)\n bucket_interval = self.rules.get('bucket_interval')\n if bucket_interval:\n if 'seconds' in bucket_interval:\n self.rules['bucket_interval_period'] = str(bucket_interval['seconds']) + 's'\n elif 'minutes' in bucket_interval:\n self.rules['bucket_interval_period'] = str(bucket_interval['minutes']) + 'm'\n elif 'hours' in bucket_interval:\n self.rules['bucket_interval_period'] = str(bucket_interval['hours']) + 'h'\n elif 'days' in bucket_interval:\n self.rules['bucket_interval_period'] = str(bucket_interval['days']) + 'd'\n elif 'weeks' in bucket_interval:\n self.rules['bucket_interval_period'] = str(bucket_interval['weeks']) + 'w'\n else:\n raise EAException(\"Unsupported window size\")\n\n if self.rules.get('use_run_every_query_size'):\n if total_seconds(self.rules['run_every']) % total_seconds(self.rules['bucket_interval_timedelta']) != 0:\n raise EAException(\"run_every must be evenly divisible by bucket_interval if specified\")\n else:\n if total_seconds(self.rules['buffer_time']) % total_seconds(self.rules['bucket_interval_timedelta']) != 0:\n raise EAException(\"Buffer_time must be evenly divisible by bucket_interval if specified\")\n\n def generate_aggregation_query(self):\n raise NotImplementedError()\n\n def add_aggregation_data(self, payload):\n for timestamp, payload_data in payload.items():\n if 'interval_aggs' in payload_data:\n self.unwrap_interval_buckets(timestamp, None, payload_data['interval_aggs']['buckets'])\n elif 'bucket_aggs' in payload_data:\n self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']['buckets'])\n else:\n self.check_matches(timestamp, None, payload_data)\n\n def unwrap_interval_buckets(self, timestamp, query_key, interval_buckets):\n for interval_data in interval_buckets:\n # Use bucket key here instead of start_time for more accurate match timestamp\n self.check_matches(ts_to_dt(interval_data['key_as_string']), query_key, interval_data)\n\n def unwrap_term_buckets(self, timestamp, term_buckets):\n for term_data in term_buckets:\n if 'interval_aggs' in term_data:\n self.unwrap_interval_buckets(timestamp, term_data['key'], term_data['interval_aggs']['buckets'])\n else:\n self.check_matches(timestamp, term_data['key'], term_data)\n\n def check_matches(self, timestamp, query_key, aggregation_data):\n raise NotImplementedError()\n\n\nclass MetricAggregationRule(BaseAggregationRule):\n \"\"\" A rule that matches when there is a low number of events given a timeframe. \"\"\"\n required_options = frozenset(['metric_agg_key', 'metric_agg_type'])\n allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count'])\n\n def __init__(self, *args):\n super(MetricAggregationRule, self).__init__(*args)\n self.ts_field = self.rules.get('timestamp_field', '@timestamp')\n if 'max_threshold' not in self.rules and 'min_threshold' not in self.rules:\n raise EAException(\"MetricAggregationRule must have at least one of either max_threshold or min_threshold\")\n\n self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type']\n\n if not self.rules['metric_agg_type'] in self.allowed_aggregations:\n raise EAException(\"metric_agg_type must be one of %s\" % (str(self.allowed_aggregations)))\n\n self.rules['aggregation_query_element'] = self.generate_aggregation_query()\n\n def get_match_str(self, match):\n message = 'Threshold violation, %s:%s %s (min: %s max : %s) \\n\\n' % (\n self.rules['metric_agg_type'],\n self.rules['metric_agg_key'],\n match[self.metric_key],\n self.rules.get('min_threshold'),\n self.rules.get('max_threshold')\n )\n return message\n\n def generate_aggregation_query(self):\n return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}}\n\n def check_matches(self, timestamp, query_key, aggregation_data):\n if \"compound_query_key\" in self.rules:\n self.check_matches_recursive(timestamp, query_key, aggregation_data, self.rules['compound_query_key'], dict())\n\n else:\n metric_val = aggregation_data[self.metric_key]['value']\n if self.crossed_thresholds(metric_val):\n match = {self.rules['timestamp_field']: timestamp,\n self.metric_key: metric_val}\n if query_key is not None:\n match[self.rules['query_key']] = query_key\n self.add_match(match)\n\n def check_matches_recursive(self, timestamp, query_key, aggregation_data, compound_keys, match_data):\n if len(compound_keys) < 1:\n # shouldn't get to this point, but checking for safety\n return\n\n match_data[compound_keys[0]] = aggregation_data['key']\n if 'bucket_aggs' in aggregation_data:\n for result in aggregation_data['bucket_aggs']['buckets']:\n self.check_matches_recursive(timestamp,\n query_key,\n result,\n compound_keys[1:],\n match_data)\n\n else:\n metric_val = aggregation_data[self.metric_key]['value']\n if self.crossed_thresholds(metric_val):\n match_data[self.rules['timestamp_field']] = timestamp\n match_data[self.metric_key] = metric_val\n\n # add compound key to payload to allow alerts to trigger for every unique occurence\n compound_value = [match_data[key] for key in self.rules['compound_query_key']]\n match_data[self.rules['query_key']] = \",\".join([str(value) for value in compound_value])\n\n self.add_match(match_data)\n\n def crossed_thresholds(self, metric_value):\n if metric_value is None:\n return False\n if 'max_threshold' in self.rules and metric_value > self.rules['max_threshold']:\n return True\n if 'min_threshold' in self.rules and metric_value < self.rules['min_threshold']:\n return True\n return False\n\n\nclass SpikeMetricAggregationRule(BaseAggregationRule, SpikeRule):\n \"\"\" A rule that matches when there is a spike in an aggregated event compared to its reference point \"\"\"\n required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'spike_height', 'spike_type'])\n allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count'])\n\n def __init__(self, *args):\n # We inherit everything from BaseAggregation and Spike, overwrite only what we need in functions below\n super(SpikeMetricAggregationRule, self).__init__(*args)\n\n # MetricAgg alert things\n self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type']\n if not self.rules['metric_agg_type'] in self.allowed_aggregations:\n raise EAException(\"metric_agg_type must be one of %s\" % (str(self.allowed_aggregations)))\n\n # Disabling bucket intervals (doesn't make sense in context of spike to split up your time period)\n if self.rules.get('bucket_interval'):\n raise EAException(\"bucket intervals are not supported for spike aggregation alerts\")\n\n self.rules['aggregation_query_element'] = self.generate_aggregation_query()\n\n def generate_aggregation_query(self):\n \"\"\"Lifted from MetricAggregationRule, added support for scripted fields\"\"\"\n if self.rules.get('metric_agg_script'):\n return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}}\n return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}}\n\n def add_aggregation_data(self, payload):\n \"\"\"\n BaseAggregationRule.add_aggregation_data unpacks our results and runs checks directly against hardcoded cutoffs.\n We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from\n the aggregation's \"value\" key to determine spikes from aggregations\n \"\"\"\n for timestamp, payload_data in payload.items():\n if 'bucket_aggs' in payload_data:\n self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs'])\n else:\n # no time / term split, just focus on the agg\n event = {self.ts_field: timestamp}\n agg_value = payload_data[self.metric_key]['value']\n self.handle_event(event, agg_value, 'all')\n return\n\n def unwrap_term_buckets(self, timestamp, term_buckets, qk=[]):\n \"\"\"\n create separate spike event trackers for each term,\n handle compound query keys\n \"\"\"\n for term_data in term_buckets['buckets']:\n qk.append(term_data['key'])\n\n # handle compound query keys (nested aggregations)\n if term_data.get('bucket_aggs'):\n self.unwrap_term_buckets(timestamp, term_data['bucket_aggs'], qk)\n # reset the query key to consider the proper depth for N > 2\n del qk[-1]\n continue\n\n qk_str = ','.join(qk)\n agg_value = term_data[self.metric_key]['value']\n event = {self.ts_field: timestamp,\n self.rules['query_key']: qk_str}\n # pass to SpikeRule's tracker\n self.handle_event(event, agg_value, qk_str)\n\n # handle unpack of lowest level\n del qk[-1]\n return\n\n def get_match_str(self, match):\n \"\"\"\n Overwrite SpikeRule's message to relate to the aggregation type & field instead of count\n \"\"\"\n message = 'An abnormal {0} of {1} ({2}) occurred around {3}.\\n'.format(\n self.rules['metric_agg_type'], self.rules['metric_agg_key'], round(match['spike_count'], 2),\n pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time'))\n )\n message += 'Preceding that time, there was a {0} of {1} of ({2}) within {3}\\n\\n'.format(\n self.rules['metric_agg_type'], self.rules['metric_agg_key'],\n round(match['reference_count'], 2), self.rules['timeframe'])\n return message\n\n\nclass PercentageMatchRule(BaseAggregationRule):\n required_options = frozenset(['match_bucket_filter'])\n\n def __init__(self, *args):\n super(PercentageMatchRule, self).__init__(*args)\n self.ts_field = self.rules.get('timestamp_field', '@timestamp')\n if 'max_percentage' not in self.rules and 'min_percentage' not in self.rules:\n raise EAException(\"PercentageMatchRule must have at least one of either min_percentage or max_percentage\")\n\n self.min_denominator = self.rules.get('min_denominator', 0)\n self.match_bucket_filter = self.rules['match_bucket_filter']\n self.rules['aggregation_query_element'] = self.generate_aggregation_query()\n\n def get_match_str(self, match):\n percentage_format_string = self.rules.get('percentage_format_string', None)\n message = 'Percentage violation, value: %s (min: %s max : %s) of %s items\\n\\n' % (\n percentage_format_string % (match['percentage']) if percentage_format_string else match['percentage'],\n self.rules.get('min_percentage'),\n self.rules.get('max_percentage'),\n match['denominator']\n )\n return message\n\n def generate_aggregation_query(self):\n return {\n 'percentage_match_aggs': {\n 'filters': {\n 'other_bucket': True,\n 'filters': {\n 'match_bucket': {\n 'bool': {\n 'must': self.match_bucket_filter\n }\n }\n }\n }\n }\n }\n\n def check_matches(self, timestamp, query_key, aggregation_data):\n match_bucket_count = aggregation_data['percentage_match_aggs']['buckets']['match_bucket']['doc_count']\n other_bucket_count = aggregation_data['percentage_match_aggs']['buckets']['_other_']['doc_count']\n\n if match_bucket_count is None or other_bucket_count is None:\n return\n else:\n total_count = other_bucket_count + match_bucket_count\n if total_count == 0 or total_count < self.min_denominator:\n return\n else:\n match_percentage = (match_bucket_count * 1.0) / (total_count * 1.0) * 100\n if self.percentage_violation(match_percentage):\n match = {self.rules['timestamp_field']: timestamp, 'percentage': match_percentage, 'denominator': total_count}\n if query_key is not None:\n match[self.rules['query_key']] = query_key\n self.add_match(match)\n\n def percentage_violation(self, match_percentage):\n if 'max_percentage' in self.rules and match_percentage > self.rules['max_percentage']:\n return True\n if 'min_percentage' in self.rules and match_percentage < self.rules['min_percentage']:\n return True\n return False\n" }, { "path": "elastalert/schema.yaml", "content": "$schema: http://json-schema.org/draft-07/schema#\ndefinitions:\n\n # Either a single string OR an array of strings\n arrayOfStrings: &arrayOfString\n type: [string, array]\n items: {type: string}\n\n # Either a single string OR an array of strings OR an array of ararys\n arrayOfStringsOrOtherArrays: &arrayOfStringsOrOtherArray\n type: [string, array]\n items: {type: [string, array]}\n\n timedelta: &timedelta\n type: object\n additionalProperties: false\n properties:\n days: {type: number}\n weeks: {type: number}\n hours: {type: number}\n minutes: {type: number}\n seconds: {type: number}\n milliseconds: {type: number}\n\n timeFrame: &timeframe\n type: object\n additionalProperties: false\n properties:\n days: {type: number}\n weeks: {type: number}\n hours: {type: number}\n minutes: {type: number}\n seconds: {type: number}\n milliseconds: {type: number}\n schedule: {type: string}\n\n filter: &filter {}\n\n mattermostField: &mattermostField\n type: object\n additionalProperties: false\n properties:\n title: {type: string}\n value: {type: string}\n args: *arrayOfString\n short: {type: boolean}\n\nrequired: [type, index, alert]\ntype: object\n\n### Rule Types section\noneOf:\n - title: Any\n properties:\n type: {enum: [any]}\n\n - title: Blacklist\n required: [blacklist, compare_key]\n properties:\n type: {enum: [blacklist]}\n compare_key: {'items': {'type': 'string'},'type': ['string', 'array']}\n blacklist: {type: array, items: {type: string}}\n\n - title: Whitelist\n required: [whitelist, compare_key, ignore_null]\n properties:\n type: {enum: [whitelist]}\n compare_key: {'items': {'type': 'string'},'type': ['string', 'array']}\n whitelist: {type: array, items: {type: string}}\n ignore_null: {type: boolean}\n\n - title: Change\n required: [query_key, compare_key, ignore_null]\n properties:\n type: {enum: [change]}\n compare_key: {'items': {'type': 'string'},'type': ['string', 'array']}\n ignore_null: {type: boolean}\n timeframe: *timeframe\n\n - title: Frequency\n required: [num_events, timeframe]\n properties:\n type: {enum: [frequency]}\n num_events: {type: integer}\n timeframe: *timeframe\n use_count_query: {type: boolean}\n doc_type: {type: string}\n use_terms_query: {type: boolean}\n terms_size: {type: integer}\n attach_related: {type: boolean}\n\n - title: Spike\n required: [spike_height, spike_type, timeframe]\n properties:\n type: {enum: [spike]}\n spike_height: {type: number}\n spike_type: {enum: [\"up\", \"down\", \"both\"]}\n timeframe: *timeframe\n use_count_query: {type: boolean}\n doc_type: {type: string}\n use_terms_query: {type: boolean}\n terms_size: {type: integer}\n alert_on_new_data: {type: boolean}\n threshold_ref: {type: integer}\n threshold_cur: {type: integer}\n\n - title: Spike Aggregation\n required: [spike_height, spike_type, timeframe]\n properties:\n type: {enum: [spike_aggregation]}\n spike_height: {type: number}\n spike_type: {enum: [\"up\", \"down\", \"both\"]}\n metric_agg_type: {enum: [\"min\", \"max\", \"avg\", \"sum\", \"cardinality\", \"value_count\"]}\n timeframe: *timeframe\n use_count_query: {type: boolean}\n doc_type: {type: string}\n use_terms_query: {type: boolean}\n terms_size: {type: integer}\n alert_on_new_data: {type: boolean}\n threshold_ref: {type: number}\n threshold_cur: {type: number}\n min_doc_count: {type: integer}\n\n - title: Flatline\n required: [threshold, timeframe]\n properties:\n type: {enum: [flatline]}\n timeframe: *timeframe\n threshold: {type: integer}\n use_count_query: {type: boolean}\n doc_type: {type: string}\n\n - title: New Term\n required: []\n properties:\n type: {enum: [new_term]}\n fields: *arrayOfStringsOrOtherArray\n terms_window_size: *timeframe\n alert_on_missing_field: {type: boolean}\n use_terms_query: {type: boolean}\n terms_size: {type: integer}\n\n - title: Cardinality\n required: [cardinality_field, timeframe]\n properties:\n type: {enum: [cardinality]}\n max_cardinality: {type: integer}\n min_cardinality: {type: integer}\n cardinality_field: {type: string}\n timeframe: *timeframe\n\n - title: Metric Aggregation\n required: [metric_agg_key,metric_agg_type]\n properties:\n type: {enum: [metric_aggregation]}\n metric_agg_type: {enum: [\"min\", \"max\", \"avg\", \"sum\", \"cardinality\", \"value_count\"]}\n #timeframe: *timeframe\n\n - title: Percentage Match\n required: [match_bucket_filter]\n properties:\n type: {enum: [percentage_match]}\n\n - title: Custom Rule from Module\n properties:\n # custom rules include a period in the rule type\n type: {pattern: \"[.]\"}\n\nproperties:\n\n # Common Settings\n es_host: {type: string}\n es_port: {type: integer}\n index: {type: string}\n name: {type: string}\n\n use_ssl: {type: boolean}\n verify_certs: {type: boolean}\n es_username: {type: string}\n es_password: {type: string}\n use_strftime_index: {type: boolean}\n\n # Optional Settings\n import: {type: string}\n aggregation: *timeframe\n realert: *timeframe\n exponential_realert: *timeframe\n\n buffer_time: *timeframe\n query_delay: *timeframe\n max_query_size: {type: integer}\n max_scrolling: {type: integer}\n\n owner: {type: string}\n priority: {type: integer}\n\n filter :\n type: [array, object]\n items: *filter\n additionalProperties: false\n properties:\n download_dashboard: {type: string}\n\n include: {type: array, items: {type: string}}\n top_count_keys: {type: array, items: {type: string}}\n top_count_number: {type: integer}\n raw_count_keys: {type: boolean}\n generate_kibana_link: {type: boolean}\n kibana_dashboard: {type: string}\n use_kibana_dashboard: {type: string}\n use_local_time: {type: boolean}\n match_enhancements: {type: array, items: {type: string}}\n query_key: *arrayOfString\n replace_dots_in_field_names: {type: boolean}\n scan_entire_timeframe: {type: boolean}\n\n ### Kibana Discover App Link\n generate_kibana_discover_url: {type: boolean}\n kibana_discover_app_url: {type: string, format: uri}\n kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']}\n kibana_discover_index_pattern_id: {type: string, minLength: 1}\n kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1}\n kibana_discover_from_timedelta: *timedelta\n kibana_discover_to_timedelta: *timedelta\n\n # Alert Content\n alert_text: {type: string} # Python format string\n alert_text_args: {type: array, items: {type: string}}\n alert_text_kw: {type: object}\n alert_text_type: {enum: [alert_text_only, exclude_fields, aggregation_summary_only]}\n alert_missing_value: {type: string}\n timestamp_field: {type: string}\n field: {}\n\n ### Commands\n command: *arrayOfString\n pipe_match_json: {type: boolean}\n fail_on_non_zero_exit: {type: boolean}\n\n ### Email\n email: *arrayOfString\n email_reply_to: {type: string}\n notify_email: *arrayOfString # if rule is slow or erroring, send to this email\n smtp_host: {type: string}\n from_addr: {type: string}\n\n ### JIRA\n jira_server: {type: string}\n jira_project: {type: string}\n jira_issuetype: {type: string}\n jira_account_file: {type: string} # a Yaml file that includes the keys {user:, password:}\n\n jira_assignee: {type: string}\n jira_component: *arrayOfString\n jira_components: *arrayOfString\n jira_label: *arrayOfString\n jira_labels: *arrayOfString\n jira_bump_tickets: {type: boolean}\n jira_bump_in_statuses: *arrayOfString\n jira_bump_not_in_statuses: *arrayOfString\n jira_max_age: {type: number}\n jira_watchers: *arrayOfString\n\n ### HipChat\n hipchat_auth_token: {type: string}\n hipchat_room_id: {type: [string, integer]}\n hipchat_domain: {type: string}\n hipchat_ignore_ssl_errors: {type: boolean}\n hipchat_notify: {type: boolean}\n hipchat_from: {type: string}\n hipchat_mentions: {type: array, items: {type: string}}\n\n ### Stride\n stride_access_token: {type: string}\n stride_cloud_id: {type: string}\n stride_conversation_id: {type: string}\n stride_ignore_ssl_errors: {type: boolean}\n\n ### Slack\n slack_webhook_url: *arrayOfString\n slack_username_override: {type: string}\n slack_emoji_override: {type: string}\n slack_icon_url_override: {type: string}\n slack_msg_color: {enum: [good, warning, danger]}\n slack_parse_override: {enum: [none, full]}\n slack_text_string: {type: string}\n slack_ignore_ssl_errors: {type: boolean}\n slack_ca_certs: {type: string}\n slack_attach_kibana_discover_url: {type: boolean}\n slack_kibana_discover_color: {type: string}\n slack_kibana_discover_title: {type: string}\n\n ### Mattermost\n mattermost_webhook_url: *arrayOfString\n mattermost_proxy: {type: string}\n mattermost_ignore_ssl_errors: {type: boolean}\n mattermost_username_override: {type: string}\n mattermost_icon_url_override: {type: string}\n mattermost_channel_override: {type: string}\n mattermost_msg_color: {enum: [good, warning, danger]}\n mattermost_msg_pretext: {type: string}\n mattermost_msg_fields: *mattermostField\n\n ## Opsgenie\n opsgenie_details:\n type: object\n minProperties: 1\n patternProperties:\n \"^.+$\":\n oneOf:\n - type: string\n - type: object\n additionalProperties: false\n required: [field]\n properties:\n field: {type: string, minLength: 1}\n\n ### PagerDuty\n pagerduty_service_key: {type: string}\n pagerduty_client_name: {type: string}\n pagerduty_event_type: {enum: [none, trigger, resolve, acknowledge]}\n\n### PagerTree\n pagertree_integration_url: {type: string}\n\n\n ### Exotel\n exotel_account_sid: {type: string}\n exotel_auth_token: {type: string}\n exotel_to_number: {type: string}\n exotel_from_number: {type: string}\n\n ### Twilio\n twilio_account_sid: {type: string}\n twilio_auth_token: {type: string}\n twilio_to_number: {type: string}\n twilio_from_number: {type: string}\n\n ### VictorOps\n victorops_api_key: {type: string}\n victorops_routing_key: {type: string}\n victorops_message_type: {enum: [INFO, WARNING, ACKNOWLEDGEMENT, CRITICAL, RECOVERY]}\n victorops_entity_id: {type: string}\n victorops_entity_display_name: {type: string}\n\n ### Telegram\n telegram_bot_token: {type: string}\n telegram_room_id: {type: string}\n telegram_api_url: {type: string}\n\n ### Gitter\n gitter_webhook_url: {type: string}\n gitter_proxy: {type: string}\n gitter_msg_level: {enum: [info, error]}\n\n ### Alerta\n alerta_api_url: {type: string}\n alerta_api_key: {type: string}\n alerta_severity: {enum: [unknown, security, debug, informational, ok, normal, cleared, indeterminate, warning, minor, major, critical]}\n alerta_resource: {type: string} # Python format string\n alerta_environment: {type: string} # Python format string\n alerta_origin: {type: string} # Python format string\n alerta_group: {type: string} # Python format string\n alerta_service: {type: array, items: {type: string}} # Python format string\n alerta_service: {type: array, items: {type: string}} # Python format string\n alerta_correlate: {type: array, items: {type: string}} # Python format string\n alerta_tags: {type: array, items: {type: string}} # Python format string\n alerta_event: {type: string} # Python format string\n alerta_customer: {type: string}\n alerta_text: {type: string} # Python format string\n alerta_type: {type: string}\n alerta_value: {type: string} # Python format string\n alerta_attributes_keys: {type: array, items: {type: string}}\n alerta_attributes_values: {type: array, items: {type: string}} # Python format string\n alerta_new_style_string_format: {type: boolean}\n\n\n ### Simple\n simple_webhook_url: *arrayOfString\n simple_proxy: {type: string}\n\n ### LineNotify\n linenotify_access_token: {type: string}\n\n ### Zabbix\n zbx_sender_host: {type: string}\n zbx_sender_port: {type: integer}\n zbx_host: {type: string}\n zbx_item: {type: string}\n" }, { "path": "elastalert/test_rule.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport argparse\nimport copy\nimport datetime\nimport json\nimport logging\nimport random\nimport re\nimport string\nimport sys\n\nimport mock\n\nfrom elastalert.config import load_conf\nfrom elastalert.elastalert import ElastAlerter\nfrom elastalert.util import EAException\nfrom elastalert.util import elasticsearch_client\nfrom elastalert.util import lookup_es_key\nfrom elastalert.util import ts_now\nfrom elastalert.util import ts_to_dt\n\nlogging.getLogger().setLevel(logging.INFO)\nlogging.getLogger('elasticsearch').setLevel(logging.WARNING)\n\n\"\"\"\nError Codes:\n 1: Error connecting to ElasticSearch\n 2: Error querying ElasticSearch\n 3: Invalid Rule\n 4: Missing/invalid timestamp\n\"\"\"\n\n\ndef print_terms(terms, parent):\n \"\"\" Prints a list of flattened dictionary keys \"\"\"\n for term in terms:\n if type(terms[term]) != dict:\n print('\\t' + parent + term)\n else:\n print_terms(terms[term], parent + term + '.')\n\n\nclass MockElastAlerter(object):\n def __init__(self):\n self.data = []\n self.formatted_output = {}\n\n def test_file(self, conf, args):\n \"\"\" Loads a rule config file, performs a query over the last day (args.days), lists available keys\n and prints the number of results. \"\"\"\n if args.schema_only:\n return []\n\n # Set up Elasticsearch client and query\n es_client = elasticsearch_client(conf)\n\n try:\n ElastAlerter.modify_rule_for_ES5(conf)\n except EAException as ea:\n print('Invalid filter provided:', str(ea), file=sys.stderr)\n if args.stop_error:\n exit(3)\n return None\n except Exception as e:\n print(\"Error connecting to ElasticSearch:\", file=sys.stderr)\n print(repr(e)[:2048], file=sys.stderr)\n if args.stop_error:\n exit(1)\n return None\n start_time = ts_now() - datetime.timedelta(days=args.days)\n end_time = ts_now()\n ts = conf.get('timestamp_field', '@timestamp')\n query = ElastAlerter.get_query(\n conf['filter'],\n starttime=start_time,\n endtime=end_time,\n timestamp_field=ts,\n to_ts_func=conf['dt_to_ts'],\n five=conf['five']\n )\n index = ElastAlerter.get_index(conf, start_time, end_time)\n\n # Get one document for schema\n try:\n res = es_client.search(index, size=1, body=query, ignore_unavailable=True)\n except Exception as e:\n print(\"Error running your filter:\", file=sys.stderr)\n print(repr(e)[:2048], file=sys.stderr)\n if args.stop_error:\n exit(3)\n return None\n num_hits = len(res['hits']['hits'])\n if not num_hits:\n print(\"Didn't get any results.\")\n return []\n\n terms = res['hits']['hits'][0]['_source']\n doc_type = res['hits']['hits'][0]['_type']\n\n # Get a count of all docs\n count_query = ElastAlerter.get_query(\n conf['filter'],\n starttime=start_time,\n endtime=end_time,\n timestamp_field=ts,\n to_ts_func=conf['dt_to_ts'],\n sort=False,\n five=conf['five']\n )\n try:\n res = es_client.count(index, doc_type=doc_type, body=count_query, ignore_unavailable=True)\n except Exception as e:\n print(\"Error querying Elasticsearch:\", file=sys.stderr)\n print(repr(e)[:2048], file=sys.stderr)\n if args.stop_error:\n exit(2)\n return None\n\n num_hits = res['count']\n\n if args.formatted_output:\n self.formatted_output['hits'] = num_hits\n self.formatted_output['days'] = args.days\n self.formatted_output['terms'] = list(terms.keys())\n self.formatted_output['result'] = terms\n else:\n print(\"Got %s hits from the last %s day%s\" % (num_hits, args.days, 's' if args.days > 1 else ''))\n print(\"\\nAvailable terms in first hit:\")\n print_terms(terms, '')\n\n # Check for missing keys\n pk = conf.get('primary_key')\n ck = conf.get('compare_key')\n if pk and not lookup_es_key(terms, pk):\n print(\"Warning: primary key %s is either missing or null!\", file=sys.stderr)\n if ck and not lookup_es_key(terms, ck):\n print(\"Warning: compare key %s is either missing or null!\", file=sys.stderr)\n\n include = conf.get('include')\n if include:\n for term in include:\n if not lookup_es_key(terms, term) and '*' not in term:\n print(\"Included term %s may be missing or null\" % (term), file=sys.stderr)\n\n for term in conf.get('top_count_keys', []):\n # If the index starts with 'logstash', fields with .raw will be available but won't in _source\n if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')):\n print(\"top_count_key %s may be missing\" % (term), file=sys.stderr)\n if not args.formatted_output:\n print('') # Newline\n\n # Download up to max_query_size (defaults to 10,000) documents to save\n if (args.save or args.formatted_output) and not args.count:\n try:\n res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True)\n except Exception as e:\n print(\"Error running your filter:\", file=sys.stderr)\n print(repr(e)[:2048], file=sys.stderr)\n if args.stop_error:\n exit(2)\n return None\n num_hits = len(res['hits']['hits'])\n\n if args.save:\n print(\"Downloaded %s documents to save\" % (num_hits))\n return res['hits']['hits']\n\n def mock_count(self, rule, start, end, index):\n \"\"\" Mocks the effects of get_hits_count using global data instead of Elasticsearch \"\"\"\n count = 0\n for doc in self.data:\n if start <= ts_to_dt(doc[rule['timestamp_field']]) < end:\n count += 1\n return {end: count}\n\n def mock_hits(self, rule, start, end, index, scroll=False):\n \"\"\" Mocks the effects of get_hits using global data instead of Elasticsearch. \"\"\"\n docs = []\n for doc in self.data:\n if start <= ts_to_dt(doc[rule['timestamp_field']]) < end:\n docs.append(doc)\n\n # Remove all fields which don't match 'include'\n for doc in docs:\n fields_to_remove = []\n for field in doc:\n if field != '_id':\n if not any([re.match(incl.replace('*', '.*'), field) for incl in rule['include']]):\n fields_to_remove.append(field)\n list(map(doc.pop, fields_to_remove))\n\n # Separate _source and _id, convert timestamps\n resp = [{'_source': doc, '_id': doc['_id']} for doc in docs]\n for doc in resp:\n doc['_source'].pop('_id')\n return ElastAlerter.process_hits(rule, resp)\n\n def mock_terms(self, rule, start, end, index, key, qk=None, size=None):\n \"\"\" Mocks the effects of get_hits_terms using global data instead of Elasticsearch. \"\"\"\n if key.endswith('.raw'):\n key = key[:-4]\n buckets = {}\n for doc in self.data:\n if key not in doc:\n continue\n if start <= ts_to_dt(doc[rule['timestamp_field']]) < end:\n if qk is None or doc[rule['query_key']] == qk:\n buckets.setdefault(doc[key], 0)\n buckets[doc[key]] += 1\n counts = list(buckets.items())\n counts.sort(key=lambda x: x[1], reverse=True)\n if size:\n counts = counts[:size]\n buckets = [{'key': value, 'doc_count': count} for value, count in counts]\n return {end: buckets}\n\n def mock_elastalert(self, elastalert):\n \"\"\" Replaces elastalert's get_hits functions with mocks. \"\"\"\n elastalert.get_hits_count = self.mock_count\n elastalert.get_hits_terms = self.mock_terms\n elastalert.get_hits = self.mock_hits\n elastalert.elasticsearch_client = mock.Mock()\n\n def run_elastalert(self, rule, conf, args):\n \"\"\" Creates an ElastAlert instance and run's over for a specific rule using either real or mock data. \"\"\"\n\n # Load and instantiate rule\n # Pass an args containing the context of whether we're alerting or not\n # It is needed to prevent unnecessary initialization of unused alerters\n load_modules_args = argparse.Namespace()\n load_modules_args.debug = not args.alert\n conf['rules_loader'].load_modules(rule, load_modules_args)\n\n # If using mock data, make sure it's sorted and find appropriate time range\n timestamp_field = rule.get('timestamp_field', '@timestamp')\n if args.json:\n if not self.data:\n return None\n try:\n self.data.sort(key=lambda x: x[timestamp_field])\n starttime = ts_to_dt(self.data[0][timestamp_field])\n endtime = self.data[-1][timestamp_field]\n endtime = ts_to_dt(endtime) + datetime.timedelta(seconds=1)\n except KeyError as e:\n print(\"All documents must have a timestamp and _id: %s\" % (e), file=sys.stderr)\n if args.stop_error:\n exit(4)\n return None\n\n # Create mock _id for documents if it's missing\n used_ids = []\n\n def get_id():\n _id = ''.join([random.choice(string.ascii_letters) for i in range(16)])\n if _id in used_ids:\n return get_id()\n used_ids.append(_id)\n return _id\n\n for doc in self.data:\n doc.update({'_id': doc.get('_id', get_id())})\n else:\n if args.end:\n if args.end == 'NOW':\n endtime = ts_now()\n else:\n try:\n endtime = ts_to_dt(args.end)\n except (TypeError, ValueError):\n self.handle_error(\"%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)\" % (args.end))\n exit(4)\n else:\n endtime = ts_now()\n if args.start:\n try:\n starttime = ts_to_dt(args.start)\n except (TypeError, ValueError):\n self.handle_error(\"%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)\" % (args.start))\n exit(4)\n else:\n # if days given as command line argument\n if args.days > 0:\n starttime = endtime - datetime.timedelta(days=args.days)\n else:\n # if timeframe is given in rule\n if 'timeframe' in rule:\n starttime = endtime - datetime.timedelta(seconds=rule['timeframe'].total_seconds() * 1.01)\n # default is 1 days / 24 hours\n else:\n starttime = endtime - datetime.timedelta(days=1)\n\n # Set run_every to cover the entire time range unless count query, terms query or agg query used\n # This is to prevent query segmenting which unnecessarily slows down tests\n if not rule.get('use_terms_query') and not rule.get('use_count_query') and not rule.get('aggregation_query_element'):\n conf['run_every'] = endtime - starttime\n\n # Instantiate ElastAlert to use mock config and special rule\n with mock.patch.object(conf['rules_loader'], 'get_hashes'):\n with mock.patch.object(conf['rules_loader'], 'load') as load_rules:\n load_rules.return_value = [rule]\n with mock.patch('elastalert.elastalert.load_conf') as load_conf:\n load_conf.return_value = conf\n if args.alert:\n client = ElastAlerter(['--verbose'])\n else:\n client = ElastAlerter(['--debug'])\n\n # Replace get_hits_* functions to use mock data\n if args.json:\n self.mock_elastalert(client)\n\n # Mock writeback to return empty results\n client.writeback_es = mock.MagicMock()\n client.writeback_es.search.return_value = {\"hits\": {\"hits\": []}}\n\n with mock.patch.object(client, 'writeback') as mock_writeback:\n client.run_rule(rule, endtime, starttime)\n\n if mock_writeback.call_count:\n\n if args.formatted_output:\n self.formatted_output['writeback'] = {}\n else:\n print(\"\\nWould have written the following documents to writeback index (default is elastalert_status):\\n\")\n\n errors = False\n for call in mock_writeback.call_args_list:\n if args.formatted_output:\n self.formatted_output['writeback'][call[0][0]] = json.loads(json.dumps(call[0][1], default=str))\n else:\n print(\"%s - %s\\n\" % (call[0][0], call[0][1]))\n\n if call[0][0] == 'elastalert_error':\n errors = True\n if errors and args.stop_error:\n exit(2)\n\n def run_rule_test(self):\n \"\"\"\n Uses args to run the various components of MockElastAlerter such as loading the file, saving data, loading data, and running.\n \"\"\"\n parser = argparse.ArgumentParser(description='Validate a rule configuration')\n parser.add_argument('file', metavar='rule', type=str, help='rule configuration filename')\n parser.add_argument('--schema-only', action='store_true', help='Show only schema errors; do not run query')\n parser.add_argument('--days', type=int, default=0, action='store', help='Query the previous N days with this rule')\n parser.add_argument('--start', dest='start', help='YYYY-MM-DDTHH:MM:SS Start querying from this timestamp.')\n parser.add_argument('--end', dest='end', help='YYYY-MM-DDTHH:MM:SS Query to this timestamp. (Default: present) '\n 'Use \"NOW\" to start from current time. (Default: present)')\n parser.add_argument('--stop-error', action='store_true', help='Stop the entire test right after the first error')\n parser.add_argument('--formatted-output', action='store_true', help='Output results in formatted JSON')\n parser.add_argument(\n '--data',\n type=str,\n metavar='FILENAME',\n action='store',\n dest='json',\n help='A JSON file containing data to run the rule against')\n parser.add_argument('--alert', action='store_true', help='Use actual alerts instead of debug output')\n parser.add_argument(\n '--save-json',\n type=str,\n metavar='FILENAME',\n action='store',\n dest='save',\n help='A file to which documents from the last day or --days will be saved')\n parser.add_argument(\n '--use-downloaded',\n action='store_true',\n dest='use_downloaded',\n help='Use the downloaded '\n )\n parser.add_argument(\n '--max-query-size',\n type=int,\n default=10000,\n action='store',\n dest='max_query_size',\n help='Maximum size of any query')\n parser.add_argument(\n '--count-only',\n action='store_true',\n dest='count',\n help='Only display the number of documents matching the filter')\n parser.add_argument('--config', action='store', dest='config', help='Global config file.')\n args = parser.parse_args()\n\n defaults = {\n 'rules_folder': 'rules',\n 'es_host': 'localhost',\n 'es_port': 14900,\n 'writeback_index': 'wb',\n 'writeback_alias': 'wb_a',\n 'max_query_size': 10000,\n 'alert_time_limit': {'hours': 24},\n 'old_query_limit': {'weeks': 1},\n 'run_every': {'minutes': 5},\n 'disable_rules_on_error': False,\n 'buffer_time': {'minutes': 45},\n 'scroll_keepalive': '30s'\n }\n overwrites = {\n 'rules_loader': 'file',\n }\n\n # Set arguments that ElastAlerter needs\n args.verbose = args.alert\n args.debug = not args.alert\n args.es_debug = False\n args.es_debug_trace = False\n\n conf = load_conf(args, defaults, overwrites)\n rule_yaml = conf['rules_loader'].load_yaml(args.file)\n conf['rules_loader'].load_options(rule_yaml, conf, args.file)\n\n if args.json:\n with open(args.json, 'r') as data_file:\n self.data = json.loads(data_file.read())\n else:\n hits = self.test_file(copy.deepcopy(rule_yaml), args)\n if hits and args.formatted_output:\n self.formatted_output['results'] = json.loads(json.dumps(hits))\n if hits and args.save:\n with open(args.save, 'wb') as data_file:\n # Add _id to _source for dump\n [doc['_source'].update({'_id': doc['_id']}) for doc in hits]\n data_file.write(json.dumps([doc['_source'] for doc in hits], indent=4))\n if args.use_downloaded:\n if hits:\n args.json = args.save\n with open(args.json, 'r') as data_file:\n self.data = json.loads(data_file.read())\n else:\n self.data = []\n\n if not args.schema_only and not args.count:\n self.run_elastalert(rule_yaml, conf, args)\n\n if args.formatted_output:\n print(json.dumps(self.formatted_output))\n\n\ndef main():\n test_instance = MockElastAlerter()\n test_instance.run_rule_test()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "elastalert/util.py", "content": "# -*- coding: utf-8 -*-\nimport collections\nimport datetime\nimport logging\nimport os\nimport re\nimport sys\n\nimport dateutil.parser\nimport pytz\nfrom six import string_types\n\nfrom . import ElasticSearchClient\nfrom .auth import Auth\n\nlogging.basicConfig()\nelastalert_logger = logging.getLogger('elastalert')\n\n\ndef get_module(module_name):\n \"\"\" Loads a module and returns a specific object.\n module_name should 'module.file.object'.\n Returns object or raises EAException on error. \"\"\"\n sys.path.append(os.getcwd())\n try:\n module_path, module_class = module_name.rsplit('.', 1)\n base_module = __import__(module_path, globals(), locals(), [module_class])\n module = getattr(base_module, module_class)\n except (ImportError, AttributeError, ValueError) as e:\n raise EAException(\"Could not import module %s: %s\" % (module_name, e)).with_traceback(sys.exc_info()[2])\n return module\n\n\ndef new_get_event_ts(ts_field):\n \"\"\" Constructs a lambda that may be called to extract the timestamp field\n from a given event.\n\n :returns: A callable function that takes an event and outputs that event's\n timestamp field.\n \"\"\"\n return lambda event: lookup_es_key(event[0], ts_field)\n\n\ndef _find_es_dict_by_key(lookup_dict, term):\n \"\"\" Performs iterative dictionary search based upon the following conditions:\n\n 1. Subkeys may either appear behind a full stop (.) or at one lookup_dict level lower in the tree.\n 2. No wildcards exist within the provided ES search terms (these are treated as string literals)\n\n This is necessary to get around inconsistencies in ES data.\n\n For example:\n {'ad.account_name': 'bob'}\n Or:\n {'csp_report': {'blocked_uri': 'bob.com'}}\n And even:\n {'juniper_duo.geoip': {'country_name': 'Democratic People's Republic of Korea'}}\n\n We want a search term of form \"key.subkey.subsubkey\" to match in all cases.\n :returns: A tuple with the first element being the dict that contains the key and the second\n element which is the last subkey used to access the target specified by the term. None is\n returned for both if the key can not be found.\n \"\"\"\n if term in lookup_dict:\n return lookup_dict, term\n # If the term does not match immediately, perform iterative lookup:\n # 1. Split the search term into tokens\n # 2. Recurrently concatenate these together to traverse deeper into the dictionary,\n # clearing the subkey at every successful lookup.\n #\n # This greedy approach is correct because subkeys must always appear in order,\n # preferring full stops and traversal interchangeably.\n #\n # Subkeys will NEVER be duplicated between an alias and a traversal.\n #\n # For example:\n # {'foo.bar': {'bar': 'ray'}} to look up foo.bar will return {'bar': 'ray'}, not 'ray'\n dict_cursor = lookup_dict\n\n while term:\n split_results = re.split(r'\\[(\\d)\\]', term, maxsplit=1)\n if len(split_results) == 3:\n sub_term, index, term = split_results\n index = int(index)\n else:\n sub_term, index, term = split_results + [None, '']\n\n subkeys = sub_term.split('.')\n\n subkey = ''\n\n while len(subkeys) > 0:\n if not dict_cursor:\n return {}, None\n\n subkey += subkeys.pop(0)\n\n if subkey in dict_cursor:\n if len(subkeys) == 0:\n break\n dict_cursor = dict_cursor[subkey]\n subkey = ''\n elif len(subkeys) == 0:\n # If there are no keys left to match, return None values\n dict_cursor = None\n subkey = None\n else:\n subkey += '.'\n\n if index is not None and subkey:\n dict_cursor = dict_cursor[subkey]\n if type(dict_cursor) == list and len(dict_cursor) > index:\n subkey = index\n if term:\n dict_cursor = dict_cursor[subkey]\n else:\n return {}, None\n\n return dict_cursor, subkey\n\n\ndef set_es_key(lookup_dict, term, value):\n \"\"\" Looks up the location that the term maps to and sets it to the given value.\n :returns: True if the value was set successfully, False otherwise.\n \"\"\"\n value_dict, value_key = _find_es_dict_by_key(lookup_dict, term)\n\n if value_dict is not None:\n value_dict[value_key] = value\n return True\n\n return False\n\n\ndef lookup_es_key(lookup_dict, term):\n \"\"\" Performs iterative dictionary search for the given term.\n :returns: The value identified by term or None if it cannot be found.\n \"\"\"\n value_dict, value_key = _find_es_dict_by_key(lookup_dict, term)\n return None if value_key is None else value_dict[value_key]\n\n\ndef ts_to_dt(timestamp):\n if isinstance(timestamp, datetime.datetime):\n return timestamp\n dt = dateutil.parser.parse(timestamp)\n # Implicitly convert local timestamps to UTC\n if dt.tzinfo is None:\n dt = dt.replace(tzinfo=pytz.utc)\n return dt\n\n\ndef dt_to_ts(dt):\n if not isinstance(dt, datetime.datetime):\n logging.warning('Expected datetime, got %s' % (type(dt)))\n return dt\n ts = dt.isoformat()\n # Round microseconds to milliseconds\n if dt.tzinfo is None:\n # Implicitly convert local times to UTC\n return ts + 'Z'\n # isoformat() uses microsecond accuracy and timezone offsets\n # but we should try to use millisecond accuracy and Z to indicate UTC\n return ts.replace('000+00:00', 'Z').replace('+00:00', 'Z')\n\n\ndef ts_to_dt_with_format(timestamp, ts_format):\n if isinstance(timestamp, datetime.datetime):\n return timestamp\n dt = datetime.datetime.strptime(timestamp, ts_format)\n # Implicitly convert local timestamps to UTC\n if dt.tzinfo is None:\n dt = dt.replace(tzinfo=dateutil.tz.tzutc())\n return dt\n\n\ndef dt_to_ts_with_format(dt, ts_format):\n if not isinstance(dt, datetime.datetime):\n logging.warning('Expected datetime, got %s' % (type(dt)))\n return dt\n ts = dt.strftime(ts_format)\n return ts\n\n\ndef ts_now():\n return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())\n\n\ndef inc_ts(timestamp, milliseconds=1):\n \"\"\"Increment a timestamp by milliseconds.\"\"\"\n dt = ts_to_dt(timestamp)\n dt += datetime.timedelta(milliseconds=milliseconds)\n return dt_to_ts(dt)\n\n\ndef pretty_ts(timestamp, tz=True):\n \"\"\"Pretty-format the given timestamp (to be printed or logged hereafter).\n If tz, the timestamp will be converted to local time.\n Format: YYYY-MM-DD HH:MM TZ\"\"\"\n dt = timestamp\n if not isinstance(timestamp, datetime.datetime):\n dt = ts_to_dt(timestamp)\n if tz:\n dt = dt.astimezone(dateutil.tz.tzlocal())\n return dt.strftime('%Y-%m-%d %H:%M %Z')\n\n\ndef ts_add(ts, td):\n \"\"\" Allows a timedelta (td) add operation on a string timestamp (ts) \"\"\"\n return dt_to_ts(ts_to_dt(ts) + td)\n\n\ndef hashable(obj):\n \"\"\" Convert obj to a hashable obj.\n We use the value of some fields from Elasticsearch as keys for dictionaries. This means\n that whatever Elasticsearch returns must be hashable, and it sometimes returns a list or dict.\"\"\"\n if not obj.__hash__:\n return str(obj)\n return obj\n\n\ndef format_index(index, start, end, add_extra=False):\n \"\"\" Takes an index, specified using strftime format, start and end time timestamps,\n and outputs a wildcard based index string to match all possible timestamps. \"\"\"\n # Convert to UTC\n start -= start.utcoffset()\n end -= end.utcoffset()\n original_start = start\n indices = set()\n while start.date() <= end.date():\n indices.add(start.strftime(index))\n start += datetime.timedelta(days=1)\n num = len(indices)\n if add_extra:\n while len(indices) == num:\n original_start -= datetime.timedelta(days=1)\n new_index = original_start.strftime(index)\n assert new_index != index, \"You cannot use a static index with search_extra_index\"\n indices.add(new_index)\n\n return ','.join(indices)\n\n\nclass EAException(Exception):\n pass\n\n\ndef seconds(td):\n return td.seconds + td.days * 24 * 3600\n\n\ndef total_seconds(dt):\n # For python 2.6 compatability\n if dt is None:\n return 0\n elif hasattr(dt, 'total_seconds'):\n return dt.total_seconds()\n else:\n return (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 10**6) / 10**6\n\n\ndef dt_to_int(dt):\n dt = dt.replace(tzinfo=None)\n return int(total_seconds((dt - datetime.datetime.utcfromtimestamp(0))) * 1000)\n\n\ndef unixms_to_dt(ts):\n return unix_to_dt(float(ts) / 1000)\n\n\ndef unix_to_dt(ts):\n dt = datetime.datetime.utcfromtimestamp(float(ts))\n dt = dt.replace(tzinfo=dateutil.tz.tzutc())\n return dt\n\n\ndef dt_to_unix(dt):\n return int(total_seconds(dt - datetime.datetime(1970, 1, 1, tzinfo=dateutil.tz.tzutc())))\n\n\ndef dt_to_unixms(dt):\n return int(dt_to_unix(dt) * 1000)\n\n\ndef cronite_datetime_to_timestamp(self, d):\n \"\"\"\n Converts a `datetime` object `d` into a UNIX timestamp.\n \"\"\"\n if d.tzinfo is not None:\n d = d.replace(tzinfo=None) - d.utcoffset()\n\n return total_seconds((d - datetime.datetime(1970, 1, 1)))\n\n\ndef add_raw_postfix(field, is_five_or_above):\n if is_five_or_above:\n end = '.keyword'\n else:\n end = '.raw'\n if not field.endswith(end):\n field += end\n return field\n\n\ndef replace_dots_in_field_names(document):\n \"\"\" This method destructively modifies document by replacing any dots in\n field names with an underscore. \"\"\"\n for key, value in list(document.items()):\n if isinstance(value, dict):\n value = replace_dots_in_field_names(value)\n if isinstance(key, string_types) and key.find('.') != -1:\n del document[key]\n document[key.replace('.', '_')] = value\n return document\n\n\ndef elasticsearch_client(conf):\n \"\"\" returns an :class:`ElasticSearchClient` instance configured using an es_conn_config \"\"\"\n es_conn_conf = build_es_conn_config(conf)\n auth = Auth()\n es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'],\n username=es_conn_conf['es_username'],\n password=es_conn_conf['es_password'],\n aws_region=es_conn_conf['aws_region'],\n profile_name=es_conn_conf['profile'])\n\n return ElasticSearchClient(es_conn_conf)\n\n\ndef build_es_conn_config(conf):\n \"\"\" Given a conf dictionary w/ raw config properties 'use_ssl', 'es_host', 'es_port'\n 'es_username' and 'es_password', this will return a new dictionary\n with properly initialized values for 'es_host', 'es_port', 'use_ssl' and 'http_auth' which\n will be a basicauth username:password formatted string \"\"\"\n parsed_conf = {}\n parsed_conf['use_ssl'] = os.environ.get('ES_USE_SSL', False)\n parsed_conf['verify_certs'] = True\n parsed_conf['ca_certs'] = None\n parsed_conf['client_cert'] = None\n parsed_conf['client_key'] = None\n parsed_conf['http_auth'] = None\n parsed_conf['es_username'] = None\n parsed_conf['es_password'] = None\n parsed_conf['aws_region'] = None\n parsed_conf['profile'] = None\n parsed_conf['es_host'] = os.environ.get('ES_HOST', conf['es_host'])\n parsed_conf['es_port'] = int(os.environ.get('ES_PORT', conf['es_port']))\n parsed_conf['es_url_prefix'] = ''\n parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20)\n parsed_conf['send_get_body_as'] = conf.get('es_send_get_body_as', 'GET')\n\n if os.environ.get('ES_USERNAME'):\n parsed_conf['es_username'] = os.environ.get('ES_USERNAME')\n parsed_conf['es_password'] = os.environ.get('ES_PASSWORD')\n elif 'es_username' in conf:\n parsed_conf['es_username'] = conf['es_username']\n parsed_conf['es_password'] = conf['es_password']\n\n if 'aws_region' in conf:\n parsed_conf['aws_region'] = conf['aws_region']\n\n # Deprecated\n if 'boto_profile' in conf:\n logging.warning('Found deprecated \"boto_profile\", use \"profile\" instead!')\n parsed_conf['profile'] = conf['boto_profile']\n\n if 'profile' in conf:\n parsed_conf['profile'] = conf['profile']\n\n if 'use_ssl' in conf:\n parsed_conf['use_ssl'] = conf['use_ssl']\n\n if 'verify_certs' in conf:\n parsed_conf['verify_certs'] = conf['verify_certs']\n\n if 'ca_certs' in conf:\n parsed_conf['ca_certs'] = conf['ca_certs']\n\n if 'client_cert' in conf:\n parsed_conf['client_cert'] = conf['client_cert']\n\n if 'client_key' in conf:\n parsed_conf['client_key'] = conf['client_key']\n\n if 'es_url_prefix' in conf:\n parsed_conf['es_url_prefix'] = conf['es_url_prefix']\n\n return parsed_conf\n\n\ndef pytzfy(dt):\n # apscheduler requires pytz timezone objects\n # This function will replace a dateutil.tz one with a pytz one\n if dt.tzinfo is not None:\n new_tz = pytz.timezone(dt.tzinfo.tzname('Y is this even required??'))\n return dt.replace(tzinfo=new_tz)\n return dt\n\n\ndef parse_duration(value):\n \"\"\"Convert ``unit=num`` spec into a ``timedelta`` object.\"\"\"\n unit, num = value.split('=')\n return datetime.timedelta(**{unit: int(num)})\n\n\ndef parse_deadline(value):\n \"\"\"Convert ``unit=num`` spec into a ``datetime`` object.\"\"\"\n duration = parse_duration(value)\n return ts_now() + duration\n\n\ndef flatten_dict(dct, delim='.', prefix=''):\n ret = {}\n for key, val in list(dct.items()):\n if type(val) == dict:\n ret.update(flatten_dict(val, prefix=prefix + key + delim))\n else:\n ret[prefix + key] = val\n return ret\n\n\ndef resolve_string(string, match, missing_text=''):\n \"\"\"\n Given a python string that may contain references to fields on the match dictionary,\n the strings are replaced using the corresponding values.\n However, if the referenced field is not found on the dictionary,\n it is replaced by a default string.\n Strings can be formatted using the old-style format ('%(field)s') or\n the new-style format ('{match[field]}').\n\n :param string: A string that may contain references to values of the 'match' dictionary.\n :param match: A dictionary with the values to replace where referenced by keys in the string.\n :param missing_text: The default text to replace a formatter with if the field doesnt exist.\n \"\"\"\n flat_match = flatten_dict(match)\n flat_match.update(match)\n dd_match = collections.defaultdict(lambda: missing_text, flat_match)\n dd_match['_missing_value'] = missing_text\n while True:\n try:\n string = string % dd_match\n string = string.format(**dd_match)\n break\n except KeyError as e:\n if '{%s}' % str(e).strip(\"'\") not in string:\n break\n string = string.replace('{%s}' % str(e).strip(\"'\"), '{_missing_value}')\n\n return string\n\n\ndef should_scrolling_continue(rule_conf):\n \"\"\"\n Tells about a rule config if it can scroll still or should stop the scrolling.\n\n :param: rule_conf as dict\n :rtype: bool\n \"\"\"\n max_scrolling = rule_conf.get('max_scrolling_count')\n stop_the_scroll = 0 < max_scrolling <= rule_conf.get('scrolling_cycle')\n\n return not stop_the_scroll\n" }, { "path": "elastalert/zabbix.py", "content": "from alerts import Alerter # , BasicMatchString\nimport logging\nfrom pyzabbix.api import ZabbixAPI\nfrom pyzabbix import ZabbixSender, ZabbixMetric\nfrom datetime import datetime\n\n\nclass ZabbixClient(ZabbixAPI):\n\n def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', sender_host='localhost',\n sender_port=10051):\n self.url = url\n self.use_authenticate = use_authenticate\n self.sender_host = sender_host\n self.sender_port = sender_port\n self.metrics_chunk_size = 200\n self.aggregated_metrics = []\n self.logger = logging.getLogger(self.__class__.__name__)\n super(ZabbixClient, self).__init__(url=self.url, use_authenticate=self.use_authenticate, user=user, password=password)\n\n def send_metric(self, hostname, key, data):\n zm = ZabbixMetric(hostname, key, data)\n if self.send_aggregated_metrics:\n\n self.aggregated_metrics.append(zm)\n if len(self.aggregated_metrics) > self.metrics_chunk_size:\n self.logger.info(\"Sending: %s metrics\" % (len(self.aggregated_metrics)))\n try:\n ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(self.aggregated_metrics)\n self.aggregated_metrics = []\n except Exception as e:\n self.logger.exception(e)\n pass\n else:\n try:\n ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(zm)\n except Exception as e:\n self.logger.exception(e)\n pass\n\n\nclass ZabbixAlerter(Alerter):\n\n # By setting required_options to a set of strings\n # You can ensure that the rule config file specifies all\n # of the options. Otherwise, ElastAlert will throw an exception\n # when trying to load the rule.\n required_options = frozenset(['zbx_sender_host', 'zbx_sender_port', 'zbx_host', 'zbx_key'])\n\n def __init__(self, *args):\n super(ZabbixAlerter, self).__init__(*args)\n\n self.zbx_sender_host = self.rule.get('zbx_sender_host', 'localhost')\n self.zbx_sender_port = self.rule.get('zbx_sender_port', 10051)\n self.zbx_host = self.rule.get('zbx_host')\n self.zbx_key = self.rule.get('zbx_key')\n\n # Alert is called\n def alert(self, matches):\n\n # Matches is a list of match dictionaries.\n # It contains more than one match when the alert has\n # the aggregation option set\n zm = []\n for match in matches:\n ts_epoch = int(datetime.strptime(match['@timestamp'], \"%Y-%m-%dT%H:%M:%S.%fZ\").strftime('%s'))\n zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch))\n\n ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm)\n\n # get_info is called after an alert is sent to get data that is written back\n # to Elasticsearch in the field \"alert_info\"\n # It should return a dict of information relevant to what the alert does\n def get_info(self):\n return {'type': 'zabbix Alerter'}\n" }, { "path": "example_rules/example_cardinality.yaml", "content": "# Alert when the rate of events exceeds a threshold\n\n# (Optional)\n# Elasticsearch host\n# es_host: elasticsearch.example.com\n\n# (Optional)\n# Elasticsearch port\n# es_port: 14900\n\n# (Required)\n# Index to search, wildcard supported\nindex: logstash-*\n\n# (OptionaL) Connect with SSL to Elasticsearch\n#use_ssl: True\n\n# (Optional) basic-auth username and password for Elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# (Required)\n# Rule name, must be unique\nname: Example cardinality rule\n\n# (Required)\n# Type of alert.\n# the frequency rule type alerts when num_events events occur with timeframe time\ntype: cardinality\n\n# (Required, cardinality specific)\n# Count the number of unique values for this field\ncardinality_field: \"Hostname\"\n\n# (Required, frequency specific)\n# Alert when there less than 15 unique hostnames\nmin_cardinality: 15\n\n# (Required, frequency specific)\n# The cardinality is defined as the number of unique values for the most recent 4 hours\ntimeframe:\n hours: 4\n\n# (Required)\n# A list of Elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n- term:\n status: \"active\"\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"email\"\n\n# (required, email specific)\n# a list of email addresses to send alerts to\nemail:\n- \"elastalert@example.com\"\n" }, { "path": "example_rules/example_change.yaml", "content": "# Alert when some field changes between documents\n# This rule would alert on documents similar to the following:\n# {'username': 'bob', 'country_name': 'USA', '@timestamp': '2014-10-15T00:00:00'}\n# {'username': 'bob', 'country_name': 'Russia', '@timestamp': '2014-10-15T05:00:00'}\n# Because the user (query_key) bob logged in from different countries (compare_key) in the same day (timeframe)\n\n# (Optional)\n# Elasticsearch host\n# es_host: elasticsearch.example.com\n\n# (Optional)\n# Elasticsearch port\n# es_port: 14900\n\n# (Optional) Connect with SSL to Elasticsearch\n#use_ssl: True\n\n# (Optional) basic-auth username and password for elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# (Required)\n# Rule name, must be unique\nname: New country login\n\n# (Required)\n# Type of alert.\n# the change rule will alert when a certain field changes in two documents within a timeframe\ntype: change\n\n# (Required)\n# Index to search, wildcard supported\nindex: logstash-*\n\n# (Required, change specific)\n# The field to look for changes in\ncompare_key: country_name\n\n# (Required, change specific)\n# Ignore documents without the compare_key (country_name) field\nignore_null: true\n\n# (Required, change specific)\n# The change must occur in two documents with the same query_key\nquery_key: username\n\n# (Required, change specific)\n# The value of compare_key must change in two events that are less than timeframe apart to trigger an alert\ntimeframe:\n days: 1\n\n# (Required)\n# A list of Elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n- query:\n query_string:\n query: \"document_type: login\"\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"email\"\n\n# (required, email specific)\n# a list of email addresses to send alerts to\nemail:\n- \"elastalert@example.com\"\n" }, { "path": "example_rules/example_frequency.yaml", "content": "# Alert when the rate of events exceeds a threshold\n\n# (Optional)\n# Elasticsearch host\n# es_host: elasticsearch.example.com\n\n# (Optional)\n# Elasticsearch port\n# es_port: 14900\n\n# (OptionaL) Connect with SSL to Elasticsearch\n#use_ssl: True\n\n# (Optional) basic-auth username and password for Elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# (Required)\n# Rule name, must be unique\nname: Example frequency rule\n\n# (Required)\n# Type of alert.\n# the frequency rule type alerts when num_events events occur with timeframe time\ntype: frequency\n\n# (Required)\n# Index to search, wildcard supported\nindex: logstash-*\n\n# (Required, frequency specific)\n# Alert when this many documents matching the query occur within a timeframe\nnum_events: 50\n\n# (Required, frequency specific)\n# num_events must occur within this amount of time to trigger an alert\ntimeframe:\n hours: 4\n\n# (Required)\n# A list of Elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n- term:\n some_field: \"some_value\"\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"email\"\n\n# (required, email specific)\n# a list of email addresses to send alerts to\nemail:\n- \"elastalert@example.com\"\n" }, { "path": "example_rules/example_new_term.yaml", "content": "# Alert when a login event is detected for user \"admin\" never before seen IP\n# In this example, \"login\" logs contain which user has logged in from what IP\n\n# (Optional)\n# Elasticsearch host\n# es_host: elasticsearch.example.com\n\n# (Optional)\n# Elasticsearch port\n# es_port: 14900\n\n# (OptionaL) Connect with SSL to Elasticsearch\n#use_ssl: True\n\n# (Optional) basic-auth username and password for Elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# (Required)\n# Rule name, must be unique\nname: Example new term rule\n\n# (Required)\n# Type of alert.\n# the frequency rule type alerts when num_events events occur with timeframe time\ntype: new_term\n\n# (Required)\n# Index to search, wildcard supported\nindex: logstash-*\n\n# (Required, new_term specific)\n# Monitor the field ip_address\nfields:\n - \"ip_address\"\n\n# (Optional, new_term specific)\n# This means that we will query 90 days worth of data when ElastAlert starts to find which values of ip_address already exist\n# If they existed in the last 90 days, no alerts will be triggered for them when they appear\nterms_window_size:\n days: 90\n\n# (Required)\n# A list of Elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\n# We are filtering for only \"login_event\" type documents with username \"admin\"\nfilter:\n- term:\n _type: \"login_event\"\n- term:\n username: admin\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"email\"\n\n# (required, email specific)\n# a list of email addresses to send alerts to\nemail:\n- \"elastalert@example.com\"\n" }, { "path": "example_rules/example_opsgenie_frequency.yaml", "content": "# Alert when the rate of events exceeds a threshold\n\n# (Optional)\n# Elasticsearch host\n#es_host: localhost\n\n# (Optional)\n# Elasticsearch port\n#es_port: 9200\n\n# (Required)\n# OpsGenie credentials\nopsgenie_key: ogkey\n\n# (Optional)\n# OpsGenie user account that the alert will show as created by\n#opsgenie_account: neh\n\n# (Optional)\n# OpsGenie recipients of the alert\n#opsgenie_recipients:\n# - \"neh\"\n\n# (Optional)\n# OpsGenie recipients with args\n# opsgenie_recipients:\n# - {recipient} \n# opsgenie_recipients_args:\n# team_prefix:'user.email'\n\n# (Optional)\n# OpsGenie teams to notify\n#opsgenie_teams:\n# - \"Infrastructure\"\n\n# (Optional)\n# OpsGenie teams with args\n# opsgenie_teams:\n# - {team_prefix}-Team \n# opsgenie_teams_args:\n# team_prefix:'team'\n\n# (Optional)\n# OpsGenie alert tags\nopsgenie_tags:\n - \"Production\"\n\n# (OptionaL) Connect with SSL to Elasticsearch\n#use_ssl: True\n\n# (Optional) basic-auth username and password for Elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# (Required)\n# Rule name, must be unique\nname: opsgenie_rule\n\n# (Required)\n# Type of alert.\n# the frequency rule type alerts when num_events events occur with timeframe time\ntype: frequency\n\n# (Required)\n# Index to search, wildcard supported\nindex: logstash-*\n\n#doc_type: \"golog\"\n\n# (Required, frequency specific)\n# Alert when this many documents matching the query occur within a timeframe\nnum_events: 50\n\n# (Required, frequency specific)\n# num_events must occur within this amount of time to trigger an alert\ntimeframe:\n hours: 2\n\n# (Required)\n# A list of Elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n- query:\n query_string:\n query: \"@message: *hihi*\"\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"opsgenie\"\n" }, { "path": "example_rules/example_percentage_match.yaml", "content": "name: Example Percentage Match\ntype: percentage_match\n\n#es_host: localhost\n#es_port: 9200\n\nindex: logstash-http-request-*\ndescription: \"95% of all http requests should be successful\"\n\nfilter:\n- term:\n _type: http_request\n\nbuffer_time:\n minutes: 5\n\nquery_key: Hostname.keyword\ndoc_type: http_request\n\nmatch_bucket_filter:\n- terms:\n ResponseStatus: [200]\n\nmin_percentage: 95\n#max_percentage: 60\n \n#bucket_interval:\n# minutes: 1\n \n#sync_bucket_interval: true\n#allow_buffer_time_overlap: true\n#use_run_every_query_size: true\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"debug\"\n\n" }, { "path": "example_rules/example_single_metric_agg.yaml", "content": "name: Metricbeat CPU Spike Rule\ntype: metric_aggregation\n\n#es_host: localhost\n#es_port: 9200\n\nindex: metricbeat-*\n\nbuffer_time:\n hours: 1\n\nmetric_agg_key: system.cpu.user.pct\nmetric_agg_type: avg\nquery_key: beat.hostname\ndoc_type: metricsets\n \nbucket_interval:\n minutes: 5\n \nsync_bucket_interval: true\n#allow_buffer_time_overlap: true\n#use_run_every_query_size: true\n\nmin_threshold: 0.1\nmax_threshold: 0.8\n\nfilter:\n- term:\n metricset.name: cpu\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"debug\"\n\n" }, { "path": "example_rules/example_spike.yaml", "content": "# Alert when there is a sudden spike in the volume of events\n\n# (Optional)\n# Elasticsearch host\n# es_host: elasticsearch.example.com\n\n# (Optional)\n# Elasticsearch port\n# es_port: 14900\n\n# (Optional) Connect with SSL to Elasticsearch\n#use_ssl: True\n\n# (Optional) basic-auth username and password for Elasticsearch\n#es_username: someusername\n#es_password: somepassword\n\n# (Required)\n# Rule name, must be unique\nname: Event spike\n\n# (Required)\n# Type of alert.\n# the spike rule type compares the number of events within two sliding windows to each other\ntype: spike\n\n# (Required)\n# Index to search, wildcard supported\nindex: logstash-*\n\n# (Required one of _cur or _ref, spike specific)\n# The minimum number of events that will trigger an alert\n# For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00\n# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height)\nthreshold_cur: 5\n#threshold_ref: 5\n\n# (Required, spike specific)\n# The size of the window used to determine average event frequency\n# We use two sliding windows each of size timeframe\n# To measure the 'reference' rate and the current rate\ntimeframe:\n hours: 2\n\n# (Required, spike specific)\n# The spike rule matches when the current window contains spike_height times more\n# events than the reference window\nspike_height: 3\n\n# (Required, spike specific)\n# The direction of the spike\n# 'up' matches only spikes, 'down' matches only troughs\n# 'both' matches both spikes and troughs\nspike_type: \"up\"\n\n# (Required)\n# A list of Elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n- query:\n query_string:\n query: \"field: value\"\n- type:\n value: \"some_doc_type\"\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"email\"\n\n# (required, email specific)\n# a list of email addresses to send alerts to\nemail:\n- \"elastalert@example.com\"\n" }, { "path": "example_rules/example_spike_single_metric_agg.yaml", "content": "name: Metricbeat Average CPU Spike Rule\ntype: spike_aggregation\n\n#es_host: localhost\n#es_port: 9200\n\nindex: metricbeat-*\n\ntimeframe:\n hours: 4\n\nbuffer_time:\n hours: 1\n\nmetric_agg_key: system.cpu.user.pct\nmetric_agg_type: avg\nquery_key: beat.hostname\ndoc_type: metricsets\n\n#allow_buffer_time_overlap: true\n#use_run_every_query_size: true\n\n# (Required one of _cur or _ref, spike specific)\n# The minimum value of the aggregation that will trigger the alert\n# For example, if we're tracking the average for a metric whose average is 0.4 between 12:00 and 2:00\n# and 0.95 between 2:00 and 4:00 with spike_height set to 2 and threshhold_cur set to 0.9:\n# _ref is 0.4 and _cur is 0.95, and the alert WILL fire\n# because 0.95 is greater than threshold_cur (0.9) and (_ref * spike_height (.4 * 2))\nthreshold_cur: 0.9\n\n# (Optional, min_doc_count)\n# for rules using a per-term aggregation via query_key, the minimum number of events\n# over the past buffer_time needed to update the spike tracker\nmin_doc_count: 5\n\n# (Required, spike specific)\n# The spike aggregation rule matches when the current window contains spike_height times higher aggregated value\n# than the reference window\nspike_height: 2\n\n# (Required, spike specific)\n# The direction of the spike\n# 'up' matches only spikes, 'down' matches only troughs\n# 'both' matches both spikes and troughs\nspike_type: \"up\"\n\nfilter:\n- term:\n metricset.name: cpu\n\n# (Required)\n# The alert is use when a match is found\nalert:\n- \"debug\"\n\n" }, { "path": "example_rules/jira_acct.txt", "content": "# Example jira_account information file\n# You should make sure that this file is not globally readable or version controlled! (Except for this example)\n\n# Jira username\nuser: elastalert-jira\n# Jira password\npassword: p455w0rd\n" }, { "path": "example_rules/ssh-repeat-offender.yaml", "content": "# Rule name, must be unique\nname: SSH abuse - reapeat offender\n\n# Alert on x events in y seconds\ntype: frequency\n\n# Alert when this many documents matching the query occur within a timeframe\nnum_events: 2\n\n# num_events must occur within this amount of time to trigger an alert\ntimeframe:\n weeks: 1\n\n# A list of elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n - term:\n rule_name: \"SSH abuse\"\n\nindex: elastalert\n\n# When the attacker continues, send a new alert after x minutes\nrealert:\n weeks: 4\n\nquery_key:\n - match_body.source.ip\n\ninclude:\n - match_body.host.hostname\n - match_body.user.name\n - match_body.source.ip\n\nalert_subject: \"SSH abuse (repeat offender) on <{}> | <{}|Show Dashboard>\"\nalert_subject_args:\n - match_body.host.hostname\n - kibana_link\n\nalert_text: |-\n An reapeat offender has been active on {}.\n\n IP: {}\n User: {}\nalert_text_args:\n - match_body.host.hostname\n - match_body.user.name\n - match_body.source.ip\n\n# The alert is use when a match is found\nalert:\n - slack\n\nslack_webhook_url: \"https://hooks.slack.com/services/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\"\nslack_username_override: \"ElastAlert\"\n\n# Alert body only cointains a title and text\nalert_text_type: alert_text_only\n\n# Link to BitSensor Kibana Dashboard\nuse_kibana4_dashboard: \"https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb\"\n" }, { "path": "example_rules/ssh.yaml", "content": "# Rule name, must be unique\n name: SSH abuse (ElastAlert 3.0.1) - 2\n\n# Alert on x events in y seconds\ntype: frequency\n\n# Alert when this many documents matching the query occur within a timeframe\nnum_events: 20\n\n# num_events must occur within this amount of time to trigger an alert\ntimeframe:\n minutes: 60\n\n# A list of elasticsearch filters used for find events\n# These filters are joined with AND and nested in a filtered query\n# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html\nfilter:\n- query:\n query_string:\n query: \"event.type:authentication_failure\"\n\nindex: auditbeat-*\n\n# When the attacker continues, send a new alert after x minutes\nrealert:\n minutes: 1\n\nquery_key:\n - source.ip\n\ninclude:\n - host.hostname\n - user.name\n - source.ip\n\ninclude_match_in_root: true\n\nalert_subject: \"SSH abuse on <{}> | <{}|Show Dashboard>\"\nalert_subject_args:\n - host.hostname\n - kibana_link\n\nalert_text: |-\n An attack on {} is detected.\n The attacker looks like:\n User: {}\n IP: {}\nalert_text_args:\n - host.hostname\n - user.name\n - source.ip\n\n# The alert is use when a match is found\nalert:\n - debug\n\nslack_webhook_url: \"https://hooks.slack.com/services/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\"\nslack_username_override: \"ElastAlert\"\n\n# Alert body only cointains a title and text\nalert_text_type: alert_text_only\n\n# Link to BitSensor Kibana Dashboard\nuse_kibana4_dashboard: \"https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb\"\n" }, { "path": "pytest.ini", "content": "[pytest]\nmarkers =\n elasticsearch: mark a test as using elasticsearch.\n" }, { "path": "requirements-dev.txt", "content": "-r requirements.txt\ncoverage==4.5.4\nflake8\npre-commit\npylint<1.4\npytest<3.3.0\nsetuptools\nsphinx_rtd_theme\ntox<2.0\n" }, { "path": "requirements.txt", "content": "apscheduler>=3.3.0\naws-requests-auth>=0.3.0\nblist>=1.3.6\nboto3>=1.4.4\ncffi>=1.11.5\nconfigparser>=3.5.0\ncroniter>=0.3.16\nelasticsearch>=7.0.0\nenvparse>=0.2.0\nexotel>=0.1.3\njira>=1.0.10,<1.0.15\njsonschema>=3.0.2\nmock>=2.0.0\nprison>=0.1.2\npy-zabbix==1.1.3\nPyStaticConfiguration>=0.10.3\npython-dateutil>=2.6.0,<2.7.0\nPyYAML>=5.1\nrequests>=2.0.0\nstomp.py>=4.1.17\ntexttable>=0.8.8\ntwilio==6.0.0\n" }, { "path": "setup.cfg", "content": "[flake8]\nexclude = .git,__pycache__,.tox,docs,virtualenv_run,modules,venv,env\nmax-line-length = 140\n" }, { "path": "setup.py", "content": "# -*- coding: utf-8 -*-\nimport os\n\nfrom setuptools import find_packages\nfrom setuptools import setup\n\n\nbase_dir = os.path.dirname(__file__)\nsetup(\n name='elastalert',\n version='0.2.4',\n description='Runs custom filters on Elasticsearch and alerts on matches',\n author='Quentin Long',\n author_email='qlo@yelp.com',\n setup_requires='setuptools',\n license='Copyright 2014 Yelp',\n classifiers=[\n 'Programming Language :: Python :: 3.6',\n 'License :: OSI Approved :: Apache Software License',\n 'Operating System :: OS Independent',\n ],\n entry_points={\n 'console_scripts': ['elastalert-create-index=elastalert.create_index:main',\n 'elastalert-test-rule=elastalert.test_rule:main',\n 'elastalert-rule-from-kibana=elastalert.rule_from_kibana:main',\n 'elastalert=elastalert.elastalert:main']},\n packages=find_packages(),\n package_data={'elastalert': ['schema.yaml', 'es_mappings/**/*.json']},\n install_requires=[\n 'apscheduler>=3.3.0',\n 'aws-requests-auth>=0.3.0',\n 'blist>=1.3.6',\n 'boto3>=1.4.4',\n 'configparser>=3.5.0',\n 'croniter>=0.3.16',\n 'elasticsearch==7.0.0',\n 'envparse>=0.2.0',\n 'exotel>=0.1.3',\n 'jira>=2.0.0',\n 'jsonschema>=3.0.2',\n 'mock>=2.0.0',\n 'prison>=0.1.2',\n 'PyStaticConfiguration>=0.10.3',\n 'python-dateutil>=2.6.0,<2.7.0',\n 'PyYAML>=3.12',\n 'requests>=2.10.0',\n 'stomp.py>=4.1.17',\n 'texttable>=0.8.8',\n 'twilio>=6.0.0,<6.1',\n 'cffi>=1.11.5'\n ]\n)\n" }, { "path": "supervisord.conf.example", "content": "[unix_http_server]\nfile=/var/run/elastalert_supervisor.sock\n\n[supervisord]\nlogfile=/var/log/elastalert_supervisord.log\nlogfile_maxbytes=1MB\nlogfile_backups=2\nloglevel=debug\nnodaemon=false\ndirectory=%(here)s\n\n[rpcinterface:supervisor]\nsupervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface\n\n[supervisorctl]\nserverurl=unix:///var/run/elastalert_supervisor.sock\n\n[program:elastalert]\n# running globally\ncommand =\n python elastalert.py\n --verbose\n# (alternative) using virtualenv\n# command=/path/to/venv/bin/elastalert --config /path/to/config.yaml --verbose \nprocess_name=elastalert\nautorestart=true\nstartsecs=15\nstopsignal=INT\nstopasgroup=true\nkillasgroup=true\nstderr_logfile=/var/log/elastalert_stderr.log\nstderr_logfile_maxbytes=5MB\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/alerts_test.py", "content": "# -*- coding: utf-8 -*-\nimport base64\nimport datetime\nimport json\nimport subprocess\n\nimport mock\nimport pytest\nfrom jira.exceptions import JIRAError\n\nfrom elastalert.alerts import AlertaAlerter\nfrom elastalert.alerts import Alerter\nfrom elastalert.alerts import BasicMatchString\nfrom elastalert.alerts import CommandAlerter\nfrom elastalert.alerts import EmailAlerter\nfrom elastalert.alerts import HipChatAlerter\nfrom elastalert.alerts import HTTPPostAlerter\nfrom elastalert.alerts import JiraAlerter\nfrom elastalert.alerts import JiraFormattedMatchString\nfrom elastalert.alerts import MsTeamsAlerter\nfrom elastalert.alerts import PagerDutyAlerter\nfrom elastalert.alerts import SlackAlerter\nfrom elastalert.alerts import StrideAlerter\nfrom elastalert.loaders import FileRulesLoader\nfrom elastalert.opsgenie import OpsGenieAlerter\nfrom elastalert.util import ts_add\nfrom elastalert.util import ts_now\n\n\nclass mock_rule:\n def get_match_str(self, event):\n return str(event)\n\n\ndef test_basic_match_string(ea):\n ea.rules[0]['top_count_keys'] = ['username']\n match = {'@timestamp': '1918-01-17', 'field': 'value', 'top_events_username': {'bob': 10, 'mallory': 5}}\n alert_text = str(BasicMatchString(ea.rules[0], match))\n assert 'anytest' in alert_text\n assert 'some stuff happened' in alert_text\n assert 'username' in alert_text\n assert 'bob: 10' in alert_text\n assert 'field: value' in alert_text\n\n # Non serializable objects don't cause errors\n match['non-serializable'] = {open: 10}\n alert_text = str(BasicMatchString(ea.rules[0], match))\n\n # unicode objects dont cause errors\n match['snowman'] = 'ā˜ƒ'\n alert_text = str(BasicMatchString(ea.rules[0], match))\n\n # Pretty printed objects\n match.pop('non-serializable')\n match['object'] = {'this': {'that': [1, 2, \"3\"]}}\n alert_text = str(BasicMatchString(ea.rules[0], match))\n assert '\"this\": {\\n \"that\": [\\n 1,\\n 2,\\n \"3\"\\n ]\\n }' in alert_text\n\n ea.rules[0]['alert_text'] = 'custom text'\n alert_text = str(BasicMatchString(ea.rules[0], match))\n assert 'custom text' in alert_text\n assert 'anytest' not in alert_text\n\n ea.rules[0]['alert_text_type'] = 'alert_text_only'\n alert_text = str(BasicMatchString(ea.rules[0], match))\n assert 'custom text' in alert_text\n assert 'some stuff happened' not in alert_text\n assert 'username' not in alert_text\n assert 'field: value' not in alert_text\n\n ea.rules[0]['alert_text_type'] = 'exclude_fields'\n alert_text = str(BasicMatchString(ea.rules[0], match))\n assert 'custom text' in alert_text\n assert 'some stuff happened' in alert_text\n assert 'username' in alert_text\n assert 'field: value' not in alert_text\n\n\ndef test_jira_formatted_match_string(ea):\n match = {'foo': {'bar': ['one', 2, 'three']}, 'top_events_poof': 'phew'}\n alert_text = str(JiraFormattedMatchString(ea.rules[0], match))\n tab = 4 * ' '\n expected_alert_text_snippet = '{code}{\\n' \\\n + tab + '\"foo\": {\\n' \\\n + 2 * tab + '\"bar\": [\\n' \\\n + 3 * tab + '\"one\",\\n' \\\n + 3 * tab + '2,\\n' \\\n + 3 * tab + '\"three\"\\n' \\\n + 2 * tab + ']\\n' \\\n + tab + '}\\n' \\\n + '}{code}'\n assert expected_alert_text_snippet in alert_text\n\n\ndef test_email():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value',\n 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': 'ā˜ƒ'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n body = mock_smtp.mock_calls[4][1][2]\n\n assert 'Reply-To: test@example.com' in body\n assert 'To: testing@test.test' in body\n assert 'From: testfrom@test.test' in body\n assert 'Subject: Test alert for test_value, owned by owner_value' in body\n\n\ndef test_email_from_field():\n rule = {'name': 'test alert', 'email': ['testing@test.test'], 'email_add_domain': 'example.com',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_from_field': 'data.user', 'owner': 'owner_value'}\n # Found, without @\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n alert.alert([{'data': {'user': 'qlo'}}])\n assert mock_smtp.mock_calls[4][1][1] == ['qlo@example.com']\n\n # Found, with @\n rule['email_add_domain'] = '@example.com'\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n alert.alert([{'data': {'user': 'qlo'}}])\n assert mock_smtp.mock_calls[4][1][1] == ['qlo@example.com']\n\n # Found, list\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n alert.alert([{'data': {'user': ['qlo', 'foo']}}])\n assert mock_smtp.mock_calls[4][1][1] == ['qlo@example.com', 'foo@example.com']\n\n # Not found\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n alert.alert([{'data': {'foo': 'qlo'}}])\n assert mock_smtp.mock_calls[4][1][1] == ['testing@test.test']\n\n # Found, wrong type\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n alert.alert([{'data': {'user': 17}}])\n assert mock_smtp.mock_calls[4][1][1] == ['testing@test.test']\n\n\ndef test_email_with_unicode_strings():\n rule = {'name': 'test alert', 'email': 'testing@test.test', 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value',\n 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': 'ā˜ƒ'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().sendmail(mock.ANY, ['testing@test.test'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n body = mock_smtp.mock_calls[4][1][2]\n\n assert 'Reply-To: test@example.com' in body\n assert 'To: testing@test.test' in body\n assert 'From: testfrom@test.test' in body\n assert 'Subject: Test alert for test_value, owned by owner_value' in body\n\n\ndef test_email_with_auth():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com',\n 'alert_subject': 'Test alert for {0}', 'alert_subject_args': ['test_term'], 'smtp_auth_file': 'file.txt',\n 'rule_file': '/tmp/foo.yaml'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n with mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'someone', 'password': 'hunter2'}\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().login('someone', 'hunter2'),\n mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n\ndef test_email_with_cert_key():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com',\n 'alert_subject': 'Test alert for {0}', 'alert_subject_args': ['test_term'], 'smtp_auth_file': 'file.txt',\n 'smtp_cert_file': 'dummy/cert.crt', 'smtp_key_file': 'dummy/client.key', 'rule_file': '/tmp/foo.yaml'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n with mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'someone', 'password': 'hunter2'}\n mock_smtp.return_value = mock.Mock()\n alert = EmailAlerter(rule)\n\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile='dummy/cert.crt', keyfile='dummy/client.key'),\n mock.call().login('someone', 'hunter2'),\n mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n\ndef test_email_with_cc():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com',\n 'cc': 'tester@testing.testing'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test', 'tester@testing.testing'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n body = mock_smtp.mock_calls[4][1][2]\n\n assert 'Reply-To: test@example.com' in body\n assert 'To: testing@test.test' in body\n assert 'CC: tester@testing.testing' in body\n assert 'From: testfrom@test.test' in body\n\n\ndef test_email_with_bcc():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com',\n 'bcc': 'tester@testing.testing'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test', 'tester@testing.testing'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n body = mock_smtp.mock_calls[4][1][2]\n\n assert 'Reply-To: test@example.com' in body\n assert 'To: testing@test.test' in body\n assert 'CC: tester@testing.testing' not in body\n assert 'From: testfrom@test.test' in body\n\n\ndef test_email_with_cc_and_bcc():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com',\n 'cc': ['test1@test.com', 'test2@test.com'], 'bcc': 'tester@testing.testing'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value'}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().sendmail(\n mock.ANY,\n [\n 'testing@test.test',\n 'test@test.test',\n 'test1@test.com',\n 'test2@test.com',\n 'tester@testing.testing'\n ],\n mock.ANY\n ),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n body = mock_smtp.mock_calls[4][1][2]\n\n assert 'Reply-To: test@example.com' in body\n assert 'To: testing@test.test' in body\n assert 'CC: test1@test.com,test2@test.com' in body\n assert 'From: testfrom@test.test' in body\n\n\ndef test_email_with_args():\n rule = {\n 'name': 'test alert',\n 'email': ['testing@test.test', 'test@test.test'],\n 'from_addr': 'testfrom@test.test',\n 'type': mock_rule(),\n 'timestamp_field': '@timestamp',\n 'email_reply_to': 'test@example.com',\n 'alert_subject': 'Test alert for {0} {1}',\n 'alert_subject_args': ['test_term', 'test.term'],\n 'alert_text': 'Test alert for {0} and {1} {2}',\n 'alert_text_args': ['test_arg1', 'test_arg2', 'test.arg3'],\n 'alert_missing_value': ''\n }\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': 'ā˜ƒ'}}])\n expected = [mock.call('localhost'),\n mock.call().ehlo(),\n mock.call().has_extn('STARTTLS'),\n mock.call().starttls(certfile=None, keyfile=None),\n mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY),\n mock.call().quit()]\n assert mock_smtp.mock_calls == expected\n\n body = mock_smtp.mock_calls[4][1][2]\n # Extract the MIME encoded message body\n body_text = base64.b64decode(body.split('\\n\\n')[-1][:-1]).decode('utf-8')\n\n assert 'testing' in body_text\n assert '' in body_text\n assert 'ā˜ƒ' in body_text\n\n assert 'Reply-To: test@example.com' in body\n assert 'To: testing@test.test' in body\n assert 'From: testfrom@test.test' in body\n assert 'Subject: Test alert for test_value :)' in body\n\n\ndef test_email_query_key_in_subject():\n rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'],\n 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com',\n 'query_key': 'username'}\n with mock.patch('elastalert.alerts.SMTP') as mock_smtp:\n mock_smtp.return_value = mock.Mock()\n\n alert = EmailAlerter(rule)\n alert.alert([{'test_term': 'test_value', 'username': 'werbenjagermanjensen'}])\n\n body = mock_smtp.mock_calls[4][1][2]\n lines = body.split('\\n')\n found_subject = False\n for line in lines:\n if line.startswith('Subject'):\n assert 'werbenjagermanjensen' in line\n found_subject = True\n assert found_subject\n\n\ndef test_opsgenie_basic():\n rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey',\n 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts',\n 'opsgenie_recipients': ['lytics'], 'type': mock_rule()}\n with mock.patch('requests.post') as mock_post:\n\n alert = OpsGenieAlerter(rule)\n alert.alert([{'@timestamp': '2014-10-31T00:00:00'}])\n print((\"mock_post: {0}\".format(mock_post._mock_call_args_list)))\n mcal = mock_post._mock_call_args_list\n print(('mcal: {0}'.format(mcal[0])))\n assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts')\n\n assert mock_post.called\n\n assert mcal[0][1]['headers']['Authorization'] == 'GenieKey ogkey'\n assert mcal[0][1]['json']['source'] == 'ElastAlert'\n assert mcal[0][1]['json']['responders'] == [{'username': 'lytics', 'type': 'user'}]\n assert mcal[0][1]['json']['source'] == 'ElastAlert'\n\n\ndef test_opsgenie_frequency():\n rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey',\n 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts',\n 'opsgenie_recipients': ['lytics'], 'type': mock_rule(),\n 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}],\n 'alert': 'opsgenie'}\n with mock.patch('requests.post') as mock_post:\n\n alert = OpsGenieAlerter(rule)\n alert.alert([{'@timestamp': '2014-10-31T00:00:00'}])\n\n assert alert.get_info()['recipients'] == rule['opsgenie_recipients']\n\n print((\"mock_post: {0}\".format(mock_post._mock_call_args_list)))\n mcal = mock_post._mock_call_args_list\n print(('mcal: {0}'.format(mcal[0])))\n assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts')\n\n assert mock_post.called\n\n assert mcal[0][1]['headers']['Authorization'] == 'GenieKey ogkey'\n assert mcal[0][1]['json']['source'] == 'ElastAlert'\n assert mcal[0][1]['json']['responders'] == [{'username': 'lytics', 'type': 'user'}]\n assert mcal[0][1]['json']['source'] == 'ElastAlert'\n assert mcal[0][1]['json']['source'] == 'ElastAlert'\n\n\ndef test_opsgenie_alert_routing():\n rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey',\n 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts',\n 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX': 'recipient'},\n 'type': mock_rule(),\n 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}],\n 'alert': 'opsgenie',\n 'opsgenie_teams': ['{TEAM_PREFIX}-Team'], 'opsgenie_teams_args': {'TEAM_PREFIX': 'team'}}\n with mock.patch('requests.post'):\n\n alert = OpsGenieAlerter(rule)\n alert.alert([{'@timestamp': '2014-10-31T00:00:00', 'team': \"Test\", 'recipient': \"lytics\"}])\n\n assert alert.get_info()['teams'] == ['Test-Team']\n assert alert.get_info()['recipients'] == ['lytics']\n\n\ndef test_opsgenie_default_alert_routing():\n rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey',\n 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts',\n 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX': 'recipient'},\n 'type': mock_rule(),\n 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}],\n 'alert': 'opsgenie',\n 'opsgenie_teams': ['{TEAM_PREFIX}-Team'],\n 'opsgenie_default_receipients': [\"devops@test.com\"], 'opsgenie_default_teams': [\"Test\"]\n }\n with mock.patch('requests.post'):\n\n alert = OpsGenieAlerter(rule)\n alert.alert([{'@timestamp': '2014-10-31T00:00:00', 'team': \"Test\"}])\n\n assert alert.get_info()['teams'] == ['{TEAM_PREFIX}-Team']\n assert alert.get_info()['recipients'] == ['devops@test.com']\n\n\ndef test_opsgenie_details_with_constant_value():\n rule = {\n 'name': 'Opsgenie Details',\n 'type': mock_rule(),\n 'opsgenie_account': 'genies',\n 'opsgenie_key': 'ogkey',\n 'opsgenie_details': {'Foo': 'Bar'}\n }\n match = {\n '@timestamp': '2014-10-31T00:00:00'\n }\n alert = OpsGenieAlerter(rule)\n\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n 'https://api.opsgenie.com/v2/alerts',\n headers={\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey ogkey'\n },\n json=mock.ANY,\n proxies=None\n )\n\n expected_json = {\n 'description': BasicMatchString(rule, match).__str__(),\n 'details': {'Foo': 'Bar'},\n 'message': 'ElastAlert: Opsgenie Details',\n 'priority': None,\n 'source': 'ElastAlert',\n 'tags': ['ElastAlert', 'Opsgenie Details'],\n 'user': 'genies'\n }\n actual_json = mock_post_request.call_args_list[0][1]['json']\n assert expected_json == actual_json\n\n\ndef test_opsgenie_details_with_field():\n rule = {\n 'name': 'Opsgenie Details',\n 'type': mock_rule(),\n 'opsgenie_account': 'genies',\n 'opsgenie_key': 'ogkey',\n 'opsgenie_details': {'Foo': {'field': 'message'}}\n }\n match = {\n 'message': 'Bar',\n '@timestamp': '2014-10-31T00:00:00'\n }\n alert = OpsGenieAlerter(rule)\n\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n 'https://api.opsgenie.com/v2/alerts',\n headers={\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey ogkey'\n },\n json=mock.ANY,\n proxies=None\n )\n\n expected_json = {\n 'description': BasicMatchString(rule, match).__str__(),\n 'details': {'Foo': 'Bar'},\n 'message': 'ElastAlert: Opsgenie Details',\n 'priority': None,\n 'source': 'ElastAlert',\n 'tags': ['ElastAlert', 'Opsgenie Details'],\n 'user': 'genies'\n }\n actual_json = mock_post_request.call_args_list[0][1]['json']\n assert expected_json == actual_json\n\n\ndef test_opsgenie_details_with_nested_field():\n rule = {\n 'name': 'Opsgenie Details',\n 'type': mock_rule(),\n 'opsgenie_account': 'genies',\n 'opsgenie_key': 'ogkey',\n 'opsgenie_details': {'Foo': {'field': 'nested.field'}}\n }\n match = {\n 'nested': {\n 'field': 'Bar'\n },\n '@timestamp': '2014-10-31T00:00:00'\n }\n alert = OpsGenieAlerter(rule)\n\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n 'https://api.opsgenie.com/v2/alerts',\n headers={\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey ogkey'\n },\n json=mock.ANY,\n proxies=None\n )\n\n expected_json = {\n 'description': BasicMatchString(rule, match).__str__(),\n 'details': {'Foo': 'Bar'},\n 'message': 'ElastAlert: Opsgenie Details',\n 'priority': None,\n 'source': 'ElastAlert',\n 'tags': ['ElastAlert', 'Opsgenie Details'],\n 'user': 'genies'\n }\n actual_json = mock_post_request.call_args_list[0][1]['json']\n assert expected_json == actual_json\n\n\ndef test_opsgenie_details_with_non_string_field():\n rule = {\n 'name': 'Opsgenie Details',\n 'type': mock_rule(),\n 'opsgenie_account': 'genies',\n 'opsgenie_key': 'ogkey',\n 'opsgenie_details': {\n 'Age': {'field': 'age'},\n 'Message': {'field': 'message'}\n }\n }\n match = {\n 'age': 10,\n 'message': {\n 'format': 'The cow goes %s!',\n 'arg0': 'moo'\n }\n }\n alert = OpsGenieAlerter(rule)\n\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n 'https://api.opsgenie.com/v2/alerts',\n headers={\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey ogkey'\n },\n json=mock.ANY,\n proxies=None\n )\n\n expected_json = {\n 'description': BasicMatchString(rule, match).__str__(),\n 'details': {\n 'Age': '10',\n 'Message': \"{'format': 'The cow goes %s!', 'arg0': 'moo'}\"\n },\n 'message': 'ElastAlert: Opsgenie Details',\n 'priority': None,\n 'source': 'ElastAlert',\n 'tags': ['ElastAlert', 'Opsgenie Details'],\n 'user': 'genies'\n }\n actual_json = mock_post_request.call_args_list[0][1]['json']\n assert expected_json == actual_json\n\n\ndef test_opsgenie_details_with_missing_field():\n rule = {\n 'name': 'Opsgenie Details',\n 'type': mock_rule(),\n 'opsgenie_account': 'genies',\n 'opsgenie_key': 'ogkey',\n 'opsgenie_details': {\n 'Message': {'field': 'message'},\n 'Missing': {'field': 'missing'}\n }\n }\n match = {\n 'message': 'Testing',\n '@timestamp': '2014-10-31T00:00:00'\n }\n alert = OpsGenieAlerter(rule)\n\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n 'https://api.opsgenie.com/v2/alerts',\n headers={\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey ogkey'\n },\n json=mock.ANY,\n proxies=None\n )\n\n expected_json = {\n 'description': BasicMatchString(rule, match).__str__(),\n 'details': {'Message': 'Testing'},\n 'message': 'ElastAlert: Opsgenie Details',\n 'priority': None,\n 'source': 'ElastAlert',\n 'tags': ['ElastAlert', 'Opsgenie Details'],\n 'user': 'genies'\n }\n actual_json = mock_post_request.call_args_list[0][1]['json']\n assert expected_json == actual_json\n\n\ndef test_opsgenie_details_with_environment_variable_replacement(environ):\n environ.update({\n 'TEST_VAR': 'Bar'\n })\n rule = {\n 'name': 'Opsgenie Details',\n 'type': mock_rule(),\n 'opsgenie_account': 'genies',\n 'opsgenie_key': 'ogkey',\n 'opsgenie_details': {'Foo': '$TEST_VAR'}\n }\n match = {\n '@timestamp': '2014-10-31T00:00:00'\n }\n alert = OpsGenieAlerter(rule)\n\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n 'https://api.opsgenie.com/v2/alerts',\n headers={\n 'Content-Type': 'application/json',\n 'Authorization': 'GenieKey ogkey'\n },\n json=mock.ANY,\n proxies=None\n )\n\n expected_json = {\n 'description': BasicMatchString(rule, match).__str__(),\n 'details': {'Foo': 'Bar'},\n 'message': 'ElastAlert: Opsgenie Details',\n 'priority': None,\n 'source': 'ElastAlert',\n 'tags': ['ElastAlert', 'Opsgenie Details'],\n 'user': 'genies'\n }\n actual_json = mock_post_request.call_args_list[0][1]['json']\n assert expected_json == actual_json\n\n\ndef test_jira():\n description_txt = \"Description stuff goes here like a runbook link.\"\n rule = {\n 'name': 'test alert',\n 'jira_account_file': 'jirafile',\n 'type': mock_rule(),\n 'jira_project': 'testproject',\n 'jira_priority': 0,\n 'jira_issuetype': 'testtype',\n 'jira_server': 'jiraserver',\n 'jira_label': 'testlabel',\n 'jira_component': 'testcomponent',\n 'jira_description': description_txt,\n 'jira_watchers': ['testwatcher1', 'testwatcher2'],\n 'timestamp_field': '@timestamp',\n 'alert_subject': 'Issue {0} occurred at {1}',\n 'alert_subject_args': ['test_term', '@timestamp'],\n 'rule_file': '/tmp/foo.yaml'\n }\n\n mock_priority = mock.Mock(id='5')\n\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = []\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n\n expected = [\n mock.call('jiraserver', basic_auth=('jirauser', 'jirapassword')),\n mock.call().priorities(),\n mock.call().fields(),\n mock.call().create_issue(\n issuetype={'name': 'testtype'},\n priority={'id': '5'},\n project={'key': 'testproject'},\n labels=['testlabel'],\n components=[{'name': 'testcomponent'}],\n description=mock.ANY,\n summary='Issue test_value occurred at 2014-10-31T00:00:00',\n ),\n mock.call().add_watcher(mock.ANY, 'testwatcher1'),\n mock.call().add_watcher(mock.ANY, 'testwatcher2'),\n ]\n\n # We don't care about additional calls to mock_jira, such as __str__\n assert mock_jira.mock_calls[:6] == expected\n assert mock_jira.mock_calls[3][2]['description'].startswith(description_txt)\n\n # Search called if jira_bump_tickets\n rule['jira_bump_tickets'] = True\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value = mock.Mock()\n mock_jira.return_value.search_issues.return_value = []\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = []\n\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n\n expected.insert(3, mock.call().search_issues(mock.ANY))\n assert mock_jira.mock_calls == expected\n\n # Remove a field if jira_ignore_in_title set\n rule['jira_ignore_in_title'] = 'test_term'\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value = mock.Mock()\n mock_jira.return_value.search_issues.return_value = []\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = []\n\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n\n assert 'test_value' not in mock_jira.mock_calls[3][1][0]\n\n # Issue is still created if search_issues throws an exception\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value = mock.Mock()\n mock_jira.return_value.search_issues.side_effect = JIRAError\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = []\n\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n\n assert mock_jira.mock_calls == expected\n\n # Only bump after 3d of inactivity\n rule['jira_bump_after_inactivity'] = 3\n mock_issue = mock.Mock()\n\n # Check ticket is bumped if it is updated 4 days ago\n mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4))\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value = mock.Mock()\n mock_jira.return_value.search_issues.return_value = [mock_issue]\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = []\n\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n # Check add_comment is called\n assert len(mock_jira.mock_calls) == 5\n assert '().add_comment' == mock_jira.mock_calls[4][0]\n\n # Check ticket is bumped is not bumped if ticket is updated right now\n mock_issue.fields.updated = str(ts_now())\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value = mock.Mock()\n mock_jira.return_value.search_issues.return_value = [mock_issue]\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = []\n\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n # Only 4 calls for mock_jira since add_comment is not called\n assert len(mock_jira.mock_calls) == 4\n\n # Test match resolved values\n rule = {\n 'name': 'test alert',\n 'jira_account_file': 'jirafile',\n 'type': mock_rule(),\n 'owner': 'the_owner',\n 'jira_project': 'testproject',\n 'jira_issuetype': 'testtype',\n 'jira_server': 'jiraserver',\n 'jira_label': 'testlabel',\n 'jira_component': 'testcomponent',\n 'jira_description': \"DESC\",\n 'jira_watchers': ['testwatcher1', 'testwatcher2'],\n 'timestamp_field': '@timestamp',\n 'jira_affected_user': \"#gmail.the_user\",\n 'rule_file': '/tmp/foo.yaml'\n }\n mock_issue = mock.Mock()\n mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4))\n mock_fields = [\n {'name': 'affected user', 'id': 'affected_user_id', 'schema': {'type': 'string'}}\n ]\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value = mock.Mock()\n mock_jira.return_value.search_issues.return_value = [mock_issue]\n mock_jira.return_value.fields.return_value = mock_fields\n mock_jira.return_value.priorities.return_value = [mock_priority]\n alert = JiraAlerter(rule)\n alert.alert([{'gmail.the_user': 'jdoe', '@timestamp': '2014-10-31T00:00:00'}])\n assert mock_jira.mock_calls[4][2]['affected_user_id'] == \"jdoe\"\n\n\ndef test_jira_arbitrary_field_support():\n description_txt = \"Description stuff goes here like a runbook link.\"\n rule = {\n 'name': 'test alert',\n 'jira_account_file': 'jirafile',\n 'type': mock_rule(),\n 'owner': 'the_owner',\n 'jira_project': 'testproject',\n 'jira_issuetype': 'testtype',\n 'jira_server': 'jiraserver',\n 'jira_label': 'testlabel',\n 'jira_component': 'testcomponent',\n 'jira_description': description_txt,\n 'jira_watchers': ['testwatcher1', 'testwatcher2'],\n 'jira_arbitrary_reference_string_field': '$owner$',\n 'jira_arbitrary_string_field': 'arbitrary_string_value',\n 'jira_arbitrary_string_array_field': ['arbitrary_string_value1', 'arbitrary_string_value2'],\n 'jira_arbitrary_string_array_field_provided_as_single_value': 'arbitrary_string_value_in_array_field',\n 'jira_arbitrary_number_field': 1,\n 'jira_arbitrary_number_array_field': [2, 3],\n 'jira_arbitrary_number_array_field_provided_as_single_value': 1,\n 'jira_arbitrary_complex_field': 'arbitrary_complex_value',\n 'jira_arbitrary_complex_array_field': ['arbitrary_complex_value1', 'arbitrary_complex_value2'],\n 'jira_arbitrary_complex_array_field_provided_as_single_value': 'arbitrary_complex_value_in_array_field',\n 'timestamp_field': '@timestamp',\n 'alert_subject': 'Issue {0} occurred at {1}',\n 'alert_subject_args': ['test_term', '@timestamp'],\n 'rule_file': '/tmp/foo.yaml'\n }\n\n mock_priority = mock.MagicMock(id='5')\n\n mock_fields = [\n {'name': 'arbitrary reference string field', 'id': 'arbitrary_reference_string_field', 'schema': {'type': 'string'}},\n {'name': 'arbitrary string field', 'id': 'arbitrary_string_field', 'schema': {'type': 'string'}},\n {'name': 'arbitrary string array field', 'id': 'arbitrary_string_array_field', 'schema': {'type': 'array', 'items': 'string'}},\n {\n 'name': 'arbitrary string array field provided as single value',\n 'id': 'arbitrary_string_array_field_provided_as_single_value',\n 'schema': {'type': 'array', 'items': 'string'}\n },\n {'name': 'arbitrary number field', 'id': 'arbitrary_number_field', 'schema': {'type': 'number'}},\n {'name': 'arbitrary number array field', 'id': 'arbitrary_number_array_field', 'schema': {'type': 'array', 'items': 'number'}},\n {\n 'name': 'arbitrary number array field provided as single value',\n 'id': 'arbitrary_number_array_field_provided_as_single_value',\n 'schema': {'type': 'array', 'items': 'number'}\n },\n {'name': 'arbitrary complex field', 'id': 'arbitrary_complex_field', 'schema': {'type': 'ArbitraryType'}},\n {\n 'name': 'arbitrary complex array field',\n 'id': 'arbitrary_complex_array_field',\n 'schema': {'type': 'array', 'items': 'ArbitraryType'}\n },\n {\n 'name': 'arbitrary complex array field provided as single value',\n 'id': 'arbitrary_complex_array_field_provided_as_single_value',\n 'schema': {'type': 'array', 'items': 'ArbitraryType'}\n },\n ]\n\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = mock_fields\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n\n expected = [\n mock.call('jiraserver', basic_auth=('jirauser', 'jirapassword')),\n mock.call().priorities(),\n mock.call().fields(),\n mock.call().create_issue(\n issuetype={'name': 'testtype'},\n project={'key': 'testproject'},\n labels=['testlabel'],\n components=[{'name': 'testcomponent'}],\n description=mock.ANY,\n summary='Issue test_value occurred at 2014-10-31T00:00:00',\n arbitrary_reference_string_field='the_owner',\n arbitrary_string_field='arbitrary_string_value',\n arbitrary_string_array_field=['arbitrary_string_value1', 'arbitrary_string_value2'],\n arbitrary_string_array_field_provided_as_single_value=['arbitrary_string_value_in_array_field'],\n arbitrary_number_field=1,\n arbitrary_number_array_field=[2, 3],\n arbitrary_number_array_field_provided_as_single_value=[1],\n arbitrary_complex_field={'name': 'arbitrary_complex_value'},\n arbitrary_complex_array_field=[{'name': 'arbitrary_complex_value1'}, {'name': 'arbitrary_complex_value2'}],\n arbitrary_complex_array_field_provided_as_single_value=[{'name': 'arbitrary_complex_value_in_array_field'}],\n ),\n mock.call().add_watcher(mock.ANY, 'testwatcher1'),\n mock.call().add_watcher(mock.ANY, 'testwatcher2'),\n ]\n\n # We don't care about additional calls to mock_jira, such as __str__\n assert mock_jira.mock_calls[:6] == expected\n assert mock_jira.mock_calls[3][2]['description'].startswith(description_txt)\n\n # Reference an arbitrary string field that is not defined on the JIRA server\n rule['jira_nonexistent_field'] = 'nonexistent field value'\n\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = mock_fields\n\n with pytest.raises(Exception) as exception:\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n assert \"Could not find a definition for the jira field 'nonexistent field'\" in str(exception)\n\n del rule['jira_nonexistent_field']\n\n # Reference a watcher that does not exist\n rule['jira_watchers'] = 'invalid_watcher'\n\n with mock.patch('elastalert.alerts.JIRA') as mock_jira, \\\n mock.patch('elastalert.alerts.yaml_loader') as mock_open:\n mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'}\n mock_jira.return_value.priorities.return_value = [mock_priority]\n mock_jira.return_value.fields.return_value = mock_fields\n\n # Cause add_watcher to raise, which most likely means that the user did not exist\n mock_jira.return_value.add_watcher.side_effect = Exception()\n\n with pytest.raises(Exception) as exception:\n alert = JiraAlerter(rule)\n alert.alert([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n assert \"Exception encountered when trying to add 'invalid_watcher' as a watcher. Does the user exist?\" in str(exception)\n\n\ndef test_kibana(ea):\n rule = {'filter': [{'query': {'query_string': {'query': 'xy:z'}}}],\n 'name': 'Test rule!',\n 'es_host': 'test.testing',\n 'es_port': 12345,\n 'timeframe': datetime.timedelta(hours=1),\n 'index': 'logstash-test',\n 'include': ['@timestamp'],\n 'timestamp_field': '@timestamp'}\n match = {'@timestamp': '2014-10-10T00:00:00'}\n with mock.patch(\"elastalert.elastalert.elasticsearch_client\") as mock_es:\n mock_create = mock.Mock(return_value={'_id': 'ABCDEFGH'})\n mock_es_inst = mock.Mock()\n mock_es_inst.index = mock_create\n mock_es_inst.host = 'test.testing'\n mock_es_inst.port = 12345\n mock_es.return_value = mock_es_inst\n link = ea.generate_kibana_db(rule, match)\n\n assert 'http://test.testing:12345/_plugin/kibana/#/dashboard/temp/ABCDEFGH' == link\n\n # Name and index\n dashboard = json.loads(mock_create.call_args_list[0][1]['body']['dashboard'])\n assert dashboard['index']['default'] == 'logstash-test'\n assert 'Test rule!' in dashboard['title']\n\n # Filters and time range\n filters = dashboard['services']['filter']['list']\n assert 'xy:z' in filters['1']['query']\n assert filters['1']['type'] == 'querystring'\n time_range = filters['0']\n assert time_range['from'] == ts_add(match['@timestamp'], -rule['timeframe'])\n assert time_range['to'] == ts_add(match['@timestamp'], datetime.timedelta(minutes=10))\n\n # Included fields active in table\n assert dashboard['rows'][1]['panels'][0]['fields'] == ['@timestamp']\n\n\ndef test_command():\n # Test command as list with a formatted arg\n rule = {'command': ['/bin/test/', '--arg', '%(somefield)s']}\n alert = CommandAlerter(rule)\n match = {'@timestamp': '2014-01-01T00:00:00',\n 'somefield': 'foobarbaz',\n 'nested': {'field': 1}}\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)\n\n # Test command as string with formatted arg (old-style string format)\n rule = {'command': '/bin/test/ --arg %(somefield)s'}\n alert = CommandAlerter(rule)\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with('/bin/test --arg foobarbaz', stdin=subprocess.PIPE, shell=False)\n\n # Test command as string without formatted arg (old-style string format)\n rule = {'command': '/bin/test/foo.sh'}\n alert = CommandAlerter(rule)\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True)\n\n # Test command as string with formatted arg (new-style string format)\n rule = {'command': '/bin/test/ --arg {match[somefield]}', 'new_style_string_format': True}\n alert = CommandAlerter(rule)\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with('/bin/test --arg foobarbaz', stdin=subprocess.PIPE, shell=False)\n\n rule = {'command': '/bin/test/ --arg {match[nested][field]}', 'new_style_string_format': True}\n alert = CommandAlerter(rule)\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with('/bin/test --arg 1', stdin=subprocess.PIPE, shell=False)\n\n # Test command as string without formatted arg (new-style string format)\n rule = {'command': '/bin/test/foo.sh', 'new_style_string_format': True}\n alert = CommandAlerter(rule)\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True)\n\n rule = {'command': '/bin/test/foo.sh {{bar}}', 'new_style_string_format': True}\n alert = CommandAlerter(rule)\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n alert.alert([match])\n assert mock_popen.called_with('/bin/test/foo.sh {bar}', stdin=subprocess.PIPE, shell=True)\n\n # Test command with pipe_match_json\n rule = {'command': ['/bin/test/', '--arg', '%(somefield)s'],\n 'pipe_match_json': True}\n alert = CommandAlerter(rule)\n match = {'@timestamp': '2014-01-01T00:00:00',\n 'somefield': 'foobarbaz'}\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n mock_subprocess = mock.Mock()\n mock_popen.return_value = mock_subprocess\n mock_subprocess.communicate.return_value = (None, None)\n alert.alert([match])\n assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)\n assert mock_subprocess.communicate.called_with(input=json.dumps(match))\n\n # Test command with fail_on_non_zero_exit\n rule = {'command': ['/bin/test/', '--arg', '%(somefield)s'],\n 'fail_on_non_zero_exit': True}\n alert = CommandAlerter(rule)\n match = {'@timestamp': '2014-01-01T00:00:00',\n 'somefield': 'foobarbaz'}\n with pytest.raises(Exception) as exception:\n with mock.patch(\"elastalert.alerts.subprocess.Popen\") as mock_popen:\n mock_subprocess = mock.Mock()\n mock_popen.return_value = mock_subprocess\n mock_subprocess.wait.return_value = 1\n alert.alert([match])\n assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False)\n assert \"Non-zero exit code while running command\" in str(exception)\n\n\ndef test_ms_teams():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'ms_teams_webhook_url': 'http://test.webhook.url',\n 'ms_teams_alert_summary': 'Alert from ElastAlert',\n 'alert_subject': 'Cool subject',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = MsTeamsAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n '@type': 'MessageCard',\n '@context': 'http://schema.org/extensions',\n 'summary': rule['ms_teams_alert_summary'],\n 'title': rule['alert_subject'],\n 'text': BasicMatchString(rule, match).__str__()\n }\n mock_post_request.assert_called_once_with(\n rule['ms_teams_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_ms_teams_uses_color_and_fixed_width_text():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'ms_teams_webhook_url': 'http://test.webhook.url',\n 'ms_teams_alert_summary': 'Alert from ElastAlert',\n 'ms_teams_alert_fixed_width': True,\n 'ms_teams_theme_color': '#124578',\n 'alert_subject': 'Cool subject',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = MsTeamsAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n body = BasicMatchString(rule, match).__str__()\n body = body.replace('`', \"'\")\n body = \"```{0}```\".format('```\\n\\n```'.join(x for x in body.split('\\n'))).replace('\\n``````', '')\n expected_data = {\n '@type': 'MessageCard',\n '@context': 'http://schema.org/extensions',\n 'summary': rule['ms_teams_alert_summary'],\n 'title': rule['alert_subject'],\n 'themeColor': '#124578',\n 'text': body\n }\n mock_post_request.assert_called_once_with(\n rule['ms_teams_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_slack_uses_custom_title():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_webhook_url': 'http://please.dontgohere.slack',\n 'alert_subject': 'Cool subject',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'channel': '',\n 'icon_emoji': ':ghost:',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': rule['alert_subject'],\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'text': '',\n 'parse': 'none'\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_slack_uses_custom_timeout():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_webhook_url': 'http://please.dontgohere.slack',\n 'alert_subject': 'Cool subject',\n 'alert': [],\n 'slack_timeout': 20\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'channel': '',\n 'icon_emoji': ':ghost:',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': rule['alert_subject'],\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'text': '',\n 'parse': 'none'\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=20\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_slack_uses_rule_name_when_custom_title_is_not_provided():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_webhook_url': ['http://please.dontgohere.slack'],\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'channel': '',\n 'icon_emoji': ':ghost:',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': rule['name'],\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'text': '',\n 'parse': 'none',\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'][0],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_slack_uses_custom_slack_channel():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_webhook_url': ['http://please.dontgohere.slack'],\n 'slack_channel_override': '#test-alert',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'channel': '#test-alert',\n 'icon_emoji': ':ghost:',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': rule['name'],\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'text': '',\n 'parse': 'none',\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'][0],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_slack_uses_list_of_custom_slack_channel():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_webhook_url': ['http://please.dontgohere.slack'],\n 'slack_channel_override': ['#test-alert', '#test-alert2'],\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data1 = {\n 'username': 'elastalert',\n 'channel': '#test-alert',\n 'icon_emoji': ':ghost:',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': rule['name'],\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'text': '',\n 'parse': 'none'\n }\n expected_data2 = {\n 'username': 'elastalert',\n 'channel': '#test-alert2',\n 'icon_emoji': ':ghost:',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': rule['name'],\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'text': '',\n 'parse': 'none'\n }\n mock_post_request.assert_called_with(\n rule['slack_webhook_url'][0],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n assert expected_data1 == json.loads(mock_post_request.call_args_list[0][1]['data'])\n assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data'])\n\n\ndef test_slack_attach_kibana_discover_url_when_generated():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_attach_kibana_discover_url': True,\n 'slack_webhook_url': 'http://please.dontgohere.slack',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'kibana_discover_url': 'http://kibana#discover'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'parse': 'none',\n 'text': '',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': 'Test Rule',\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n },\n {\n 'color': '#ec4b98',\n 'title': 'Discover in Kibana',\n 'title_link': 'http://kibana#discover'\n }\n ],\n 'icon_emoji': ':ghost:',\n 'channel': ''\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n actual_data = json.loads(mock_post_request.call_args_list[0][1]['data'])\n assert expected_data == actual_data\n\n\ndef test_slack_attach_kibana_discover_url_when_not_generated():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_attach_kibana_discover_url': True,\n 'slack_webhook_url': 'http://please.dontgohere.slack',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'parse': 'none',\n 'text': '',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': 'Test Rule',\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n }\n ],\n 'icon_emoji': ':ghost:',\n 'channel': ''\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n actual_data = json.loads(mock_post_request.call_args_list[0][1]['data'])\n assert expected_data == actual_data\n\n\ndef test_slack_kibana_discover_title():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_attach_kibana_discover_url': True,\n 'slack_kibana_discover_title': 'Click to discover in Kibana',\n 'slack_webhook_url': 'http://please.dontgohere.slack',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'kibana_discover_url': 'http://kibana#discover'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'parse': 'none',\n 'text': '',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': 'Test Rule',\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n },\n {\n 'color': '#ec4b98',\n 'title': 'Click to discover in Kibana',\n 'title_link': 'http://kibana#discover'\n }\n ],\n 'icon_emoji': ':ghost:',\n 'channel': ''\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n actual_data = json.loads(mock_post_request.call_args_list[0][1]['data'])\n assert expected_data == actual_data\n\n\ndef test_slack_kibana_discover_color():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'slack_attach_kibana_discover_url': True,\n 'slack_kibana_discover_color': 'blue',\n 'slack_webhook_url': 'http://please.dontgohere.slack',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = SlackAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'kibana_discover_url': 'http://kibana#discover'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n 'username': 'elastalert',\n 'parse': 'none',\n 'text': '',\n 'attachments': [\n {\n 'color': 'danger',\n 'title': 'Test Rule',\n 'text': BasicMatchString(rule, match).__str__(),\n 'mrkdwn_in': ['text', 'pretext'],\n 'fields': []\n },\n {\n 'color': 'blue',\n 'title': 'Discover in Kibana',\n 'title_link': 'http://kibana#discover'\n }\n ],\n 'icon_emoji': ':ghost:',\n 'channel': ''\n }\n mock_post_request.assert_called_once_with(\n rule['slack_webhook_url'],\n data=mock.ANY,\n headers={'content-type': 'application/json'},\n proxies=None,\n verify=False,\n timeout=10\n )\n actual_data = json.loads(mock_post_request.call_args_list[0][1]['data'])\n assert expected_data == actual_data\n\n\ndef test_http_alerter_with_payload():\n rule = {\n 'name': 'Test HTTP Post Alerter With Payload',\n 'type': 'any',\n 'http_post_url': 'http://test.webhook.url',\n 'http_post_payload': {'posted_name': 'somefield'},\n 'http_post_static_payload': {'name': 'somestaticname'},\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = HTTPPostAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'posted_name': 'foobarbaz',\n 'name': 'somestaticname'\n }\n mock_post_request.assert_called_once_with(\n rule['http_post_url'],\n data=mock.ANY,\n headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'},\n proxies=None,\n timeout=10\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_http_alerter_with_payload_all_values():\n rule = {\n 'name': 'Test HTTP Post Alerter With Payload',\n 'type': 'any',\n 'http_post_url': 'http://test.webhook.url',\n 'http_post_payload': {'posted_name': 'somefield'},\n 'http_post_static_payload': {'name': 'somestaticname'},\n 'http_post_all_values': True,\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = HTTPPostAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'posted_name': 'foobarbaz',\n 'name': 'somestaticname',\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n mock_post_request.assert_called_once_with(\n rule['http_post_url'],\n data=mock.ANY,\n headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'},\n proxies=None,\n timeout=10\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_http_alerter_without_payload():\n rule = {\n 'name': 'Test HTTP Post Alerter Without Payload',\n 'type': 'any',\n 'http_post_url': 'http://test.webhook.url',\n 'http_post_static_payload': {'name': 'somestaticname'},\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = HTTPPostAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz',\n 'name': 'somestaticname'\n }\n mock_post_request.assert_called_once_with(\n rule['http_post_url'],\n data=mock.ANY,\n headers={'Content-Type': 'application/json', 'Accept': 'application/json;charset=utf-8'},\n proxies=None,\n timeout=10\n )\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_client_name': 'ponies inc.',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'description': 'Test PD Rule',\n 'details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: foobarbaz\\n'\n },\n 'event_type': 'trigger',\n 'incident_key': '',\n 'service_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with('https://events.pagerduty.com/generic/2010-04-15/create_event.json',\n data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter_v2():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_client_name': 'ponies inc.',\n 'pagerduty_api_version': 'v2',\n 'pagerduty_v2_payload_class': 'ping failure',\n 'pagerduty_v2_payload_component': 'mysql',\n 'pagerduty_v2_payload_group': 'app-stack',\n 'pagerduty_v2_payload_severity': 'error',\n 'pagerduty_v2_payload_source': 'mysql.host.name',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'payload': {\n 'class': 'ping failure',\n 'component': 'mysql',\n 'group': 'app-stack',\n 'severity': 'error',\n 'source': 'mysql.host.name',\n 'summary': 'Test PD Rule',\n 'custom_details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: foobarbaz\\n'\n },\n 'timestamp': '2017-01-01T00:00:00'\n },\n 'event_action': 'trigger',\n 'dedup_key': '',\n 'routing_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with('https://events.pagerduty.com/v2/enqueue',\n data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter_custom_incident_key():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_client_name': 'ponies inc.',\n 'pagerduty_incident_key': 'custom key',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'description': 'Test PD Rule',\n 'details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: foobarbaz\\n'\n },\n 'event_type': 'trigger',\n 'incident_key': 'custom key',\n 'service_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with(alert.url, data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter_custom_incident_key_with_args():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_client_name': 'ponies inc.',\n 'pagerduty_incident_key': 'custom {0}',\n 'pagerduty_incident_key_args': ['somefield'],\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'description': 'Test PD Rule',\n 'details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: foobarbaz\\n'\n },\n 'event_type': 'trigger',\n 'incident_key': 'custom foobarbaz',\n 'service_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with(alert.url, data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter_custom_alert_subject():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'alert_subject': 'Hungry kittens',\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_client_name': 'ponies inc.',\n 'pagerduty_incident_key': 'custom {0}',\n 'pagerduty_incident_key_args': ['somefield'],\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'description': 'Hungry kittens',\n 'details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: foobarbaz\\n'\n },\n 'event_type': 'trigger',\n 'incident_key': 'custom foobarbaz',\n 'service_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with(alert.url, data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter_custom_alert_subject_with_args():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'alert_subject': '{0} kittens',\n 'alert_subject_args': ['somefield'],\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_client_name': 'ponies inc.',\n 'pagerduty_incident_key': 'custom {0}',\n 'pagerduty_incident_key_args': ['someotherfield'],\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'Stinky',\n 'someotherfield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'description': 'Stinky kittens',\n 'details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: Stinky\\nsomeotherfield: foobarbaz\\n'\n },\n 'event_type': 'trigger',\n 'incident_key': 'custom foobarbaz',\n 'service_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with(alert.url, data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_pagerduty_alerter_custom_alert_subject_with_args_specifying_trigger():\n rule = {\n 'name': 'Test PD Rule',\n 'type': 'any',\n 'alert_subject': '{0} kittens',\n 'alert_subject_args': ['somefield'],\n 'pagerduty_service_key': 'magicalbadgers',\n 'pagerduty_event_type': 'trigger',\n 'pagerduty_client_name': 'ponies inc.',\n 'pagerduty_incident_key': 'custom {0}',\n 'pagerduty_incident_key_args': ['someotherfield'],\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = PagerDutyAlerter(rule)\n match = {\n '@timestamp': '2017-01-01T00:00:00',\n 'somefield': 'Stinkiest',\n 'someotherfield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n expected_data = {\n 'client': 'ponies inc.',\n 'description': 'Stinkiest kittens',\n 'details': {\n 'information': 'Test PD Rule\\n\\n@timestamp: 2017-01-01T00:00:00\\nsomefield: Stinkiest\\nsomeotherfield: foobarbaz\\n'\n },\n 'event_type': 'trigger',\n 'incident_key': 'custom foobarbaz',\n 'service_key': 'magicalbadgers',\n }\n mock_post_request.assert_called_once_with(alert.url, data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None)\n assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_alert_text_kw(ea):\n rule = ea.rules[0].copy()\n rule['alert_text'] = '{field} at {time}'\n rule['alert_text_kw'] = {\n '@timestamp': 'time',\n 'field': 'field',\n }\n match = {'@timestamp': '1918-01-17', 'field': 'value'}\n alert_text = str(BasicMatchString(rule, match))\n body = '{field} at {@timestamp}'.format(**match)\n assert body in alert_text\n\n\ndef test_alert_text_global_substitution(ea):\n rule = ea.rules[0].copy()\n rule['owner'] = 'the owner from rule'\n rule['priority'] = 'priority from rule'\n rule['abc'] = 'abc from rule'\n rule['alert_text'] = 'Priority: {0}; Owner: {1}; Abc: {2}'\n rule['alert_text_args'] = ['priority', 'owner', 'abc']\n\n match = {\n '@timestamp': '2016-01-01',\n 'field': 'field_value',\n 'abc': 'abc from match',\n }\n\n alert_text = str(BasicMatchString(rule, match))\n assert 'Priority: priority from rule' in alert_text\n assert 'Owner: the owner from rule' in alert_text\n\n # When the key exists in both places, it will come from the match\n assert 'Abc: abc from match' in alert_text\n\n\ndef test_alert_text_kw_global_substitution(ea):\n rule = ea.rules[0].copy()\n rule['foo_rule'] = 'foo from rule'\n rule['owner'] = 'the owner from rule'\n rule['abc'] = 'abc from rule'\n rule['alert_text'] = 'Owner: {owner}; Foo: {foo}; Abc: {abc}'\n rule['alert_text_kw'] = {\n 'owner': 'owner',\n 'foo_rule': 'foo',\n 'abc': 'abc',\n }\n\n match = {\n '@timestamp': '2016-01-01',\n 'field': 'field_value',\n 'abc': 'abc from match',\n }\n\n alert_text = str(BasicMatchString(rule, match))\n assert 'Owner: the owner from rule' in alert_text\n assert 'Foo: foo from rule' in alert_text\n\n # When the key exists in both places, it will come from the match\n assert 'Abc: abc from match' in alert_text\n\n\ndef test_resolving_rule_references(ea):\n rule = {\n 'name': 'test_rule',\n 'type': mock_rule(),\n 'owner': 'the_owner',\n 'priority': 2,\n 'list_of_things': [\n '1',\n '$owner$',\n [\n '11',\n '$owner$',\n ],\n ],\n 'nested_dict': {\n 'nested_one': '1',\n 'nested_owner': '$owner$',\n },\n 'resolved_string_reference': '$owner$',\n 'resolved_int_reference': '$priority$',\n 'unresolved_reference': '$foo$',\n }\n alert = Alerter(rule)\n assert 'the_owner' == alert.rule['resolved_string_reference']\n assert 2 == alert.rule['resolved_int_reference']\n assert '$foo$' == alert.rule['unresolved_reference']\n assert 'the_owner' == alert.rule['list_of_things'][1]\n assert 'the_owner' == alert.rule['list_of_things'][2][1]\n assert 'the_owner' == alert.rule['nested_dict']['nested_owner']\n\n\ndef test_stride_plain_text():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'stride_access_token': 'token',\n 'stride_cloud_id': 'cloud_id',\n 'stride_conversation_id': 'conversation_id',\n 'alert_subject': 'Cool subject',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = StrideAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n body = \"{0}\\n\\n@timestamp: {1}\\nsomefield: {2}\".format(\n rule['name'], match['@timestamp'], match['somefield']\n )\n expected_data = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': [\n {'type': 'text', 'text': body}\n ]}\n ]}\n ]}}\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])},\n verify=True,\n proxies=None\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_stride_underline_text():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'stride_access_token': 'token',\n 'stride_cloud_id': 'cloud_id',\n 'stride_conversation_id': 'conversation_id',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Underline Text',\n 'alert_text_type': 'alert_text_only',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = StrideAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n body = \"Underline Text\"\n expected_data = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': [\n {'type': 'text', 'text': body, 'marks': [\n {'type': 'underline'}\n ]}\n ]}\n ]}\n ]}}\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])},\n verify=True,\n proxies=None\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_stride_bold_text():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'stride_access_token': 'token',\n 'stride_cloud_id': 'cloud_id',\n 'stride_conversation_id': 'conversation_id',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Bold Text',\n 'alert_text_type': 'alert_text_only',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = StrideAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n body = \"Bold Text\"\n expected_data = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': [\n {'type': 'text', 'text': body, 'marks': [\n {'type': 'strong'}\n ]}\n ]}\n ]}\n ]}}\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])},\n verify=True,\n proxies=None\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_stride_strong_text():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'stride_access_token': 'token',\n 'stride_cloud_id': 'cloud_id',\n 'stride_conversation_id': 'conversation_id',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Bold Text',\n 'alert_text_type': 'alert_text_only',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = StrideAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n body = \"Bold Text\"\n expected_data = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': [\n {'type': 'text', 'text': body, 'marks': [\n {'type': 'strong'}\n ]}\n ]}\n ]}\n ]}}\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])},\n verify=True,\n proxies=None\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_stride_hyperlink():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'stride_access_token': 'token',\n 'stride_cloud_id': 'cloud_id',\n 'stride_conversation_id': 'conversation_id',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Link',\n 'alert_text_type': 'alert_text_only',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = StrideAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n body = \"Link\"\n expected_data = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': [\n {'type': 'text', 'text': body, 'marks': [\n {'type': 'link', 'attrs': {'href': 'http://stride.com'}}\n ]}\n ]}\n ]}\n ]}}\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])},\n verify=True,\n proxies=None\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_stride_html():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'stride_access_token': 'token',\n 'stride_cloud_id': 'cloud_id',\n 'stride_conversation_id': 'conversation_id',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Alert: we found something. Link',\n 'alert_text_type': 'alert_text_only',\n 'alert': []\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = StrideAlerter(rule)\n match = {\n '@timestamp': '2016-01-01T00:00:00',\n 'somefield': 'foobarbaz'\n }\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {'body': {'version': 1, 'type': \"doc\", 'content': [\n {'type': \"panel\", 'attrs': {'panelType': \"warning\"}, 'content': [\n {'type': 'paragraph', 'content': [\n {'type': 'text', 'text': 'Alert', 'marks': [\n {'type': 'strong'}\n ]},\n {'type': 'text', 'text': ': we found something. '},\n {'type': 'text', 'text': 'Link', 'marks': [\n {'type': 'link', 'attrs': {'href': 'http://stride.com'}}\n ]}\n ]}\n ]}\n ]}}\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])},\n verify=True,\n proxies=None\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_hipchat_body_size_limit_text():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'hipchat_auth_token': 'token',\n 'hipchat_room_id': 'room_id',\n 'hipchat_message_format': 'text',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Alert: we found something.\\n\\n{message}',\n 'alert_text_type': 'alert_text_only',\n 'alert': [],\n 'alert_text_kw': {\n '@timestamp': 'time',\n 'message': 'message',\n },\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = HipChatAlerter(rule)\n match = {\n '@timestamp': '2018-01-01T00:00:00',\n 'message': 'foo bar\\n' * 5000,\n }\n body = alert.create_alert_body([match])\n\n assert len(body) <= 10000\n\n\ndef test_hipchat_body_size_limit_html():\n rule = {\n 'name': 'Test Rule',\n 'type': 'any',\n 'hipchat_auth_token': 'token',\n 'hipchat_room_id': 'room_id',\n 'hipchat_message_format': 'html',\n 'alert_subject': 'Cool subject',\n 'alert_text': 'Alert: we found something.\\n\\n{message}',\n 'alert_text_type': 'alert_text_only',\n 'alert': [],\n 'alert_text_kw': {\n '@timestamp': 'time',\n 'message': 'message',\n },\n }\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = HipChatAlerter(rule)\n match = {\n '@timestamp': '2018-01-01T00:00:00',\n 'message': 'foo bar\\n' * 5000,\n }\n\n body = alert.create_alert_body([match])\n\n assert len(body) <= 10000\n\n\ndef test_alerta_no_auth(ea):\n rule = {\n 'name': 'Test Alerta rule!',\n 'alerta_api_url': 'http://elastalerthost:8080/api/alert',\n 'timeframe': datetime.timedelta(hours=1),\n 'timestamp_field': '@timestamp',\n 'alerta_api_skip_ssl': True,\n 'alerta_attributes_keys': [\"hostname\", \"TimestampEvent\", \"senderIP\"],\n 'alerta_attributes_values': [\"%(key)s\", \"%(logdate)s\", \"%(sender_ip)s\"],\n 'alerta_correlate': [\"ProbeUP\", \"ProbeDOWN\"],\n 'alerta_event': \"ProbeUP\",\n 'alerta_group': \"Health\",\n 'alerta_origin': \"Elastalert\",\n 'alerta_severity': \"debug\",\n 'alerta_text': \"Probe %(hostname)s is UP at %(logdate)s GMT\",\n 'alerta_value': \"UP\",\n 'type': 'any',\n 'alerta_use_match_timestamp': True,\n 'alert': 'alerta'\n }\n\n match = {\n '@timestamp': '2014-10-10T00:00:00',\n # 'key': ---- missing field on purpose, to verify that simply the text is left empty\n # 'logdate': ---- missing field on purpose, to verify that simply the text is left empty\n 'sender_ip': '1.1.1.1',\n 'hostname': 'aProbe'\n }\n\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = AlertaAlerter(rule)\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n \"origin\": \"Elastalert\",\n \"resource\": \"elastalert\",\n \"severity\": \"debug\",\n \"service\": [\"elastalert\"],\n \"tags\": [],\n \"text\": \"Probe aProbe is UP at GMT\",\n \"value\": \"UP\",\n \"createTime\": \"2014-10-10T00:00:00.000000Z\",\n \"environment\": \"Production\",\n \"rawData\": \"Test Alerta rule!\\n\\n@timestamp: 2014-10-10T00:00:00\\nhostname: aProbe\\nsender_ip: 1.1.1.1\\n\",\n \"timeout\": 86400,\n \"correlate\": [\"ProbeUP\", \"ProbeDOWN\"],\n \"group\": \"Health\",\n \"attributes\": {\"senderIP\": \"1.1.1.1\", \"hostname\": \"\", \"TimestampEvent\": \"\"},\n \"type\": \"elastalert\",\n \"event\": \"ProbeUP\"\n }\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n headers={\n 'content-type': 'application/json'},\n verify=False\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_alerta_auth(ea):\n rule = {\n 'name': 'Test Alerta rule!',\n 'alerta_api_url': 'http://elastalerthost:8080/api/alert',\n 'alerta_api_key': '123456789ABCDEF',\n 'timeframe': datetime.timedelta(hours=1),\n 'timestamp_field': '@timestamp',\n 'alerta_severity': \"debug\",\n 'type': 'any',\n 'alerta_use_match_timestamp': True,\n 'alert': 'alerta'\n }\n\n match = {\n '@timestamp': '2014-10-10T00:00:00',\n 'sender_ip': '1.1.1.1',\n 'hostname': 'aProbe'\n }\n\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = AlertaAlerter(rule)\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n verify=True,\n headers={\n 'content-type': 'application/json',\n 'Authorization': 'Key {}'.format(rule['alerta_api_key'])})\n\n\ndef test_alerta_new_style(ea):\n rule = {\n 'name': 'Test Alerta rule!',\n 'alerta_api_url': 'http://elastalerthost:8080/api/alert',\n 'timeframe': datetime.timedelta(hours=1),\n 'timestamp_field': '@timestamp',\n 'alerta_attributes_keys': [\"hostname\", \"TimestampEvent\", \"senderIP\"],\n 'alerta_attributes_values': [\"{hostname}\", \"{logdate}\", \"{sender_ip}\"],\n 'alerta_correlate': [\"ProbeUP\", \"ProbeDOWN\"],\n 'alerta_event': \"ProbeUP\",\n 'alerta_group': \"Health\",\n 'alerta_origin': \"Elastalert\",\n 'alerta_severity': \"debug\",\n 'alerta_text': \"Probe {hostname} is UP at {logdate} GMT\",\n 'alerta_value': \"UP\",\n 'alerta_new_style_string_format': True,\n 'type': 'any',\n 'alerta_use_match_timestamp': True,\n 'alert': 'alerta'\n }\n\n match = {\n '@timestamp': '2014-10-10T00:00:00',\n # 'key': ---- missing field on purpose, to verify that simply the text is left empty\n # 'logdate': ---- missing field on purpose, to verify that simply the text is left empty\n 'sender_ip': '1.1.1.1',\n 'hostname': 'aProbe'\n }\n\n rules_loader = FileRulesLoader({})\n rules_loader.load_modules(rule)\n alert = AlertaAlerter(rule)\n with mock.patch('requests.post') as mock_post_request:\n alert.alert([match])\n\n expected_data = {\n \"origin\": \"Elastalert\",\n \"resource\": \"elastalert\",\n \"severity\": \"debug\",\n \"service\": [\"elastalert\"],\n \"tags\": [],\n \"text\": \"Probe aProbe is UP at GMT\",\n \"value\": \"UP\",\n \"createTime\": \"2014-10-10T00:00:00.000000Z\",\n \"environment\": \"Production\",\n \"rawData\": \"Test Alerta rule!\\n\\n@timestamp: 2014-10-10T00:00:00\\nhostname: aProbe\\nsender_ip: 1.1.1.1\\n\",\n \"timeout\": 86400,\n \"correlate\": [\"ProbeUP\", \"ProbeDOWN\"],\n \"group\": \"Health\",\n \"attributes\": {\"senderIP\": \"1.1.1.1\", \"hostname\": \"aProbe\", \"TimestampEvent\": \"\"},\n \"type\": \"elastalert\",\n \"event\": \"ProbeUP\"\n }\n\n mock_post_request.assert_called_once_with(\n alert.url,\n data=mock.ANY,\n verify=True,\n headers={\n 'content-type': 'application/json'}\n )\n assert expected_data == json.loads(\n mock_post_request.call_args_list[0][1]['data'])\n\n\ndef test_alert_subject_size_limit_no_args(ea):\n rule = {\n 'name': 'test_rule',\n 'type': mock_rule(),\n 'owner': 'the_owner',\n 'priority': 2,\n 'alert_subject': 'A very long subject',\n 'alert_subject_max_len': 5\n }\n alert = Alerter(rule)\n alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n assert 5 == len(alertSubject)\n\n\ndef test_alert_subject_size_limit_with_args(ea):\n rule = {\n 'name': 'test_rule',\n 'type': mock_rule(),\n 'owner': 'the_owner',\n 'priority': 2,\n 'alert_subject': 'Test alert for {0} {1}',\n 'alert_subject_args': ['test_term', 'test.term'],\n 'alert_subject_max_len': 6\n }\n alert = Alerter(rule)\n alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])\n assert 6 == len(alertSubject)\n" }, { "path": "tests/auth_test.py", "content": "# -*- coding: utf-8 -*-\nfrom elastalert.auth import Auth, RefeshableAWSRequestsAuth\n\n\ndef test_auth_none():\n\n auth = Auth()(\n host='localhost:8080',\n username=None,\n password=None,\n aws_region=None,\n profile_name=None\n )\n\n assert not auth\n\n\ndef test_auth_username_password():\n\n auth = Auth()(\n host='localhost:8080',\n username='user',\n password='password',\n aws_region=None,\n profile_name=None\n )\n\n assert auth == 'user:password'\n\n\ndef test_auth_aws_region():\n\n auth = Auth()(\n host='localhost:8080',\n username=None,\n password=None,\n aws_region='us-east-1',\n profile_name=None\n )\n\n assert type(auth) == RefeshableAWSRequestsAuth\n assert auth.aws_region == 'us-east-1'\n" }, { "path": "tests/base_test.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport datetime\nimport json\nimport threading\n\nimport elasticsearch\nimport mock\nimport pytest\nfrom elasticsearch.exceptions import ConnectionError\nfrom elasticsearch.exceptions import ElasticsearchException\n\nfrom elastalert.enhancements import BaseEnhancement\nfrom elastalert.enhancements import DropMatchException\nfrom elastalert.kibana import dashboard_temp\nfrom elastalert.util import dt_to_ts\nfrom elastalert.util import dt_to_unix\nfrom elastalert.util import dt_to_unixms\nfrom elastalert.util import EAException\nfrom elastalert.util import ts_now\nfrom elastalert.util import ts_to_dt\nfrom elastalert.util import unix_to_dt\n\nSTART_TIMESTAMP = '2014-09-26T12:34:45Z'\nEND_TIMESTAMP = '2014-09-27T12:34:45Z'\nSTART = ts_to_dt(START_TIMESTAMP)\nEND = ts_to_dt(END_TIMESTAMP)\n\n\ndef _set_hits(ea_inst, hits):\n res = {'hits': {'total': len(hits), 'hits': hits}}\n ea_inst.client_es.return_value = res\n\n\ndef generate_hits(timestamps, **kwargs):\n hits = []\n for i, ts in enumerate(timestamps):\n data = {'_id': 'id{}'.format(i),\n '_source': {'@timestamp': ts},\n '_type': 'logs',\n '_index': 'idx'}\n for key, item in kwargs.items():\n data['_source'][key] = item\n # emulate process_hits(), add metadata to _source\n for field in ['_id', '_type', '_index']:\n data['_source'][field] = data[field]\n hits.append(data)\n return {'hits': {'total': len(hits), 'hits': hits}}\n\n\ndef assert_alerts(ea_inst, calls):\n \"\"\" Takes a list of lists of timestamps. Asserts that an alert was called for each list, containing those timestamps. \"\"\"\n assert ea_inst.rules[0]['alert'][0].alert.call_count == len(calls)\n for call_num, call_args in enumerate(ea_inst.rules[0]['alert'][0].alert.call_args_list):\n assert not any([match['@timestamp'] not in calls[call_num] for match in call_args[0][0]])\n assert len(call_args[0][0]) == len(calls[call_num])\n\n\ndef test_starttime(ea):\n invalid = ['2014-13-13',\n '2014-11-24T30:00:00',\n 'Not A Timestamp']\n for ts in invalid:\n with pytest.raises((TypeError, ValueError)):\n ts_to_dt(ts)\n\n\ndef test_init_rule(ea):\n # Simulate state of a rule just loaded from a file\n ea.rules[0]['minimum_starttime'] = datetime.datetime.now()\n new_rule = copy.copy(ea.rules[0])\n list(map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']))\n\n # Properties are copied from ea.rules[0]\n ea.rules[0]['starttime'] = '2014-01-02T00:11:22'\n ea.rules[0]['processed_hits'] = ['abcdefg']\n new_rule = ea.init_rule(new_rule, False)\n for prop in ['starttime', 'agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime', 'run_every']:\n assert new_rule[prop] == ea.rules[0][prop]\n\n # Properties are fresh\n new_rule = ea.init_rule(new_rule, True)\n new_rule.pop('starttime')\n assert 'starttime' not in new_rule\n assert new_rule['processed_hits'] == {}\n\n # Assert run_every is unique\n new_rule['run_every'] = datetime.timedelta(seconds=17)\n new_rule = ea.init_rule(new_rule, True)\n assert new_rule['run_every'] == datetime.timedelta(seconds=17)\n\n\ndef test_query(ea):\n ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.run_query(ea.rules[0], START, END)\n ea.thread_data.current_es.search.assert_called_with(body={\n 'query': {'filtered': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'],\n ignore_unavailable=True,\n size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive'])\n\n\ndef test_query_sixsix(ea_sixsix):\n ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea_sixsix.run_query(ea_sixsix.rules[0], START, END)\n ea_sixsix.thread_data.current_es.search.assert_called_with(body={\n 'query': {'bool': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'],\n ignore_unavailable=True,\n size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive'])\n\n\ndef test_query_with_fields(ea):\n ea.rules[0]['_source_enabled'] = False\n ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.run_query(ea.rules[0], START, END)\n ea.thread_data.current_es.search.assert_called_with(body={\n 'query': {'filtered': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['@timestamp']}, index='idx', ignore_unavailable=True,\n size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive'])\n\n\ndef test_query_sixsix_with_fields(ea_sixsix):\n ea_sixsix.rules[0]['_source_enabled'] = False\n ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea_sixsix.run_query(ea_sixsix.rules[0], START, END)\n ea_sixsix.thread_data.current_es.search.assert_called_with(body={\n 'query': {'bool': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}], 'stored_fields': ['@timestamp']}, index='idx',\n ignore_unavailable=True,\n size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive'])\n\n\ndef test_query_with_unix(ea):\n ea.rules[0]['timestamp_type'] = 'unix'\n ea.rules[0]['dt_to_ts'] = dt_to_unix\n ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.run_query(ea.rules[0], START, END)\n start_unix = dt_to_unix(START)\n end_unix = dt_to_unix(END)\n ea.thread_data.current_es.search.assert_called_with(\n body={'query': {'filtered': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'],\n ignore_unavailable=True,\n size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive'])\n\n\ndef test_query_sixsix_with_unix(ea_sixsix):\n ea_sixsix.rules[0]['timestamp_type'] = 'unix'\n ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unix\n ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea_sixsix.run_query(ea_sixsix.rules[0], START, END)\n start_unix = dt_to_unix(START)\n end_unix = dt_to_unix(END)\n ea_sixsix.thread_data.current_es.search.assert_called_with(\n body={'query': {'bool': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'],\n ignore_unavailable=True,\n size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive'])\n\n\ndef test_query_with_unixms(ea):\n ea.rules[0]['timestamp_type'] = 'unixms'\n ea.rules[0]['dt_to_ts'] = dt_to_unixms\n ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.run_query(ea.rules[0], START, END)\n start_unix = dt_to_unixms(START)\n end_unix = dt_to_unixms(END)\n ea.thread_data.current_es.search.assert_called_with(\n body={'query': {'filtered': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'],\n ignore_unavailable=True,\n size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive'])\n\n\ndef test_query_sixsix_with_unixms(ea_sixsix):\n ea_sixsix.rules[0]['timestamp_type'] = 'unixms'\n ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unixms\n ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea_sixsix.run_query(ea_sixsix.rules[0], START, END)\n start_unix = dt_to_unixms(START)\n end_unix = dt_to_unixms(END)\n ea_sixsix.thread_data.current_es.search.assert_called_with(\n body={'query': {'bool': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}},\n 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'],\n ignore_unavailable=True,\n size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive'])\n\n\ndef test_no_hits(ea):\n ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.run_query(ea.rules[0], START, END)\n assert ea.rules[0]['type'].add_data.call_count == 0\n\n\ndef test_no_terms_hits(ea):\n ea.rules[0]['use_terms_query'] = True\n ea.rules[0]['query_key'] = 'QWERTY'\n ea.rules[0]['doc_type'] = 'uiop'\n ea.thread_data.current_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.run_query(ea.rules[0], START, END)\n assert ea.rules[0]['type'].add_terms_data.call_count == 0\n\n\ndef test_some_hits(ea):\n hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP])\n hits_dt = generate_hits([START, END])\n ea.thread_data.current_es.search.return_value = hits\n ea.run_query(ea.rules[0], START, END)\n assert ea.rules[0]['type'].add_data.call_count == 1\n ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']])\n\n\ndef test_some_hits_unix(ea):\n ea.rules[0]['timestamp_type'] = 'unix'\n ea.rules[0]['dt_to_ts'] = dt_to_unix\n ea.rules[0]['ts_to_dt'] = unix_to_dt\n hits = generate_hits([dt_to_unix(START), dt_to_unix(END)])\n hits_dt = generate_hits([START, END])\n ea.thread_data.current_es.search.return_value = copy.deepcopy(hits)\n ea.run_query(ea.rules[0], START, END)\n assert ea.rules[0]['type'].add_data.call_count == 1\n ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']])\n\n\ndef _duplicate_hits_generator(timestamps, **kwargs):\n \"\"\"Generator repeatedly returns identical hits dictionaries\n \"\"\"\n while True:\n yield generate_hits(timestamps, **kwargs)\n\n\ndef test_duplicate_timestamps(ea):\n ea.thread_data.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate')\n ea.run_query(ea.rules[0], START, ts_to_dt('2014-01-01T00:00:00Z'))\n\n assert len(ea.rules[0]['type'].add_data.call_args_list[0][0][0]) == 3\n assert ea.rules[0]['type'].add_data.call_count == 1\n\n # Run the query again, duplicates will be removed and not added\n ea.run_query(ea.rules[0], ts_to_dt('2014-01-01T00:00:00Z'), END)\n assert ea.rules[0]['type'].add_data.call_count == 1\n\n\ndef test_match(ea):\n hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP])\n ea.thread_data.current_es.search.return_value = hits\n ea.rules[0]['type'].matches = [{'@timestamp': END}]\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n\n ea.rules[0]['alert'][0].alert.called_with({'@timestamp': END_TIMESTAMP})\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n\ndef test_run_rule_calls_garbage_collect(ea):\n start_time = '2014-09-26T00:00:00Z'\n end_time = '2014-09-26T12:00:00Z'\n ea.buffer_time = datetime.timedelta(hours=1)\n ea.run_every = datetime.timedelta(hours=1)\n with mock.patch.object(ea.rules[0]['type'], 'garbage_collect') as mock_gc, \\\n mock.patch.object(ea, 'run_query'):\n ea.run_rule(ea.rules[0], ts_to_dt(end_time), ts_to_dt(start_time))\n\n # Running ElastAlert every hour for 12 hours, we should see self.garbage_collect called 12 times.\n assert mock_gc.call_count == 12\n\n # The calls should be spaced 1 hour apart\n expected_calls = [ts_to_dt(start_time) + datetime.timedelta(hours=i) for i in range(1, 13)]\n for e in expected_calls:\n mock_gc.assert_any_call(e)\n\n\ndef run_rule_query_exception(ea, mock_es):\n with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es_init:\n mock_es_init.return_value = mock_es\n ea.run_rule(ea.rules[0], END, START)\n\n # Assert neither add_data nor garbage_collect were called\n # and that starttime did not change\n assert ea.rules[0].get('starttime') == START\n assert ea.rules[0]['type'].add_data.call_count == 0\n assert ea.rules[0]['type'].garbage_collect.call_count == 0\n assert ea.rules[0]['type'].add_count_data.call_count == 0\n\n\ndef test_query_exception(ea):\n mock_es = mock.Mock()\n mock_es.search.side_effect = ElasticsearchException\n run_rule_query_exception(ea, mock_es)\n\n\ndef test_query_exception_count_query(ea):\n ea.rules[0]['use_count_query'] = True\n ea.rules[0]['doc_type'] = 'blahblahblahblah'\n mock_es = mock.Mock()\n mock_es.count.side_effect = ElasticsearchException\n run_rule_query_exception(ea, mock_es)\n\n\ndef test_match_with_module(ea):\n mod = BaseEnhancement(ea.rules[0])\n mod.process = mock.Mock()\n ea.rules[0]['match_enhancements'] = [mod]\n test_match(ea)\n mod.process.assert_called_with({'@timestamp': END, 'num_hits': 0, 'num_matches': 1})\n\n\ndef test_match_with_module_from_pending(ea):\n mod = BaseEnhancement(ea.rules[0])\n mod.process = mock.Mock()\n ea.rules[0]['match_enhancements'] = [mod]\n ea.rules[0].pop('aggregation')\n pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'],\n 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP}\n # First call, return the pending alert, second, no associated aggregated alerts\n ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}},\n {'hits': {'hits': []}}]\n ea.send_pending_alerts()\n assert mod.process.call_count == 0\n\n # If aggregation is set, enhancement IS called\n pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'],\n 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP}\n ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}},\n {'hits': {'hits': []}}]\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)\n ea.send_pending_alerts()\n assert mod.process.call_count == 1\n\n\ndef test_match_with_module_with_agg(ea):\n mod = BaseEnhancement(ea.rules[0])\n mod.process = mock.Mock()\n ea.rules[0]['match_enhancements'] = [mod]\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15)\n hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP])\n ea.thread_data.current_es.search.return_value = hits\n ea.rules[0]['type'].matches = [{'@timestamp': END}]\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert mod.process.call_count == 0\n\n\ndef test_match_with_enhancements_first(ea):\n mod = BaseEnhancement(ea.rules[0])\n mod.process = mock.Mock()\n ea.rules[0]['match_enhancements'] = [mod]\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15)\n ea.rules[0]['run_enhancements_first'] = True\n hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP])\n ea.thread_data.current_es.search.return_value = hits\n ea.rules[0]['type'].matches = [{'@timestamp': END}]\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n with mock.patch.object(ea, 'add_aggregated_alert') as add_alert:\n ea.run_rule(ea.rules[0], END, START)\n mod.process.assert_called_with({'@timestamp': END, 'num_hits': 0, 'num_matches': 1})\n assert add_alert.call_count == 1\n\n # Assert that dropmatchexception behaves properly\n mod.process = mock.MagicMock(side_effect=DropMatchException)\n ea.rules[0]['type'].matches = [{'@timestamp': END}]\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n with mock.patch.object(ea, 'add_aggregated_alert') as add_alert:\n ea.run_rule(ea.rules[0], END, START)\n mod.process.assert_called_with({'@timestamp': END, 'num_hits': 0, 'num_matches': 1})\n assert add_alert.call_count == 0\n\n\ndef test_agg_matchtime(ea):\n ea.max_aggregation = 1337\n hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']\n alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10))\n hits = generate_hits(hits_timestamps)\n ea.thread_data.current_es.search.return_value = hits\n with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:\n # Aggregate first two, query over full range\n mock_es.return_value = ea.thread_data.current_es\n ea.rules[0]['aggregate_by_match_time'] = True\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)\n ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]\n ea.run_rule(ea.rules[0], END, START)\n\n # Assert that the three matches were added to Elasticsearch\n call1 = ea.writeback_es.index.call_args_list[0][1]['body']\n call2 = ea.writeback_es.index.call_args_list[1][1]['body']\n call3 = ea.writeback_es.index.call_args_list[2][1]['body']\n assert call1['match_body']['@timestamp'] == '2014-09-26T12:34:45'\n assert not call1['alert_sent']\n assert 'aggregate_id' not in call1\n assert call1['alert_time'] == alerttime1\n\n assert call2['match_body']['@timestamp'] == '2014-09-26T12:40:45'\n assert not call2['alert_sent']\n assert call2['aggregate_id'] == 'ABCD'\n\n assert call3['match_body']['@timestamp'] == '2014-09-26T12:47:45'\n assert not call3['alert_sent']\n assert 'aggregate_id' not in call3\n\n # First call - Find all pending alerts (only entries without agg_id)\n # Second call - Find matches with agg_id == 'ABCD'\n # Third call - Find matches with agg_id == 'CDEF'\n ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1},\n {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}},\n {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}},\n {'hits': {'total': 0, 'hits': []}}]\n\n with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:\n ea.send_pending_alerts()\n # Assert that current_es was refreshed from the aggregate rules\n assert mock_es.called_with(host='', port='')\n assert mock_es.call_count == 2\n assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]])\n\n call1 = ea.writeback_es.deprecated_search.call_args_list[7][1]['body']\n call2 = ea.writeback_es.deprecated_search.call_args_list[8][1]['body']\n call3 = ea.writeback_es.deprecated_search.call_args_list[9][1]['body']\n call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body']\n\n assert 'alert_time' in call2['filter']['range']\n assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD'\n assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF'\n assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337\n\n\ndef test_agg_not_matchtime(ea):\n ea.max_aggregation = 1337\n hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']\n match_time = ts_to_dt('2014-09-26T12:55:00Z')\n hits = generate_hits(hits_timestamps)\n ea.thread_data.current_es.search.return_value = hits\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n with mock.patch('elastalert.elastalert.ts_now', return_value=match_time):\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)\n ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]\n ea.run_rule(ea.rules[0], END, START)\n\n # Assert that the three matches were added to Elasticsearch\n call1 = ea.writeback_es.index.call_args_list[0][1]['body']\n call2 = ea.writeback_es.index.call_args_list[1][1]['body']\n call3 = ea.writeback_es.index.call_args_list[2][1]['body']\n assert call1['match_body']['@timestamp'] == '2014-09-26T12:34:45'\n assert not call1['alert_sent']\n assert 'aggregate_id' not in call1\n assert call1['alert_time'] == dt_to_ts(match_time + datetime.timedelta(minutes=10))\n\n assert call2['match_body']['@timestamp'] == '2014-09-26T12:40:45'\n assert not call2['alert_sent']\n assert call2['aggregate_id'] == 'ABCD'\n\n assert call3['match_body']['@timestamp'] == '2014-09-26T12:47:45'\n assert not call3['alert_sent']\n assert call3['aggregate_id'] == 'ABCD'\n\n\ndef test_agg_cron(ea):\n ea.max_aggregation = 1337\n hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45']\n hits = generate_hits(hits_timestamps)\n ea.thread_data.current_es.search.return_value = hits\n alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00'))\n alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00'))\n\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n with mock.patch('elastalert.elastalert.croniter.get_next') as mock_ts:\n # Aggregate first two, query over full range\n mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')),\n dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))]\n ea.rules[0]['aggregation'] = {'schedule': '*/5 * * * *'}\n ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]\n ea.run_rule(ea.rules[0], END, START)\n\n # Assert that the three matches were added to Elasticsearch\n call1 = ea.writeback_es.index.call_args_list[0][1]['body']\n call2 = ea.writeback_es.index.call_args_list[1][1]['body']\n call3 = ea.writeback_es.index.call_args_list[2][1]['body']\n\n assert call1['match_body']['@timestamp'] == '2014-09-26T12:34:45'\n assert not call1['alert_sent']\n assert 'aggregate_id' not in call1\n assert call1['alert_time'] == alerttime1\n\n assert call2['match_body']['@timestamp'] == '2014-09-26T12:40:45'\n assert not call2['alert_sent']\n assert call2['aggregate_id'] == 'ABCD'\n\n assert call3['match_body']['@timestamp'] == '2014-09-26T12:47:45'\n assert call3['alert_time'] == alerttime2\n assert not call3['alert_sent']\n assert 'aggregate_id' not in call3\n\n\ndef test_agg_no_writeback_connectivity(ea):\n \"\"\" Tests that if writeback_es throws an exception, the matches will be added to 'agg_matches' and when\n run again, that they will be passed again to add_aggregated_alert \"\"\"\n hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'\n hits = generate_hits([hit1, hit2, hit3])\n ea.thread_data.current_es.search.return_value = hits\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)\n ea.rules[0]['type'].matches = [{'@timestamp': hit1},\n {'@timestamp': hit2},\n {'@timestamp': hit3}]\n ea.writeback_es.index.side_effect = elasticsearch.exceptions.ElasticsearchException('Nope')\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n with mock.patch.object(ea, 'find_pending_aggregate_alert', return_value=None):\n ea.run_rule(ea.rules[0], END, START)\n\n assert ea.rules[0]['agg_matches'] == [{'@timestamp': hit1, 'num_hits': 0, 'num_matches': 3},\n {'@timestamp': hit2, 'num_hits': 0, 'num_matches': 3},\n {'@timestamp': hit3, 'num_hits': 0, 'num_matches': 3}]\n\n ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}}\n ea.add_aggregated_alert = mock.Mock()\n\n with mock.patch.object(ea, 'run_query'):\n ea.run_rule(ea.rules[0], END, START)\n\n ea.add_aggregated_alert.assert_any_call({'@timestamp': hit1, 'num_hits': 0, 'num_matches': 3}, ea.rules[0])\n ea.add_aggregated_alert.assert_any_call({'@timestamp': hit2, 'num_hits': 0, 'num_matches': 3}, ea.rules[0])\n ea.add_aggregated_alert.assert_any_call({'@timestamp': hit3, 'num_hits': 0, 'num_matches': 3}, ea.rules[0])\n\n\ndef test_agg_with_aggregation_key(ea):\n ea.max_aggregation = 1337\n hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45']\n match_time = ts_to_dt('2014-09-26T12:45:00Z')\n hits = generate_hits(hits_timestamps)\n ea.thread_data.current_es.search.return_value = hits\n with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:\n mock_es.return_value = ea.thread_data.current_es\n with mock.patch('elastalert.elastalert.ts_now', return_value=match_time):\n ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10)\n ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps]\n # Hit1 and Hit3 should be aggregated together, since they have same query_key value\n ea.rules[0]['type'].matches[0]['key'] = 'Key Value 1'\n ea.rules[0]['type'].matches[1]['key'] = 'Key Value 2'\n ea.rules[0]['type'].matches[2]['key'] = 'Key Value 1'\n ea.rules[0]['aggregation_key'] = 'key'\n ea.run_rule(ea.rules[0], END, START)\n\n # Assert that the three matches were added to elasticsearch\n call1 = ea.writeback_es.index.call_args_list[0][1]['body']\n call2 = ea.writeback_es.index.call_args_list[1][1]['body']\n call3 = ea.writeback_es.index.call_args_list[2][1]['body']\n assert call1['match_body']['key'] == 'Key Value 1'\n assert not call1['alert_sent']\n assert 'aggregate_id' not in call1\n assert 'aggregation_key' in call1\n assert call1['aggregation_key'] == 'Key Value 1'\n assert call1['alert_time'] == dt_to_ts(match_time + datetime.timedelta(minutes=10))\n\n assert call2['match_body']['key'] == 'Key Value 2'\n assert not call2['alert_sent']\n assert 'aggregate_id' not in call2\n assert 'aggregation_key' in call2\n assert call2['aggregation_key'] == 'Key Value 2'\n assert call2['alert_time'] == dt_to_ts(match_time + datetime.timedelta(minutes=10))\n\n assert call3['match_body']['key'] == 'Key Value 1'\n assert not call3['alert_sent']\n # Call3 should have it's aggregate_id set to call1's _id\n # It should also have the same alert_time as call1\n assert call3['aggregate_id'] == 'ABCD'\n assert 'aggregation_key' in call3\n assert call3['aggregation_key'] == 'Key Value 1'\n assert call3['alert_time'] == dt_to_ts(match_time + datetime.timedelta(minutes=10))\n\n # First call - Find all pending alerts (only entries without agg_id)\n # Second call - Find matches with agg_id == 'ABCD'\n # Third call - Find matches with agg_id == 'CDEF'\n ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1},\n {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}},\n {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}},\n {'hits': {'total': 0, 'hits': []}}]\n\n with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es:\n mock_es.return_value = ea.thread_data.current_es\n ea.send_pending_alerts()\n # Assert that current_es was refreshed from the aggregate rules\n assert mock_es.called_with(host='', port='')\n assert mock_es.call_count == 2\n assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]])\n\n call1 = ea.writeback_es.deprecated_search.call_args_list[7][1]['body']\n call2 = ea.writeback_es.deprecated_search.call_args_list[8][1]['body']\n call3 = ea.writeback_es.deprecated_search.call_args_list[9][1]['body']\n call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body']\n\n assert 'alert_time' in call2['filter']['range']\n assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD'\n assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF'\n assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337\n\n\ndef test_silence(ea):\n # Silence test rule for 4 hours\n ea.args.rule = 'test_rule.yaml' # Not a real name, just has to be set\n ea.args.silence = 'hours=4'\n ea.silence()\n\n # Don't alert even with a match\n match = [{'@timestamp': '2014-11-17T00:00:00'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 0\n\n # Mock ts_now() to +5 hours, alert on match\n match = [{'@timestamp': '2014-11-17T00:00:00'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.ts_now') as mock_ts:\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n # Converted twice to add tzinfo\n mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n\ndef test_compound_query_key(ea):\n ea.rules[0]['query_key'] = 'this,that,those'\n ea.rules[0]['compound_query_key'] = ['this', 'that', 'those']\n hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that='ā˜ƒ', those=4)\n ea.thread_data.current_es.search.return_value = hits\n ea.run_query(ea.rules[0], START, END)\n call_args = ea.rules[0]['type'].add_data.call_args_list[0]\n assert 'this,that,those' in call_args[0][0][0]\n assert call_args[0][0][0]['this,that,those'] == 'abc, ā˜ƒ, 4'\n\n\ndef test_silence_query_key(ea):\n # Silence test rule for 4 hours\n ea.args.rule = 'test_rule.yaml' # Not a real name, just has to be set\n ea.args.silence = 'hours=4'\n ea.silence('anytest.qlo')\n\n # Don't alert even with a match\n match = [{'@timestamp': '2014-11-17T00:00:00', 'username': 'qlo'}]\n ea.rules[0]['type'].matches = match\n ea.rules[0]['query_key'] = 'username'\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 0\n\n # If there is a new record with a different value for the query_key, we should get an alert\n match = [{'@timestamp': '2014-11-17T00:00:01', 'username': 'dpopes'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n # Mock ts_now() to +5 hours, alert on match\n match = [{'@timestamp': '2014-11-17T00:00:00', 'username': 'qlo'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.ts_now') as mock_ts:\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n # Converted twice to add tzinfo\n mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5)))\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 2\n\n\ndef test_realert(ea):\n hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)]\n matches = [{'@timestamp': x} for x in hits]\n ea.thread_data.current_es.search.return_value = hits\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.rules[0]['realert'] = datetime.timedelta(seconds=50)\n ea.rules[0]['type'].matches = matches\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n # Doesn't alert again\n matches = [{'@timestamp': x} for x in hits]\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n ea.rules[0]['type'].matches = matches\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n # mock ts_now() to past the realert time\n matches = [{'@timestamp': hits[0]}]\n with mock.patch('elastalert.elastalert.ts_now') as mock_ts:\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n # mock_ts is converted twice to add tzinfo\n mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(minutes=10)))\n ea.rules[0]['type'].matches = matches\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 2\n\n\ndef test_realert_with_query_key(ea):\n ea.rules[0]['query_key'] = 'username'\n ea.rules[0]['realert'] = datetime.timedelta(minutes=10)\n\n # Alert and silence username: qlo\n match = [{'@timestamp': '2014-11-17T00:00:00', 'username': 'qlo'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n # Dont alert again for same username\n match = [{'@timestamp': '2014-11-17T00:05:00', 'username': 'qlo'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n # Do alert with a different value\n match = [{'@timestamp': '2014-11-17T00:05:00', 'username': ''}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 2\n\n # Alert with query_key missing\n match = [{'@timestamp': '2014-11-17T00:05:00'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 3\n\n # Still alert with a different value\n match = [{'@timestamp': '2014-11-17T00:05:00', 'username': 'ghengis_khan'}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 4\n\n\ndef test_realert_with_nested_query_key(ea):\n ea.rules[0]['query_key'] = 'user.name'\n ea.rules[0]['realert'] = datetime.timedelta(minutes=10)\n\n # Alert and silence username: qlo\n match = [{'@timestamp': '2014-11-17T00:00:00', 'user': {'name': 'qlo'}}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n # Dont alert again for same username\n match = [{'@timestamp': '2014-11-17T00:05:00', 'user': {'name': 'qlo'}}]\n ea.rules[0]['type'].matches = match\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.run_rule(ea.rules[0], END, START)\n assert ea.rules[0]['alert'][0].alert.call_count == 1\n\n\ndef test_count(ea):\n ea.rules[0]['use_count_query'] = True\n ea.rules[0]['doc_type'] = 'doctype'\n with mock.patch('elastalert.elastalert.elasticsearch_client'), \\\n mock.patch.object(ea, 'get_hits_count') as mock_hits:\n ea.run_rule(ea.rules[0], END, START)\n\n # Assert that es.count is run against every run_every timeframe between START and END\n start = START\n query = {\n 'query': {'filtered': {\n 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}}\n while END - start > ea.run_every:\n end = start + ea.run_every\n query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end)\n query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start)\n mock_hits.assert_any_call(mock.ANY, start, end, mock.ANY)\n start = start + ea.run_every\n\n\ndef run_and_assert_segmented_queries(ea, start, end, segment_size):\n with mock.patch.object(ea, 'run_query') as mock_run_query:\n ea.run_rule(ea.rules[0], end, start)\n original_end, original_start = end, start\n for call_args in mock_run_query.call_args_list:\n end = min(start + segment_size, original_end)\n assert call_args[0][1:3] == (start, end)\n start += segment_size\n\n # Assert elastalert_status was created for the entire time range\n assert ea.writeback_es.index.call_args_list[-1][1]['body']['starttime'] == dt_to_ts(original_start)\n if ea.rules[0].get('aggregation_query_element'):\n assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(\n original_end - (original_end - end))\n assert original_end - end < segment_size\n else:\n assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end)\n\n\ndef test_query_segmenting_reset_num_hits(ea):\n # Tests that num_hits gets reset every time run_query is run\n def assert_num_hits_reset():\n assert ea.thread_data.num_hits == 0\n ea.thread_data.num_hits += 10\n with mock.patch.object(ea, 'run_query') as mock_run_query:\n mock_run_query.side_effect = assert_num_hits_reset()\n ea.run_rule(ea.rules[0], END, START)\n assert mock_run_query.call_count > 1\n\n\ndef test_query_segmenting(ea):\n # buffer_time segments with normal queries\n ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=53)\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n run_and_assert_segmented_queries(ea, START, END, ea.rules[0]['buffer_time'])\n\n # run_every segments with count queries\n ea.rules[0]['use_count_query'] = True\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n run_and_assert_segmented_queries(ea, START, END, ea.run_every)\n\n # run_every segments with terms queries\n ea.rules[0].pop('use_count_query')\n ea.rules[0]['use_terms_query'] = True\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n run_and_assert_segmented_queries(ea, START, END, ea.run_every)\n\n # buffer_time segments with terms queries\n ea.rules[0].pop('use_terms_query')\n ea.rules[0]['aggregation_query_element'] = {'term': 'term_val'}\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=30)\n run_and_assert_segmented_queries(ea, START, END, ea.rules[0]['buffer_time'])\n\n # partial segment size scenario\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=53)\n run_and_assert_segmented_queries(ea, START, END, ea.rules[0]['buffer_time'])\n\n # run every segmenting\n ea.rules[0]['use_run_every_query_size'] = True\n with mock.patch('elastalert.elastalert.elasticsearch_client'):\n run_and_assert_segmented_queries(ea, START, END, ea.run_every)\n\n\ndef test_get_starttime(ea):\n endtime = '2015-01-01T00:00:00Z'\n mock_es = mock.Mock()\n mock_es.search.return_value = {'hits': {'hits': [{'_source': {'endtime': endtime}}]}}\n mock_es.info.return_value = {'version': {'number': '2.0'}}\n ea.writeback_es = mock_es\n\n # 4 days old, will return endtime\n with mock.patch('elastalert.elastalert.ts_now') as mock_ts:\n mock_ts.return_value = ts_to_dt('2015-01-05T00:00:00Z') # 4 days ahead of the endtime\n assert ea.get_starttime(ea.rules[0]) == ts_to_dt(endtime)\n\n # 10 days old, will return None\n with mock.patch('elastalert.elastalert.ts_now') as mock_ts:\n mock_ts.return_value = ts_to_dt('2015-01-11T00:00:00Z') # 10 days ahead of the endtime\n assert ea.get_starttime(ea.rules[0]) is None\n\n\ndef test_set_starttime(ea):\n # standard query, no starttime, no last run\n end = ts_to_dt('2014-10-10T10:10:10')\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = None\n ea.set_starttime(ea.rules[0], end)\n assert mock_gs.call_count == 1\n assert ea.rules[0]['starttime'] == end - ea.buffer_time\n\n # Standard query, no starttime, rule specific buffer_time\n ea.rules[0].pop('starttime')\n ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=37)\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = None\n ea.set_starttime(ea.rules[0], end)\n assert mock_gs.call_count == 1\n assert ea.rules[0]['starttime'] == end - datetime.timedelta(minutes=37)\n ea.rules[0].pop('buffer_time')\n\n # Standard query, no starttime, last run\n ea.rules[0].pop('starttime')\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = ts_to_dt('2014-10-10T00:00:00')\n ea.set_starttime(ea.rules[0], end)\n assert mock_gs.call_count == 1\n assert ea.rules[0]['starttime'] == ts_to_dt('2014-10-10T00:00:00')\n\n # Standard query, no starttime, last run, assure buffer_time doesn't go past\n ea.rules[0].pop('starttime')\n ea.rules[0]['buffer_time'] = datetime.timedelta(weeks=1000)\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = ts_to_dt('2014-10-09T00:00:00')\n # First call sets minumum_time\n ea.set_starttime(ea.rules[0], end)\n # Second call uses buffer_time, but it goes past minimum\n ea.set_starttime(ea.rules[0], end)\n assert ea.rules[0]['starttime'] == ts_to_dt('2014-10-09T00:00:00')\n\n # Standard query, starttime\n ea.rules[0].pop('buffer_time')\n ea.rules[0].pop('minimum_starttime')\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = None\n ea.set_starttime(ea.rules[0], end)\n assert mock_gs.call_count == 0\n assert ea.rules[0]['starttime'] == end - ea.buffer_time\n\n # Count query, starttime, no previous endtime\n ea.rules[0]['use_count_query'] = True\n ea.rules[0]['doc_type'] = 'blah'\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = None\n ea.set_starttime(ea.rules[0], end)\n assert mock_gs.call_count == 0\n assert ea.rules[0]['starttime'] == end - ea.run_every\n\n # Count query, with previous endtime\n with mock.patch('elastalert.elastalert.elasticsearch_client'), \\\n mock.patch.object(ea, 'get_hits_count'):\n ea.run_rule(ea.rules[0], END, START)\n ea.set_starttime(ea.rules[0], end)\n assert ea.rules[0]['starttime'] == END\n\n # buffer_time doesn't go past previous endtime\n ea.rules[0].pop('use_count_query')\n ea.rules[0]['previous_endtime'] = end - ea.buffer_time * 2\n ea.set_starttime(ea.rules[0], end)\n assert ea.rules[0]['starttime'] == ea.rules[0]['previous_endtime']\n\n # Make sure starttime is updated if previous_endtime isn't used\n ea.rules[0]['previous_endtime'] = end - ea.buffer_time / 2\n ea.rules[0]['starttime'] = ts_to_dt('2014-10-09T00:00:01')\n ea.set_starttime(ea.rules[0], end)\n assert ea.rules[0]['starttime'] == end - ea.buffer_time\n\n # scan_entire_timeframe\n ea.rules[0].pop('previous_endtime')\n ea.rules[0].pop('starttime')\n ea.rules[0]['timeframe'] = datetime.timedelta(days=3)\n ea.rules[0]['scan_entire_timeframe'] = True\n with mock.patch.object(ea, 'get_starttime') as mock_gs:\n mock_gs.return_value = None\n ea.set_starttime(ea.rules[0], end)\n assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3)\n\n\ndef test_kibana_dashboard(ea):\n match = {'@timestamp': '2014-10-11T00:00:00'}\n mock_es = mock.Mock()\n ea.rules[0]['use_kibana_dashboard'] = 'my dashboard'\n with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es_init:\n mock_es_init.return_value = mock_es\n\n # No dashboard found\n mock_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}}\n with pytest.raises(EAException):\n ea.use_kibana_link(ea.rules[0], match)\n mock_call = mock_es.deprecated_search.call_args_list[0][1]\n assert mock_call['body'] == {'query': {'term': {'_id': 'my dashboard'}}}\n\n # Dashboard found\n mock_es.index.return_value = {'_id': 'ABCDEFG'}\n mock_es.deprecated_search.return_value = {'hits': {'hits': [{'_source': {'dashboard': json.dumps(dashboard_temp)}}]}}\n url = ea.use_kibana_link(ea.rules[0], match)\n assert 'ABCDEFG' in url\n db = json.loads(mock_es.index.call_args_list[0][1]['body']['dashboard'])\n assert 'anytest' in db['title']\n\n # Query key filtering added\n ea.rules[0]['query_key'] = 'foobar'\n match['foobar'] = 'baz'\n url = ea.use_kibana_link(ea.rules[0], match)\n db = json.loads(mock_es.index.call_args_list[-1][1]['body']['dashboard'])\n assert db['services']['filter']['list']['1']['field'] == 'foobar'\n assert db['services']['filter']['list']['1']['query'] == '\"baz\"'\n\n # Compound query key\n ea.rules[0]['query_key'] = 'foo,bar'\n ea.rules[0]['compound_query_key'] = ['foo', 'bar']\n match['foo'] = 'cat'\n match['bar'] = 'dog'\n match['foo,bar'] = 'cat, dog'\n url = ea.use_kibana_link(ea.rules[0], match)\n db = json.loads(mock_es.index.call_args_list[-1][1]['body']['dashboard'])\n found_filters = 0\n for filter_id, filter_dict in list(db['services']['filter']['list'].items()):\n if (filter_dict['field'] == 'foo' and filter_dict['query'] == '\"cat\"') or \\\n (filter_dict['field'] == 'bar' and filter_dict['query'] == '\"dog\"'):\n found_filters += 1\n continue\n assert found_filters == 2\n\n\ndef test_rule_changes(ea):\n ea.rule_hashes = {'rules/rule1.yaml': 'ABC',\n 'rules/rule2.yaml': 'DEF'}\n run_every = datetime.timedelta(seconds=1)\n ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': [],\n 'run_every': run_every},\n {'rule_file': 'rules/rule2.yaml', 'name': 'rule2', 'filter': [],\n 'run_every': run_every}]]\n ea.rules[1]['processed_hits'] = ['save me']\n new_hashes = {'rules/rule1.yaml': 'ABC',\n 'rules/rule3.yaml': 'XXX',\n 'rules/rule2.yaml': '!@#$'}\n\n with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes:\n with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load:\n mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml', 'run_every': run_every},\n {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml', 'run_every': run_every}]\n mock_hashes.return_value = new_hashes\n ea.load_rule_changes()\n\n # All 3 rules still exist\n assert ea.rules[0]['name'] == 'rule1'\n assert ea.rules[1]['name'] == 'rule2'\n assert ea.rules[1]['processed_hits'] == ['save me']\n assert ea.rules[2]['name'] == 'rule3'\n\n # Assert 2 and 3 were reloaded\n assert mock_load.call_count == 2\n mock_load.assert_any_call('rules/rule2.yaml', ea.conf)\n mock_load.assert_any_call('rules/rule3.yaml', ea.conf)\n\n # A new rule with a conflicting name wont load\n new_hashes = copy.copy(new_hashes)\n new_hashes.update({'rules/rule4.yaml': 'asdf'})\n with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes:\n with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load:\n with mock.patch.object(ea, 'send_notification_email') as mock_send:\n mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff',\n 'rule_file': 'rules/rule4.yaml', 'run_every': run_every}\n mock_hashes.return_value = new_hashes\n ea.load_rule_changes()\n mock_send.assert_called_once_with(exception=mock.ANY, rule_file='rules/rule4.yaml')\n assert len(ea.rules) == 3\n assert not any(['new' in rule for rule in ea.rules])\n\n # A new rule with is_enabled=False wont load\n new_hashes = copy.copy(new_hashes)\n new_hashes.update({'rules/rule4.yaml': 'asdf'})\n with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes:\n with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load:\n mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False,\n 'rule_file': 'rules/rule4.yaml', 'run_every': run_every}\n mock_hashes.return_value = new_hashes\n ea.load_rule_changes()\n assert len(ea.rules) == 3\n assert not any(['new' in rule for rule in ea.rules])\n\n # An old rule which didn't load gets reloaded\n new_hashes = copy.copy(new_hashes)\n new_hashes['rules/rule4.yaml'] = 'qwerty'\n with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes:\n with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load:\n mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml',\n 'run_every': run_every}\n mock_hashes.return_value = new_hashes\n ea.load_rule_changes()\n assert len(ea.rules) == 4\n\n # Disable a rule by removing the file\n new_hashes.pop('rules/rule4.yaml')\n with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes:\n with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load:\n mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml',\n 'run_every': run_every}\n mock_hashes.return_value = new_hashes\n ea.load_rule_changes()\n ea.scheduler.remove_job.assert_called_with(job_id='rule4')\n\n\ndef test_strf_index(ea):\n \"\"\" Test that the get_index function properly generates indexes spanning days \"\"\"\n ea.rules[0]['index'] = 'logstash-%Y.%m.%d'\n ea.rules[0]['use_strftime_index'] = True\n\n # Test formatting with times\n start = ts_to_dt('2015-01-02T12:34:45Z')\n end = ts_to_dt('2015-01-02T16:15:14Z')\n assert ea.get_index(ea.rules[0], start, end) == 'logstash-2015.01.02'\n end = ts_to_dt('2015-01-03T01:02:03Z')\n assert set(ea.get_index(ea.rules[0], start, end).split(',')) == set(['logstash-2015.01.02', 'logstash-2015.01.03'])\n\n # Test formatting for wildcard\n assert ea.get_index(ea.rules[0]) == 'logstash-*'\n ea.rules[0]['index'] = 'logstash-%Y.%m'\n assert ea.get_index(ea.rules[0]) == 'logstash-*'\n ea.rules[0]['index'] = 'logstash-%Y.%m-stuff'\n assert ea.get_index(ea.rules[0]) == 'logstash-*-stuff'\n\n\ndef test_count_keys(ea):\n ea.rules[0]['timeframe'] = datetime.timedelta(minutes=60)\n ea.rules[0]['top_count_keys'] = ['this', 'that']\n ea.rules[0]['type'].matches = {'@timestamp': END}\n ea.rules[0]['doc_type'] = 'blah'\n buckets = [{'aggregations': {\n 'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}},\n {'aggregations': {'filtered': {\n 'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}]\n ea.thread_data.current_es.deprecated_search.side_effect = buckets\n counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that'])\n calls = ea.thread_data.current_es.deprecated_search.call_args_list\n assert calls[0][1]['search_type'] == 'count'\n assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5,\n 'min_doc_count': 1}\n assert counts['top_events_this'] == {'a': 10, 'b': 5}\n assert counts['top_events_that'] == {'d': 10, 'c': 12}\n\n\ndef test_exponential_realert(ea):\n ea.rules[0]['exponential_realert'] = datetime.timedelta(days=1) # 1 day ~ 10 * 2**13 seconds\n ea.rules[0]['realert'] = datetime.timedelta(seconds=10)\n\n until = ts_to_dt('2015-03-24T00:00:00')\n ts5s = until + datetime.timedelta(seconds=5)\n ts15s = until + datetime.timedelta(seconds=15)\n ts1m = until + datetime.timedelta(minutes=1)\n ts5m = until + datetime.timedelta(minutes=5)\n ts4h = until + datetime.timedelta(hours=4)\n\n test_values = [(ts5s, until, 0), # Exp will increase to 1, 10*2**0 = 10s\n (ts15s, until, 0), # Exp will stay at 0, 10*2**0 = 10s\n (ts15s, until, 1), # Exp will increase to 2, 10*2**1 = 20s\n (ts1m, until, 2), # Exp will decrease to 1, 10*2**2 = 40s\n (ts1m, until, 3), # Exp will increase to 4, 10*2**3 = 1m20s\n (ts5m, until, 1), # Exp will lower back to 0, 10*2**1 = 20s\n (ts4h, until, 9), # Exp will lower back to 0, 10*2**9 = 1h25m\n (ts4h, until, 10), # Exp will lower back to 9, 10*2**10 = 2h50m\n (ts4h, until, 11)] # Exp will increase to 12, 10*2**11 = 5h\n results = (1, 0, 2, 1, 4, 0, 0, 9, 12)\n next_res = iter(results)\n for args in test_values:\n ea.silence_cache[ea.rules[0]['name']] = (args[1], args[2])\n next_alert, exponent = ea.next_alert_time(ea.rules[0], ea.rules[0]['name'], args[0])\n assert exponent == next(next_res)\n\n\ndef test_wait_until_responsive(ea):\n \"\"\"Unblock as soon as ElasticSearch becomes responsive.\"\"\"\n\n # Takes a while before becoming responsive.\n ea.writeback_es.indices.exists.side_effect = [\n ConnectionError(), # ES is not yet responsive.\n False, # index does not yet exist.\n True,\n ]\n\n clock = mock.MagicMock()\n clock.side_effect = [0.0, 1.0, 2.0, 3.0, 4.0]\n timeout = datetime.timedelta(seconds=3.5)\n with mock.patch('time.sleep') as sleep:\n ea.wait_until_responsive(timeout=timeout, clock=clock)\n\n # Sleep as little as we can.\n sleep.mock_calls == [\n mock.call(1.0),\n ]\n\n\ndef test_wait_until_responsive_timeout_es_not_available(ea, capsys):\n \"\"\"Bail out if ElasticSearch doesn't (quickly) become responsive.\"\"\"\n\n # Never becomes responsive :-)\n ea.writeback_es.ping.return_value = False\n ea.writeback_es.indices.exists.return_value = False\n\n clock = mock.MagicMock()\n clock.side_effect = [0.0, 1.0, 2.0, 3.0]\n timeout = datetime.timedelta(seconds=2.5)\n with mock.patch('time.sleep') as sleep:\n with pytest.raises(SystemExit) as exc:\n ea.wait_until_responsive(timeout=timeout, clock=clock)\n assert exc.value.code == 1\n\n # Ensure we get useful diagnostics.\n output, errors = capsys.readouterr()\n assert 'Could not reach ElasticSearch at \"es:14900\".' in errors\n\n # Slept until we passed the deadline.\n sleep.mock_calls == [\n mock.call(1.0),\n mock.call(1.0),\n mock.call(1.0),\n ]\n\n\ndef test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys):\n \"\"\"Bail out if ElasticSearch doesn't (quickly) become responsive.\"\"\"\n\n # Never becomes responsive :-)\n ea.writeback_es.ping.return_value = True\n ea.writeback_es.indices.exists.return_value = False\n\n clock = mock.MagicMock()\n clock.side_effect = [0.0, 1.0, 2.0, 3.0]\n timeout = datetime.timedelta(seconds=2.5)\n with mock.patch('time.sleep') as sleep:\n with pytest.raises(SystemExit) as exc:\n ea.wait_until_responsive(timeout=timeout, clock=clock)\n assert exc.value.code == 1\n\n # Ensure we get useful diagnostics.\n output, errors = capsys.readouterr()\n assert 'Writeback alias \"wb_a\" does not exist, did you run `elastalert-create-index`?' in errors\n\n # Slept until we passed the deadline.\n sleep.mock_calls == [\n mock.call(1.0),\n mock.call(1.0),\n mock.call(1.0),\n ]\n\n\ndef test_stop(ea):\n \"\"\" The purpose of this test is to make sure that calling ElastAlerter.stop() will break it\n out of a ElastAlerter.start() loop. This method exists to provide a mechanism for running\n ElastAlert with threads and thus must be tested with threads. mock_loop verifies the loop\n is running and will call stop after several iterations. \"\"\"\n\n # Exit the thread on the fourth iteration\n def mock_loop():\n for i in range(3):\n assert ea.running\n yield\n ea.stop()\n\n with mock.patch.object(ea, 'sleep_for', return_value=None):\n with mock.patch.object(ea, 'sleep_for') as mock_run:\n mock_run.side_effect = mock_loop()\n start_thread = threading.Thread(target=ea.start)\n # Set as daemon to prevent a failed test from blocking exit\n start_thread.daemon = True\n start_thread.start()\n\n # Give it a few seconds to run the loop\n start_thread.join(5)\n\n assert not ea.running\n assert not start_thread.is_alive()\n assert mock_run.call_count == 4\n\n\ndef test_notify_email(ea):\n mock_smtp = mock.Mock()\n ea.rules[0]['notify_email'] = ['foo@foo.foo', 'bar@bar.bar']\n with mock.patch('elastalert.elastalert.SMTP') as mock_smtp_f:\n mock_smtp_f.return_value = mock_smtp\n\n # Notify_email from rules, array\n ea.send_notification_email('omg', rule=ea.rules[0])\n assert set(mock_smtp.sendmail.call_args_list[0][0][1]) == set(ea.rules[0]['notify_email'])\n\n # With ea.notify_email\n ea.notify_email = ['baz@baz.baz']\n ea.send_notification_email('omg', rule=ea.rules[0])\n assert set(mock_smtp.sendmail.call_args_list[1][0][1]) == set(['baz@baz.baz'] + ea.rules[0]['notify_email'])\n\n # With ea.notify email but as single string\n ea.rules[0]['notify_email'] = 'foo@foo.foo'\n ea.send_notification_email('omg', rule=ea.rules[0])\n assert set(mock_smtp.sendmail.call_args_list[2][0][1]) == set(['baz@baz.baz', 'foo@foo.foo'])\n\n # None from rule\n ea.rules[0].pop('notify_email')\n ea.send_notification_email('omg', rule=ea.rules[0])\n assert set(mock_smtp.sendmail.call_args_list[3][0][1]) == set(['baz@baz.baz'])\n\n\ndef test_uncaught_exceptions(ea):\n e = Exception(\"Errors yo!\")\n\n # With disabling set to false\n ea.disable_rules_on_error = False\n ea.handle_uncaught_exception(e, ea.rules[0])\n assert len(ea.rules) == 1\n assert len(ea.disabled_rules) == 0\n\n # With disabling set to true\n ea.disable_rules_on_error = True\n ea.handle_uncaught_exception(e, ea.rules[0])\n assert len(ea.rules) == 0\n assert len(ea.disabled_rules) == 1\n\n # Changing the file should re-enable it\n ea.rule_hashes = {'blah.yaml': 'abc'}\n new_hashes = {'blah.yaml': 'def'}\n with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes:\n with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load:\n mock_load.side_effect = [ea.disabled_rules[0]]\n mock_hashes.return_value = new_hashes\n ea.load_rule_changes()\n assert len(ea.rules) == 1\n assert len(ea.disabled_rules) == 0\n\n # Notify email is sent\n ea.notify_email = 'qlo@example.com'\n with mock.patch.object(ea, 'send_notification_email') as mock_email:\n ea.handle_uncaught_exception(e, ea.rules[0])\n assert mock_email.call_args_list[0][1] == {'exception': e, 'rule': ea.disabled_rules[0]}\n\n\ndef test_get_top_counts_handles_no_hits_returned(ea):\n with mock.patch.object(ea, 'get_hits_terms') as mock_hits:\n mock_hits.return_value = None\n\n rule = ea.rules[0]\n starttime = datetime.datetime.now() - datetime.timedelta(minutes=10)\n endtime = datetime.datetime.now()\n keys = ['foo']\n\n all_counts = ea.get_top_counts(rule, starttime, endtime, keys)\n assert all_counts == {'top_events_foo': {}}\n\n\ndef test_remove_old_events(ea):\n now = ts_now()\n minute = datetime.timedelta(minutes=1)\n ea.rules[0]['processed_hits'] = {'foo': now - minute,\n 'bar': now - minute * 5,\n 'baz': now - minute * 15}\n ea.rules[0]['buffer_time'] = datetime.timedelta(minutes=10)\n\n # With a query delay, only events older than 20 minutes will be removed (none)\n ea.rules[0]['query_delay'] = datetime.timedelta(minutes=10)\n ea.remove_old_events(ea.rules[0])\n assert len(ea.rules[0]['processed_hits']) == 3\n\n # With no query delay, the 15 minute old event will be removed\n ea.rules[0].pop('query_delay')\n ea.remove_old_events(ea.rules[0])\n assert len(ea.rules[0]['processed_hits']) == 2\n assert 'baz' not in ea.rules[0]['processed_hits']\n\n\ndef test_query_with_whitelist_filter_es(ea):\n ea.rules[0]['_source_enabled'] = False\n ea.rules[0]['five'] = False\n ea.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}]\n ea.rules[0]['compare_key'] = \"username\"\n ea.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1']\n new_rule = copy.copy(ea.rules[0])\n ea.init_rule(new_rule, True)\n assert 'NOT username:\"xudan1\" AND NOT username:\"xudan12\" AND NOT username:\"aa1\"' \\\n in new_rule['filter'][-1]['query']['query_string']['query']\n\n\ndef test_query_with_whitelist_filter_es_five(ea_sixsix):\n ea_sixsix.rules[0]['_source_enabled'] = False\n ea_sixsix.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}]\n ea_sixsix.rules[0]['compare_key'] = \"username\"\n ea_sixsix.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1']\n new_rule = copy.copy(ea_sixsix.rules[0])\n ea_sixsix.init_rule(new_rule, True)\n assert 'NOT username:\"xudan1\" AND NOT username:\"xudan12\" AND NOT username:\"aa1\"' in \\\n new_rule['filter'][-1]['query_string']['query']\n\n\ndef test_query_with_blacklist_filter_es(ea):\n ea.rules[0]['_source_enabled'] = False\n ea.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}]\n ea.rules[0]['compare_key'] = \"username\"\n ea.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1']\n new_rule = copy.copy(ea.rules[0])\n ea.init_rule(new_rule, True)\n assert 'username:\"xudan1\" OR username:\"xudan12\" OR username:\"aa1\"' in \\\n new_rule['filter'][-1]['query']['query_string']['query']\n\n\ndef test_query_with_blacklist_filter_es_five(ea_sixsix):\n ea_sixsix.rules[0]['_source_enabled'] = False\n ea_sixsix.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}]\n ea_sixsix.rules[0]['compare_key'] = \"username\"\n ea_sixsix.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1']\n ea_sixsix.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1']\n new_rule = copy.copy(ea_sixsix.rules[0])\n ea_sixsix.init_rule(new_rule, True)\n assert 'username:\"xudan1\" OR username:\"xudan12\" OR username:\"aa1\"' in new_rule['filter'][-1]['query_string'][\n 'query']\n" }, { "path": "tests/conftest.py", "content": "# -*- coding: utf-8 -*-\nimport datetime\nimport logging\nimport os\n\nimport mock\nimport pytest\n\nimport elastalert.elastalert\nimport elastalert.util\nfrom elastalert.util import dt_to_ts\nfrom elastalert.util import ts_to_dt\n\nwriteback_index = 'wb'\n\n\ndef pytest_addoption(parser):\n parser.addoption(\n \"--runelasticsearch\", action=\"store_true\", default=False, help=\"run elasticsearch tests\"\n )\n\n\ndef pytest_collection_modifyitems(config, items):\n if config.getoption(\"--runelasticsearch\"):\n # --runelasticsearch given in cli: run elasticsearch tests, skip ordinary unit tests\n skip_unit_tests = pytest.mark.skip(reason=\"not running when --runelasticsearch option is used to run\")\n for item in items:\n if \"elasticsearch\" not in item.keywords:\n item.add_marker(skip_unit_tests)\n else:\n # skip elasticsearch tests\n skip_elasticsearch = pytest.mark.skip(reason=\"need --runelasticsearch option to run\")\n for item in items:\n if \"elasticsearch\" in item.keywords:\n item.add_marker(skip_elasticsearch)\n\n\n@pytest.fixture(scope='function', autouse=True)\ndef reset_loggers():\n \"\"\"Prevent logging handlers from capturing temporary file handles.\n\n For example, a test that uses the `capsys` fixture and calls\n `logging.exception()` will initialize logging with a default handler that\n captures `sys.stderr`. When the test ends, the file handles will be closed\n and `sys.stderr` will be returned to its original handle, but the logging\n will have a dangling reference to the temporary handle used in the `capsys`\n fixture.\n\n \"\"\"\n logger = logging.getLogger()\n for handler in logger.handlers:\n logger.removeHandler(handler)\n\n\nclass mock_es_indices_client(object):\n def __init__(self):\n self.exists = mock.Mock(return_value=True)\n\n\nclass mock_es_client(object):\n def __init__(self, host='es', port=14900):\n self.host = host\n self.port = port\n self.return_hits = []\n self.search = mock.Mock()\n self.deprecated_search = mock.Mock()\n self.create = mock.Mock()\n self.index = mock.Mock()\n self.delete = mock.Mock()\n self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '2.0'}})\n self.ping = mock.Mock(return_value=True)\n self.indices = mock_es_indices_client()\n self.es_version = mock.Mock(return_value='2.0')\n self.is_atleastfive = mock.Mock(return_value=False)\n self.is_atleastsix = mock.Mock(return_value=False)\n self.is_atleastsixtwo = mock.Mock(return_value=False)\n self.is_atleastsixsix = mock.Mock(return_value=False)\n self.is_atleastseven = mock.Mock(return_value=False)\n self.resolve_writeback_index = mock.Mock(return_value=writeback_index)\n\n\nclass mock_es_sixsix_client(object):\n def __init__(self, host='es', port=14900):\n self.host = host\n self.port = port\n self.return_hits = []\n self.search = mock.Mock()\n self.deprecated_search = mock.Mock()\n self.create = mock.Mock()\n self.index = mock.Mock()\n self.delete = mock.Mock()\n self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}})\n self.ping = mock.Mock(return_value=True)\n self.indices = mock_es_indices_client()\n self.es_version = mock.Mock(return_value='6.6.0')\n self.is_atleastfive = mock.Mock(return_value=True)\n self.is_atleastsix = mock.Mock(return_value=True)\n self.is_atleastsixtwo = mock.Mock(return_value=False)\n self.is_atleastsixsix = mock.Mock(return_value=True)\n self.is_atleastseven = mock.Mock(return_value=False)\n\n def writeback_index_side_effect(index, doc_type):\n if doc_type == 'silence':\n return index + '_silence'\n elif doc_type == 'past_elastalert':\n return index + '_past'\n elif doc_type == 'elastalert_status':\n return index + '_status'\n elif doc_type == 'elastalert_error':\n return index + '_error'\n return index\n\n self.resolve_writeback_index = mock.Mock(side_effect=writeback_index_side_effect)\n\n\nclass mock_rule_loader(object):\n def __init__(self, conf):\n self.base_config = conf\n self.load = mock.Mock()\n self.get_hashes = mock.Mock()\n self.load_configuration = mock.Mock()\n\n\nclass mock_ruletype(object):\n def __init__(self):\n self.add_data = mock.Mock()\n self.add_count_data = mock.Mock()\n self.add_terms_data = mock.Mock()\n self.matches = []\n self.get_match_data = lambda x: x\n self.get_match_str = lambda x: \"some stuff happened\"\n self.garbage_collect = mock.Mock()\n\n\nclass mock_alert(object):\n def __init__(self):\n self.alert = mock.Mock()\n\n def get_info(self):\n return {'type': 'mock'}\n\n\n@pytest.fixture\ndef ea():\n rules = [{'es_host': '',\n 'es_port': 14900,\n 'name': 'anytest',\n 'index': 'idx',\n 'filter': [],\n 'include': ['@timestamp'],\n 'aggregation': datetime.timedelta(0),\n 'realert': datetime.timedelta(0),\n 'processed_hits': {},\n 'timestamp_field': '@timestamp',\n 'match_enhancements': [],\n 'rule_file': 'blah.yaml',\n 'max_query_size': 10000,\n 'ts_to_dt': ts_to_dt,\n 'dt_to_ts': dt_to_ts,\n '_source_enabled': True,\n 'run_every': datetime.timedelta(seconds=15)}]\n conf = {'rules_folder': 'rules',\n 'run_every': datetime.timedelta(minutes=10),\n 'buffer_time': datetime.timedelta(minutes=5),\n 'alert_time_limit': datetime.timedelta(hours=24),\n 'es_host': 'es',\n 'es_port': 14900,\n 'writeback_index': 'wb',\n 'writeback_alias': 'wb_a',\n 'rules': rules,\n 'max_query_size': 10000,\n 'old_query_limit': datetime.timedelta(weeks=1),\n 'disable_rules_on_error': False,\n 'scroll_keepalive': '30s'}\n elastalert.util.elasticsearch_client = mock_es_client\n conf['rules_loader'] = mock_rule_loader(conf)\n elastalert.elastalert.elasticsearch_client = mock_es_client\n with mock.patch('elastalert.elastalert.load_conf') as load_conf:\n with mock.patch('elastalert.elastalert.BackgroundScheduler'):\n load_conf.return_value = conf\n conf['rules_loader'].load.return_value = rules\n conf['rules_loader'].get_hashes.return_value = {}\n ea = elastalert.elastalert.ElastAlerter(['--pin_rules'])\n ea.rules[0]['type'] = mock_ruletype()\n ea.rules[0]['alert'] = [mock_alert()]\n ea.writeback_es = mock_es_client()\n ea.writeback_es.search.return_value = {'hits': {'hits': []}, 'total': 0}\n ea.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}}\n ea.writeback_es.index.return_value = {'_id': 'ABCD', 'created': True}\n ea.current_es = mock_es_client('', '')\n ea.thread_data.current_es = ea.current_es\n ea.thread_data.num_hits = 0\n ea.thread_data.num_dupes = 0\n return ea\n\n\n@pytest.fixture\ndef ea_sixsix():\n rules = [{'es_host': '',\n 'es_port': 14900,\n 'name': 'anytest',\n 'index': 'idx',\n 'filter': [],\n 'include': ['@timestamp'],\n 'run_every': datetime.timedelta(seconds=1),\n 'aggregation': datetime.timedelta(0),\n 'realert': datetime.timedelta(0),\n 'processed_hits': {},\n 'timestamp_field': '@timestamp',\n 'match_enhancements': [],\n 'rule_file': 'blah.yaml',\n 'max_query_size': 10000,\n 'ts_to_dt': ts_to_dt,\n 'dt_to_ts': dt_to_ts,\n '_source_enabled': True}]\n conf = {'rules_folder': 'rules',\n 'run_every': datetime.timedelta(minutes=10),\n 'buffer_time': datetime.timedelta(minutes=5),\n 'alert_time_limit': datetime.timedelta(hours=24),\n 'es_host': 'es',\n 'es_port': 14900,\n 'writeback_index': writeback_index,\n 'writeback_alias': 'wb_a',\n 'rules': rules,\n 'max_query_size': 10000,\n 'old_query_limit': datetime.timedelta(weeks=1),\n 'disable_rules_on_error': False,\n 'scroll_keepalive': '30s'}\n conf['rules_loader'] = mock_rule_loader(conf)\n elastalert.elastalert.elasticsearch_client = mock_es_sixsix_client\n elastalert.util.elasticsearch_client = mock_es_sixsix_client\n with mock.patch('elastalert.elastalert.load_conf') as load_conf:\n with mock.patch('elastalert.elastalert.BackgroundScheduler'):\n load_conf.return_value = conf\n conf['rules_loader'].load.return_value = rules\n conf['rules_loader'].get_hashes.return_value = {}\n ea_sixsix = elastalert.elastalert.ElastAlerter(['--pin_rules'])\n ea_sixsix.rules[0]['type'] = mock_ruletype()\n ea_sixsix.rules[0]['alert'] = [mock_alert()]\n ea_sixsix.writeback_es = mock_es_sixsix_client()\n ea_sixsix.writeback_es.search.return_value = {'hits': {'hits': []}}\n ea_sixsix.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}}\n ea_sixsix.writeback_es.index.return_value = {'_id': 'ABCD'}\n ea_sixsix.current_es = mock_es_sixsix_client('', -1)\n return ea_sixsix\n\n\n@pytest.fixture(scope='function')\ndef environ():\n \"\"\"py.test fixture to get a fresh mutable environment.\"\"\"\n old_env = os.environ\n new_env = dict(list(old_env.items()))\n os.environ = new_env\n yield os.environ\n os.environ = old_env\n" }, { "path": "tests/create_index_test.py", "content": "# -*- coding: utf-8 -*-\nimport json\n\nimport pytest\n\nimport elastalert.create_index\n\nes_mappings = [\n 'elastalert',\n 'elastalert_error',\n 'elastalert_status',\n 'past_elastalert',\n 'silence'\n]\n\n\n@pytest.mark.parametrize('es_mapping', es_mappings)\ndef test_read_default_index_mapping(es_mapping):\n mapping = elastalert.create_index.read_es_index_mapping(es_mapping)\n assert es_mapping not in mapping\n print((json.dumps(mapping, indent=2)))\n\n\n@pytest.mark.parametrize('es_mapping', es_mappings)\ndef test_read_es_5_index_mapping(es_mapping):\n mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 5)\n assert es_mapping in mapping\n print((json.dumps(mapping, indent=2)))\n\n\n@pytest.mark.parametrize('es_mapping', es_mappings)\ndef test_read_es_6_index_mapping(es_mapping):\n mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 6)\n assert es_mapping not in mapping\n print((json.dumps(mapping, indent=2)))\n\n\ndef test_read_default_index_mappings():\n mappings = elastalert.create_index.read_es_index_mappings()\n assert len(mappings) == len(es_mappings)\n print((json.dumps(mappings, indent=2)))\n\n\ndef test_read_es_5_index_mappings():\n mappings = elastalert.create_index.read_es_index_mappings(5)\n assert len(mappings) == len(es_mappings)\n print((json.dumps(mappings, indent=2)))\n\n\ndef test_read_es_6_index_mappings():\n mappings = elastalert.create_index.read_es_index_mappings(6)\n assert len(mappings) == len(es_mappings)\n print((json.dumps(mappings, indent=2)))\n" }, { "path": "tests/elasticsearch_test.py", "content": "# -*- coding: utf-8 -*-\nimport datetime\nimport json\nimport time\n\nimport dateutil\nimport pytest\n\nimport elastalert.create_index\nimport elastalert.elastalert\nfrom elastalert import ElasticSearchClient\nfrom elastalert.util import build_es_conn_config\nfrom tests.conftest import ea # noqa: F401\n\ntest_index = 'test_index'\n\nes_host = '127.0.0.1'\nes_port = 9200\nes_timeout = 10\n\n\n@pytest.fixture\ndef es_client():\n es_conn_config = build_es_conn_config({'es_host': es_host, 'es_port': es_port, 'es_conn_timeout': es_timeout})\n return ElasticSearchClient(es_conn_config)\n\n\n@pytest.mark.elasticsearch\nclass TestElasticsearch(object):\n # TODO perform teardown removing data inserted into Elasticsearch\n # Warning!!!: Test class is not erasing its testdata on the Elasticsearch server.\n # This is not a problem as long as the data is manually removed or the test environment\n # is torn down after the test run(eg. running tests in a test environment such as Travis)\n def test_create_indices(self, es_client):\n elastalert.create_index.create_index_mappings(es_client=es_client, ea_index=test_index)\n indices_mappings = es_client.indices.get_mapping(test_index + '*')\n print(('-' * 50))\n print((json.dumps(indices_mappings, indent=2)))\n print(('-' * 50))\n if es_client.is_atleastsix():\n assert test_index in indices_mappings\n assert test_index + '_error' in indices_mappings\n assert test_index + '_status' in indices_mappings\n assert test_index + '_silence' in indices_mappings\n assert test_index + '_past' in indices_mappings\n else:\n assert 'elastalert' in indices_mappings[test_index]['mappings']\n assert 'elastalert_error' in indices_mappings[test_index]['mappings']\n assert 'elastalert_status' in indices_mappings[test_index]['mappings']\n assert 'silence' in indices_mappings[test_index]['mappings']\n assert 'past_elastalert' in indices_mappings[test_index]['mappings']\n\n @pytest.mark.usefixtures(\"ea\")\n def test_aggregated_alert(self, ea, es_client): # noqa: F811\n match_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta(\n days=1)\n ea.rules[0]['aggregate_by_match_time'] = True\n match = {'@timestamp': match_timestamp,\n 'num_hits': 0,\n 'num_matches': 3\n }\n ea.writeback_es = es_client\n res = ea.add_aggregated_alert(match, ea.rules[0])\n if ea.writeback_es.is_atleastsix():\n assert res['result'] == 'created'\n else:\n assert res['created'] is True\n # Make sure added data is available for querying\n time.sleep(2)\n # Now lets find the pending aggregated alert\n assert ea.find_pending_aggregate_alert(ea.rules[0])\n\n @pytest.mark.usefixtures(\"ea\")\n def test_silenced(self, ea, es_client): # noqa: F811\n until_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta(\n days=1)\n ea.writeback_es = es_client\n res = ea.set_realert(ea.rules[0]['name'], until_timestamp, 0)\n if ea.writeback_es.is_atleastsix():\n assert res['result'] == 'created'\n else:\n assert res['created'] is True\n # Make sure added data is available for querying\n time.sleep(2)\n # Force lookup in elasticsearch\n ea.silence_cache = {}\n # Now lets check if our rule is reported as silenced\n assert ea.is_silenced(ea.rules[0]['name'])\n\n @pytest.mark.usefixtures(\"ea\")\n def test_get_hits(self, ea, es_client): # noqa: F811\n start = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0)\n end = start + datetime.timedelta(days=1)\n ea.current_es = es_client\n if ea.current_es.is_atleastfive():\n ea.rules[0]['five'] = True\n else:\n ea.rules[0]['five'] = False\n ea.thread_data.current_es = ea.current_es\n hits = ea.get_hits(ea.rules[0], start, end, test_index)\n\n assert isinstance(hits, list)\n" }, { "path": "tests/kibana_discover_test.py", "content": "# -*- coding: utf-8 -*-\nfrom datetime import timedelta\nimport pytest\n\nfrom elastalert.kibana_discover import generate_kibana_discover_url\n\n\n@pytest.mark.parametrize(\"kibana_version\", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8'])\ndef test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version):\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': kibana_version,\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\n@pytest.mark.parametrize(\"kibana_version\", ['7.0', '7.1', '7.2', '7.3'])\ndef test_generate_kibana_discover_url_with_kibana_7x(kibana_version):\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': kibana_version,\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'filters%3A%21%28%29%2C'\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_missing_kibana_discover_version():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_index_pattern_id': 'logs',\n 'timestamp_field': 'timestamp',\n 'name': 'test'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n assert url is None\n\n\ndef test_generate_kibana_discover_url_with_missing_kibana_discover_app_url():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs',\n 'timestamp_field': 'timestamp',\n 'name': 'test'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n assert url is None\n\n\ndef test_generate_kibana_discover_url_with_missing_kibana_discover_index_pattern_id():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'timestamp_field': 'timestamp',\n 'name': 'test'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n assert url is None\n\n\ndef test_generate_kibana_discover_url_with_invalid_kibana_version():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '4.5',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n assert url is None\n\n\ndef test_generate_kibana_discover_url_with_kibana_discover_app_url_env_substitution(environ):\n environ.update({\n 'KIBANA_HOST': 'kibana',\n 'KIBANA_PORT': '5601',\n })\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_from_timedelta():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '7.3',\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'kibana_discover_from_timedelta': timedelta(hours=1),\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T04:00:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'filters%3A%21%28%29%2C'\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C'\n + 'to%3A%272019-09-01T04%3A10%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_from_timedelta_and_timeframe():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '7.3',\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'kibana_discover_from_timedelta': timedelta(hours=1),\n 'timeframe': timedelta(minutes=20),\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T04:00:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'filters%3A%21%28%29%2C'\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C'\n + 'to%3A%272019-09-01T04%3A20%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_to_timedelta():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '7.3',\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'kibana_discover_to_timedelta': timedelta(hours=1),\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T04:00:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'filters%3A%21%28%29%2C'\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T03%3A50%3A00Z%27%2C'\n + 'to%3A%272019-09-01T05%3A00%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_to_timedelta_and_timeframe():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '7.3',\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'kibana_discover_to_timedelta': timedelta(hours=1),\n 'timeframe': timedelta(minutes=20),\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T04:00:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'filters%3A%21%28%29%2C'\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C'\n + 'to%3A%272019-09-01T05%3A00%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_timeframe():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '7.3',\n 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a',\n 'timeframe': timedelta(minutes=20),\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T04:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'filters%3A%21%28%29%2C'\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T04%3A10%3A00Z%27%2C'\n + 'to%3A%272019-09-01T04%3A50%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_custom_columns():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'kibana_discover_columns': ['level', 'message'],\n 'timestamp_field': 'timestamp'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28level%2Cmessage%29%2C'\n + 'filters%3A%21%28%29%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_single_filter():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'filter': [\n {'term': {'level': 30}}\n ]\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3Afilter%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Abool%2C'\n + 'negate%3A%21f%2C'\n + 'type%3Acustom%2C'\n + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27'\n + '%29' # meta end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_multiple_filters():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247',\n 'timestamp_field': 'timestamp',\n 'filter': [\n {'term': {'app': 'test'}},\n {'term': {'level': 30}}\n ]\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3Afilter%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C'\n + 'key%3Abool%2C'\n + 'negate%3A%21f%2C'\n + 'type%3Acustom%2C'\n + 'value%3A%27%7B%22must%22%3A%5B' # value start\n + '%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D'\n + '%5D%7D%27' # value end\n + '%29' # meta end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_int_query_key():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'query_key': 'geo.dest'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z',\n 'geo.dest': 200\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Ageo.dest%2C'\n + 'negate%3A%21f%2C'\n + 'params%3A%28query%3A200%2C' # params start\n + 'type%3Aphrase'\n + '%29%2C' # params end\n + 'type%3Aphrase%2C'\n + 'value%3A%27200%27'\n + '%29%2C' # meta end\n + 'query%3A%28' # query start\n + 'match%3A%28' # match start\n + 'geo.dest%3A%28' # reponse start\n + 'query%3A200%2C'\n + 'type%3Aphrase'\n + '%29' # geo.dest end\n + '%29' # match end\n + '%29' # query end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_str_query_key():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'query_key': 'geo.dest'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z',\n 'geo': {\n 'dest': 'ok'\n }\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Ageo.dest%2C'\n + 'negate%3A%21f%2C'\n + 'params%3A%28query%3Aok%2C' # params start\n + 'type%3Aphrase'\n + '%29%2C' # params end\n + 'type%3Aphrase%2C'\n + 'value%3Aok'\n + '%29%2C' # meta end\n + 'query%3A%28' # query start\n + 'match%3A%28' # match start\n + 'geo.dest%3A%28' # geo.dest start\n + 'query%3Aok%2C'\n + 'type%3Aphrase'\n + '%29' # geo.dest end\n + '%29' # match end\n + '%29' # query end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_null_query_key_value():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'query_key': 'status'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z',\n 'status': None\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'exists%3A%28field%3Astatus%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Astatus%2C'\n + 'negate%3A%21t%2C'\n + 'type%3Aexists%2C'\n + 'value%3Aexists'\n + '%29' # meta end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_missing_query_key_value():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'query_key': 'status'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'exists%3A%28field%3Astatus%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Astatus%2C'\n + 'negate%3A%21t%2C'\n + 'type%3Aexists%2C'\n + 'value%3Aexists'\n + '%29' # meta end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_compound_query_key():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'compound_query_key': ['geo.src', 'geo.dest'],\n 'query_key': 'geo.src,geo.dest'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z',\n 'geo': {\n 'src': 'CA',\n 'dest': 'US'\n }\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # geo.src filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Ageo.src%2C'\n + 'negate%3A%21f%2C'\n + 'params%3A%28query%3ACA%2C' # params start\n + 'type%3Aphrase'\n + '%29%2C' # params end\n + 'type%3Aphrase%2C'\n + 'value%3ACA'\n + '%29%2C' # meta end\n + 'query%3A%28' # query start\n + 'match%3A%28' # match start\n + 'geo.src%3A%28' # reponse start\n + 'query%3ACA%2C'\n + 'type%3Aphrase'\n + '%29' # geo.src end\n + '%29' # match end\n + '%29' # query end\n + '%29%2C' # geo.src filter end\n\n + '%28' # geo.dest filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Ageo.dest%2C'\n + 'negate%3A%21f%2C'\n + 'params%3A%28query%3AUS%2C' # params start\n + 'type%3Aphrase'\n + '%29%2C' # params end\n + 'type%3Aphrase%2C'\n + 'value%3AUS'\n + '%29%2C' # meta end\n + 'query%3A%28' # query start\n + 'match%3A%28' # match start\n + 'geo.dest%3A%28' # geo.dest start\n + 'query%3AUS%2C'\n + 'type%3Aphrase'\n + '%29' # geo.dest end\n + '%29' # match end\n + '%29' # query end\n + '%29' # geo.dest filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n\n\ndef test_generate_kibana_discover_url_with_filter_and_query_key():\n url = generate_kibana_discover_url(\n rule={\n 'kibana_discover_app_url': 'http://kibana:5601/#/discover',\n 'kibana_discover_version': '6.8',\n 'kibana_discover_index_pattern_id': 'logs-*',\n 'timestamp_field': 'timestamp',\n 'filter': [\n {'term': {'level': 30}}\n ],\n 'query_key': 'status'\n },\n match={\n 'timestamp': '2019-09-01T00:30:00Z',\n 'status': 'ok'\n }\n )\n expectedUrl = (\n 'http://kibana:5601/#/discover'\n + '?_g=%28' # global start\n + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C'\n + 'time%3A%28' # time start\n + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C'\n + 'mode%3Aabsolute%2C'\n + 'to%3A%272019-09-01T00%3A40%3A00Z%27'\n + '%29' # time end\n + '%29' # global end\n + '&_a=%28' # app start\n + 'columns%3A%21%28_source%29%2C'\n + 'filters%3A%21%28' # filters start\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3Afilter%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Abool%2C'\n + 'negate%3A%21f%2C'\n + 'type%3Acustom%2C'\n + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27'\n + '%29' # meta end\n + '%29%2C' # filter end\n\n + '%28' # filter start\n + '%27%24state%27%3A%28store%3AappState%29%2C'\n + 'meta%3A%28' # meta start\n + 'alias%3A%21n%2C'\n + 'disabled%3A%21f%2C'\n + 'index%3A%27logs-%2A%27%2C'\n + 'key%3Astatus%2C'\n + 'negate%3A%21f%2C'\n + 'params%3A%28query%3Aok%2C' # params start\n + 'type%3Aphrase'\n + '%29%2C' # params end\n + 'type%3Aphrase%2C'\n + 'value%3Aok'\n + '%29%2C' # meta end\n + 'query%3A%28' # query start\n + 'match%3A%28' # match start\n + 'status%3A%28' # status start\n + 'query%3Aok%2C'\n + 'type%3Aphrase'\n + '%29' # status end\n + '%29' # match end\n + '%29' # query end\n + '%29' # filter end\n\n + '%29%2C' # filters end\n + 'index%3A%27logs-%2A%27%2C'\n + 'interval%3Aauto'\n + '%29' # app end\n )\n assert url == expectedUrl\n" }, { "path": "tests/kibana_test.py", "content": "import copy\nimport json\n\nfrom elastalert.kibana import add_filter\nfrom elastalert.kibana import dashboard_temp\nfrom elastalert.kibana import filters_from_dashboard\nfrom elastalert.kibana import kibana4_dashboard_link\n\n\n# Dashboard schema with only filters section\ntest_dashboard = '''{\n \"title\": \"AD Lock Outs\",\n \"services\": {\n \"filter\": {\n \"list\": {\n \"0\": {\n \"type\": \"time\",\n \"field\": \"@timestamp\",\n \"from\": \"now-7d\",\n \"to\": \"now\",\n \"mandate\": \"must\",\n \"active\": true,\n \"alias\": \"\",\n \"id\": 0\n },\n \"1\": {\n \"type\": \"field\",\n \"field\": \"_log_type\",\n \"query\": \"\\\\\"active_directory\\\\\"\",\n \"mandate\": \"must\",\n \"active\": true,\n \"alias\": \"\",\n \"id\": 1\n },\n \"2\": {\n \"type\": \"querystring\",\n \"query\": \"ad.security_auditing_code:4740\",\n \"mandate\": \"must\",\n \"active\": true,\n \"alias\": \"\",\n \"id\": 2\n }\n },\n \"ids\": [\n 0,\n 1,\n 2\n ]\n }\n }\n}'''\ntest_dashboard = json.loads(test_dashboard)\n\n\ndef test_filters_from_dashboard():\n filters = filters_from_dashboard(test_dashboard)\n assert {'term': {'_log_type': '\"active_directory\"'}} in filters\n assert {'query': {'query_string': {'query': 'ad.security_auditing_code:4740'}}} in filters\n\n\ndef test_add_filter():\n basic_filter = {\"term\": {\"this\": \"that\"}}\n db = copy.deepcopy(dashboard_temp)\n add_filter(db, basic_filter)\n assert db['services']['filter']['list']['1'] == {\n 'field': 'this',\n 'alias': '',\n 'mandate': 'must',\n 'active': True,\n 'query': '\"that\"',\n 'type': 'field',\n 'id': 1\n }\n\n list_filter = {\"term\": {\"this\": [\"that\", \"those\"]}}\n db = copy.deepcopy(dashboard_temp)\n add_filter(db, list_filter)\n assert db['services']['filter']['list']['1'] == {\n 'field': 'this',\n 'alias': '',\n 'mandate': 'must',\n 'active': True,\n 'query': '(\"that\" AND \"those\")',\n 'type': 'field',\n 'id': 1\n }\n\n\ndef test_url_encoded():\n url = kibana4_dashboard_link('example.com/#/Dashboard', '2015-01-01T00:00:00Z', '2017-01-01T00:00:00Z')\n assert not any([special_char in url for special_char in [\"',\\\":;?&=()\"]])\n\n\ndef test_url_env_substitution(environ):\n environ.update({\n 'KIBANA_HOST': 'kibana',\n 'KIBANA_PORT': '5601',\n })\n url = kibana4_dashboard_link(\n 'http://$KIBANA_HOST:$KIBANA_PORT/#/Dashboard',\n '2015-01-01T00:00:00Z',\n '2017-01-01T00:00:00Z',\n )\n assert url.startswith('http://kibana:5601/#/Dashboard')\n" }, { "path": "tests/loaders_test.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport datetime\nimport os\n\nimport mock\nimport pytest\n\nimport elastalert.alerts\nimport elastalert.ruletypes\nfrom elastalert.config import load_conf\nfrom elastalert.loaders import FileRulesLoader\nfrom elastalert.util import EAException\n\ntest_config = {'rules_folder': 'test_folder',\n 'run_every': {'minutes': 10},\n 'buffer_time': {'minutes': 10},\n 'es_host': 'elasticsearch.test',\n 'es_port': 12345,\n 'writeback_index': 'test_index',\n 'writeback_alias': 'test_alias'}\n\ntest_rule = {'es_host': 'test_host',\n 'es_port': 12345,\n 'name': 'testrule',\n 'type': 'spike',\n 'spike_height': 2,\n 'spike_type': 'up',\n 'timeframe': {'minutes': 10},\n 'index': 'test_index',\n 'query_key': 'testkey',\n 'compare_key': 'comparekey',\n 'filter': [{'term': {'key': 'value'}}],\n 'alert': 'email',\n 'use_count_query': True,\n 'doc_type': 'blsh',\n 'email': 'test@test.test',\n 'aggregation': {'hours': 2},\n 'include': ['comparekey', '@timestamp']}\n\ntest_args = mock.Mock()\ntest_args.config = 'test_config'\ntest_args.rule = None\ntest_args.debug = False\ntest_args.es_debug_trace = None\n\n\ndef test_import_rules():\n rules_loader = FileRulesLoader(test_config)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy['type'] = 'testing.test.RuleType'\n with mock.patch.object(rules_loader, 'load_yaml') as mock_open:\n mock_open.return_value = test_rule_copy\n\n # Test that type is imported\n with mock.patch('builtins.__import__') as mock_import:\n mock_import.return_value = elastalert.ruletypes\n rules_loader.load_configuration('test_config', test_config)\n assert mock_import.call_args_list[0][0][0] == 'testing.test'\n assert mock_import.call_args_list[0][0][3] == ['RuleType']\n\n # Test that alerts are imported\n test_rule_copy = copy.deepcopy(test_rule)\n mock_open.return_value = test_rule_copy\n test_rule_copy['alert'] = 'testing2.test2.Alerter'\n with mock.patch('builtins.__import__') as mock_import:\n mock_import.return_value = elastalert.alerts\n rules_loader.load_configuration('test_config', test_config)\n assert mock_import.call_args_list[0][0][0] == 'testing2.test2'\n assert mock_import.call_args_list[0][0][3] == ['Alerter']\n\n\ndef test_import_import():\n rules_loader = FileRulesLoader(test_config)\n import_rule = copy.deepcopy(test_rule)\n del(import_rule['es_host'])\n del(import_rule['es_port'])\n import_rule['import'] = 'importme.ymlt'\n import_me = {\n 'es_host': 'imported_host',\n 'es_port': 12349,\n 'email': 'ignored@email', # overwritten by the email in import_rule\n }\n\n with mock.patch.object(rules_loader, 'get_yaml') as mock_open:\n mock_open.side_effect = [import_rule, import_me]\n rules = rules_loader.load_configuration('blah.yaml', test_config)\n assert mock_open.call_args_list[0][0] == ('blah.yaml',)\n assert mock_open.call_args_list[1][0] == ('importme.ymlt',)\n assert len(mock_open.call_args_list) == 2\n assert rules['es_port'] == 12349\n assert rules['es_host'] == 'imported_host'\n assert rules['email'] == ['test@test.test']\n assert rules['filter'] == import_rule['filter']\n\n # check global import_rule dependency\n assert rules_loader.import_rules == {'blah.yaml': ['importme.ymlt']}\n\n\ndef test_import_absolute_import():\n rules_loader = FileRulesLoader(test_config)\n import_rule = copy.deepcopy(test_rule)\n del(import_rule['es_host'])\n del(import_rule['es_port'])\n import_rule['import'] = '/importme.ymlt'\n import_me = {\n 'es_host': 'imported_host',\n 'es_port': 12349,\n 'email': 'ignored@email', # overwritten by the email in import_rule\n }\n\n with mock.patch.object(rules_loader, 'get_yaml') as mock_open:\n mock_open.side_effect = [import_rule, import_me]\n rules = rules_loader.load_configuration('blah.yaml', test_config)\n assert mock_open.call_args_list[0][0] == ('blah.yaml',)\n assert mock_open.call_args_list[1][0] == ('/importme.ymlt',)\n assert len(mock_open.call_args_list) == 2\n assert rules['es_port'] == 12349\n assert rules['es_host'] == 'imported_host'\n assert rules['email'] == ['test@test.test']\n assert rules['filter'] == import_rule['filter']\n\n\ndef test_import_filter():\n # Check that if a filter is specified the rules are merged:\n\n rules_loader = FileRulesLoader(test_config)\n import_rule = copy.deepcopy(test_rule)\n del(import_rule['es_host'])\n del(import_rule['es_port'])\n import_rule['import'] = 'importme.ymlt'\n import_me = {\n 'es_host': 'imported_host',\n 'es_port': 12349,\n 'filter': [{'term': {'ratchet': 'clank'}}],\n }\n\n with mock.patch.object(rules_loader, 'get_yaml') as mock_open:\n mock_open.side_effect = [import_rule, import_me]\n rules = rules_loader.load_configuration('blah.yaml', test_config)\n assert rules['filter'] == [{'term': {'ratchet': 'clank'}}, {'term': {'key': 'value'}}]\n\n\ndef test_load_inline_alert_rule():\n rules_loader = FileRulesLoader(test_config)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy['alert'] = [\n {\n 'email': {\n 'email': 'foo@bar.baz'\n }\n },\n {\n 'email': {\n 'email': 'baz@foo.bar'\n }\n }\n ]\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch.object(rules_loader, 'get_yaml') as mock_open:\n mock_open.side_effect = [test_config_copy, test_rule_copy]\n rules_loader.load_modules(test_rule_copy)\n assert isinstance(test_rule_copy['alert'][0], elastalert.alerts.EmailAlerter)\n assert isinstance(test_rule_copy['alert'][1], elastalert.alerts.EmailAlerter)\n assert 'foo@bar.baz' in test_rule_copy['alert'][0].rule['email']\n assert 'baz@foo.bar' in test_rule_copy['alert'][1].rule['email']\n\n\ndef test_file_rules_loader_get_names_recursive():\n conf = {'scan_subdirectories': True, 'rules_folder': 'root'}\n rules_loader = FileRulesLoader(conf)\n walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)),\n ('root/folder_a', (), ('a.yaml', 'ab.yaml')),\n ('root/folder_b', (), ('b.yaml',)))\n with mock.patch('os.walk') as mock_walk:\n mock_walk.return_value = walk_paths\n paths = rules_loader.get_names(conf)\n\n paths = [p.replace(os.path.sep, '/') for p in paths]\n\n assert 'root/rule.yaml' in paths\n assert 'root/folder_a/a.yaml' in paths\n assert 'root/folder_a/ab.yaml' in paths\n assert 'root/folder_b/b.yaml' in paths\n assert len(paths) == 4\n\n\ndef test_file_rules_loader_get_names():\n # Check for no subdirectory\n conf = {'scan_subdirectories': False, 'rules_folder': 'root'}\n rules_loader = FileRulesLoader(conf)\n files = ['badfile', 'a.yaml', 'b.yaml']\n\n with mock.patch('os.listdir') as mock_list:\n with mock.patch('os.path.isfile') as mock_path:\n mock_path.return_value = True\n mock_list.return_value = files\n paths = rules_loader.get_names(conf)\n\n paths = [p.replace(os.path.sep, '/') for p in paths]\n\n assert 'root/a.yaml' in paths\n assert 'root/b.yaml' in paths\n assert len(paths) == 2\n\n\ndef test_load_rules():\n test_rule_copy = copy.deepcopy(test_rule)\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n\n with mock.patch('os.walk') as mock_ls:\n mock_ls.return_value = [('', [], ['testrule.yaml'])]\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType)\n assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter)\n assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta)\n assert isinstance(rules['run_every'], datetime.timedelta)\n for included_key in ['comparekey', 'testkey', '@timestamp']:\n assert included_key in rules['rules'][0]['include']\n\n # Assert include doesn't contain duplicates\n assert rules['rules'][0]['include'].count('@timestamp') == 1\n assert rules['rules'][0]['include'].count('comparekey') == 1\n\n\ndef test_load_default_host_port():\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('es_host')\n test_rule_copy.pop('es_port')\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n\n with mock.patch('os.walk') as mock_ls:\n mock_ls.return_value = [('', [], ['testrule.yaml'])]\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n\n # Assert include doesn't contain duplicates\n assert rules['es_port'] == 12345\n assert rules['es_host'] == 'elasticsearch.test'\n\n\ndef test_load_ssl_env_false():\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('es_host')\n test_rule_copy.pop('es_port')\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n\n with mock.patch('os.listdir') as mock_ls:\n with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}):\n mock_ls.return_value = ['testrule.yaml']\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n\n assert rules['use_ssl'] is False\n\n\ndef test_load_ssl_env_true():\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('es_host')\n test_rule_copy.pop('es_port')\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n\n with mock.patch('os.listdir') as mock_ls:\n with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}):\n mock_ls.return_value = ['testrule.yaml']\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n\n assert rules['use_ssl'] is True\n\n\ndef test_load_url_prefix_env():\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('es_host')\n test_rule_copy.pop('es_port')\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n\n with mock.patch('os.listdir') as mock_ls:\n with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}):\n mock_ls.return_value = ['testrule.yaml']\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n\n assert rules['es_url_prefix'] == 'es/'\n\n\ndef test_load_disabled_rules():\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy['is_enabled'] = False\n test_config_copy = copy.deepcopy(test_config)\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n\n with mock.patch('os.listdir') as mock_ls:\n mock_ls.return_value = ['testrule.yaml']\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n # The rule is not loaded for it has \"is_enabled=False\"\n assert len(rules['rules']) == 0\n\n\ndef test_raises_on_missing_config():\n optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name')\n test_rule_copy = copy.deepcopy(test_rule)\n for key in list(test_rule_copy.keys()):\n test_rule_copy = copy.deepcopy(test_rule)\n test_config_copy = copy.deepcopy(test_config)\n test_rule_copy.pop(key)\n\n # Non required keys\n if key in optional_keys:\n continue\n\n with mock.patch('elastalert.config.yaml_loader') as mock_conf_open:\n mock_conf_open.return_value = test_config_copy\n with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open:\n mock_rule_open.return_value = test_rule_copy\n with mock.patch('os.walk') as mock_walk:\n mock_walk.return_value = [('', [], ['testrule.yaml'])]\n with pytest.raises(EAException, message='key %s should be required' % key):\n rules = load_conf(test_args)\n rules['rules'] = rules['rules_loader'].load(rules)\n\n\ndef test_compound_query_key():\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('use_count_query')\n test_rule_copy['query_key'] = ['field1', 'field2']\n rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml')\n assert 'field1' in test_rule_copy['include']\n assert 'field2' in test_rule_copy['include']\n assert test_rule_copy['query_key'] == 'field1,field2'\n assert test_rule_copy['compound_query_key'] == ['field1', 'field2']\n\n\ndef test_query_key_with_single_value():\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('use_count_query')\n test_rule_copy['query_key'] = ['field1']\n rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml')\n assert 'field1' in test_rule_copy['include']\n assert test_rule_copy['query_key'] == 'field1'\n assert 'compound_query_key' not in test_rule_copy\n\n\ndef test_query_key_with_no_values():\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('use_count_query')\n test_rule_copy['query_key'] = []\n rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml')\n assert 'query_key' not in test_rule_copy\n assert 'compound_query_key' not in test_rule_copy\n\n\ndef test_name_inference():\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy.pop('name')\n rules_loader.load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml')\n assert test_rule_copy['name'] == 'msmerc woz ere'\n\n\ndef test_raises_on_bad_generate_kibana_filters():\n test_rule['generate_kibana_link'] = True\n bad_filters = [[{'not': {'terms': {'blah': 'blah'}}}],\n [{'terms': {'blah': 'blah'}}],\n [{'query': {'not_querystring': 'this:that'}}],\n [{'query': {'wildcard': 'this*that'}}],\n [{'blah': 'blah'}]]\n good_filters = [[{'term': {'field': 'value'}}],\n [{'not': {'term': {'this': 'that'}}}],\n [{'not': {'query': {'query_string': {'query': 'this:that'}}}}],\n [{'query': {'query_string': {'query': 'this:that'}}}],\n [{'range': {'blah': {'from': 'a', 'to': 'b'}}}],\n [{'not': {'range': {'blah': {'from': 'a', 'to': 'b'}}}}]]\n\n # Test that all the good filters work, but fail with a bad filter added\n for good in good_filters:\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy['filter'] = good\n with mock.patch.object(rules_loader, 'get_yaml') as mock_open:\n mock_open.return_value = test_rule_copy\n rules_loader.load_configuration('blah', test_config)\n for bad in bad_filters:\n test_rule_copy['filter'] = good + bad\n with pytest.raises(EAException):\n rules_loader.load_configuration('blah', test_config)\n\n\ndef test_kibana_discover_from_timedelta():\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy['kibana_discover_from_timedelta'] = {'minutes': 2}\n rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml')\n assert isinstance(test_rule_copy['kibana_discover_from_timedelta'], datetime.timedelta)\n assert test_rule_copy['kibana_discover_from_timedelta'] == datetime.timedelta(minutes=2)\n\n\ndef test_kibana_discover_to_timedelta():\n test_config_copy = copy.deepcopy(test_config)\n rules_loader = FileRulesLoader(test_config_copy)\n test_rule_copy = copy.deepcopy(test_rule)\n test_rule_copy['kibana_discover_to_timedelta'] = {'minutes': 2}\n rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml')\n assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta)\n assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2)\n" }, { "path": "tests/rules_test.py", "content": "# -*- coding: utf-8 -*-\nimport copy\nimport datetime\n\nimport mock\nimport pytest\n\nfrom elastalert.ruletypes import AnyRule\nfrom elastalert.ruletypes import BaseAggregationRule\nfrom elastalert.ruletypes import BlacklistRule\nfrom elastalert.ruletypes import CardinalityRule\nfrom elastalert.ruletypes import ChangeRule\nfrom elastalert.ruletypes import EventWindow\nfrom elastalert.ruletypes import FlatlineRule\nfrom elastalert.ruletypes import FrequencyRule\nfrom elastalert.ruletypes import MetricAggregationRule\nfrom elastalert.ruletypes import NewTermsRule\nfrom elastalert.ruletypes import PercentageMatchRule\nfrom elastalert.ruletypes import SpikeRule\nfrom elastalert.ruletypes import WhitelistRule\nfrom elastalert.util import dt_to_ts\nfrom elastalert.util import EAException\nfrom elastalert.util import ts_now\nfrom elastalert.util import ts_to_dt\n\n\ndef hits(size, **kwargs):\n ret = []\n for n in range(size):\n ts = ts_to_dt('2014-09-26T12:%s:%sZ' % (n / 60, n % 60))\n n += 1\n event = create_event(ts, **kwargs)\n ret.append(event)\n return ret\n\n\ndef create_event(timestamp, timestamp_field='@timestamp', **kwargs):\n event = {timestamp_field: timestamp}\n event.update(**kwargs)\n return event\n\n\ndef create_bucket_aggregation(agg_name, buckets):\n agg = {agg_name: {'buckets': buckets}}\n return agg\n\n\ndef create_percentage_match_agg(match_count, other_count):\n agg = create_bucket_aggregation(\n 'percentage_match_aggs', {\n 'match_bucket': {\n 'doc_count': match_count\n },\n '_other_': {\n 'doc_count': other_count\n }\n }\n )\n return agg\n\n\ndef assert_matches_have(matches, terms):\n assert len(matches) == len(terms)\n for match, term in zip(matches, terms):\n assert term[0] in match\n assert match[term[0]] == term[1]\n if len(term) > 2:\n assert match[term[2]] == term[3]\n\n\ndef test_any():\n event = hits(1)\n rule = AnyRule({})\n rule.add_data([event])\n assert rule.matches == [event]\n\n\ndef test_freq():\n events = hits(60, timestamp_field='blah', username='qlo')\n rules = {'num_events': 59,\n 'timeframe': datetime.timedelta(hours=1),\n 'timestamp_field': 'blah'}\n rule = FrequencyRule(rules)\n rule.add_data(events)\n assert len(rule.matches) == 1\n\n # Test wit query_key\n events = hits(60, timestamp_field='blah', username='qlo')\n rules['query_key'] = 'username'\n rule = FrequencyRule(rules)\n rule.add_data(events)\n assert len(rule.matches) == 1\n\n # Doesn't match\n events = hits(60, timestamp_field='blah', username='qlo')\n rules['num_events'] = 61\n rule = FrequencyRule(rules)\n rule.add_data(events)\n assert len(rule.matches) == 0\n\n # garbage collection\n assert 'qlo' in rule.occurrences\n rule.garbage_collect(ts_to_dt('2014-09-28T12:0:0'))\n assert rule.occurrences == {}\n\n\ndef test_freq_count():\n rules = {'num_events': 100,\n 'timeframe': datetime.timedelta(hours=1),\n 'use_count_query': True}\n # Normal match\n rule = FrequencyRule(rules)\n rule.add_count_data({ts_to_dt('2014-10-10T00:00:00'): 75})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T00:15:00'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T00:25:00'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T00:45:00'): 6})\n assert len(rule.matches) == 1\n\n # First data goes out of timeframe first\n rule = FrequencyRule(rules)\n rule.add_count_data({ts_to_dt('2014-10-10T00:00:00'): 75})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T00:45:00'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T00:55:00'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T01:05:00'): 6})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-10T01:00:00'): 75})\n assert len(rule.matches) == 1\n\n\ndef test_freq_out_of_order():\n events = hits(60, timestamp_field='blah', username='qlo')\n rules = {'num_events': 59,\n 'timeframe': datetime.timedelta(hours=1),\n 'timestamp_field': 'blah'}\n rule = FrequencyRule(rules)\n rule.add_data(events[:10])\n assert len(rule.matches) == 0\n\n # Try to add events from before the first occurrence\n rule.add_data([{'blah': ts_to_dt('2014-09-26T11:00:00'), 'username': 'qlo'}] * 50)\n assert len(rule.matches) == 0\n\n rule.add_data(events[15:20])\n assert len(rule.matches) == 0\n rule.add_data(events[10:15])\n assert len(rule.matches) == 0\n rule.add_data(events[20:55])\n rule.add_data(events[57:])\n assert len(rule.matches) == 0\n rule.add_data(events[55:57])\n assert len(rule.matches) == 1\n\n\ndef test_freq_terms():\n rules = {'num_events': 10,\n 'timeframe': datetime.timedelta(hours=1),\n 'query_key': 'username'}\n rule = FrequencyRule(rules)\n\n terms1 = {ts_to_dt('2014-01-01T00:01:00Z'): [{'key': 'userA', 'doc_count': 1},\n {'key': 'userB', 'doc_count': 5}]}\n terms2 = {ts_to_dt('2014-01-01T00:10:00Z'): [{'key': 'userA', 'doc_count': 8},\n {'key': 'userB', 'doc_count': 5}]}\n terms3 = {ts_to_dt('2014-01-01T00:25:00Z'): [{'key': 'userA', 'doc_count': 3},\n {'key': 'userB', 'doc_count': 0}]}\n # Initial data\n rule.add_terms_data(terms1)\n assert len(rule.matches) == 0\n\n # Match for user B\n rule.add_terms_data(terms2)\n assert len(rule.matches) == 1\n assert rule.matches[0].get('username') == 'userB'\n\n # Match for user A\n rule.add_terms_data(terms3)\n assert len(rule.matches) == 2\n assert rule.matches[1].get('username') == 'userA'\n\n\ndef test_eventwindow():\n timeframe = datetime.timedelta(minutes=10)\n window = EventWindow(timeframe)\n timestamps = [ts_to_dt(x) for x in ['2014-01-01T10:00:00',\n '2014-01-01T10:05:00',\n '2014-01-01T10:03:00',\n '2014-01-01T09:55:00',\n '2014-01-01T10:09:00']]\n for ts in timestamps:\n window.append([{'@timestamp': ts}, 1])\n\n timestamps.sort()\n for exp, actual in zip(timestamps[1:], window.data):\n assert actual[0]['@timestamp'] == exp\n\n window.append([{'@timestamp': ts_to_dt('2014-01-01T10:14:00')}, 1])\n timestamps.append(ts_to_dt('2014-01-01T10:14:00'))\n for exp, actual in zip(timestamps[3:], window.data):\n assert actual[0]['@timestamp'] == exp\n\n\ndef test_spike_count():\n rules = {'threshold_ref': 10,\n 'spike_height': 2,\n 'timeframe': datetime.timedelta(seconds=10),\n 'spike_type': 'both',\n 'timestamp_field': '@timestamp'}\n rule = SpikeRule(rules)\n\n # Double rate of events at 20 seconds\n rule.add_count_data({ts_to_dt('2014-09-26T00:00:00'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-09-26T00:00:10'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-09-26T00:00:20'): 20})\n assert len(rule.matches) == 1\n\n # Downward spike\n rule = SpikeRule(rules)\n rule.add_count_data({ts_to_dt('2014-09-26T00:00:00'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-09-26T00:00:10'): 10})\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-09-26T00:00:20'): 0})\n assert len(rule.matches) == 1\n\n\ndef test_spike_deep_key():\n rules = {'threshold_ref': 10,\n 'spike_height': 2,\n 'timeframe': datetime.timedelta(seconds=10),\n 'spike_type': 'both',\n 'timestamp_field': '@timestamp',\n 'query_key': 'foo.bar.baz'}\n rule = SpikeRule(rules)\n rule.add_data([{'@timestamp': ts_to_dt('2015'), 'foo': {'bar': {'baz': 'LOL'}}}])\n assert 'LOL' in rule.cur_windows\n\n\ndef test_spike():\n # Events are 1 per second\n events = hits(100, timestamp_field='ts')\n\n # Constant rate, doesn't match\n rules = {'threshold_ref': 10,\n 'spike_height': 2,\n 'timeframe': datetime.timedelta(seconds=10),\n 'spike_type': 'both',\n 'use_count_query': False,\n 'timestamp_field': 'ts'}\n rule = SpikeRule(rules)\n rule.add_data(events)\n assert len(rule.matches) == 0\n\n # Double the rate of events after [50:]\n events2 = events[:50]\n for event in events[50:]:\n events2.append(event)\n events2.append({'ts': event['ts'] + datetime.timedelta(milliseconds=1)})\n rules['spike_type'] = 'up'\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 1\n\n # Doesn't match\n rules['spike_height'] = 3\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 0\n\n # Downward spike\n events = events[:50] + events[75:]\n rules['spike_type'] = 'down'\n rule = SpikeRule(rules)\n rule.add_data(events)\n assert len(rule.matches) == 1\n\n # Doesn't meet threshold_ref\n # When ref hits 11, cur is only 20\n rules['spike_height'] = 2\n rules['threshold_ref'] = 11\n rules['spike_type'] = 'up'\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 0\n\n # Doesn't meet threshold_cur\n # Maximum rate of events is 20 per 10 seconds\n rules['threshold_ref'] = 10\n rules['threshold_cur'] = 30\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 0\n\n # Alert on new data\n # (At least 25 events occur before 30 seconds has elapsed)\n rules.pop('threshold_ref')\n rules['timeframe'] = datetime.timedelta(seconds=30)\n rules['threshold_cur'] = 25\n rules['spike_height'] = 2\n rules['alert_on_new_data'] = True\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 1\n\n\ndef test_spike_query_key():\n events = hits(100, timestamp_field='ts', username='qlo')\n # Constant rate, doesn't match\n rules = {'threshold_ref': 10,\n 'spike_height': 2,\n 'timeframe': datetime.timedelta(seconds=10),\n 'spike_type': 'both',\n 'use_count_query': False,\n 'timestamp_field': 'ts',\n 'query_key': 'username'}\n rule = SpikeRule(rules)\n rule.add_data(events)\n assert len(rule.matches) == 0\n\n # Double the rate of events, but with a different usename\n events_bob = hits(100, timestamp_field='ts', username='bob')\n events2 = events[:50]\n for num in range(50, 99):\n events2.append(events_bob[num])\n events2.append(events[num])\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 0\n\n # Double the rate of events, with the same username\n events2 = events[:50]\n for num in range(50, 99):\n events2.append(events_bob[num])\n events2.append(events[num])\n events2.append(events[num])\n rule = SpikeRule(rules)\n rule.add_data(events2)\n assert len(rule.matches) == 1\n\n\ndef test_spike_terms():\n rules = {'threshold_ref': 5,\n 'spike_height': 2,\n 'timeframe': datetime.timedelta(minutes=10),\n 'spike_type': 'both',\n 'use_count_query': False,\n 'timestamp_field': 'ts',\n 'query_key': 'username',\n 'use_term_query': True}\n terms1 = {ts_to_dt('2014-01-01T00:01:00Z'): [{'key': 'userA', 'doc_count': 10},\n {'key': 'userB', 'doc_count': 5}]}\n terms2 = {ts_to_dt('2014-01-01T00:10:00Z'): [{'key': 'userA', 'doc_count': 22},\n {'key': 'userB', 'doc_count': 5}]}\n terms3 = {ts_to_dt('2014-01-01T00:25:00Z'): [{'key': 'userA', 'doc_count': 25},\n {'key': 'userB', 'doc_count': 27}]}\n terms4 = {ts_to_dt('2014-01-01T00:27:00Z'): [{'key': 'userA', 'doc_count': 10},\n {'key': 'userB', 'doc_count': 12},\n {'key': 'userC', 'doc_count': 100}]}\n terms5 = {ts_to_dt('2014-01-01T00:30:00Z'): [{'key': 'userD', 'doc_count': 100},\n {'key': 'userC', 'doc_count': 100}]}\n\n rule = SpikeRule(rules)\n\n # Initial input\n rule.add_terms_data(terms1)\n assert len(rule.matches) == 0\n\n # No spike for UserA because windows not filled\n rule.add_terms_data(terms2)\n assert len(rule.matches) == 0\n\n # Spike for userB only\n rule.add_terms_data(terms3)\n assert len(rule.matches) == 1\n assert rule.matches[0].get('username') == 'userB'\n\n # Test no alert for new user over threshold\n rules.pop('threshold_ref')\n rules['threshold_cur'] = 50\n rule = SpikeRule(rules)\n rule.add_terms_data(terms1)\n rule.add_terms_data(terms2)\n rule.add_terms_data(terms3)\n rule.add_terms_data(terms4)\n assert len(rule.matches) == 0\n\n # Test alert_on_new_data\n rules['alert_on_new_data'] = True\n rule = SpikeRule(rules)\n rule.add_terms_data(terms1)\n rule.add_terms_data(terms2)\n rule.add_terms_data(terms3)\n rule.add_terms_data(terms4)\n assert len(rule.matches) == 1\n\n # Test that another alert doesn't fire immediately for userC but it does for userD\n rule.matches = []\n rule.add_terms_data(terms5)\n assert len(rule.matches) == 1\n assert rule.matches[0]['username'] == 'userD'\n\n\ndef test_spike_terms_query_key_alert_on_new_data():\n rules = {'spike_height': 1.5,\n 'timeframe': datetime.timedelta(minutes=10),\n 'spike_type': 'both',\n 'use_count_query': False,\n 'timestamp_field': 'ts',\n 'query_key': 'username',\n 'use_term_query': True,\n 'alert_on_new_data': True}\n\n terms1 = {ts_to_dt('2014-01-01T00:01:00Z'): [{'key': 'userA', 'doc_count': 10}]}\n terms2 = {ts_to_dt('2014-01-01T00:06:00Z'): [{'key': 'userA', 'doc_count': 10}]}\n terms3 = {ts_to_dt('2014-01-01T00:11:00Z'): [{'key': 'userA', 'doc_count': 10}]}\n terms4 = {ts_to_dt('2014-01-01T00:21:00Z'): [{'key': 'userA', 'doc_count': 20}]}\n terms5 = {ts_to_dt('2014-01-01T00:26:00Z'): [{'key': 'userA', 'doc_count': 20}]}\n terms6 = {ts_to_dt('2014-01-01T00:31:00Z'): [{'key': 'userA', 'doc_count': 20}]}\n terms7 = {ts_to_dt('2014-01-01T00:36:00Z'): [{'key': 'userA', 'doc_count': 20}]}\n terms8 = {ts_to_dt('2014-01-01T00:41:00Z'): [{'key': 'userA', 'doc_count': 20}]}\n\n rule = SpikeRule(rules)\n\n # Initial input\n rule.add_terms_data(terms1)\n assert len(rule.matches) == 0\n\n # No spike for UserA because windows not filled\n rule.add_terms_data(terms2)\n assert len(rule.matches) == 0\n\n rule.add_terms_data(terms3)\n assert len(rule.matches) == 0\n\n rule.add_terms_data(terms4)\n assert len(rule.matches) == 0\n\n # Spike\n rule.add_terms_data(terms5)\n assert len(rule.matches) == 1\n\n rule.matches[:] = []\n\n # There will be no more spikes since all terms have the same doc_count\n rule.add_terms_data(terms6)\n assert len(rule.matches) == 0\n\n rule.add_terms_data(terms7)\n assert len(rule.matches) == 0\n\n rule.add_terms_data(terms8)\n assert len(rule.matches) == 0\n\n\ndef test_blacklist():\n events = [{'@timestamp': ts_to_dt('2014-09-26T12:34:56Z'), 'term': 'good'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:57Z'), 'term': 'bad'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:58Z'), 'term': 'also good'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:59Z'), 'term': 'really bad'},\n {'@timestamp': ts_to_dt('2014-09-26T12:35:00Z'), 'no_term': 'bad'}]\n rules = {'blacklist': ['bad', 'really bad'],\n 'compare_key': 'term',\n 'timestamp_field': '@timestamp'}\n rule = BlacklistRule(rules)\n rule.add_data(events)\n assert_matches_have(rule.matches, [('term', 'bad'), ('term', 'really bad')])\n\n\ndef test_whitelist():\n events = [{'@timestamp': ts_to_dt('2014-09-26T12:34:56Z'), 'term': 'good'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:57Z'), 'term': 'bad'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:58Z'), 'term': 'also good'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:59Z'), 'term': 'really bad'},\n {'@timestamp': ts_to_dt('2014-09-26T12:35:00Z'), 'no_term': 'bad'}]\n rules = {'whitelist': ['good', 'also good'],\n 'compare_key': 'term',\n 'ignore_null': True,\n 'timestamp_field': '@timestamp'}\n rule = WhitelistRule(rules)\n rule.add_data(events)\n assert_matches_have(rule.matches, [('term', 'bad'), ('term', 'really bad')])\n\n\ndef test_whitelist_dont_ignore_nulls():\n events = [{'@timestamp': ts_to_dt('2014-09-26T12:34:56Z'), 'term': 'good'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:57Z'), 'term': 'bad'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:58Z'), 'term': 'also good'},\n {'@timestamp': ts_to_dt('2014-09-26T12:34:59Z'), 'term': 'really bad'},\n {'@timestamp': ts_to_dt('2014-09-26T12:35:00Z'), 'no_term': 'bad'}]\n rules = {'whitelist': ['good', 'also good'],\n 'compare_key': 'term',\n 'ignore_null': True,\n 'timestamp_field': '@timestamp'}\n rules['ignore_null'] = False\n rule = WhitelistRule(rules)\n rule.add_data(events)\n assert_matches_have(rule.matches, [('term', 'bad'), ('term', 'really bad'), ('no_term', 'bad')])\n\n\ndef test_change():\n events = hits(10, username='qlo', term='good', second_term='yes')\n events[8].pop('term')\n events[8].pop('second_term')\n events[9]['term'] = 'bad'\n events[9]['second_term'] = 'no'\n rules = {'compound_compare_key': ['term', 'second_term'],\n 'query_key': 'username',\n 'ignore_null': True,\n 'timestamp_field': '@timestamp'}\n rule = ChangeRule(rules)\n rule.add_data(events)\n assert_matches_have(rule.matches, [('term', 'bad', 'second_term', 'no')])\n\n # Unhashable QK\n events2 = hits(10, username=['qlo'], term='good', second_term='yes')\n events2[9]['term'] = 'bad'\n events2[9]['second_term'] = 'no'\n rule = ChangeRule(rules)\n rule.add_data(events2)\n assert_matches_have(rule.matches, [('term', 'bad', 'second_term', 'no')])\n\n # Don't ignore nulls\n rules['ignore_null'] = False\n rule = ChangeRule(rules)\n rule.add_data(events)\n assert_matches_have(rule.matches, [('username', 'qlo'), ('term', 'bad', 'second_term', 'no')])\n\n # With timeframe\n rules['timeframe'] = datetime.timedelta(seconds=2)\n rules['ignore_null'] = True\n rule = ChangeRule(rules)\n rule.add_data(events)\n assert_matches_have(rule.matches, [('term', 'bad', 'second_term', 'no')])\n\n # With timeframe, doesn't match\n events = events[:8] + events[9:]\n rules['timeframe'] = datetime.timedelta(seconds=1)\n rule = ChangeRule(rules)\n rule.add_data(events)\n assert rule.matches == []\n\n\ndef test_new_term():\n rules = {'fields': ['a', 'b'],\n 'timestamp_field': '@timestamp',\n 'es_host': 'example.com', 'es_port': 10, 'index': 'logstash',\n 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts}\n mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},\n {'key': 'key2', 'doc_count': 5}]}}}}\n\n with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:\n mock_es.return_value = mock.Mock()\n mock_es.return_value.search.return_value = mock_res\n mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}\n call_args = []\n\n # search is called with a mutable dict containing timestamps, this is required to test\n def record_args(*args, **kwargs):\n call_args.append((copy.deepcopy(args), copy.deepcopy(kwargs)))\n return mock_res\n\n mock_es.return_value.search.side_effect = record_args\n rule = NewTermsRule(rules)\n\n # 30 day default range, 1 day default step, times 2 fields\n assert rule.es.search.call_count == 60\n\n # Assert that all calls have the proper ordering of time ranges\n old_ts = '2010-01-01T00:00:00Z'\n old_field = ''\n for call in call_args:\n field = call[1]['body']['aggs']['filtered']['aggs']['values']['terms']['field']\n if old_field != field:\n old_field = field\n old_ts = '2010-01-01T00:00:00Z'\n gte = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gte']\n assert gte > old_ts\n lt = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lt']\n assert lt > gte\n old_ts = gte\n\n # Key1 and key2 shouldn't cause a match\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])\n assert rule.matches == []\n\n # Neither will missing values\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])\n assert rule.matches == []\n\n # Key3 causes an alert for field b\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])\n assert len(rule.matches) == 1\n assert rule.matches[0]['new_field'] == 'b'\n assert rule.matches[0]['b'] == 'key3'\n rule.matches = []\n\n # Key3 doesn't cause another alert for field b\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])\n assert rule.matches == []\n\n # Missing_field\n rules['alert_on_missing_field'] = True\n with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:\n mock_es.return_value = mock.Mock()\n mock_es.return_value.search.return_value = mock_res\n mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}\n rule = NewTermsRule(rules)\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])\n assert len(rule.matches) == 1\n assert rule.matches[0]['missing_field'] == 'b'\n\n\ndef test_new_term_nested_field():\n\n rules = {'fields': ['a', 'b.c'],\n 'timestamp_field': '@timestamp',\n 'es_host': 'example.com', 'es_port': 10, 'index': 'logstash',\n 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts}\n mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},\n {'key': 'key2', 'doc_count': 5}]}}}}\n with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:\n mock_es.return_value = mock.Mock()\n mock_es.return_value.search.return_value = mock_res\n mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}\n rule = NewTermsRule(rules)\n\n assert rule.es.search.call_count == 60\n\n # Key3 causes an alert for nested field b.c\n rule.add_data([{'@timestamp': ts_now(), 'b': {'c': 'key3'}}])\n assert len(rule.matches) == 1\n assert rule.matches[0]['new_field'] == 'b.c'\n assert rule.matches[0]['b']['c'] == 'key3'\n rule.matches = []\n\n\ndef test_new_term_with_terms():\n rules = {'fields': ['a'],\n 'timestamp_field': '@timestamp',\n 'es_host': 'example.com', 'es_port': 10, 'index': 'logstash', 'query_key': 'a',\n 'window_step_size': {'days': 2},\n 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts}\n mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},\n {'key': 'key2', 'doc_count': 5}]}}}}\n\n with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:\n mock_es.return_value = mock.Mock()\n mock_es.return_value.search.return_value = mock_res\n mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}\n rule = NewTermsRule(rules)\n\n # Only 15 queries because of custom step size\n assert rule.es.search.call_count == 15\n\n # Key1 and key2 shouldn't cause a match\n terms = {ts_now(): [{'key': 'key1', 'doc_count': 1},\n {'key': 'key2', 'doc_count': 1}]}\n rule.add_terms_data(terms)\n assert rule.matches == []\n\n # Key3 causes an alert for field a\n terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}\n rule.add_terms_data(terms)\n assert len(rule.matches) == 1\n assert rule.matches[0]['new_field'] == 'a'\n assert rule.matches[0]['a'] == 'key3'\n rule.matches = []\n\n # Key3 doesn't cause another alert\n terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}\n rule.add_terms_data(terms)\n assert rule.matches == []\n\n\ndef test_new_term_with_composite_fields():\n rules = {'fields': [['a', 'b', 'c'], ['d', 'e.f']],\n 'timestamp_field': '@timestamp',\n 'es_host': 'example.com', 'es_port': 10, 'index': 'logstash',\n 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts}\n\n mock_res = {\n 'aggregations': {\n 'filtered': {\n 'values': {\n 'buckets': [\n {\n 'key': 'key1',\n 'doc_count': 5,\n 'values': {\n 'buckets': [\n {\n 'key': 'key2',\n 'doc_count': 5,\n 'values': {\n 'buckets': [\n {\n 'key': 'key3',\n 'doc_count': 3,\n },\n {\n 'key': 'key4',\n 'doc_count': 2,\n },\n ]\n }\n }\n ]\n }\n }\n ]\n }\n }\n }\n }\n\n with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:\n mock_es.return_value = mock.Mock()\n mock_es.return_value.search.return_value = mock_res\n mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}\n rule = NewTermsRule(rules)\n\n assert rule.es.search.call_count == 60\n\n # key3 already exists, and thus shouldn't cause a match\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key3'}])\n assert rule.matches == []\n\n # key5 causes an alert for composite field [a, b, c]\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key5'}])\n assert len(rule.matches) == 1\n assert rule.matches[0]['new_field'] == ('a', 'b', 'c')\n assert rule.matches[0]['a'] == 'key1'\n assert rule.matches[0]['b'] == 'key2'\n assert rule.matches[0]['c'] == 'key5'\n rule.matches = []\n\n # New values in other fields that are not part of the composite key should not cause an alert\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key4', 'd': 'unrelated_value'}])\n assert len(rule.matches) == 0\n rule.matches = []\n\n # Verify nested fields work properly\n # Key6 causes an alert for nested field e.f\n rule.add_data([{'@timestamp': ts_now(), 'd': 'key4', 'e': {'f': 'key6'}}])\n assert len(rule.matches) == 1\n assert rule.matches[0]['new_field'] == ('d', 'e.f')\n assert rule.matches[0]['d'] == 'key4'\n assert rule.matches[0]['e']['f'] == 'key6'\n rule.matches = []\n\n # Missing_fields\n rules['alert_on_missing_field'] = True\n with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:\n mock_es.return_value = mock.Mock()\n mock_es.return_value.search.return_value = mock_res\n mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}\n rule = NewTermsRule(rules)\n rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])\n assert len(rule.matches) == 2\n # This means that any one of the three n composite fields were not present\n assert rule.matches[0]['missing_field'] == ('a', 'b', 'c')\n assert rule.matches[1]['missing_field'] == ('d', 'e.f')\n\n\ndef test_flatline():\n events = hits(40)\n rules = {\n 'timeframe': datetime.timedelta(seconds=30),\n 'threshold': 2,\n 'timestamp_field': '@timestamp',\n }\n\n rule = FlatlineRule(rules)\n\n # 1 hit should cause an alert until after at least 30 seconds pass\n rule.add_data(hits(1))\n assert rule.matches == []\n\n # Add hits with timestamps 2014-09-26T12:00:00 --> 2014-09-26T12:00:09\n rule.add_data(events[0:10])\n\n # This will be run at the end of the hits\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:11Z'))\n assert rule.matches == []\n\n # This would be run if the query returned nothing for a future timestamp\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:45Z'))\n assert len(rule.matches) == 1\n\n # After another garbage collection, since there are still no events, a new match is added\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:50Z'))\n assert len(rule.matches) == 2\n\n # Add hits with timestamps 2014-09-26T12:00:30 --> 2014-09-26T12:00:39\n rule.add_data(events[30:])\n\n # Now that there is data in the last 30 minutes, no more matches should be added\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:55Z'))\n assert len(rule.matches) == 2\n\n # After that window passes with no more data, a new match is added\n rule.garbage_collect(ts_to_dt('2014-09-26T12:01:11Z'))\n assert len(rule.matches) == 3\n\n\ndef test_flatline_no_data():\n rules = {\n 'timeframe': datetime.timedelta(seconds=30),\n 'threshold': 2,\n 'timestamp_field': '@timestamp',\n }\n\n rule = FlatlineRule(rules)\n\n # Initial lack of data\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:00Z'))\n assert len(rule.matches) == 0\n\n # Passed the timeframe, still no events\n rule.garbage_collect(ts_to_dt('2014-09-26T12:35:00Z'))\n assert len(rule.matches) == 1\n\n\ndef test_flatline_count():\n rules = {'timeframe': datetime.timedelta(seconds=30),\n 'threshold': 1,\n 'timestamp_field': '@timestamp'}\n rule = FlatlineRule(rules)\n rule.add_count_data({ts_to_dt('2014-10-11T00:00:00'): 1})\n rule.garbage_collect(ts_to_dt('2014-10-11T00:00:10'))\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-11T00:00:15'): 0})\n rule.garbage_collect(ts_to_dt('2014-10-11T00:00:20'))\n assert len(rule.matches) == 0\n rule.add_count_data({ts_to_dt('2014-10-11T00:00:35'): 0})\n assert len(rule.matches) == 1\n\n\ndef test_flatline_query_key():\n rules = {'timeframe': datetime.timedelta(seconds=30),\n 'threshold': 1,\n 'use_query_key': True,\n 'query_key': 'qk',\n 'timestamp_field': '@timestamp'}\n\n rule = FlatlineRule(rules)\n\n # Adding two separate query keys, the flatline rule should trigger for both\n rule.add_data(hits(1, qk='key1'))\n rule.add_data(hits(1, qk='key2'))\n rule.add_data(hits(1, qk='key3'))\n assert rule.matches == []\n\n # This will be run at the end of the hits\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:11Z'))\n assert rule.matches == []\n\n # Add new data from key3. It will not immediately cause an alert\n rule.add_data([create_event(ts_to_dt('2014-09-26T12:00:20Z'), qk='key3')])\n\n # key1 and key2 have not had any new data, so they will trigger the flatline alert\n timestamp = '2014-09-26T12:00:45Z'\n rule.garbage_collect(ts_to_dt(timestamp))\n assert len(rule.matches) == 2\n assert set(['key1', 'key2']) == set([m['key'] for m in rule.matches if m['@timestamp'] == timestamp])\n\n # Next time the rule runs, all 3 keys still have no data, so all three will cause an alert\n timestamp = '2014-09-26T12:01:20Z'\n rule.garbage_collect(ts_to_dt(timestamp))\n assert len(rule.matches) == 5\n assert set(['key1', 'key2', 'key3']) == set([m['key'] for m in rule.matches if m['@timestamp'] == timestamp])\n\n\ndef test_flatline_forget_query_key():\n rules = {'timeframe': datetime.timedelta(seconds=30),\n 'threshold': 1,\n 'query_key': 'qk',\n 'forget_keys': True,\n 'timestamp_field': '@timestamp'}\n\n rule = FlatlineRule(rules)\n\n # Adding two separate query keys, the flatline rule should trigger for both\n rule.add_data(hits(1, qk='key1'))\n assert rule.matches == []\n\n # This will be run at the end of the hits\n rule.garbage_collect(ts_to_dt('2014-09-26T12:00:11Z'))\n assert rule.matches == []\n\n # Key1 should not alert\n timestamp = '2014-09-26T12:00:45Z'\n rule.garbage_collect(ts_to_dt(timestamp))\n assert len(rule.matches) == 1\n rule.matches = []\n\n # key1 was forgotten, so no more matches\n rule.garbage_collect(ts_to_dt('2014-09-26T12:01:11Z'))\n assert rule.matches == []\n\n\ndef test_cardinality_max():\n rules = {'max_cardinality': 4,\n 'timeframe': datetime.timedelta(minutes=10),\n 'cardinality_field': 'user',\n 'timestamp_field': '@timestamp'}\n rule = CardinalityRule(rules)\n\n # Add 4 different usernames\n users = ['bill', 'coach', 'zoey', 'louis']\n for user in users:\n event = {'@timestamp': datetime.datetime.now(), 'user': user}\n rule.add_data([event])\n assert len(rule.matches) == 0\n rule.garbage_collect(datetime.datetime.now())\n\n # Add a duplicate, stay at 4 cardinality\n event = {'@timestamp': datetime.datetime.now(), 'user': 'coach'}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now())\n assert len(rule.matches) == 0\n\n # Next unique will trigger\n event = {'@timestamp': datetime.datetime.now(), 'user': 'francis'}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now())\n assert len(rule.matches) == 1\n rule.matches = []\n\n # 15 minutes later, adding more will not trigger an alert\n users = ['nick', 'rochelle', 'ellis']\n for user in users:\n event = {'@timestamp': datetime.datetime.now() + datetime.timedelta(minutes=15), 'user': user}\n rule.add_data([event])\n assert len(rule.matches) == 0\n\n\ndef test_cardinality_min():\n rules = {'min_cardinality': 4,\n 'timeframe': datetime.timedelta(minutes=10),\n 'cardinality_field': 'user',\n 'timestamp_field': '@timestamp'}\n rule = CardinalityRule(rules)\n\n # Add 2 different usernames, no alert because time hasn't elapsed\n users = ['foo', 'bar']\n for user in users:\n event = {'@timestamp': datetime.datetime.now(), 'user': user}\n rule.add_data([event])\n assert len(rule.matches) == 0\n rule.garbage_collect(datetime.datetime.now())\n\n # Add 3 more unique ad t+5 mins\n users = ['faz', 'fuz', 'fiz']\n for user in users:\n event = {'@timestamp': datetime.datetime.now() + datetime.timedelta(minutes=5), 'user': user}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now() + datetime.timedelta(minutes=5))\n assert len(rule.matches) == 0\n\n # Adding the same one again at T+15 causes an alert\n user = 'faz'\n event = {'@timestamp': datetime.datetime.now() + datetime.timedelta(minutes=15), 'user': user}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now() + datetime.timedelta(minutes=15))\n assert len(rule.matches) == 1\n\n\ndef test_cardinality_qk():\n rules = {'max_cardinality': 2,\n 'timeframe': datetime.timedelta(minutes=10),\n 'cardinality_field': 'foo',\n 'timestamp_field': '@timestamp',\n 'query_key': 'user'}\n rule = CardinalityRule(rules)\n\n # Add 3 different usernames, one value each\n users = ['foo', 'bar', 'baz']\n for user in users:\n event = {'@timestamp': datetime.datetime.now(), 'user': user, 'foo': 'foo' + user}\n rule.add_data([event])\n assert len(rule.matches) == 0\n rule.garbage_collect(datetime.datetime.now())\n\n # Add 2 more unique for \"baz\", one alert per value\n values = ['faz', 'fuz', 'fiz']\n for value in values:\n event = {'@timestamp': datetime.datetime.now() + datetime.timedelta(minutes=5), 'user': 'baz', 'foo': value}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now() + datetime.timedelta(minutes=5))\n assert len(rule.matches) == 2\n assert rule.matches[0]['user'] == 'baz'\n assert rule.matches[1]['user'] == 'baz'\n assert rule.matches[0]['foo'] == 'fuz'\n assert rule.matches[1]['foo'] == 'fiz'\n\n\ndef test_cardinality_nested_cardinality_field():\n rules = {'max_cardinality': 4,\n 'timeframe': datetime.timedelta(minutes=10),\n 'cardinality_field': 'd.ip',\n 'timestamp_field': '@timestamp'}\n rule = CardinalityRule(rules)\n\n # Add 4 different IPs\n ips = ['10.0.0.1', '10.0.0.2', '10.0.0.3', '10.0.0.4']\n for ip in ips:\n event = {'@timestamp': datetime.datetime.now(), 'd': {'ip': ip}}\n rule.add_data([event])\n assert len(rule.matches) == 0\n rule.garbage_collect(datetime.datetime.now())\n\n # Add a duplicate, stay at 4 cardinality\n event = {'@timestamp': datetime.datetime.now(), 'd': {'ip': '10.0.0.4'}}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now())\n assert len(rule.matches) == 0\n\n # Add an event with no IP, stay at 4 cardinality\n event = {'@timestamp': datetime.datetime.now()}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now())\n assert len(rule.matches) == 0\n\n # Next unique will trigger\n event = {'@timestamp': datetime.datetime.now(), 'd': {'ip': '10.0.0.5'}}\n rule.add_data([event])\n rule.garbage_collect(datetime.datetime.now())\n assert len(rule.matches) == 1\n rule.matches = []\n\n # 15 minutes later, adding more will not trigger an alert\n ips = ['10.0.0.6', '10.0.0.7', '10.0.0.8']\n for ip in ips:\n event = {'@timestamp': datetime.datetime.now() + datetime.timedelta(minutes=15), 'd': {'ip': ip}}\n rule.add_data([event])\n assert len(rule.matches) == 0\n\n\ndef test_base_aggregation_constructor():\n rules = {'bucket_interval_timedelta': datetime.timedelta(seconds=10),\n 'buffer_time': datetime.timedelta(minutes=1),\n 'timestamp_field': '@timestamp'}\n\n # Test time period constructor logic\n rules['bucket_interval'] = {'seconds': 10}\n rule = BaseAggregationRule(rules)\n assert rule.rules['bucket_interval_period'] == '10s'\n\n rules['bucket_interval'] = {'minutes': 5}\n rule = BaseAggregationRule(rules)\n assert rule.rules['bucket_interval_period'] == '5m'\n\n rules['bucket_interval'] = {'hours': 4}\n rule = BaseAggregationRule(rules)\n assert rule.rules['bucket_interval_period'] == '4h'\n\n rules['bucket_interval'] = {'days': 2}\n rule = BaseAggregationRule(rules)\n assert rule.rules['bucket_interval_period'] == '2d'\n\n rules['bucket_interval'] = {'weeks': 1}\n rule = BaseAggregationRule(rules)\n assert rule.rules['bucket_interval_period'] == '1w'\n\n # buffer_time evenly divisible by bucket_interval\n with pytest.raises(EAException):\n rules['bucket_interval_timedelta'] = datetime.timedelta(seconds=13)\n rule = BaseAggregationRule(rules)\n\n # run_every evenly divisible by bucket_interval\n rules['use_run_every_query_size'] = True\n rules['run_every'] = datetime.timedelta(minutes=2)\n rules['bucket_interval_timedelta'] = datetime.timedelta(seconds=10)\n rule = BaseAggregationRule(rules)\n\n with pytest.raises(EAException):\n rules['bucket_interval_timedelta'] = datetime.timedelta(seconds=13)\n rule = BaseAggregationRule(rules)\n\n\ndef test_base_aggregation_payloads():\n with mock.patch.object(BaseAggregationRule, 'check_matches', return_value=None) as mock_check_matches:\n rules = {'bucket_interval': {'seconds': 10},\n 'bucket_interval_timedelta': datetime.timedelta(seconds=10),\n 'buffer_time': datetime.timedelta(minutes=5),\n 'timestamp_field': '@timestamp'}\n\n timestamp = datetime.datetime.now()\n interval_agg = create_bucket_aggregation('interval_aggs', [{'key_as_string': '2014-01-01T00:00:00Z'}])\n rule = BaseAggregationRule(rules)\n\n # Payload not wrapped\n rule.add_aggregation_data({timestamp: {}})\n mock_check_matches.assert_called_once_with(timestamp, None, {})\n mock_check_matches.reset_mock()\n\n # Payload wrapped by date_histogram\n interval_agg_data = {timestamp: interval_agg}\n rule.add_aggregation_data(interval_agg_data)\n mock_check_matches.assert_called_once_with(ts_to_dt('2014-01-01T00:00:00Z'), None, {'key_as_string': '2014-01-01T00:00:00Z'})\n mock_check_matches.reset_mock()\n\n # Payload wrapped by terms\n bucket_agg_data = {timestamp: create_bucket_aggregation('bucket_aggs', [{'key': 'qk'}])}\n rule.add_aggregation_data(bucket_agg_data)\n mock_check_matches.assert_called_once_with(timestamp, 'qk', {'key': 'qk'})\n mock_check_matches.reset_mock()\n\n # Payload wrapped by terms and date_histogram\n bucket_interval_agg_data = {\n timestamp: create_bucket_aggregation('bucket_aggs', [{'key': 'qk', 'interval_aggs': interval_agg['interval_aggs']}])\n }\n rule.add_aggregation_data(bucket_interval_agg_data)\n mock_check_matches.assert_called_once_with(ts_to_dt('2014-01-01T00:00:00Z'), 'qk', {'key_as_string': '2014-01-01T00:00:00Z'})\n mock_check_matches.reset_mock()\n\n\ndef test_metric_aggregation():\n rules = {'buffer_time': datetime.timedelta(minutes=5),\n 'timestamp_field': '@timestamp',\n 'metric_agg_type': 'avg',\n 'metric_agg_key': 'cpu_pct'}\n\n # Check threshold logic\n with pytest.raises(EAException):\n rule = MetricAggregationRule(rules)\n\n rules['min_threshold'] = 0.1\n rules['max_threshold'] = 0.8\n\n rule = MetricAggregationRule(rules)\n\n assert rule.rules['aggregation_query_element'] == {'metric_cpu_pct_avg': {'avg': {'field': 'cpu_pct'}}}\n\n assert rule.crossed_thresholds(None) is False\n assert rule.crossed_thresholds(0.09) is True\n assert rule.crossed_thresholds(0.10) is False\n assert rule.crossed_thresholds(0.79) is False\n assert rule.crossed_thresholds(0.81) is True\n\n rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': None}})\n rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.5}})\n assert len(rule.matches) == 0\n\n rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.05}})\n rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.95}})\n assert len(rule.matches) == 2\n\n rules['query_key'] = 'qk'\n rule = MetricAggregationRule(rules)\n rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}})\n assert rule.matches[0]['qk'] == 'qk_val'\n\n\ndef test_metric_aggregation_complex_query_key():\n rules = {'buffer_time': datetime.timedelta(minutes=5),\n 'timestamp_field': '@timestamp',\n 'metric_agg_type': 'avg',\n 'metric_agg_key': 'cpu_pct',\n 'compound_query_key': ['qk', 'sub_qk'],\n 'query_key': 'qk,sub_qk',\n 'max_threshold': 0.8}\n\n query = {\"bucket_aggs\": {\"buckets\": [\n {\"metric_cpu_pct_avg\": {\"value\": 0.91}, \"key\": \"sub_qk_val1\"},\n {\"metric_cpu_pct_avg\": {\"value\": 0.95}, \"key\": \"sub_qk_val2\"},\n {\"metric_cpu_pct_avg\": {\"value\": 0.89}, \"key\": \"sub_qk_val3\"}]\n }, \"key\": \"qk_val\"}\n\n rule = MetricAggregationRule(rules)\n rule.check_matches(datetime.datetime.now(), 'qk_val', query)\n assert len(rule.matches) == 3\n assert rule.matches[0]['qk'] == 'qk_val'\n assert rule.matches[1]['qk'] == 'qk_val'\n assert rule.matches[0]['sub_qk'] == 'sub_qk_val1'\n assert rule.matches[1]['sub_qk'] == 'sub_qk_val2'\n\n\ndef test_percentage_match():\n rules = {'match_bucket_filter': {'term': 'term_val'},\n 'buffer_time': datetime.timedelta(minutes=5),\n 'timestamp_field': '@timestamp'}\n\n # Check threshold logic\n with pytest.raises(EAException):\n rule = PercentageMatchRule(rules)\n\n rules['min_percentage'] = 25\n rules['max_percentage'] = 75\n rule = PercentageMatchRule(rules)\n\n assert rule.rules['aggregation_query_element'] == {\n 'percentage_match_aggs': {\n 'filters': {\n 'other_bucket': True,\n 'filters': {\n 'match_bucket': {\n 'bool': {\n 'must': {\n 'term': 'term_val'\n }\n }\n }\n }\n }\n }\n }\n\n assert rule.percentage_violation(25) is False\n assert rule.percentage_violation(50) is False\n assert rule.percentage_violation(75) is False\n assert rule.percentage_violation(24.9) is True\n assert rule.percentage_violation(75.1) is True\n\n rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(0, 0))\n rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(None, 100))\n rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(26, 74))\n rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(74, 26))\n\n assert len(rule.matches) == 0\n\n rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(24, 76))\n rule.check_matches(datetime.datetime.now(), None, create_percentage_match_agg(76, 24))\n assert len(rule.matches) == 2\n\n rules['query_key'] = 'qk'\n rule = PercentageMatchRule(rules)\n rule.check_matches(datetime.datetime.now(), 'qk_val', create_percentage_match_agg(76.666666667, 24))\n assert rule.matches[0]['qk'] == 'qk_val'\n assert '76.1589403974' in rule.get_match_str(rule.matches[0])\n rules['percentage_format_string'] = '%.2f'\n assert '76.16' in rule.get_match_str(rule.matches[0])\n" }, { "path": "tests/util_test.py", "content": "# -*- coding: utf-8 -*-\nfrom datetime import datetime\nfrom datetime import timedelta\n\nimport mock\nimport pytest\nfrom dateutil.parser import parse as dt\n\nfrom elastalert.util import add_raw_postfix\nfrom elastalert.util import format_index\nfrom elastalert.util import lookup_es_key\nfrom elastalert.util import parse_deadline\nfrom elastalert.util import parse_duration\nfrom elastalert.util import replace_dots_in_field_names\nfrom elastalert.util import resolve_string\nfrom elastalert.util import set_es_key\nfrom elastalert.util import should_scrolling_continue\n\n\n@pytest.mark.parametrize('spec, expected_delta', [\n ('hours=2', timedelta(hours=2)),\n ('minutes=30', timedelta(minutes=30)),\n ('seconds=45', timedelta(seconds=45)),\n])\ndef test_parse_duration(spec, expected_delta):\n \"\"\"``unit=num`` specs can be translated into ``timedelta`` instances.\"\"\"\n assert parse_duration(spec) == expected_delta\n\n\n@pytest.mark.parametrize('spec, expected_deadline', [\n ('hours=2', dt('2017-07-07T12:00:00.000Z')),\n ('minutes=30', dt('2017-07-07T10:30:00.000Z')),\n ('seconds=45', dt('2017-07-07T10:00:45.000Z')),\n])\ndef test_parse_deadline(spec, expected_deadline):\n \"\"\"``unit=num`` specs can be translated into ``datetime`` instances.\"\"\"\n\n # Note: Can't mock ``utcnow`` directly because ``datetime`` is a built-in.\n class MockDatetime(datetime):\n @staticmethod\n def utcnow():\n return dt('2017-07-07T10:00:00.000Z')\n\n with mock.patch('datetime.datetime', MockDatetime):\n assert parse_deadline(spec) == expected_deadline\n\n\ndef test_setting_keys(ea):\n expected = 12467267\n record = {\n 'Message': '12345',\n 'Fields': {\n 'ts': 'fail',\n 'severity': 'large',\n 'user': 'jimmay'\n }\n }\n\n # Set the value\n assert set_es_key(record, 'Fields.ts', expected)\n\n # Get the value again\n assert lookup_es_key(record, 'Fields.ts') == expected\n\n\ndef test_looking_up_missing_keys(ea):\n record = {\n 'Message': '12345',\n 'Fields': {\n 'severity': 'large',\n 'user': 'jimmay',\n 'null': None\n }\n }\n\n assert lookup_es_key(record, 'Fields.ts') is None\n\n assert lookup_es_key(record, 'Fields.null.foo') is None\n\n\ndef test_looking_up_nested_keys(ea):\n expected = 12467267\n record = {\n 'Message': '12345',\n 'Fields': {\n 'ts': expected,\n 'severity': 'large',\n 'user': 'jimmay'\n }\n }\n\n assert lookup_es_key(record, 'Fields.ts') == expected\n\n\ndef test_looking_up_nested_composite_keys(ea):\n expected = 12467267\n record = {\n 'Message': '12345',\n 'Fields': {\n 'ts.value': expected,\n 'severity': 'large',\n 'user': 'jimmay'\n }\n }\n\n assert lookup_es_key(record, 'Fields.ts.value') == expected\n\n\ndef test_looking_up_arrays(ea):\n record = {\n 'flags': [1, 2, 3],\n 'objects': [\n {'foo': 'bar'},\n {'foo': [{'bar': 'baz'}]},\n {'foo': {'bar': 'baz'}}\n ]\n }\n assert lookup_es_key(record, 'flags[0]') == 1\n assert lookup_es_key(record, 'flags[1]') == 2\n assert lookup_es_key(record, 'objects[0]foo') == 'bar'\n assert lookup_es_key(record, 'objects[1]foo[0]bar') == 'baz'\n assert lookup_es_key(record, 'objects[2]foo.bar') == 'baz'\n assert lookup_es_key(record, 'objects[1]foo[1]bar') is None\n assert lookup_es_key(record, 'objects[1]foo[0]baz') is None\n\n\ndef test_add_raw_postfix(ea):\n expected = 'foo.raw'\n assert add_raw_postfix('foo', False) == expected\n assert add_raw_postfix('foo.raw', False) == expected\n expected = 'foo.keyword'\n assert add_raw_postfix('foo', True) == expected\n assert add_raw_postfix('foo.keyword', True) == expected\n\n\ndef test_replace_dots_in_field_names(ea):\n actual = {\n 'a': {\n 'b.c': 'd',\n 'e': {\n 'f': {\n 'g.h': 0\n }\n }\n },\n 'i.j.k': 1,\n 'l': {\n 'm': 2\n }\n }\n expected = {\n 'a': {\n 'b_c': 'd',\n 'e': {\n 'f': {\n 'g_h': 0\n }\n }\n },\n 'i_j_k': 1,\n 'l': {\n 'm': 2\n }\n }\n assert replace_dots_in_field_names(actual) == expected\n assert replace_dots_in_field_names({'a': 0, 1: 2}) == {'a': 0, 1: 2}\n\n\ndef test_resolve_string(ea):\n match = {\n 'name': 'mySystem',\n 'temperature': 45,\n 'humidity': 80.56,\n 'sensors': ['outsideSensor', 'insideSensor'],\n 'foo': {'bar': 'baz'}\n }\n\n expected_outputs = [\n \"mySystem is online \",\n \"Sensors ['outsideSensor', 'insideSensor'] in the have temp 45 and 80.56 humidity\",\n \"Actuator in the has temp \",\n 'Something baz']\n old_style_strings = [\n \"%(name)s is online %(noKey)s\",\n \"Sensors %(sensors)s in the %(noPlace)s have temp %(temperature)s and %(humidity)s humidity\",\n \"Actuator %(noKey)s in the %(noPlace)s has temp %(noKey)s\",\n 'Something %(foo.bar)s']\n\n assert resolve_string(old_style_strings[0], match) == expected_outputs[0]\n assert resolve_string(old_style_strings[1], match) == expected_outputs[1]\n assert resolve_string(old_style_strings[2], match) == expected_outputs[2]\n assert resolve_string(old_style_strings[3], match) == expected_outputs[3]\n\n new_style_strings = [\n \"{name} is online {noKey}\",\n \"Sensors {sensors} in the {noPlace} have temp {temperature} and {humidity} humidity\",\n \"Actuator {noKey} in the {noPlace} has temp {noKey}\",\n \"Something {foo[bar]}\"]\n\n assert resolve_string(new_style_strings[0], match) == expected_outputs[0]\n assert resolve_string(new_style_strings[1], match) == expected_outputs[1]\n assert resolve_string(new_style_strings[2], match) == expected_outputs[2]\n assert resolve_string(new_style_strings[3], match) == expected_outputs[3]\n\n\ndef test_format_index():\n pattern = 'logstash-%Y.%m.%d'\n pattern2 = 'logstash-%Y.%W'\n date = dt('2018-06-25T12:00:00Z')\n date2 = dt('2018-06-26T12:00:00Z')\n assert sorted(format_index(pattern, date, date).split(',')) == ['logstash-2018.06.25']\n assert sorted(format_index(pattern, date, date2).split(',')) == ['logstash-2018.06.25', 'logstash-2018.06.26']\n assert sorted(format_index(pattern, date, date2, True).split(',')) == ['logstash-2018.06.24',\n 'logstash-2018.06.25',\n 'logstash-2018.06.26']\n assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26']\n\n\ndef test_should_scrolling_continue():\n rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1}\n rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2}\n rule_before_first_run = {'max_scrolling_count': 0, 'scrolling_cycle': 0}\n rule_before_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 1}\n rule_over_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 3}\n\n assert should_scrolling_continue(rule_no_max_scrolling) is True\n assert should_scrolling_continue(rule_reached_max_scrolling) is False\n assert should_scrolling_continue(rule_before_first_run) is True\n assert should_scrolling_continue(rule_before_max_scrolling) is True\n assert should_scrolling_continue(rule_over_max_scrolling) is False\n" }, { "path": "tox.ini", "content": "[tox]\nproject = elastalert\nenvlist = py36,docs\n\n[testenv]\ndeps = -rrequirements-dev.txt\ncommands =\n coverage run --source=elastalert/,tests/ -m pytest --strict {posargs}\n coverage report -m\n flake8 .\n\n[testenv:lint]\ndeps = {[testenv]deps}\n pylint\ncommands =\n pylint --rcfile=.pylintrc elastalert\n pylint --rcfile=.pylintrc tests\n\n[testenv:devenv]\nenvdir = virtualenv_run\ncommands =\n\n[pytest]\nnorecursedirs = .* virtualenv_run docs build venv env\n\n[testenv:docs]\ndeps = {[testenv]deps}\n sphinx==1.6.6\nchangedir = docs\ncommands = sphinx-build -b html -d build/doctrees -W source build/html\n" } ]